From 6ef9efb443778f138b995c7c4dad2134be259166 Mon Sep 17 00:00:00 2001 From: James Hobbs Date: Wed, 26 Aug 2026 15:03:52 -0500 Subject: [PATCH 01/43] feat(streamlit): add Deepnote app helpers --- README.md | 2 + deepnote_toolkit/streamlit/__init__.py | 40 +++ deepnote_toolkit/streamlit/auth.py | 189 ++++++++++ deepnote_toolkit/streamlit/client.py | 335 +++++++++++++++++ deepnote_toolkit/streamlit/document.py | 316 ++++++++++++++++ deepnote_toolkit/streamlit/widgets.py | 118 ++++++ docs/streamlit-apps.md | 77 ++++ tests/unit/test_deepnote_streamlit_auth.py | 209 +++++++++++ tests/unit/test_deepnote_streamlit_client.py | 337 ++++++++++++++++++ .../unit/test_deepnote_streamlit_document.py | 166 +++++++++ tests/unit/test_deepnote_streamlit_widgets.py | 98 +++++ 11 files changed, 1887 insertions(+) create mode 100644 deepnote_toolkit/streamlit/__init__.py create mode 100644 deepnote_toolkit/streamlit/auth.py create mode 100644 deepnote_toolkit/streamlit/client.py create mode 100644 deepnote_toolkit/streamlit/document.py create mode 100644 deepnote_toolkit/streamlit/widgets.py create mode 100644 docs/streamlit-apps.md create mode 100644 tests/unit/test_deepnote_streamlit_auth.py create mode 100644 tests/unit/test_deepnote_streamlit_client.py create mode 100644 tests/unit/test_deepnote_streamlit_document.py create mode 100644 tests/unit/test_deepnote_streamlit_widgets.py diff --git a/README.md b/README.md index 5a8cab41..b911a97b 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,8 @@ It starts and manages Jupyter, Streamlit, and LSP servers, and provides runtime - Native **Deepnote component library** including beautiful `DataFrame` rendering and interactive inputs - **Python kernel with curated set of libraries preinstalled**, allowing you to focus on work instead of fighting with Python dependencies - Run multiple **interactive applications built with Streamlit** +- Build custom Streamlit interfaces over local Deepnote files and hosted runs with + [per-viewer authentication](docs/streamlit-apps.md) - Language Server Protocol integration for code completion and intelligence - Git integration with SSH/HTTPS authentication diff --git a/deepnote_toolkit/streamlit/__init__.py b/deepnote_toolkit/streamlit/__init__.py new file mode 100644 index 00000000..87a67393 --- /dev/null +++ b/deepnote_toolkit/streamlit/__init__.py @@ -0,0 +1,40 @@ +"""Helpers for building Streamlit apps over local Deepnote files.""" + +from .auth import ( + CurrentUserApiCredentials, + CurrentUserApiTokenError, + current_user_api_credentials, + current_user_api_token, +) +from .client import DeepnoteCloudRunner, DeepnoteRunner, RunnerError, RunnerInfo +from .document import ( + DATAFRAME_MIME, + INDEX_COLUMN, + DeepnoteDataframe, + DeepnoteDocument, + InputBlock, + NotebookOutput, + RunResult, + join_text, +) +from .widgets import render_inputs + +__all__ = [ + "DATAFRAME_MIME", + "INDEX_COLUMN", + "CurrentUserApiTokenError", + "CurrentUserApiCredentials", + "DeepnoteDataframe", + "DeepnoteCloudRunner", + "DeepnoteDocument", + "DeepnoteRunner", + "InputBlock", + "NotebookOutput", + "RunResult", + "RunnerError", + "RunnerInfo", + "current_user_api_token", + "current_user_api_credentials", + "join_text", + "render_inputs", +] diff --git a/deepnote_toolkit/streamlit/auth.py b/deepnote_toolkit/streamlit/auth.py new file mode 100644 index 00000000..c5aabdcf --- /dev/null +++ b/deepnote_toolkit/streamlit/auth.py @@ -0,0 +1,189 @@ +"""Per-viewer authentication for Streamlit apps hosted by Deepnote.""" + +from __future__ import annotations + +import json +import re +from collections.abc import Callable, Mapping +from dataclasses import dataclass +from typing import Any +from urllib.error import HTTPError, URLError +from urllib.parse import urlparse +from urllib.request import Request, urlopen + +from deepnote_toolkit.config import get_config +from deepnote_toolkit.streamlit_data_apps import ( + _read_streamlit_token_from_context, +) + +OpenUrl = Callable[..., Any] +STREAMLIT_APP_HOST_PATTERN = re.compile( + r"^streamlit-([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})\.", + re.IGNORECASE, +) + + +class CurrentUserApiTokenError(RuntimeError): + """Raised when a hosted app cannot obtain the current viewer's API token.""" + + +@dataclass(frozen=True) +class CurrentUserApiCredentials: + """A short-lived viewer-scoped public API credential.""" + + token: str + api_origin: str + expires_at_seconds: float + + +def current_user_api_token() -> str: + """Return a short-lived public API bearer for the current Streamlit viewer. + + The opaque streamlit-token cookie is exchanged for a viewer-scoped token. + It is never itself used as a public API bearer. The exchange happens on + every call so a long-lived, multi-user process does not retain credentials. + """ + + return current_user_api_credentials().token + + +def current_user_api_credentials( + *, + app_id: str | None = None, + webapp_url: str | None = None, + streamlit_token: str | None = None, + timeout: float = 10, + opener: OpenUrl = urlopen, +) -> CurrentUserApiCredentials: + """Exchange the active viewer cookie for public API credentials. + + The returned API origin must be used with the returned bearer. Hosted clients + should call this for every request, or cache it only within the current + Streamlit session until shortly before expires_at_seconds. + """ + + resolved_app_id = app_id or _read_streamlit_app_id_from_context() + if not resolved_app_id: + raise CurrentUserApiTokenError( + "Could not resolve a Deepnote Streamlit app ID from the request host." + ) + + viewer_token = streamlit_token or _read_streamlit_token_from_context() + if not viewer_token: + raise CurrentUserApiTokenError( + "Could not read the current viewer's streamlit-token cookie." + ) + + resolved_webapp_url = webapp_url or get_config().runtime.webapp_url + if not resolved_webapp_url: + raise CurrentUserApiTokenError( + "DEEPNOTE_WEBAPP_URL is required in a hosted Streamlit app." + ) + resolved_webapp_url = _validated_origin( + resolved_webapp_url, name="DEEPNOTE_WEBAPP_URL" + ) + + request = Request( + (f"{resolved_webapp_url}/api/streamlit-apps/" f"{resolved_app_id}/api-token"), + data=b"", + method="POST", + headers={ + "Accept": "application/json", + "Content-Type": "application/json", + "StreamlitToken": viewer_token, + }, + ) + try: + with opener(request, timeout=timeout) as response: + payload = json.loads(response.read()) + except HTTPError as error: + raise CurrentUserApiTokenError( + f"Current viewer API-token exchange returned HTTP {error.code}." + ) from error + except URLError as error: + raise CurrentUserApiTokenError( + "Could not reach Deepnote to exchange the current viewer's API token." + ) from error + except TimeoutError as error: + raise CurrentUserApiTokenError( + "Current viewer API-token exchange timed out." + ) from error + except (json.JSONDecodeError, UnicodeDecodeError) as error: + raise CurrentUserApiTokenError( + "Current viewer API-token exchange returned invalid JSON." + ) from error + + if not isinstance(payload, Mapping): + raise CurrentUserApiTokenError( + "Current viewer API-token exchange returned a non-object response." + ) + + token = payload.get("token") + api_origin = payload.get("apiOrigin") + expires_at_seconds = payload.get("expiresAtSeconds") + if ( + not isinstance(token, str) + or not token + or not isinstance(api_origin, str) + or not isinstance(expires_at_seconds, (int, float)) + or isinstance(expires_at_seconds, bool) + ): + raise CurrentUserApiTokenError( + "Current viewer API-token exchange response is missing required fields." + ) + + return CurrentUserApiCredentials( + token=token, + api_origin=_validated_origin(api_origin, name="apiOrigin"), + expires_at_seconds=float(expires_at_seconds), + ) + + +def _read_streamlit_app_id_from_context() -> str | None: + """Resolve the app UUID from the external Streamlit request hostname.""" + + try: + import streamlit as st # type: ignore[import-not-found] + except ImportError: + return None + + try: + headers = st.context.headers + except Exception: + return None + + if not headers: + return None + normalized_headers = {str(key).lower(): value for key, value in headers.items()} + for name in ("x-original-host", "host"): + host = normalized_headers.get(name) + if not isinstance(host, str): + continue + match = STREAMLIT_APP_HOST_PATTERN.match(host) + if match: + return match.group(1).lower() + return None + + +def _has_hosted_streamlit_context() -> bool: + """Return whether this request carries either hosted-app identity signal.""" + + return bool( + _read_streamlit_app_id_from_context() or _read_streamlit_token_from_context() + ) + + +def _validated_origin(value: str, *, name: str) -> str: + parsed = urlparse(value) + if ( + parsed.scheme not in {"http", "https"} + or not parsed.netloc + or parsed.username + or parsed.password + or parsed.path not in {"", "/"} + or parsed.params + or parsed.query + or parsed.fragment + ): + raise CurrentUserApiTokenError(f"{name} must be a valid HTTP(S) origin.") + return value.rstrip("/") diff --git a/deepnote_toolkit/streamlit/client.py b/deepnote_toolkit/streamlit/client.py new file mode 100644 index 00000000..9681a8b9 --- /dev/null +++ b/deepnote_toolkit/streamlit/client.py @@ -0,0 +1,335 @@ +"""HTTP client for the unified Deepnote app runner API.""" + +from __future__ import annotations + +import json +import os +import time +from collections.abc import Callable, Iterable, Mapping +from dataclasses import dataclass +from typing import Any +from urllib.error import HTTPError, URLError +from urllib.request import Request, urlopen + +from .auth import ( + CurrentUserApiTokenError, + _has_hosted_streamlit_context, + current_user_api_credentials, +) +from .document import InputBlock, RunResult + +OpenUrl = Callable[..., Any] +TokenProvider = Callable[[], str] +Sleep = Callable[[float], None] + +TERMINAL_RUN_STATUSES = frozenset({"success", "error", "internal_error", "stopped"}) +DEFAULT_API_ORIGIN = "https://api.deepnote.com" + + +class RunnerError(RuntimeError): + """The Deepnote runner was unavailable or rejected a request.""" + + +@dataclass(frozen=True) +class RunnerInfo: + """The target and input contract exposed by a Deepnote runner.""" + + notebook: str + inputs: tuple[InputBlock, ...] + run_target: str + + def accepts_inputs(self, inputs: Iterable[InputBlock]) -> bool: + """Return whether input variable names and block types match this runner.""" + + return _input_contract(inputs) == _input_contract(self.inputs) + + +class DeepnoteRunner: + """One client for a runner configured for Deepnote Cloud or a local kernel.""" + + def __init__( + self, + base_url: str = "http://127.0.0.1:8787", + *, + timeout: float = 600, + opener: OpenUrl = urlopen, + ): + self.base_url = base_url.rstrip("/") + self.timeout = timeout + self._open = opener + + def info(self) -> RunnerInfo: + payload = self._request("GET", "/api/info") + values = payload.get("inputs") + inputs = ( + tuple( + InputBlock.from_api(value) + for value in values + if isinstance(value, Mapping) + ) + if isinstance(values, list) + else () + ) + return RunnerInfo( + notebook=str(payload.get("notebook", "Untitled project")), + inputs=inputs, + run_target=str(payload.get("runTarget", "")), + ) + + def run(self, inputs: Mapping[str, Any]) -> RunResult: + return RunResult(self._request("POST", "/api/run", {"inputs": dict(inputs)})) + + def _request( + self, method: str, path: str, body: Mapping[str, Any] | None = None + ) -> Mapping[str, Any]: + encoded = json.dumps(body).encode() if body is not None else None + request = Request( + f"{self.base_url}{path}", + data=encoded, + method=method, + headers={"Content-Type": "application/json", "Accept": "application/json"}, + ) + try: + with self._open(request, timeout=self.timeout) as response: + payload = json.loads(response.read()) + except HTTPError as error: + detail = error.read().decode(errors="replace") + try: + parsed_detail = json.loads(detail) + message = ( + parsed_detail.get("error", detail) + if isinstance(parsed_detail, Mapping) + else detail + ) + except json.JSONDecodeError: + message = detail + raise RunnerError( + f"Deepnote runner returned HTTP {error.code}: {message}" + ) from error + except URLError as error: + raise RunnerError( + f"Could not reach Deepnote runner at {self.base_url}: {error.reason}" + ) from error + except TimeoutError as error: + raise RunnerError( + f"Deepnote runner at {self.base_url} timed out after {self.timeout:g} seconds" + ) from error + except (json.JSONDecodeError, UnicodeDecodeError) as error: + raise RunnerError( + "Deepnote runner returned an invalid JSON response" + ) from error + if not isinstance(payload, Mapping): + raise RunnerError("Deepnote runner returned a non-object response") + return payload + + +class DeepnoteCloudRunner: + """Run an existing notebook directly through the Deepnote public API. + + A token provider is called for every request, which lets long-lived Streamlit + sessions use short-lived credentials without caching them in this library. + """ + + def __init__( + self, + notebook_id: str, + *, + token: str | None = None, + token_provider: TokenProvider | None = None, + base_url: str = DEFAULT_API_ORIGIN, + timeout: float = 600, + poll_interval: float = 2, + opener: OpenUrl = urlopen, + sleep: Sleep = time.sleep, + ): + if not notebook_id: + raise ValueError("notebook_id is required") + if token is not None and token_provider is not None: + raise ValueError("Pass token or token_provider, not both") + self.notebook_id = notebook_id + self.base_url = base_url.rstrip("/") + self.timeout = timeout + self.poll_interval = poll_interval + self._static_token = token + self._token_provider = token_provider + self._open = opener + self._sleep = sleep + + def info(self) -> RunnerInfo: + payload = self._request("GET", f"/v2/notebooks/{self.notebook_id}") + notebook = payload.get("notebook") + if not isinstance(notebook, Mapping): + raise RunnerError("Deepnote API response did not include a notebook") + raw_inputs = notebook.get("inputs") + inputs = tuple( + InputBlock.from_api( + { + "variableName": value.get("name"), + "type": value.get("type"), + "value": value.get("value"), + "label": value.get("label"), + } + ) + for value in raw_inputs or [] + if isinstance(value, Mapping) and isinstance(value.get("name"), str) + ) + return RunnerInfo( + notebook=str(notebook.get("name", "Untitled notebook")), + inputs=inputs, + run_target="cloud", + ) + + def run(self, inputs: Mapping[str, Any]) -> RunResult: + started = self._run_payload( + self._request( + "POST", + "/v2/runs", + { + "notebookId": self.notebook_id, + "inputs": _normalize_cloud_inputs(inputs), + }, + ) + ) + run_id = _required_run_id(started) + deadline = time.monotonic() + self.timeout + current = started + while str(current.get("status", "")) not in TERMINAL_RUN_STATUSES: + if time.monotonic() >= deadline: + raise RunnerError( + f"Deepnote run {run_id} did not finish within {self.timeout:g} seconds" + ) + self._sleep(self.poll_interval) + current = self._run_payload( + self._request("GET", f"/v2/runs/{run_id}?snapshotDelivery=inline") + ) + + status = str(current.get("status", "")) + snapshot = current.get("snapshot") + snapshot_yaml = current.get("snapshotContent") + if snapshot_yaml is None and isinstance(snapshot, Mapping): + snapshot_yaml = snapshot.get("snapshotContent") + error = current.get("error") + if isinstance(error, Mapping): + error = error.get("message") or json.dumps(error) + return RunResult( + { + "target": "cloud", + "success": status == "success", + "runId": run_id, + "status": status, + "error": str(error) if error is not None else None, + "snapshotYaml": snapshot_yaml, + "viewUrl": current.get("viewUrl"), + } + ) + + def _request( + self, method: str, path: str, body: Mapping[str, Any] | None = None + ) -> Mapping[str, Any]: + encoded = json.dumps(body).encode() if body is not None else None + token, api_origin = self._authentication() + request = Request( + f"{api_origin}{path}", + data=encoded, + method=method, + headers={ + "Authorization": f"Bearer {token}", + "Content-Type": "application/json", + "Accept": "application/json", + }, + ) + try: + with self._open(request, timeout=min(self.timeout, 30)) as response: + payload = json.loads(response.read()) + except HTTPError as error: + detail = error.read().decode(errors="replace") + try: + parsed = json.loads(detail) + message = ( + parsed.get("message") or parsed.get("error") or detail + if isinstance(parsed, Mapping) + else detail + ) + except json.JSONDecodeError: + message = detail + raise RunnerError( + f"Deepnote API returned HTTP {error.code}: {message}" + ) from error + except URLError as error: + raise RunnerError( + f"Could not reach the Deepnote API at {api_origin}: {error.reason}" + ) from error + except TimeoutError as error: + raise RunnerError("Deepnote API request timed out") from error + except (json.JSONDecodeError, UnicodeDecodeError) as error: + raise RunnerError( + "Deepnote API returned an invalid JSON response" + ) from error + if not isinstance(payload, Mapping): + raise RunnerError("Deepnote API returned a non-object response") + return payload + + def _authentication(self) -> tuple[str, str]: + if self._token_provider is not None: + token = self._token_provider() + return self._required_token(token), self.base_url + + if self._static_token is not None: + return self._required_token(self._static_token), self.base_url + + # Hosted apps always authenticate as the current viewer. In particular, + # never fall back to a process-wide environment token when this request + # has a hosted Streamlit app hostname. + if _has_hosted_streamlit_context(): + try: + credentials = current_user_api_credentials( + timeout=min(self.timeout, 30), opener=self._open + ) + except CurrentUserApiTokenError as error: + raise RunnerError(str(error)) from error + api_origin = ( + credentials.api_origin + if self.base_url == DEFAULT_API_ORIGIN + else self.base_url + ) + return credentials.token, api_origin + + return self._required_token(os.environ.get("DEEPNOTE_TOKEN")), self.base_url + + @staticmethod + def _required_token(token: str | None) -> str: + if not token: + raise RunnerError("A Deepnote API token is required") + return token + + @staticmethod + def _run_payload(payload: Mapping[str, Any]) -> Mapping[str, Any]: + run = payload.get("run") + return run if isinstance(run, Mapping) else payload + + +def _required_run_id(run: Mapping[str, Any]) -> str: + run_id = run.get("runId") or run.get("id") + if not isinstance(run_id, str) or not run_id: + raise RunnerError("Deepnote API response did not include a run id") + return run_id + + +def _input_contract(inputs: Iterable[InputBlock]) -> tuple[tuple[str, str], ...]: + return tuple( + sorted((input_block.variable_name, input_block.type) for input_block in inputs) + ) + + +def _normalize_cloud_inputs( + inputs: Mapping[str, Any], +) -> dict[str, str | bool | list[str]]: + normalized: dict[str, str | bool | list[str]] = {} + for name, value in inputs.items(): + if isinstance(value, bool): + normalized[name] = value + elif isinstance(value, list): + normalized[name] = [str(item) for item in value] + else: + normalized[name] = str(value) + return normalized diff --git a/deepnote_toolkit/streamlit/document.py b/deepnote_toolkit/streamlit/document.py new file mode 100644 index 00000000..0829b9de --- /dev/null +++ b/deepnote_toolkit/streamlit/document.py @@ -0,0 +1,316 @@ +"""Typed, deliberately small views over `.deepnote` YAML and run responses.""" + +from __future__ import annotations + +import base64 +import binascii +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import yaml + +DATAFRAME_MIME = "application/vnd.deepnote.dataframe.v3+json" +INDEX_COLUMN = "_deepnote_index_column" + + +def join_text(value: Any) -> str: + """Normalize nbformat's string-or-list text values to one string.""" + + if isinstance(value, list): + return "".join(str(part) for part in value) + return "" if value is None else str(value) + + +@dataclass(frozen=True) +class InputBlock: + """The metadata a UI needs to render one Deepnote input block.""" + + variable_name: str + type: str + value: Any + label: str | None = None + options: tuple[str, ...] = () + multiple: bool = False + min: float | int | None = None + max: float | int | None = None + step: float | int | None = None + + @classmethod + def from_block(cls, block: Mapping[str, Any]) -> InputBlock | None: + block_type = str(block.get("type", "")) + metadata = block.get("metadata") + if not block_type.startswith("input-") or not isinstance(metadata, Mapping): + return None + variable_name = metadata.get("deepnote_variable_name") + if not isinstance(variable_name, str) or not variable_name: + return None + options = metadata.get("deepnote_variable_options") + return cls( + variable_name=variable_name, + type=block_type, + label=_optional_string(metadata.get("deepnote_input_label")), + value=metadata.get("deepnote_variable_value"), + options=( + tuple(str(option) for option in options) + if isinstance(options, list) + else () + ), + multiple=metadata.get("deepnote_allow_multiple_values") is True, + min=_optional_number(metadata.get("deepnote_slider_min_value")), + max=_optional_number(metadata.get("deepnote_slider_max_value")), + step=_optional_number(metadata.get("deepnote_slider_step")), + ) + + @classmethod + def from_api(cls, value: Mapping[str, Any]) -> InputBlock: + """Read the camelCase shape returned by `GET /api/info`.""" + + options = value.get("options") + return cls( + variable_name=str(value["variableName"]), + type=str(value["type"]), + label=_optional_string(value.get("label")), + value=value.get("value"), + options=( + tuple(str(option) for option in options) + if isinstance(options, list) + else () + ), + multiple=value.get("multiple") is True, + min=_optional_number(value.get("min")), + max=_optional_number(value.get("max")), + step=_optional_number(value.get("step")), + ) + + +@dataclass(frozen=True) +class DeepnoteDataframe: + """A structured Deepnote dataframe output, independent of pandas.""" + + columns: tuple[Mapping[str, Any], ...] + rows: tuple[Mapping[str, Any], ...] + raw: Mapping[str, Any] + + @classmethod + def from_value(cls, value: Any) -> DeepnoteDataframe | None: + if not isinstance(value, Mapping): + return None + columns = value.get("columns") + rows = value.get("rows") + if not isinstance(columns, list) or not isinstance(rows, list): + return None + if not all(isinstance(column, Mapping) for column in columns): + return None + if not all(isinstance(row, Mapping) for row in rows): + return None + return cls(columns=tuple(columns), rows=tuple(rows), raw=value) + + @property + def data_columns(self) -> tuple[str, ...]: + return tuple( + str(column["name"]) + for column in self.columns + if column.get("name") != INDEX_COLUMN + ) + + def records(self, *, include_index: bool = True) -> list[dict[str, Any]]: + """Return rows ready for `st.dataframe`, optionally omitting Deepnote's index column.""" + + if include_index: + return [dict(row) for row in self.rows] + return [ + {key: value for key, value in row.items() if key != INDEX_COLUMN} + for row in self.rows + ] + + +@dataclass(frozen=True) +class NotebookOutput: + """One nbformat-compatible output emitted by a Deepnote block.""" + + block_id: str + block_type: str | None + raw: Mapping[str, Any] + + @property + def output_type(self) -> str: + return str(self.raw.get("output_type", "")) + + @property + def data(self) -> Mapping[str, Any]: + value = self.raw.get("data") + return value if isinstance(value, Mapping) else {} + + def text(self, mime: str = "text/plain") -> str: + if self.output_type == "stream" and mime == "text/plain": + return join_text(self.raw.get("text")) + return join_text(self.data.get(mime)) + + def image_bytes(self, mime: str = "image/png") -> bytes | None: + value = self.data.get(mime) + if value is None: + return None + encoded = "".join(join_text(value).split()) + try: + return base64.b64decode(encoded, validate=True) + except (ValueError, binascii.Error): + return None + + @property + def dataframe(self) -> DeepnoteDataframe | None: + return DeepnoteDataframe.from_value(self.data.get(DATAFRAME_MIME)) + + +class OutputCollection: + """Shared output queries for a loaded document and a live run result.""" + + outputs: tuple[NotebookOutput, ...] + + def outputs_for_mime(self, mime: str) -> list[NotebookOutput]: + return [output for output in self.outputs if mime in output.data] + + def first_dataframe(self) -> DeepnoteDataframe | None: + for output in self.outputs: + if dataframe := output.dataframe: + return dataframe + return None + + def images(self, mime: str = "image/png") -> list[bytes]: + return [ + image + for output in self.outputs + if (image := output.image_bytes(mime)) is not None + ] + + def text(self, mime: str = "text/plain") -> str: + return "".join(output.text(mime) for output in self.outputs).strip() + + def agent_text(self) -> str: + chunks: list[str] = [] + for output in self.outputs: + if output.block_type != "agent": + continue + if output.output_type == "stream": + chunks.append(output.text()) + else: + chunks.append(output.text("text/markdown") or output.text()) + return "".join(chunks).strip() + + +class DeepnoteDocument(OutputCollection): + """A parsed source or snapshot `.deepnote` file.""" + + def __init__(self, raw: Mapping[str, Any]): + project = raw.get("project") + if not isinstance(project, Mapping) or not isinstance( + project.get("notebooks"), list + ): + raise ValueError("Expected a .deepnote document with project.notebooks") + self.raw = raw + self.project_name = str(project.get("name", "Untitled project")) + self.inputs, self.outputs = _read_blocks(project["notebooks"]) + + @classmethod + def load(cls, path: str | Path) -> DeepnoteDocument: + source = Path(path) + try: + raw = yaml.safe_load(source.read_text(encoding="utf-8")) + except yaml.YAMLError as error: + raise ValueError(f"Could not parse {source}: {error}") from error + if not isinstance(raw, Mapping): + raise ValueError(f"Expected {source} to contain a YAML object") + return cls(raw) + + @classmethod + def parse(cls, content: str) -> DeepnoteDocument: + try: + raw = yaml.safe_load(content) + except yaml.YAMLError as error: + raise ValueError(f"Could not parse .deepnote YAML: {error}") from error + if not isinstance(raw, Mapping): + raise ValueError("Expected .deepnote YAML to contain an object") + return cls(raw) + + +class RunResult(OutputCollection): + """The normalized result of `POST /api/run`, for either cloud or local execution.""" + + def __init__(self, raw: Mapping[str, Any]): + self.raw = raw + self.target = str(raw.get("target", "")) + self.success = raw.get("success") is True + self.run_id = _optional_string(raw.get("runId")) + self.status = _optional_string(raw.get("status")) + self.created = raw.get("created") is True + self.view_url = _optional_string(raw.get("viewUrl")) + self.error = _optional_string(raw.get("error")) + self.snapshot_yaml = _optional_string(raw.get("snapshotYaml")) + self.snapshot = ( + DeepnoteDocument.parse(self.snapshot_yaml) if self.snapshot_yaml else None + ) + if self.snapshot: + self.outputs = self.snapshot.outputs + else: + self.outputs = _outputs_from_run(raw.get("outputs")) + + +def _read_blocks( + notebooks: Sequence[Any], +) -> tuple[tuple[InputBlock, ...], tuple[NotebookOutput, ...]]: + inputs: list[InputBlock] = [] + outputs: list[NotebookOutput] = [] + for notebook in notebooks: + if not isinstance(notebook, Mapping): + continue + blocks = notebook.get("blocks") + if not isinstance(blocks, list): + continue + for block in blocks: + if not isinstance(block, Mapping): + continue + if input_block := InputBlock.from_block(block): + inputs.append(input_block) + block_outputs = block.get("outputs") + if not isinstance(block_outputs, list): + continue + block_id = str(block.get("id", "")) + block_type = _optional_string(block.get("type")) + outputs.extend( + NotebookOutput(block_id=block_id, block_type=block_type, raw=output) + for output in block_outputs + if isinstance(output, Mapping) + ) + return tuple(inputs), tuple(outputs) + + +def _outputs_from_run(value: Any) -> tuple[NotebookOutput, ...]: + if not isinstance(value, list): + return () + outputs: list[NotebookOutput] = [] + for block in value: + if not isinstance(block, Mapping): + continue + block_id = str(block.get("blockId", "")) + raw_outputs = block.get("outputs") + if not isinstance(raw_outputs, list): + continue + outputs.extend( + NotebookOutput(block_id=block_id, block_type=None, raw=output) + for output in raw_outputs + if isinstance(output, Mapping) + ) + return tuple(outputs) + + +def _optional_string(value: Any) -> str | None: + return value if isinstance(value, str) else None + + +def _optional_number(value: Any) -> float | int | None: + return ( + value + if isinstance(value, (float, int)) and not isinstance(value, bool) + else None + ) diff --git a/deepnote_toolkit/streamlit/widgets.py b/deepnote_toolkit/streamlit/widgets.py new file mode 100644 index 00000000..7ceb887a --- /dev/null +++ b/deepnote_toolkit/streamlit/widgets.py @@ -0,0 +1,118 @@ +"""Map Deepnote input blocks to native Streamlit widgets.""" + +from __future__ import annotations + +from collections.abc import Iterable +from datetime import date +from typing import Any + +from .document import InputBlock + + +def render_inputs( + inputs: Iterable[InputBlock], container: Any = None, *, key_prefix: str = "deepnote" +) -> dict[str, Any]: + """Render input blocks and return API-ready values keyed by variable name. + + `container` may be `st`, `st.sidebar`, or a fake with the same widget methods for tests. When it + is omitted, Streamlit is imported lazily so parsing and API clients work without the app extra. + """ + + if container is None: + import streamlit as container + + values: dict[str, Any] = {} + for input_block in inputs: + label = input_block.label or input_block.variable_name.replace("_", " ").title() + key = f"{key_prefix}:{input_block.variable_name}" + values[input_block.variable_name] = _render_one( + container, input_block, label, key + ) + return values + + +def _render_one(container: Any, input_block: InputBlock, label: str, key: str) -> Any: + if input_block.type == "input-checkbox": + return container.checkbox(label, value=_as_bool(input_block.value), key=key) + + if input_block.type == "input-select": + options = list(input_block.options) + if input_block.multiple: + defaults = input_block.value if isinstance(input_block.value, list) else [] + return container.multiselect(label, options, default=defaults, key=key) + index = ( + options.index(str(input_block.value)) + if str(input_block.value) in options + else 0 + ) + return ( + container.selectbox(label, options, index=index, key=key) if options else "" + ) + + if input_block.type == "input-slider": + minimum = input_block.min if input_block.min is not None else 0 + maximum = input_block.max if input_block.max is not None else 100 + step = input_block.step if input_block.step is not None else 1 + value = _as_number(input_block.value, minimum) + if any(isinstance(number, float) for number in (minimum, maximum, value, step)): + minimum, maximum, value, step = ( + float(number) for number in (minimum, maximum, value, step) + ) + return container.slider( + label, min_value=minimum, max_value=maximum, value=value, step=step, key=key + ) + + if input_block.type == "input-date": + return _serialize_date( + container.date_input(label, value=_as_date(input_block.value), key=key) + ) + + if input_block.type == "input-date-range": + raw = input_block.value if isinstance(input_block.value, list) else [] + defaults = tuple(_as_date(value) for value in raw[:2]) + selected = container.date_input(label, value=defaults, key=key) + if isinstance(selected, (list, tuple)): + serialized = [_serialize_date(value) for value in selected] + if len(serialized) == 2: + return serialized + if len(serialized) == 1: + return [serialized[0], serialized[0]] + fallback = [_serialize_date(value) for value in defaults] + return fallback if len(fallback) == 2 else [date.today().isoformat()] * 2 + return [_serialize_date(selected), _serialize_date(selected)] + + if input_block.type == "input-textarea": + return container.text_area(label, value=str(input_block.value or ""), key=key) + + return container.text_input(label, value=str(input_block.value or ""), key=key) + + +def _as_bool(value: Any) -> bool: + if isinstance(value, bool): + return value + return str(value).lower() in {"true", "1"} + + +def _as_number(value: Any, fallback: float | int) -> float | int: + try: + number = float(value) + return ( + number + if isinstance(fallback, float) or not number.is_integer() + else int(number) + ) + except (TypeError, ValueError): + return fallback + + +def _as_date(value: Any) -> date: + if isinstance(value, date): + return value + try: + return date.fromisoformat(str(value)) + except ValueError: + return date.today() + + +def _serialize_date(value: Any) -> str: + return value.isoformat() if hasattr(value, "isoformat") else str(value) diff --git a/docs/streamlit-apps.md b/docs/streamlit-apps.md new file mode 100644 index 00000000..f98adf74 --- /dev/null +++ b/docs/streamlit-apps.md @@ -0,0 +1,77 @@ +# Build Streamlit apps from Deepnote notebooks + +Deepnote Toolkit provides a small typed layer for custom Streamlit apps backed by +`.deepnote` source files and snapshots. + +```python +from pathlib import Path + +import streamlit as st +from deepnote_toolkit.streamlit import ( + DeepnoteCloudRunner, + DeepnoteDocument, + render_inputs, +) + +document = DeepnoteDocument.load(Path("report.deepnote")) +values = render_inputs(document.inputs, st.sidebar) + +if st.button("Run"): + result = DeepnoteCloudRunner("your-notebook-id").run(values) + st.dataframe(result.first_dataframe().records()) +``` + +`DeepnoteDocument` reads typed input definitions and structured notebook outputs. +`render_inputs` maps Deepnote input blocks to native Streamlit widgets. +`DeepnoteCloudRunner` calls the same public notebooks and runs API used by the +Deepnote CLI. `DeepnoteRunner` is available for the local-runner sidecar. + +## Authentication modes + +A hosted Deepnote Streamlit app needs no token configuration. For each API request, +the cloud runner: + +1. reads the current viewer's opaque `streamlit-token` cookie; +2. resolves the app ID from `x-original-host`, falling back to `host`; +3. exchanges the cookie at + `POST /api/streamlit-apps/{appId}/api-token`; and +4. calls the returned `apiOrigin` with the short-lived token as a bearer. + +The opaque cookie is never sent to the public API. Credentials are not cached in +process globals or Streamlit session state, and a hosted request never falls back to +a shared environment token. + +The exchange endpoint must return `token`, `apiOrigin`, and +`expiresAtSeconds`. Deployments must provide `DEEPNOTE_WEBAPP_URL` through the +Toolkit runtime configuration. + +For another public API client, use both values returned by +`current_user_api_credentials()`. `current_user_api_token()` is a token-provider +convenience for clients whose API origin is configured separately. + +For local development, pass a user's API token explicitly or set +`DEEPNOTE_TOKEN`: + +```python +runner = DeepnoteCloudRunner("your-notebook-id", token="your-api-token") +``` + +A callable `token_provider=` can supply a renewable token. It is invoked for every +request. `DeepnoteRunner` can instead call a local `@deepnote/local-runner` +sidecar at `http://127.0.0.1:8787`. + +Static apps only load a committed snapshot with `DeepnoteDocument`; they require +no token or network access. + +## Synchronize at deployment + +Runtime requests only read and run the existing cloud notebook. Synchronize source +in an explicit deployment step: + +```bash +deepnote run report.deepnote --cloud --notebook-id "$DEEPNOTE_NOTEBOOK_ID" --push --dry-run +deepnote run report.deepnote --cloud --notebook-id "$DEEPNOTE_NOTEBOOK_ID" --push --yes +``` + +Use `RunnerInfo.accepts_inputs(document.inputs)` before submitting values to +verify that the deployed notebook still has matching input names and block types. diff --git a/tests/unit/test_deepnote_streamlit_auth.py b/tests/unit/test_deepnote_streamlit_auth.py new file mode 100644 index 00000000..fc6cdd51 --- /dev/null +++ b/tests/unit/test_deepnote_streamlit_auth.py @@ -0,0 +1,209 @@ +from __future__ import annotations + +import io +import json +import sys +from types import SimpleNamespace +from typing import Any +from unittest.mock import patch +from urllib.error import HTTPError + +import pytest + +from deepnote_toolkit.streamlit import ( + CurrentUserApiTokenError, + current_user_api_credentials, + current_user_api_token, +) +from deepnote_toolkit.streamlit.auth import ( + _read_streamlit_app_id_from_context, +) + +APP_ID = "3853c7f5-2048-4b57-946d-6c5592c3317e" + + +class FakeResponse: + def __init__(self, payload: Any): + self.payload = payload + + def __enter__(self) -> "FakeResponse": + return self + + def __exit__(self, *_args: object) -> None: + return None + + def read(self) -> bytes: + return json.dumps(self.payload).encode() + + +def test_reads_app_id_from_original_host_before_host() -> None: + streamlit = SimpleNamespace( + context=SimpleNamespace( + headers={ + "Host": "streamlit-00000000-0000-0000-0000-000000000000.example", + "X-Original-Host": f"streamlit-{APP_ID}.deepnote.com", + } + ) + ) + + with patch.dict(sys.modules, {"streamlit": streamlit}): + assert _read_streamlit_app_id_from_context() == APP_ID + + +def test_reads_app_id_from_host_fallback() -> None: + streamlit = SimpleNamespace( + context=SimpleNamespace( + headers={"host": f"streamlit-{APP_ID}.deepnote.com:443"} + ) + ) + + with patch.dict(sys.modules, {"streamlit": streamlit}): + assert _read_streamlit_app_id_from_context() == APP_ID + + +@pytest.mark.parametrize( + "streamlit", + [ + SimpleNamespace(context=SimpleNamespace(headers={})), + SimpleNamespace(context=SimpleNamespace(headers={"host": "localhost:8501"})), + ], +) +def test_app_id_is_unavailable_outside_hosted_app(streamlit: object) -> None: + with patch.dict(sys.modules, {"streamlit": streamlit}): + assert _read_streamlit_app_id_from_context() is None + + +def test_exchanges_opaque_cookie_for_public_api_credentials() -> None: + captured = {} + + def open_request(request: Any, *, timeout: float) -> FakeResponse: + captured["url"] = request.full_url + captured["method"] = request.method + captured["headers"] = dict(request.header_items()) + captured["body"] = request.data + captured["timeout"] = timeout + return FakeResponse( + { + "token": "viewer-api-token", + "apiOrigin": "https://api.deepnote-staging.com/", + "expiresAtSeconds": 1_800_000_000, + } + ) + + credentials = current_user_api_credentials( + app_id=APP_ID, + webapp_url="https://deepnote-staging.com/", + streamlit_token="opaque-cookie", + timeout=7, + opener=open_request, + ) + + assert captured["url"] == ( + f"https://deepnote-staging.com/api/streamlit-apps/{APP_ID}/api-token" + ) + assert captured["method"] == "POST" + assert captured["body"] == b"" + assert captured["timeout"] == 7 + headers = {key.lower(): value for key, value in captured["headers"].items()} + assert headers["streamlittoken"] == "opaque-cookie" + assert "authorization" not in headers + assert credentials.token == "viewer-api-token" + assert credentials.api_origin == "https://api.deepnote-staging.com" + assert credentials.expires_at_seconds == 1_800_000_000 + + +def test_public_token_provider_exchanges_on_every_call() -> None: + with patch( + "deepnote_toolkit.streamlit.auth.current_user_api_credentials" + ) as exchange: + exchange.side_effect = [ + SimpleNamespace(token="first"), + SimpleNamespace(token="second"), + ] + + assert current_user_api_token() == "first" + assert current_user_api_token() == "second" + + assert exchange.call_count == 2 + + +def test_exchange_requires_hosted_streamlit_context() -> None: + with ( + patch( + "deepnote_toolkit.streamlit.auth._read_streamlit_app_id_from_context", + return_value=None, + ), + pytest.raises(CurrentUserApiTokenError, match="app ID"), + ): + current_user_api_token() + + +def test_exchange_requires_viewer_cookie() -> None: + with ( + patch( + "deepnote_toolkit.streamlit.auth._read_streamlit_app_id_from_context", + return_value=APP_ID, + ), + patch( + "deepnote_toolkit.streamlit.auth._read_streamlit_token_from_context", + return_value=None, + ), + pytest.raises(CurrentUserApiTokenError, match="streamlit-token"), + ): + current_user_api_token() + + +@pytest.mark.parametrize( + "payload", + [ + {}, + {"token": "token"}, + { + "token": "token", + "apiOrigin": "javascript:alert(1)", + "expiresAtSeconds": 123, + }, + { + "token": "token", + "apiOrigin": "https://api.deepnote.com/unexpected", + "expiresAtSeconds": 123, + }, + { + "token": "token", + "apiOrigin": "https://api.deepnote.com?secret=value", + "expiresAtSeconds": 123, + }, + ], +) +def test_exchange_rejects_invalid_response(payload: dict[str, Any]) -> None: + with pytest.raises(CurrentUserApiTokenError): + current_user_api_credentials( + app_id=APP_ID, + webapp_url="https://deepnote.com", + streamlit_token="opaque-cookie", + opener=lambda *_args, **_kwargs: FakeResponse(payload), + ) + + +def test_exchange_error_does_not_expose_response_body() -> None: + secret_response = "must-not-leak" + + def open_request(*_args: Any, **_kwargs: Any) -> FakeResponse: + raise HTTPError( + "https://deepnote.com/api/streamlit-apps/id/api-token", + 401, + "Unauthorized", + {}, + io.BytesIO(json.dumps({"error": secret_response}).encode()), + ) + + with pytest.raises(CurrentUserApiTokenError) as exc_info: + current_user_api_credentials( + app_id=APP_ID, + webapp_url="https://deepnote.com", + streamlit_token="opaque-cookie", + opener=open_request, + ) + + assert "HTTP 401" in str(exc_info.value) + assert secret_response not in str(exc_info.value) diff --git a/tests/unit/test_deepnote_streamlit_client.py b/tests/unit/test_deepnote_streamlit_client.py new file mode 100644 index 00000000..dca49f5b --- /dev/null +++ b/tests/unit/test_deepnote_streamlit_client.py @@ -0,0 +1,337 @@ +import io +import json +from typing import Any +from unittest.mock import MagicMock, patch +from urllib.error import HTTPError, URLError + +import pytest + +from deepnote_toolkit.streamlit import ( + CurrentUserApiCredentials, + CurrentUserApiTokenError, + DeepnoteCloudRunner, + DeepnoteRunner, + InputBlock, + RunnerError, + RunnerInfo, +) + + +class FakeResponse: + def __init__(self, payload: Any): + self.payload = payload + + def __enter__(self) -> "FakeResponse": + return self + + def __exit__(self, *_: object) -> None: + return None + + def read(self) -> bytes: + return json.dumps(self.payload).encode() + + +def test_info_parses_runner_contract() -> None: + calls = [] + + def open_request(request: Any, *, timeout: float) -> FakeResponse: + calls.append((request.full_url, request.method, timeout)) + return FakeResponse( + { + "notebook": "Revenue", + "runTarget": "cloud", + "inputs": [ + {"variableName": "region", "type": "input-select", "value": "All"} + ], + } + ) + + info = DeepnoteRunner("http://runner/", timeout=12, opener=open_request).info() + + assert calls == [("http://runner/api/info", "GET", 12)] + assert info.notebook == "Revenue" + assert info.run_target == "cloud" + assert info.inputs[0].variable_name == "region" + + +def test_runner_info_requires_matching_input_names_and_types() -> None: + info = RunnerInfo( + notebook="Revenue", + inputs=(InputBlock("region", "input-select", "All"),), + run_target="cloud", + ) + + assert info.accepts_inputs([InputBlock("region", "input-select", "Europe")]) + assert not info.accepts_inputs([InputBlock("market", "input-select", "Europe")]) + assert not info.accepts_inputs([InputBlock("region", "input-text", "Europe")]) + + +def test_run_posts_inputs_and_parses_one_result_shape() -> None: + def open_request(request: Any, *, timeout: float) -> FakeResponse: + assert timeout == 600 + assert request.method == "POST" + assert json.loads(request.data) == {"inputs": {"limit": 20}} + return FakeResponse({"target": "local", "success": True, "outputs": []}) + + result = DeepnoteRunner(opener=open_request).run({"limit": 20}) + + assert result.target == "local" + assert result.success is True + + +def test_http_error_surfaces_runner_message() -> None: + def open_request(*_: Any, **__: Any) -> FakeResponse: + raise HTTPError( + "http://runner/api/run", + 500, + "Server error", + {}, + io.BytesIO(b'{"error":"DEEPNOTE_TOKEN is required"}'), + ) + + with pytest.raises(RunnerError, match="DEEPNOTE_TOKEN is required"): + DeepnoteRunner("http://runner", opener=open_request).run({}) + + +def test_connection_error_names_runner_url() -> None: + def open_request(*_: Any, **__: Any) -> FakeResponse: + raise URLError("connection refused") + + with pytest.raises(RunnerError, match="http://runner"): + DeepnoteRunner("http://runner", opener=open_request).info() + + +def test_timeout_names_runner_url_and_duration() -> None: + def open_request(*_: Any, **__: Any) -> FakeResponse: + raise TimeoutError + + with pytest.raises(RunnerError, match="http://runner.*12 seconds"): + DeepnoteRunner("http://runner", timeout=12, opener=open_request).info() + + +def test_cloud_info_reads_public_notebook_contract() -> None: + def open_request(request: Any, *, timeout: float) -> FakeResponse: + assert request.full_url == "https://api.deepnote.com/v2/notebooks/notebook-1" + assert request.headers["Authorization"] == "Bearer token-1" + assert timeout == 30 + return FakeResponse( + { + "notebook": { + "name": "Revenue", + "inputs": [ + { + "name": "region", + "type": "input-select", + "value": "All", + "label": "Region", + } + ], + } + } + ) + + info = DeepnoteCloudRunner( + "notebook-1", token="token-1", opener=open_request + ).info() + + assert info.notebook == "Revenue" + assert info.run_target == "cloud" + assert info.inputs[0].variable_name == "region" + + +def test_cloud_run_posts_inputs_polls_and_parses_inline_snapshot() -> None: + calls = [] + responses = iter( + [ + {"run": {"runId": "run-1", "status": "pending"}}, + {"run": {"runId": "run-1", "status": "running"}}, + { + "run": { + "runId": "run-1", + "status": "success", + "snapshot": { + "snapshotContent": "project:\n name: Result\n notebooks:\n - blocks: []\n" + }, + } + }, + ] + ) + + def open_request(request: Any, *, timeout: float) -> FakeResponse: + calls.append( + ( + request.full_url, + request.method, + request.headers["Authorization"], + request.data, + timeout, + ) + ) + return FakeResponse(next(responses)) + + tokens = iter(["token-1", "token-2", "token-3"]) + sleeps = [] + result = DeepnoteCloudRunner( + "notebook-1", + token_provider=lambda: next(tokens), + opener=open_request, + sleep=sleeps.append, + poll_interval=0.25, + ).run({"limit": 20, "enabled": True, "regions": ["EU"]}) + + assert json.loads(calls[0][3]) == { + "notebookId": "notebook-1", + "inputs": {"limit": "20", "enabled": True, "regions": ["EU"]}, + } + assert calls[1][0].endswith("/v2/runs/run-1?snapshotDelivery=inline") + assert [call[2] for call in calls] == [ + "Bearer token-1", + "Bearer token-2", + "Bearer token-3", + ] + assert sleeps == [0.25, 0.25] + assert result.success is True + assert result.snapshot is not None + assert result.snapshot.project_name == "Result" + + +def test_cloud_run_surfaces_terminal_error() -> None: + def open_request(_request: Any, *, timeout: float) -> FakeResponse: + assert timeout == 30 + return FakeResponse( + { + "run": { + "id": "run-1", + "status": "error", + "error": {"message": "bad input"}, + } + } + ) + + result = DeepnoteCloudRunner("notebook-1", token="token", opener=open_request).run( + {} + ) + + assert result.success is False + assert result.error == "bad input" + + +def test_hosted_cloud_runner_exchanges_per_request_and_uses_api_origin( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("DEEPNOTE_TOKEN", "must-not-be-used") + api_calls = [] + responses = iter( + [ + {"run": {"runId": "run-1", "status": "pending"}}, + {"run": {"runId": "run-1", "status": "success"}}, + ] + ) + + def open_request(request: Any, *, timeout: float) -> FakeResponse: + api_calls.append( + ( + request.full_url, + request.headers["Authorization"], + timeout, + ) + ) + return FakeResponse(next(responses)) + + credentials = [ + CurrentUserApiCredentials( + token="viewer-token-1", + api_origin="https://api.deepnote-staging.com", + expires_at_seconds=1_800_000_000, + ), + CurrentUserApiCredentials( + token="viewer-token-2", + api_origin="https://api.deepnote-staging.com", + expires_at_seconds=1_800_000_001, + ), + ] + with ( + patch( + "deepnote_toolkit.streamlit.client._has_hosted_streamlit_context", + return_value=True, + ), + patch( + "deepnote_toolkit.streamlit.client.current_user_api_credentials", + side_effect=credentials, + ) as exchange, + ): + result = DeepnoteCloudRunner( + "notebook-1", + opener=open_request, + sleep=lambda _delay: None, + ).run({}) + + assert result.success is True + assert exchange.call_count == 2 + assert api_calls == [ + ( + "https://api.deepnote-staging.com/v2/runs", + "Bearer viewer-token-1", + 30, + ), + ( + "https://api.deepnote-staging.com/v2/runs/run-1" "?snapshotDelivery=inline", + "Bearer viewer-token-2", + 30, + ), + ] + + +def test_hosted_runner_never_falls_back_to_environment_token( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("DEEPNOTE_TOKEN", "shared-token") + opener = MagicMock() + with ( + patch( + "deepnote_toolkit.streamlit.client._has_hosted_streamlit_context", + return_value=True, + ), + patch( + "deepnote_toolkit.streamlit.client.current_user_api_credentials", + side_effect=CurrentUserApiTokenError("viewer token unavailable"), + ), + pytest.raises(RunnerError, match="viewer token unavailable"), + ): + DeepnoteCloudRunner("notebook-1", opener=opener).info() + + opener.assert_not_called() + + +def test_local_cloud_runner_uses_environment_token( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("DEEPNOTE_TOKEN", "local-token") + + def open_request(request: Any, *, timeout: float) -> FakeResponse: + assert request.headers["Authorization"] == "Bearer local-token" + assert timeout == 30 + return FakeResponse({"notebook": {"name": "Revenue", "inputs": []}}) + + with patch( + "deepnote_toolkit.streamlit.client._has_hosted_streamlit_context", + return_value=False, + ): + info = DeepnoteCloudRunner("notebook-1", opener=open_request).info() + + assert info.notebook == "Revenue" + + +def test_cloud_runner_requires_one_token_source( + monkeypatch: pytest.MonkeyPatch, +) -> None: + with pytest.raises(ValueError, match="not both"): + DeepnoteCloudRunner("notebook-1", token="token", token_provider=lambda: "other") + + monkeypatch.delenv("DEEPNOTE_TOKEN", raising=False) + with pytest.raises(RunnerError, match="token is required"): + DeepnoteCloudRunner( + "notebook-1", + token="", + opener=lambda *_args, **_kwargs: FakeResponse({}), + ).info() diff --git a/tests/unit/test_deepnote_streamlit_document.py b/tests/unit/test_deepnote_streamlit_document.py new file mode 100644 index 00000000..191b0e3f --- /dev/null +++ b/tests/unit/test_deepnote_streamlit_document.py @@ -0,0 +1,166 @@ +from pathlib import Path + +import pytest + +from deepnote_toolkit.streamlit import ( + DATAFRAME_MIME, + DeepnoteDocument, + InputBlock, + RunResult, + join_text, +) + +SNAPSHOT_YAML = """ +project: + name: Sales performance + notebooks: + - blocks: + - id: region-input + type: input-select + metadata: + deepnote_variable_name: region + deepnote_input_label: Region + deepnote_variable_value: Europe + deepnote_variable_options: [All, Europe] + - id: table + type: code + outputs: + - output_type: execute_result + data: + application/vnd.deepnote.dataframe.v3+json: + columns: + - name: _deepnote_index_column + - name: Revenue + rows: + - _deepnote_index_column: Europe + Revenue: 42 + - id: agent + type: agent + outputs: + - output_type: display_data + data: + text/markdown: "**Done**" +""" + + +def test_loads_inputs_and_structured_outputs(tmp_path: Path) -> None: + path = tmp_path / "sales.snapshot.deepnote" + path.write_text(SNAPSHOT_YAML, encoding="utf-8") + + snapshot = DeepnoteDocument.load(path) + + assert snapshot.project_name == "Sales performance" + assert snapshot.inputs == ( + InputBlock( + "region", + "input-select", + "Europe", + label="Region", + options=("All", "Europe"), + ), + ) + dataframe = snapshot.first_dataframe() + assert dataframe is not None + assert dataframe.data_columns == ("Revenue",) + assert dataframe.records(include_index=False) == [{"Revenue": 42}] + assert snapshot.agent_text() == "**Done**" + + +def test_reads_input_metadata_from_file_and_api_shapes() -> None: + file_input = InputBlock.from_block( + { + "type": "input-slider", + "metadata": { + "deepnote_variable_name": "limit", + "deepnote_input_label": "Row limit", + "deepnote_variable_value": "20", + "deepnote_slider_min_value": 10, + "deepnote_slider_max_value": 100, + "deepnote_slider_step": 10, + }, + } + ) + api_input = InputBlock.from_api( + { + "variableName": "countries", + "type": "input-select", + "label": "Countries", + "value": ["Panama"], + "options": ["Panama", "Colombia"], + "multiple": True, + } + ) + + assert file_input == InputBlock( + variable_name="limit", + type="input-slider", + label="Row limit", + value="20", + min=10, + max=100, + step=10, + ) + assert api_input.options == ("Panama", "Colombia") + assert api_input.multiple is True + + +def test_run_result_prefers_snapshot_outputs_and_preserves_cloud_fields() -> None: + result = RunResult( + { + "target": "cloud", + "success": True, + "runId": "run-1", + "status": "success", + "viewUrl": "https://deepnote.com/project/example", + "snapshotYaml": SNAPSHOT_YAML, + "outputs": [], + } + ) + + assert result.success is True + assert result.target == "cloud" + assert result.run_id == "run-1" + assert result.agent_text() == "**Done**" + + +def test_run_result_falls_back_to_inline_outputs_without_snapshot() -> None: + result = RunResult( + { + "target": "local", + "success": True, + "outputs": [ + { + "blockId": "code-1", + "outputs": [ + { + "output_type": "execute_result", + "data": { + DATAFRAME_MIME: { + "columns": [{"name": "value"}], + "rows": [{"value": 42}], + } + }, + } + ], + } + ], + } + ) + + dataframe = result.first_dataframe() + assert dataframe is not None + assert dataframe.records() == [{"value": 42}] + + +@pytest.mark.parametrize( + ("value", "expected"), + [(["hello", " ", "world"], "hello world"), ("hello", "hello"), (None, "")], +) +def test_join_text(value: object, expected: str) -> None: + assert join_text(value) == expected + + +@pytest.mark.parametrize("content", ["hello: world", "[]", ""]) +def test_rejects_non_deepnote_yaml(content: str) -> None: + with pytest.raises(ValueError): + DeepnoteDocument.parse(content) diff --git a/tests/unit/test_deepnote_streamlit_widgets.py b/tests/unit/test_deepnote_streamlit_widgets.py new file mode 100644 index 00000000..1b3f105f --- /dev/null +++ b/tests/unit/test_deepnote_streamlit_widgets.py @@ -0,0 +1,98 @@ +from datetime import date +from typing import Any + +from deepnote_toolkit.streamlit import InputBlock, render_inputs + + +class FakeContainer: + def checkbox(self, _label: str, **kwargs: Any) -> Any: + return kwargs["value"] + + def multiselect(self, _label: str, _options: list[str], **kwargs: Any) -> Any: + return kwargs["default"] + + def selectbox(self, _label: str, options: list[str], **kwargs: Any) -> Any: + return options[kwargs["index"]] + + def slider(self, _label: str, **kwargs: Any) -> Any: + return kwargs["value"] + + def date_input(self, _label: str, **kwargs: Any) -> Any: + return kwargs["value"] + + def text_area(self, _label: str, **kwargs: Any) -> Any: + return kwargs["value"] + + def text_input(self, _label: str, **kwargs: Any) -> Any: + return kwargs["value"] + + +def test_render_inputs_maps_all_deepnote_input_types_to_api_values() -> None: + inputs = [ + InputBlock("name", "input-text", "Ada"), + InputBlock("notes", "input-textarea", "Hello"), + InputBlock("enabled", "input-checkbox", True), + InputBlock("region", "input-select", "Europe", options=("All", "Europe")), + InputBlock( + "regions", + "input-select", + ["Europe"], + options=("All", "Europe"), + multiple=True, + ), + InputBlock("limit", "input-slider", "20", min=10, max=100, step=10), + InputBlock("as_of", "input-date", date(2026, 8, 17)), + InputBlock("period", "input-date-range", [date(2026, 8, 1), date(2026, 8, 17)]), + ] + + assert render_inputs(inputs, FakeContainer()) == { + "name": "Ada", + "notes": "Hello", + "enabled": True, + "region": "Europe", + "regions": ["Europe"], + "limit": 20, + "as_of": "2026-08-17", + "period": ["2026-08-01", "2026-08-17"], + } + + +def test_incomplete_date_range_is_still_valid_for_runner_contract() -> None: + class IncompleteDateContainer(FakeContainer): + def date_input(self, _label: str, **_kwargs: Any) -> Any: + return (date(2026, 8, 17),) + + values = render_inputs( + [ + InputBlock( + "period", "input-date-range", [date(2026, 8, 1), date(2026, 8, 17)] + ) + ], + IncompleteDateContainer(), + ) + + assert values == {"period": ["2026-08-17", "2026-08-17"]} + + +def test_slider_preserves_fractional_default_with_integer_bounds() -> None: + class SliderContainer(FakeContainer): + slider_kwargs: dict[str, Any] + + def slider(self, _label: str, **kwargs: Any) -> Any: + self.slider_kwargs = kwargs + return kwargs["value"] + + container = SliderContainer() + values = render_inputs( + [InputBlock("threshold", "input-slider", "20.5", min=10, max=30, step=0.5)], + container, + ) + + assert values == {"threshold": 20.5} + assert container.slider_kwargs == { + "min_value": 10.0, + "max_value": 30.0, + "value": 20.5, + "step": 0.5, + "key": "deepnote:threshold", + } From c0015ecd32ae50a67764421b846a2a2a60bd5cca Mon Sep 17 00:00:00 2001 From: James Hobbs Date: Wed, 26 Aug 2026 15:10:29 -0500 Subject: [PATCH 02/43] fix(streamlit): type lazy optional import --- deepnote_toolkit/streamlit/widgets.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/deepnote_toolkit/streamlit/widgets.py b/deepnote_toolkit/streamlit/widgets.py index 7ceb887a..c9c44217 100644 --- a/deepnote_toolkit/streamlit/widgets.py +++ b/deepnote_toolkit/streamlit/widgets.py @@ -19,7 +19,9 @@ def render_inputs( """ if container is None: - import streamlit as container + import streamlit as st # type: ignore[import-not-found] + + container = st values: dict[str, Any] = {} for input_block in inputs: From c07d5c22b567198cfeba8852fff08dd16b1b6451 Mon Sep 17 00:00:00 2001 From: James Hobbs Date: Wed, 26 Aug 2026 15:15:58 -0500 Subject: [PATCH 03/43] fix(streamlit): tolerate malformed output metadata --- deepnote_toolkit/streamlit/document.py | 13 +++--- .../unit/test_deepnote_streamlit_document.py | 45 +++++++++++++++++++ 2 files changed, 53 insertions(+), 5 deletions(-) diff --git a/deepnote_toolkit/streamlit/document.py b/deepnote_toolkit/streamlit/document.py index 0829b9de..af6225d7 100644 --- a/deepnote_toolkit/streamlit/document.py +++ b/deepnote_toolkit/streamlit/document.py @@ -110,9 +110,9 @@ def from_value(cls, value: Any) -> DeepnoteDataframe | None: @property def data_columns(self) -> tuple[str, ...]: return tuple( - str(column["name"]) + str(column.get("name")) for column in self.columns - if column.get("name") != INDEX_COLUMN + if column.get("name") not in (None, INDEX_COLUMN) ) def records(self, *, include_index: bool = True) -> list[dict[str, Any]]: @@ -247,9 +247,12 @@ def __init__(self, raw: Mapping[str, Any]): self.view_url = _optional_string(raw.get("viewUrl")) self.error = _optional_string(raw.get("error")) self.snapshot_yaml = _optional_string(raw.get("snapshotYaml")) - self.snapshot = ( - DeepnoteDocument.parse(self.snapshot_yaml) if self.snapshot_yaml else None - ) + self.snapshot = None + if self.snapshot_yaml: + try: + self.snapshot = DeepnoteDocument.parse(self.snapshot_yaml) + except ValueError: + pass if self.snapshot: self.outputs = self.snapshot.outputs else: diff --git a/tests/unit/test_deepnote_streamlit_document.py b/tests/unit/test_deepnote_streamlit_document.py index 191b0e3f..2511b548 100644 --- a/tests/unit/test_deepnote_streamlit_document.py +++ b/tests/unit/test_deepnote_streamlit_document.py @@ -66,6 +66,27 @@ def test_loads_inputs_and_structured_outputs(tmp_path: Path) -> None: assert snapshot.agent_text() == "**Done**" +def test_dataframe_ignores_columns_without_names() -> None: + dataframe = DeepnoteDocument.parse( + """ +project: + notebooks: + - blocks: + - id: table + type: code + outputs: + - output_type: execute_result + data: + application/vnd.deepnote.dataframe.v3+json: + columns: [{}, {name: value}] + rows: [{value: 42}] +""" + ).first_dataframe() + + assert dataframe is not None + assert dataframe.data_columns == ("value",) + + def test_reads_input_metadata_from_file_and_api_shapes() -> None: file_input = InputBlock.from_block( { @@ -152,6 +173,30 @@ def test_run_result_falls_back_to_inline_outputs_without_snapshot() -> None: assert dataframe.records() == [{"value": 42}] +def test_run_result_falls_back_to_inline_outputs_for_malformed_snapshot() -> None: + result = RunResult( + { + "target": "cloud", + "success": True, + "snapshotYaml": "not: a deepnote snapshot", + "outputs": [ + { + "blockId": "code-1", + "outputs": [ + { + "output_type": "stream", + "text": "fallback output", + } + ], + } + ], + } + ) + + assert result.snapshot is None + assert result.text() == "fallback output" + + @pytest.mark.parametrize( ("value", "expected"), [(["hello", " ", "world"], "hello world"), ("hello", "hello"), (None, "")], From 3988b05d847585568831d6dadd8ebd961ecc4ec6 Mon Sep 17 00:00:00 2001 From: James Hobbs Date: Wed, 26 Aug 2026 15:15:58 -0500 Subject: [PATCH 04/43] fix(streamlit): sanitize multiselect defaults --- deepnote_toolkit/streamlit/widgets.py | 9 ++++++++- tests/unit/test_deepnote_streamlit_widgets.py | 17 +++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/deepnote_toolkit/streamlit/widgets.py b/deepnote_toolkit/streamlit/widgets.py index c9c44217..8f9f04d8 100644 --- a/deepnote_toolkit/streamlit/widgets.py +++ b/deepnote_toolkit/streamlit/widgets.py @@ -40,7 +40,14 @@ def _render_one(container: Any, input_block: InputBlock, label: str, key: str) - if input_block.type == "input-select": options = list(input_block.options) if input_block.multiple: - defaults = input_block.value if isinstance(input_block.value, list) else [] + raw_defaults = ( + input_block.value if isinstance(input_block.value, list) else [] + ) + defaults = [ + normalized + for value in raw_defaults + if (normalized := str(value)) in options + ] return container.multiselect(label, options, default=defaults, key=key) index = ( options.index(str(input_block.value)) diff --git a/tests/unit/test_deepnote_streamlit_widgets.py b/tests/unit/test_deepnote_streamlit_widgets.py index 1b3f105f..4699efd6 100644 --- a/tests/unit/test_deepnote_streamlit_widgets.py +++ b/tests/unit/test_deepnote_streamlit_widgets.py @@ -96,3 +96,20 @@ def slider(self, _label: str, **kwargs: Any) -> Any: "step": 0.5, "key": "deepnote:threshold", } + + +def test_multiselect_normalizes_and_filters_stale_defaults() -> None: + values = render_inputs( + [ + InputBlock( + "regions", + "input-select", + [1, "Europe", "Missing"], + options=("1", "Europe"), + multiple=True, + ) + ], + FakeContainer(), + ) + + assert values == {"regions": ["1", "Europe"]} From 667cefe634f3044f7aafd161b448824cf4c5f992 Mon Sep 17 00:00:00 2001 From: James Hobbs Date: Wed, 26 Aug 2026 15:15:59 -0500 Subject: [PATCH 05/43] refactor(streamlit): expose viewer token reader --- deepnote_toolkit/streamlit/auth.py | 6 +++--- deepnote_toolkit/streamlit_data_apps.py | 8 +++++++- tests/unit/test_deepnote_streamlit_auth.py | 2 +- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/deepnote_toolkit/streamlit/auth.py b/deepnote_toolkit/streamlit/auth.py index c5aabdcf..36e2ed5d 100644 --- a/deepnote_toolkit/streamlit/auth.py +++ b/deepnote_toolkit/streamlit/auth.py @@ -13,7 +13,7 @@ from deepnote_toolkit.config import get_config from deepnote_toolkit.streamlit_data_apps import ( - _read_streamlit_token_from_context, + read_streamlit_token_from_context, ) OpenUrl = Callable[..., Any] @@ -68,7 +68,7 @@ def current_user_api_credentials( "Could not resolve a Deepnote Streamlit app ID from the request host." ) - viewer_token = streamlit_token or _read_streamlit_token_from_context() + viewer_token = streamlit_token or read_streamlit_token_from_context() if not viewer_token: raise CurrentUserApiTokenError( "Could not read the current viewer's streamlit-token cookie." @@ -169,7 +169,7 @@ def _has_hosted_streamlit_context() -> bool: """Return whether this request carries either hosted-app identity signal.""" return bool( - _read_streamlit_app_id_from_context() or _read_streamlit_token_from_context() + _read_streamlit_app_id_from_context() or read_streamlit_token_from_context() ) diff --git a/deepnote_toolkit/streamlit_data_apps.py b/deepnote_toolkit/streamlit_data_apps.py index 99fadeb3..fb87b6d4 100644 --- a/deepnote_toolkit/streamlit_data_apps.py +++ b/deepnote_toolkit/streamlit_data_apps.py @@ -61,7 +61,7 @@ def __init__( self.integration_name = integration_name -def _read_streamlit_token_from_context() -> Optional[str]: +def read_streamlit_token_from_context() -> Optional[str]: """Read the ``streamlit-token`` cookie from the active Streamlit context. Returns ``None`` if Streamlit is not installed, no script run is active, or the cookie @@ -87,6 +87,12 @@ def _read_streamlit_token_from_context() -> Optional[str]: return token +def _read_streamlit_token_from_context() -> Optional[str]: + """Backward-compatible private alias for the public cookie helper.""" + + return read_streamlit_token_from_context() + + def get_federated_auth_token( integration_id: str, *, diff --git a/tests/unit/test_deepnote_streamlit_auth.py b/tests/unit/test_deepnote_streamlit_auth.py index fc6cdd51..bd12caed 100644 --- a/tests/unit/test_deepnote_streamlit_auth.py +++ b/tests/unit/test_deepnote_streamlit_auth.py @@ -145,7 +145,7 @@ def test_exchange_requires_viewer_cookie() -> None: return_value=APP_ID, ), patch( - "deepnote_toolkit.streamlit.auth._read_streamlit_token_from_context", + "deepnote_toolkit.streamlit.auth.read_streamlit_token_from_context", return_value=None, ), pytest.raises(CurrentUserApiTokenError, match="streamlit-token"), From 554768050fce5878c30fe09224cbe7396e9acb12 Mon Sep 17 00:00:00 2001 From: James Hobbs Date: Wed, 26 Aug 2026 15:21:36 -0500 Subject: [PATCH 06/43] style(streamlit): apply pinned Black formatting --- tests/unit/test_deepnote_streamlit_document.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/unit/test_deepnote_streamlit_document.py b/tests/unit/test_deepnote_streamlit_document.py index 2511b548..895c0be9 100644 --- a/tests/unit/test_deepnote_streamlit_document.py +++ b/tests/unit/test_deepnote_streamlit_document.py @@ -67,8 +67,7 @@ def test_loads_inputs_and_structured_outputs(tmp_path: Path) -> None: def test_dataframe_ignores_columns_without_names() -> None: - dataframe = DeepnoteDocument.parse( - """ + dataframe = DeepnoteDocument.parse(""" project: notebooks: - blocks: @@ -80,8 +79,7 @@ def test_dataframe_ignores_columns_without_names() -> None: application/vnd.deepnote.dataframe.v3+json: columns: [{}, {name: value}] rows: [{value: 42}] -""" - ).first_dataframe() +""").first_dataframe() assert dataframe is not None assert dataframe.data_columns == ("value",) From ff164ba14c84bcf558c5a3444e93fbaa4b803d2d Mon Sep 17 00:00:00 2001 From: James Hobbs Date: Wed, 26 Aug 2026 15:55:05 -0500 Subject: [PATCH 07/43] fix(streamlit): consume sanitized run outputs --- deepnote_toolkit/streamlit/client.py | 2 + deepnote_toolkit/streamlit/document.py | 31 ++++++++- docs/streamlit-apps.md | 6 ++ tests/unit/test_deepnote_streamlit_client.py | 70 ++++++++++++++++++++ 4 files changed, 108 insertions(+), 1 deletion(-) diff --git a/deepnote_toolkit/streamlit/client.py b/deepnote_toolkit/streamlit/client.py index 9681a8b9..7e4db3c2 100644 --- a/deepnote_toolkit/streamlit/client.py +++ b/deepnote_toolkit/streamlit/client.py @@ -186,6 +186,7 @@ def run(self, inputs: Mapping[str, Any]) -> RunResult: "/v2/runs", { "notebookId": self.notebook_id, + "detached": True, "inputs": _normalize_cloud_inputs(inputs), }, ) @@ -219,6 +220,7 @@ def run(self, inputs: Mapping[str, Any]) -> RunResult: "status": status, "error": str(error) if error is not None else None, "snapshotYaml": snapshot_yaml, + "snapshotBlocks": current.get("snapshotBlocks"), "viewUrl": current.get("viewUrl"), } ) diff --git a/deepnote_toolkit/streamlit/document.py b/deepnote_toolkit/streamlit/document.py index af6225d7..3c0b9c59 100644 --- a/deepnote_toolkit/streamlit/document.py +++ b/deepnote_toolkit/streamlit/document.py @@ -256,7 +256,12 @@ def __init__(self, raw: Mapping[str, Any]): if self.snapshot: self.outputs = self.snapshot.outputs else: - self.outputs = _outputs_from_run(raw.get("outputs")) + snapshot_blocks = raw.get("snapshotBlocks") + self.outputs = ( + _outputs_from_snapshot_blocks(snapshot_blocks) + if isinstance(snapshot_blocks, list) + else _outputs_from_run(raw.get("outputs")) + ) def _read_blocks( @@ -307,6 +312,30 @@ def _outputs_from_run(value: Any) -> tuple[NotebookOutput, ...]: return tuple(outputs) +def _outputs_from_snapshot_blocks(value: Any) -> tuple[NotebookOutput, ...]: + if not isinstance(value, list): + return () + outputs: list[NotebookOutput] = [] + for block in value: + if not isinstance(block, Mapping): + continue + block_id = str(block.get("id", "")) + block_type = _optional_string(block.get("type")) + raw_outputs = block.get("outputs") + if not isinstance(raw_outputs, list): + continue + outputs.extend( + NotebookOutput( + block_id=block_id, + block_type=block_type, + raw=output, + ) + for output in raw_outputs + if isinstance(output, Mapping) + ) + return tuple(outputs) + + def _optional_string(value: Any) -> str | None: return value if isinstance(value, str) else None diff --git a/docs/streamlit-apps.md b/docs/streamlit-apps.md index f98adf74..56f6ddcd 100644 --- a/docs/streamlit-apps.md +++ b/docs/streamlit-apps.md @@ -37,6 +37,12 @@ the cloud runner: `POST /api/streamlit-apps/{appId}/api-token`; and 4. calls the returned `apiOrigin` with the short-lived token as a bearer. +Cloud runs explicitly request `detached: true`, keeping viewer-triggered work out +of the shared project session. Hosted app tokens receive sanitized +`snapshotBlocks` containing the executed notebook's outputs, not the raw +project snapshot. API-key clients remain compatible with inline +`snapshotContent` responses. + The opaque cookie is never sent to the public API. Credentials are not cached in process globals or Streamlit session state, and a hosted request never falls back to a shared environment token. diff --git a/tests/unit/test_deepnote_streamlit_client.py b/tests/unit/test_deepnote_streamlit_client.py index dca49f5b..1a07bb2f 100644 --- a/tests/unit/test_deepnote_streamlit_client.py +++ b/tests/unit/test_deepnote_streamlit_client.py @@ -181,6 +181,7 @@ def open_request(request: Any, *, timeout: float) -> FakeResponse: assert json.loads(calls[0][3]) == { "notebookId": "notebook-1", + "detached": True, "inputs": {"limit": "20", "enabled": True, "regions": ["EU"]}, } assert calls[1][0].endswith("/v2/runs/run-1?snapshotDelivery=inline") @@ -195,6 +196,75 @@ def open_request(request: Any, *, timeout: float) -> FakeResponse: assert result.snapshot.project_name == "Result" +def test_cloud_run_reads_sanitized_snapshot_blocks_without_raw_snapshot() -> None: + responses = iter( + [ + {"run": {"runId": "run-1", "status": "pending"}}, + { + "run": { + "runId": "run-1", + "status": "success", + "snapshotBlocks": [ + { + "id": "code-1", + "type": "code", + "outputs": [ + { + "output_type": "execute_result", + "data": { + "application/vnd.deepnote.dataframe.v3+json": { + "columns": [{"name": "revenue"}], + "rows": [{"revenue": 42}], + } + }, + } + ], + "metadata": {"deepnote_table_state": {}}, + }, + { + "id": "agent-1", + "type": "agent", + "outputs": [ + { + "output_type": "display_data", + "data": {"text/markdown": "**Done**"}, + } + ], + "metadata": {}, + }, + ], + } + }, + ] + ) + + def open_request(request: Any, *, timeout: float) -> FakeResponse: + assert timeout == 30 + if request.method == "POST": + assert json.loads(request.data) == { + "notebookId": "notebook-1", + "detached": True, + "inputs": {"region": "EU"}, + } + return FakeResponse(next(responses)) + + result = DeepnoteCloudRunner( + "notebook-1", + token="token", + opener=open_request, + sleep=lambda _delay: None, + ).run({"region": "EU"}) + + assert result.snapshot is None + assert result.snapshot_yaml is None + assert [output.block_id for output in result.outputs] == ["code-1", "agent-1"] + assert [output.block_type for output in result.outputs] == ["code", "agent"] + dataframe = result.first_dataframe() + assert dataframe is not None + assert dataframe.records() == [{"revenue": 42}] + assert result.agent_text() == "**Done**" + + def test_cloud_run_surfaces_terminal_error() -> None: def open_request(_request: Any, *, timeout: float) -> FakeResponse: assert timeout == 30 From 76121cd308e9138c2b5fb93773d56267a38f075b Mon Sep 17 00:00:00 2001 From: Wojciech Apanowicz Date: Thu, 17 Sep 2026 15:46:40 +0200 Subject: [PATCH 08/43] fix(streamlit): exchange the viewer token through the userpod API --- deepnote_toolkit/streamlit/auth.py | 18 ++++++------------ docs/streamlit-apps.md | 8 ++++---- tests/unit/test_deepnote_streamlit_auth.py | 7 ++----- 3 files changed, 12 insertions(+), 21 deletions(-) diff --git a/deepnote_toolkit/streamlit/auth.py b/deepnote_toolkit/streamlit/auth.py index 36e2ed5d..28ad8969 100644 --- a/deepnote_toolkit/streamlit/auth.py +++ b/deepnote_toolkit/streamlit/auth.py @@ -11,7 +11,10 @@ from urllib.parse import urlparse from urllib.request import Request, urlopen -from deepnote_toolkit.config import get_config +from deepnote_toolkit.get_webapp_url import ( + get_absolute_userpod_api_url, + get_project_auth_headers, +) from deepnote_toolkit.streamlit_data_apps import ( read_streamlit_token_from_context, ) @@ -50,7 +53,6 @@ def current_user_api_token() -> str: def current_user_api_credentials( *, app_id: str | None = None, - webapp_url: str | None = None, streamlit_token: str | None = None, timeout: float = 10, opener: OpenUrl = urlopen, @@ -74,23 +76,15 @@ def current_user_api_credentials( "Could not read the current viewer's streamlit-token cookie." ) - resolved_webapp_url = webapp_url or get_config().runtime.webapp_url - if not resolved_webapp_url: - raise CurrentUserApiTokenError( - "DEEPNOTE_WEBAPP_URL is required in a hosted Streamlit app." - ) - resolved_webapp_url = _validated_origin( - resolved_webapp_url, name="DEEPNOTE_WEBAPP_URL" - ) - request = Request( - (f"{resolved_webapp_url}/api/streamlit-apps/" f"{resolved_app_id}/api-token"), + get_absolute_userpod_api_url(f"streamlit-apps/{resolved_app_id}/api-token"), data=b"", method="POST", headers={ "Accept": "application/json", "Content-Type": "application/json", "StreamlitToken": viewer_token, + **get_project_auth_headers(), }, ) try: diff --git a/docs/streamlit-apps.md b/docs/streamlit-apps.md index 56f6ddcd..82b699e8 100644 --- a/docs/streamlit-apps.md +++ b/docs/streamlit-apps.md @@ -33,8 +33,8 @@ the cloud runner: 1. reads the current viewer's opaque `streamlit-token` cookie; 2. resolves the app ID from `x-original-host`, falling back to `host`; -3. exchanges the cookie at - `POST /api/streamlit-apps/{appId}/api-token`; and +3. exchanges the cookie through the project's userpod API at + `POST /userpod-api/streamlit-apps/{appId}/api-token`; and 4. calls the returned `apiOrigin` with the short-lived token as a bearer. Cloud runs explicitly request `detached: true`, keeping viewer-triggered work out @@ -48,8 +48,8 @@ process globals or Streamlit session state, and a hosted request never falls bac a shared environment token. The exchange endpoint must return `token`, `apiOrigin`, and -`expiresAtSeconds`. Deployments must provide `DEEPNOTE_WEBAPP_URL` through the -Toolkit runtime configuration. +`expiresAtSeconds`. The request goes through the same userpod API route as the +Toolkit's other webapp calls, so a hosted app needs no extra configuration. For another public API client, use both values returned by `current_user_api_credentials()`. `current_user_api_token()` is a token-provider diff --git a/tests/unit/test_deepnote_streamlit_auth.py b/tests/unit/test_deepnote_streamlit_auth.py index bd12caed..e325d394 100644 --- a/tests/unit/test_deepnote_streamlit_auth.py +++ b/tests/unit/test_deepnote_streamlit_auth.py @@ -92,14 +92,13 @@ def open_request(request: Any, *, timeout: float) -> FakeResponse: credentials = current_user_api_credentials( app_id=APP_ID, - webapp_url="https://deepnote-staging.com/", streamlit_token="opaque-cookie", timeout=7, opener=open_request, ) assert captured["url"] == ( - f"https://deepnote-staging.com/api/streamlit-apps/{APP_ID}/api-token" + f"http://localhost:19456/userpod-api/streamlit-apps/{APP_ID}/api-token" ) assert captured["method"] == "POST" assert captured["body"] == b"" @@ -179,7 +178,6 @@ def test_exchange_rejects_invalid_response(payload: dict[str, Any]) -> None: with pytest.raises(CurrentUserApiTokenError): current_user_api_credentials( app_id=APP_ID, - webapp_url="https://deepnote.com", streamlit_token="opaque-cookie", opener=lambda *_args, **_kwargs: FakeResponse(payload), ) @@ -190,7 +188,7 @@ def test_exchange_error_does_not_expose_response_body() -> None: def open_request(*_args: Any, **_kwargs: Any) -> FakeResponse: raise HTTPError( - "https://deepnote.com/api/streamlit-apps/id/api-token", + "http://localhost:19456/userpod-api/streamlit-apps/id/api-token", 401, "Unauthorized", {}, @@ -200,7 +198,6 @@ def open_request(*_args: Any, **_kwargs: Any) -> FakeResponse: with pytest.raises(CurrentUserApiTokenError) as exc_info: current_user_api_credentials( app_id=APP_ID, - webapp_url="https://deepnote.com", streamlit_token="opaque-cookie", opener=open_request, ) From cb3d264eb890185e4a6eb300a186cffbb1abc801 Mon Sep 17 00:00:00 2001 From: Wojciech Apanowicz Date: Thu, 17 Sep 2026 15:54:33 +0200 Subject: [PATCH 09/43] perf(streamlit): reuse viewer API credentials within a Streamlit session --- deepnote_toolkit/streamlit/auth.py | 50 +++++++++++++-- deepnote_toolkit/streamlit/client.py | 3 +- docs/streamlit-apps.md | 12 ++-- tests/unit/test_deepnote_streamlit_auth.py | 75 +++++++++++++++++++++- 4 files changed, 127 insertions(+), 13 deletions(-) diff --git a/deepnote_toolkit/streamlit/auth.py b/deepnote_toolkit/streamlit/auth.py index 28ad8969..8f58a9ee 100644 --- a/deepnote_toolkit/streamlit/auth.py +++ b/deepnote_toolkit/streamlit/auth.py @@ -2,8 +2,10 @@ from __future__ import annotations +import hashlib import json import re +import time from collections.abc import Callable, Mapping from dataclasses import dataclass from typing import Any @@ -26,6 +28,10 @@ ) +_SESSION_STATE_KEY = "_deepnote_current_user_api_credentials" +_EXPIRY_MARGIN_SECONDS = 60 + + class CurrentUserApiTokenError(RuntimeError): """Raised when a hosted app cannot obtain the current viewer's API token.""" @@ -43,8 +49,7 @@ def current_user_api_token() -> str: """Return a short-lived public API bearer for the current Streamlit viewer. The opaque streamlit-token cookie is exchanged for a viewer-scoped token. - It is never itself used as a public API bearer. The exchange happens on - every call so a long-lived, multi-user process does not retain credentials. + It is never itself used as a public API bearer. """ return current_user_api_credentials().token @@ -59,9 +64,9 @@ def current_user_api_credentials( ) -> CurrentUserApiCredentials: """Exchange the active viewer cookie for public API credentials. - The returned API origin must be used with the returned bearer. Hosted clients - should call this for every request, or cache it only within the current - Streamlit session until shortly before expires_at_seconds. + The returned API origin must be used with the returned bearer. Credentials are + reused within the current Streamlit session until shortly before they expire, + and never shared between sessions. """ resolved_app_id = app_id or _read_streamlit_app_id_from_context() @@ -76,6 +81,20 @@ def current_user_api_credentials( "Could not read the current viewer's streamlit-token cookie." ) + session_state = _read_streamlit_session_state() + cache_key = ( + resolved_app_id, + hashlib.sha256(viewer_token.encode()).hexdigest(), + ) + if session_state is not None: + cached = session_state.get(_SESSION_STATE_KEY) + if ( + isinstance(cached, tuple) + and cached[0] == cache_key + and cached[1].expires_at_seconds - _EXPIRY_MARGIN_SECONDS > time.time() + ): + return cached[1] + request = Request( get_absolute_userpod_api_url(f"streamlit-apps/{resolved_app_id}/api-token"), data=b"", @@ -126,11 +145,30 @@ def current_user_api_credentials( "Current viewer API-token exchange response is missing required fields." ) - return CurrentUserApiCredentials( + credentials = CurrentUserApiCredentials( token=token, api_origin=_validated_origin(api_origin, name="apiOrigin"), expires_at_seconds=float(expires_at_seconds), ) + if session_state is not None: + session_state[_SESSION_STATE_KEY] = (cache_key, credentials) + return credentials + + +def _read_streamlit_session_state() -> Any | None: + """Return the current session's state, or None outside a Streamlit script run.""" + + try: + import streamlit as st # type: ignore[import-not-found] + from streamlit.runtime.scriptrunner import ( # type: ignore[import-not-found] + get_script_run_ctx, + ) + except ImportError: + return None + + if get_script_run_ctx() is None: + return None + return st.session_state def _read_streamlit_app_id_from_context() -> str | None: diff --git a/deepnote_toolkit/streamlit/client.py b/deepnote_toolkit/streamlit/client.py index 7e4db3c2..15f39905 100644 --- a/deepnote_toolkit/streamlit/client.py +++ b/deepnote_toolkit/streamlit/client.py @@ -127,7 +127,8 @@ class DeepnoteCloudRunner: """Run an existing notebook directly through the Deepnote public API. A token provider is called for every request, which lets long-lived Streamlit - sessions use short-lived credentials without caching them in this library. + sessions use short-lived credentials. Hosted viewer credentials are reused + within the viewer's own Streamlit session until shortly before they expire. """ def __init__( diff --git a/docs/streamlit-apps.md b/docs/streamlit-apps.md index 82b699e8..a72f0084 100644 --- a/docs/streamlit-apps.md +++ b/docs/streamlit-apps.md @@ -28,8 +28,7 @@ Deepnote CLI. `DeepnoteRunner` is available for the local-runner sidecar. ## Authentication modes -A hosted Deepnote Streamlit app needs no token configuration. For each API request, -the cloud runner: +A hosted Deepnote Streamlit app needs no token configuration. The cloud runner: 1. reads the current viewer's opaque `streamlit-token` cookie; 2. resolves the app ID from `x-original-host`, falling back to `host`; @@ -43,9 +42,12 @@ of the shared project session. Hosted app tokens receive sanitized project snapshot. API-key clients remain compatible with inline `snapshotContent` responses. -The opaque cookie is never sent to the public API. Credentials are not cached in -process globals or Streamlit session state, and a hosted request never falls back to -a shared environment token. +The opaque cookie is never sent to the public API. The exchanged credentials are kept +in the viewer's own Streamlit session state and reused until a minute before they +expire. They are never kept in process globals or shared between sessions, and a +hosted request never falls back to a shared environment token. Deepnote rechecks +the viewer's access on every API request, so a reused bearer stops working as soon +as access is revoked. The exchange endpoint must return `token`, `apiOrigin`, and `expiresAtSeconds`. The request goes through the same userpod API route as the diff --git a/tests/unit/test_deepnote_streamlit_auth.py b/tests/unit/test_deepnote_streamlit_auth.py index e325d394..9078a7d5 100644 --- a/tests/unit/test_deepnote_streamlit_auth.py +++ b/tests/unit/test_deepnote_streamlit_auth.py @@ -3,6 +3,7 @@ import io import json import sys +import time from types import SimpleNamespace from typing import Any from unittest.mock import patch @@ -111,7 +112,79 @@ def open_request(request: Any, *, timeout: float) -> FakeResponse: assert credentials.expires_at_seconds == 1_800_000_000 -def test_public_token_provider_exchanges_on_every_call() -> None: +def _hosted_session_modules(session_state: dict[str, Any]) -> dict[str, Any]: + scriptrunner = SimpleNamespace(get_script_run_ctx=lambda: object()) + return { + "streamlit": SimpleNamespace(session_state=session_state), + "streamlit.runtime": SimpleNamespace(scriptrunner=scriptrunner), + "streamlit.runtime.scriptrunner": scriptrunner, + } + + +def _counting_opener(expires_at_seconds: float) -> tuple[list[Any], Any]: + requests: list[Any] = [] + + def open_request(request: Any, *, timeout: float) -> FakeResponse: + requests.append(request) + return FakeResponse( + { + "token": f"viewer-api-token-{len(requests)}", + "apiOrigin": "https://api.deepnote.com", + "expiresAtSeconds": expires_at_seconds, + } + ) + + return requests, open_request + + +def test_reuses_credentials_within_a_streamlit_session() -> None: + requests, open_request = _counting_opener(time.time() + 15 * 60) + + with patch.dict(sys.modules, _hosted_session_modules({})): + first = current_user_api_credentials( + app_id=APP_ID, streamlit_token="opaque-cookie", opener=open_request + ) + second = current_user_api_credentials( + app_id=APP_ID, streamlit_token="opaque-cookie", opener=open_request + ) + + assert len(requests) == 1 + assert second == first + + +def test_does_not_share_credentials_between_sessions() -> None: + requests, open_request = _counting_opener(time.time() + 15 * 60) + + for _session in range(2): + with patch.dict(sys.modules, _hosted_session_modules({})): + current_user_api_credentials( + app_id=APP_ID, streamlit_token="opaque-cookie", opener=open_request + ) + + assert len(requests) == 2 + + +@pytest.mark.parametrize( + ("expires_in_seconds", "second_cookie"), + [(30, "opaque-cookie"), (15 * 60, "another-cookie")], +) +def test_exchanges_again_near_expiry_or_for_another_cookie( + expires_in_seconds: int, second_cookie: str +) -> None: + requests, open_request = _counting_opener(time.time() + expires_in_seconds) + + with patch.dict(sys.modules, _hosted_session_modules({})): + current_user_api_credentials( + app_id=APP_ID, streamlit_token="opaque-cookie", opener=open_request + ) + current_user_api_credentials( + app_id=APP_ID, streamlit_token=second_cookie, opener=open_request + ) + + assert len(requests) == 2 + + +def test_public_token_provider_returns_the_current_credentials_token() -> None: with patch( "deepnote_toolkit.streamlit.auth.current_user_api_credentials" ) as exchange: From 733e4c1032c3e339e3421f8dece497540e6744db Mon Sep 17 00:00:00 2001 From: Wojciech Apanowicz Date: Fri, 18 Sep 2026 13:18:07 +0200 Subject: [PATCH 10/43] fix(streamlit): harden the cloud runner and input widgets per review - Refuse the DEEPNOTE_TOKEN fallback on a Streamlit thread that has no viewer request, so a worker thread cannot run as the token owner. - Read timestamp-shaped date values, keep empty dates empty, and resolve relative date ranges instead of submitting today's date. - Let DeepnoteDocument read a single notebook, and render inputs that share a variable name once. - Wait briefly for a snapshot that lags the terminal run status. - Retry transient poll failures instead of aborting the run. --- deepnote_toolkit/streamlit/auth.py | 17 +++ deepnote_toolkit/streamlit/client.py | 62 ++++++++- deepnote_toolkit/streamlit/document.py | 29 ++++- deepnote_toolkit/streamlit/widgets.py | 67 +++++++--- docs/streamlit-apps.md | 10 +- tests/unit/test_deepnote_streamlit_client.py | 123 +++++++++++++++++- .../unit/test_deepnote_streamlit_document.py | 28 ++++ tests/unit/test_deepnote_streamlit_widgets.py | 69 +++++++++- 8 files changed, 368 insertions(+), 37 deletions(-) diff --git a/deepnote_toolkit/streamlit/auth.py b/deepnote_toolkit/streamlit/auth.py index 8f58a9ee..4f0e2a98 100644 --- a/deepnote_toolkit/streamlit/auth.py +++ b/deepnote_toolkit/streamlit/auth.py @@ -205,6 +205,23 @@ def _has_hosted_streamlit_context() -> bool: ) +def _is_streamlit_thread_without_request() -> bool: + """Return whether a Streamlit server is running but this thread has no viewer request. + + Worker threads see no headers or cookies, so they look identical to a local script. + """ + + try: + from streamlit import runtime # type: ignore[import-not-found] + from streamlit.runtime.scriptrunner import ( # type: ignore[import-not-found] + get_script_run_ctx, + ) + except ImportError: + return False + + return runtime.exists() and get_script_run_ctx(suppress_warning=True) is None + + def _validated_origin(value: str, *, name: str) -> str: parsed = urlparse(value) if ( diff --git a/deepnote_toolkit/streamlit/client.py b/deepnote_toolkit/streamlit/client.py index 15f39905..7a2c025f 100644 --- a/deepnote_toolkit/streamlit/client.py +++ b/deepnote_toolkit/streamlit/client.py @@ -14,6 +14,7 @@ from .auth import ( CurrentUserApiTokenError, _has_hosted_streamlit_context, + _is_streamlit_thread_without_request, current_user_api_credentials, ) from .document import InputBlock, RunResult @@ -24,11 +25,17 @@ TERMINAL_RUN_STATUSES = frozenset({"success", "error", "internal_error", "stopped"}) DEFAULT_API_ORIGIN = "https://api.deepnote.com" +MAX_TRANSIENT_POLL_FAILURES = 5 +SNAPSHOT_SETTLE_ATTEMPTS = 3 class RunnerError(RuntimeError): """The Deepnote runner was unavailable or rejected a request.""" + def __init__(self, message: str, *, transient: bool = False): + super().__init__(message) + self.transient = transient + @dataclass(frozen=True) class RunnerInfo: @@ -195,15 +202,34 @@ def run(self, inputs: Mapping[str, Any]) -> RunResult: run_id = _required_run_id(started) deadline = time.monotonic() + self.timeout current = started + transient_failures = 0 while str(current.get("status", "")) not in TERMINAL_RUN_STATUSES: if time.monotonic() >= deadline: raise RunnerError( f"Deepnote run {run_id} did not finish within {self.timeout:g} seconds" ) self._sleep(self.poll_interval) - current = self._run_payload( - self._request("GET", f"/v2/runs/{run_id}?snapshotDelivery=inline") - ) + try: + current = self._get_run(run_id) + transient_failures = 0 + except RunnerError as error: + transient_failures += 1 + if ( + not error.transient + or transient_failures > MAX_TRANSIENT_POLL_FAILURES + ): + raise + + # The snapshot can attach shortly after the status turns terminal. + for _ in range(SNAPSHOT_SETTLE_ATTEMPTS): + if _has_snapshot(current): + break + self._sleep(self.poll_interval) + try: + current = self._get_run(run_id) + except RunnerError as error: + if not error.transient: + raise status = str(current.get("status", "")) snapshot = current.get("snapshot") @@ -226,6 +252,11 @@ def run(self, inputs: Mapping[str, Any]) -> RunResult: } ) + def _get_run(self, run_id: str) -> Mapping[str, Any]: + return self._run_payload( + self._request("GET", f"/v2/runs/{run_id}?snapshotDelivery=inline") + ) + def _request( self, method: str, path: str, body: Mapping[str, Any] | None = None ) -> Mapping[str, Any]: @@ -256,14 +287,18 @@ def _request( except json.JSONDecodeError: message = detail raise RunnerError( - f"Deepnote API returned HTTP {error.code}: {message}" + f"Deepnote API returned HTTP {error.code}: {message}", + transient=error.code == 429 or error.code >= 500, ) from error except URLError as error: raise RunnerError( - f"Could not reach the Deepnote API at {api_origin}: {error.reason}" + f"Could not reach the Deepnote API at {api_origin}: {error.reason}", + transient=True, ) from error except TimeoutError as error: - raise RunnerError("Deepnote API request timed out") from error + raise RunnerError( + "Deepnote API request timed out", transient=True + ) from error except (json.JSONDecodeError, UnicodeDecodeError) as error: raise RunnerError( "Deepnote API returned an invalid JSON response" @@ -297,6 +332,12 @@ def _authentication(self) -> tuple[str, str]: ) return credentials.token, api_origin + if _is_streamlit_thread_without_request(): + raise RunnerError( + "No viewer request is available on this thread. Call the runner from " + "the Streamlit script thread, or pass token= or token_provider=." + ) + return self._required_token(os.environ.get("DEEPNOTE_TOKEN")), self.base_url @staticmethod @@ -311,6 +352,15 @@ def _run_payload(payload: Mapping[str, Any]) -> Mapping[str, Any]: return run if isinstance(run, Mapping) else payload +def _has_snapshot(run: Mapping[str, Any]) -> bool: + snapshot = run.get("snapshot") + return bool( + run.get("snapshotContent") + or isinstance(run.get("snapshotBlocks"), list) + or (isinstance(snapshot, Mapping) and snapshot.get("snapshotContent")) + ) + + def _required_run_id(run: Mapping[str, Any]) -> str: run_id = run.get("runId") or run.get("id") if not isinstance(run_id, str) or not run_id: diff --git a/deepnote_toolkit/streamlit/document.py b/deepnote_toolkit/streamlit/document.py index 3c0b9c59..3e4504c2 100644 --- a/deepnote_toolkit/streamlit/document.py +++ b/deepnote_toolkit/streamlit/document.py @@ -200,20 +200,35 @@ def agent_text(self) -> str: class DeepnoteDocument(OutputCollection): - """A parsed source or snapshot `.deepnote` file.""" + """A parsed source or snapshot `.deepnote` file. - def __init__(self, raw: Mapping[str, Any]): + Pass `notebook_id` to read one notebook of a multi-notebook project, so the + inputs match what a runner for that notebook accepts. + """ + + def __init__(self, raw: Mapping[str, Any], *, notebook_id: str | None = None): project = raw.get("project") if not isinstance(project, Mapping) or not isinstance( project.get("notebooks"), list ): raise ValueError("Expected a .deepnote document with project.notebooks") + notebooks = project["notebooks"] + if notebook_id is not None: + notebooks = [ + notebook + for notebook in notebooks + if isinstance(notebook, Mapping) and notebook.get("id") == notebook_id + ] + if not notebooks: + raise ValueError(f"Notebook {notebook_id} is not in this document") self.raw = raw self.project_name = str(project.get("name", "Untitled project")) - self.inputs, self.outputs = _read_blocks(project["notebooks"]) + self.inputs, self.outputs = _read_blocks(notebooks) @classmethod - def load(cls, path: str | Path) -> DeepnoteDocument: + def load( + cls, path: str | Path, *, notebook_id: str | None = None + ) -> DeepnoteDocument: source = Path(path) try: raw = yaml.safe_load(source.read_text(encoding="utf-8")) @@ -221,17 +236,17 @@ def load(cls, path: str | Path) -> DeepnoteDocument: raise ValueError(f"Could not parse {source}: {error}") from error if not isinstance(raw, Mapping): raise ValueError(f"Expected {source} to contain a YAML object") - return cls(raw) + return cls(raw, notebook_id=notebook_id) @classmethod - def parse(cls, content: str) -> DeepnoteDocument: + def parse(cls, content: str, *, notebook_id: str | None = None) -> DeepnoteDocument: try: raw = yaml.safe_load(content) except yaml.YAMLError as error: raise ValueError(f"Could not parse .deepnote YAML: {error}") from error if not isinstance(raw, Mapping): raise ValueError("Expected .deepnote YAML to contain an object") - return cls(raw) + return cls(raw, notebook_id=notebook_id) class RunResult(OutputCollection): diff --git a/deepnote_toolkit/streamlit/widgets.py b/deepnote_toolkit/streamlit/widgets.py index 8f9f04d8..34e41b97 100644 --- a/deepnote_toolkit/streamlit/widgets.py +++ b/deepnote_toolkit/streamlit/widgets.py @@ -2,12 +2,21 @@ from __future__ import annotations +import calendar +import re from collections.abc import Iterable -from datetime import date +from datetime import date, timedelta from typing import Any from .document import InputBlock +_RELATIVE_RANGE_MONTHS = { + "pastMonth": 1, + "past3months": 3, + "past6months": 6, + "pastYear": 12, +} + def render_inputs( inputs: Iterable[InputBlock], container: Any = None, *, key_prefix: str = "deepnote" @@ -25,6 +34,9 @@ def render_inputs( values: dict[str, Any] = {} for input_block in inputs: + # One submitted value applies to every block sharing a variable name. + if input_block.variable_name in values: + continue label = input_block.label or input_block.variable_name.replace("_", " ").title() key = f"{key_prefix}:{input_block.variable_name}" values[input_block.variable_name] = _render_one( @@ -72,23 +84,23 @@ def _render_one(container: Any, input_block: InputBlock, label: str, key: str) - ) if input_block.type == "input-date": - return _serialize_date( + selected = _serialize_date( container.date_input(label, value=_as_date(input_block.value), key=key) ) + # Date blocks older than version 2 only parse a full timestamp. + is_timestamp = isinstance(input_block.value, str) and "T" in input_block.value + return f"{selected}T00:00:00.000Z" if selected and is_timestamp else selected if input_block.type == "input-date-range": - raw = input_block.value if isinstance(input_block.value, list) else [] - defaults = tuple(_as_date(value) for value in raw[:2]) - selected = container.date_input(label, value=defaults, key=key) - if isinstance(selected, (list, tuple)): - serialized = [_serialize_date(value) for value in selected] - if len(serialized) == 2: - return serialized - if len(serialized) == 1: - return [serialized[0], serialized[0]] - fallback = [_serialize_date(value) for value in defaults] - return fallback if len(fallback) == 2 else [date.today().isoformat()] * 2 - return [_serialize_date(selected), _serialize_date(selected)] + selected = container.date_input( + label, value=_as_date_range(input_block.value), key=key + ) + if not isinstance(selected, (list, tuple)): + selected = (selected, selected) + serialized = [_serialize_date(value) for value in selected] + if len(serialized) == 1: + return serialized * 2 + return serialized if len(serialized) == 2 else ["", ""] if input_block.type == "input-textarea": return container.text_area(label, value=str(input_block.value or ""), key=key) @@ -114,14 +126,33 @@ def _as_number(value: Any, fallback: float | int) -> float | int: return fallback -def _as_date(value: Any) -> date: +def _as_date(value: Any) -> date | None: + """Read a date or the date part of a timestamp; None leaves the widget empty.""" + if isinstance(value, date): return value try: - return date.fromisoformat(str(value)) + return date.fromisoformat(str(value)[:10]) except ValueError: - return date.today() + return None + + +def _as_date_range(value: Any) -> tuple[date, ...]: + """Resolve an absolute or relative Deepnote range; () leaves the widget empty.""" + + if isinstance(value, list): + dates = tuple(_as_date(item) for item in value[:2]) + return dates if len(dates) == 2 and None not in dates else () + + today = date.today() + if match := re.fullmatch(r"past(\d+)days|customDays(\d+)", str(value)): + return today - timedelta(days=int(match.group(1) or match.group(2))), today + if months := _RELATIVE_RANGE_MONTHS.get(str(value)): + year, month = divmod(today.year * 12 + today.month - 1 - months, 12) + last_day = calendar.monthrange(year, month + 1)[1] + return date(year, month + 1, min(today.day, last_day)), today + return () def _serialize_date(value: Any) -> str: - return value.isoformat() if hasattr(value, "isoformat") else str(value) + return value.isoformat() if isinstance(value, date) else "" diff --git a/docs/streamlit-apps.md b/docs/streamlit-apps.md index a72f0084..f21a81f8 100644 --- a/docs/streamlit-apps.md +++ b/docs/streamlit-apps.md @@ -22,6 +22,8 @@ if st.button("Run"): ``` `DeepnoteDocument` reads typed input definitions and structured notebook outputs. +In a project with several notebooks, pass the notebook you run so the inputs match +it: `DeepnoteDocument.load(path, notebook_id="your-notebook-id")`. `render_inputs` maps Deepnote input blocks to native Streamlit widgets. `DeepnoteCloudRunner` calls the same public notebooks and runs API used by the Deepnote CLI. `DeepnoteRunner` is available for the local-runner sidecar. @@ -45,7 +47,9 @@ project snapshot. API-key clients remain compatible with inline The opaque cookie is never sent to the public API. The exchanged credentials are kept in the viewer's own Streamlit session state and reused until a minute before they expire. They are never kept in process globals or shared between sessions, and a -hosted request never falls back to a shared environment token. Deepnote rechecks +hosted request never falls back to a shared environment token. Call the runner +from the Streamlit script thread: a worker thread has no viewer request, so the +runner raises instead of using `DEEPNOTE_TOKEN`. Deepnote rechecks the viewer's access on every API request, so a reused bearer stops working as soon as access is revoked. @@ -83,3 +87,7 @@ deepnote run report.deepnote --cloud --notebook-id "$DEEPNOTE_NOTEBOOK_ID" --pus Use `RunnerInfo.accepts_inputs(document.inputs)` before submitting values to verify that the deployed notebook still has matching input names and block types. + +The cloud runner retries a poll that fails with a timeout, a network error, HTTP +429 or a 5xx, up to five times in a row. After the run finishes it waits briefly +for the outputs, which can arrive after the final status. diff --git a/tests/unit/test_deepnote_streamlit_client.py b/tests/unit/test_deepnote_streamlit_client.py index 1a07bb2f..a3efe218 100644 --- a/tests/unit/test_deepnote_streamlit_client.py +++ b/tests/unit/test_deepnote_streamlit_client.py @@ -278,9 +278,9 @@ def open_request(_request: Any, *, timeout: float) -> FakeResponse: } ) - result = DeepnoteCloudRunner("notebook-1", token="token", opener=open_request).run( - {} - ) + result = DeepnoteCloudRunner( + "notebook-1", token="token", opener=open_request, sleep=lambda _delay: None + ).run({}) assert result.success is False assert result.error == "bad input" @@ -294,7 +294,7 @@ def test_hosted_cloud_runner_exchanges_per_request_and_uses_api_origin( responses = iter( [ {"run": {"runId": "run-1", "status": "pending"}}, - {"run": {"runId": "run-1", "status": "success"}}, + {"run": {"runId": "run-1", "status": "success", "snapshotBlocks": []}}, ] ) @@ -405,3 +405,118 @@ def test_cloud_runner_requires_one_token_source( token="", opener=lambda *_args, **_kwargs: FakeResponse({}), ).info() + + +def test_cloud_run_retries_transient_poll_failures() -> None: + responses = iter( + [ + {"run": {"runId": "run-1", "status": "pending"}}, + HTTPError("http://api", 503, "Unavailable", {}, io.BytesIO(b"{}")), + URLError("connection reset"), + {"run": {"runId": "run-1", "status": "success", "snapshotBlocks": []}}, + ] + ) + + def open_request(_request: Any, *, timeout: float) -> FakeResponse: + response = next(responses) + if isinstance(response, Exception): + raise response + return FakeResponse(response) + + result = DeepnoteCloudRunner( + "notebook-1", token="token", opener=open_request, sleep=lambda _delay: None + ).run({}) + + assert result.success is True + + +def test_cloud_run_raises_poll_failures_that_are_not_transient() -> None: + responses = iter( + [ + {"run": {"runId": "run-1", "status": "pending"}}, + HTTPError("http://api", 403, "Forbidden", {}, io.BytesIO(b"{}")), + ] + ) + + def open_request(_request: Any, *, timeout: float) -> FakeResponse: + response = next(responses) + if isinstance(response, Exception): + raise response + return FakeResponse(response) + + with pytest.raises(RunnerError, match="HTTP 403"): + DeepnoteCloudRunner( + "notebook-1", token="token", opener=open_request, sleep=lambda _delay: None + ).run({}) + + +def test_cloud_run_stops_retrying_after_repeated_transient_failures() -> None: + calls = [] + + def open_request(request: Any, *, timeout: float) -> FakeResponse: + calls.append(request.method) + if request.method == "POST": + return FakeResponse({"run": {"runId": "run-1", "status": "pending"}}) + raise URLError("connection reset") + + with pytest.raises(RunnerError, match="connection reset"): + DeepnoteCloudRunner( + "notebook-1", token="token", opener=open_request, sleep=lambda _delay: None + ).run({}) + + assert calls == ["POST"] + ["GET"] * 6 + + +def test_cloud_run_waits_for_a_snapshot_that_lags_the_terminal_status() -> None: + responses = iter( + [ + {"run": {"runId": "run-1", "status": "success"}}, + {"run": {"runId": "run-1", "status": "success"}}, + { + "run": { + "runId": "run-1", + "status": "success", + "snapshotBlocks": [ + { + "id": "code-1", + "type": "code", + "outputs": [{"output_type": "stream", "text": "done"}], + } + ], + } + }, + ] + ) + sleeps = [] + + result = DeepnoteCloudRunner( + "notebook-1", + token="token", + opener=lambda _request, *, timeout: FakeResponse(next(responses)), + sleep=sleeps.append, + poll_interval=0.5, + ).run({}) + + assert sleeps == [0.5, 0.5] + assert result.text() == "done" + + +def test_worker_thread_never_falls_back_to_environment_token( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("DEEPNOTE_TOKEN", "shared-token") + opener = MagicMock() + with ( + patch( + "deepnote_toolkit.streamlit.client._has_hosted_streamlit_context", + return_value=False, + ), + patch( + "deepnote_toolkit.streamlit.client._is_streamlit_thread_without_request", + return_value=True, + ), + pytest.raises(RunnerError, match="No viewer request"), + ): + DeepnoteCloudRunner("notebook-1", opener=opener).info() + + opener.assert_not_called() diff --git a/tests/unit/test_deepnote_streamlit_document.py b/tests/unit/test_deepnote_streamlit_document.py index 895c0be9..78b9be1f 100644 --- a/tests/unit/test_deepnote_streamlit_document.py +++ b/tests/unit/test_deepnote_streamlit_document.py @@ -207,3 +207,31 @@ def test_join_text(value: object, expected: str) -> None: def test_rejects_non_deepnote_yaml(content: str) -> None: with pytest.raises(ValueError): DeepnoteDocument.parse(content) + + +MULTI_NOTEBOOK_YAML = """ +project: + name: Sales + notebooks: + - id: notebook-a + blocks: + - type: input-text + metadata: {deepnote_variable_name: region, deepnote_variable_value: EU} + - id: notebook-b + blocks: + - type: input-text + metadata: {deepnote_variable_name: region, deepnote_variable_value: US} +""" + + +def test_notebook_id_scopes_inputs_to_one_notebook() -> None: + everything = DeepnoteDocument.parse(MULTI_NOTEBOOK_YAML) + scoped = DeepnoteDocument.parse(MULTI_NOTEBOOK_YAML, notebook_id="notebook-b") + + assert [input_block.value for input_block in everything.inputs] == ["EU", "US"] + assert scoped.inputs == (InputBlock("region", "input-text", "US"),) + + +def test_unknown_notebook_id_is_rejected() -> None: + with pytest.raises(ValueError, match="notebook-c is not in this document"): + DeepnoteDocument.parse(MULTI_NOTEBOOK_YAML, notebook_id="notebook-c") diff --git a/tests/unit/test_deepnote_streamlit_widgets.py b/tests/unit/test_deepnote_streamlit_widgets.py index 4699efd6..349c37e9 100644 --- a/tests/unit/test_deepnote_streamlit_widgets.py +++ b/tests/unit/test_deepnote_streamlit_widgets.py @@ -1,4 +1,4 @@ -from datetime import date +from datetime import date, timedelta from typing import Any from deepnote_toolkit.streamlit import InputBlock, render_inputs @@ -113,3 +113,70 @@ def test_multiselect_normalizes_and_filters_stale_defaults() -> None: ) assert values == {"regions": ["1", "Europe"]} + + +def test_date_reads_timestamp_default_and_keeps_its_shape() -> None: + defaults = [] + + class RecordingContainer(FakeContainer): + def date_input(self, _label: str, **kwargs: Any) -> Any: + defaults.append(kwargs["value"]) + return date(2026, 8, 20) + + values = render_inputs( + [ + InputBlock("legacy", "input-date", "2026-08-17T00:00:00.000Z"), + InputBlock("current", "input-date", "2026-08-17"), + ], + RecordingContainer(), + ) + + assert defaults == [date(2026, 8, 17), date(2026, 8, 17)] + assert values == {"legacy": "2026-08-20T00:00:00.000Z", "current": "2026-08-20"} + + +def test_empty_dates_stay_empty_instead_of_becoming_today() -> None: + values = render_inputs( + [ + InputBlock("as_of", "input-date", ""), + InputBlock("period", "input-date-range", ["", ""]), + ], + FakeContainer(), + ) + + assert values == {"as_of": "", "period": ["", ""]} + + +def test_relative_date_ranges_resolve_to_concrete_dates() -> None: + values = render_inputs( + [ + InputBlock("week", "input-date-range", "past7days"), + InputBlock("custom", "input-date-range", "customDays3"), + InputBlock("year", "input-date-range", "pastYear"), + ], + FakeContainer(), + ) + + today = date.today() + # Feb 29 has no counterpart a year earlier and clamps to Feb 28. + year_ago_day = 28 if (today.month, today.day) == (2, 29) else today.day + assert values == { + "week": [(today - timedelta(days=7)).isoformat(), today.isoformat()], + "custom": [(today - timedelta(days=3)).isoformat(), today.isoformat()], + "year": [ + today.replace(year=today.year - 1, day=year_ago_day).isoformat(), + today.isoformat(), + ], + } + + +def test_inputs_sharing_a_variable_name_render_once() -> None: + values = render_inputs( + [ + InputBlock("region", "input-text", "EU"), + InputBlock("region", "input-text", "US"), + ], + FakeContainer(), + ) + + assert values == {"region": "EU"} From e38f544a8f076ea0b80cbcd4c840705e4e94d868 Mon Sep 17 00:00:00 2001 From: Wojciech Apanowicz Date: Fri, 18 Sep 2026 13:41:22 +0200 Subject: [PATCH 11/43] fix(streamlit): retry transient token-exchange failures while polling A viewer token that expires mid-run is re-exchanged during a poll. A timeout, network error, HTTP 429 or 5xx from that exchange now counts as a transient poll failure instead of aborting the run. --- deepnote_toolkit/streamlit/auth.py | 12 +++++-- deepnote_toolkit/streamlit/client.py | 2 +- tests/unit/test_deepnote_streamlit_client.py | 35 ++++++++++++++++++++ 3 files changed, 45 insertions(+), 4 deletions(-) diff --git a/deepnote_toolkit/streamlit/auth.py b/deepnote_toolkit/streamlit/auth.py index 4f0e2a98..72340ec9 100644 --- a/deepnote_toolkit/streamlit/auth.py +++ b/deepnote_toolkit/streamlit/auth.py @@ -35,6 +35,10 @@ class CurrentUserApiTokenError(RuntimeError): """Raised when a hosted app cannot obtain the current viewer's API token.""" + def __init__(self, message: str, *, transient: bool = False): + super().__init__(message) + self.transient = transient + @dataclass(frozen=True) class CurrentUserApiCredentials: @@ -111,15 +115,17 @@ def current_user_api_credentials( payload = json.loads(response.read()) except HTTPError as error: raise CurrentUserApiTokenError( - f"Current viewer API-token exchange returned HTTP {error.code}." + f"Current viewer API-token exchange returned HTTP {error.code}.", + transient=error.code == 429 or error.code >= 500, ) from error except URLError as error: raise CurrentUserApiTokenError( - "Could not reach Deepnote to exchange the current viewer's API token." + "Could not reach Deepnote to exchange the current viewer's API token.", + transient=True, ) from error except TimeoutError as error: raise CurrentUserApiTokenError( - "Current viewer API-token exchange timed out." + "Current viewer API-token exchange timed out.", transient=True ) from error except (json.JSONDecodeError, UnicodeDecodeError) as error: raise CurrentUserApiTokenError( diff --git a/deepnote_toolkit/streamlit/client.py b/deepnote_toolkit/streamlit/client.py index 7a2c025f..8fc5113e 100644 --- a/deepnote_toolkit/streamlit/client.py +++ b/deepnote_toolkit/streamlit/client.py @@ -324,7 +324,7 @@ def _authentication(self) -> tuple[str, str]: timeout=min(self.timeout, 30), opener=self._open ) except CurrentUserApiTokenError as error: - raise RunnerError(str(error)) from error + raise RunnerError(str(error), transient=error.transient) from error api_origin = ( credentials.api_origin if self.base_url == DEFAULT_API_ORIGIN diff --git a/tests/unit/test_deepnote_streamlit_client.py b/tests/unit/test_deepnote_streamlit_client.py index a3efe218..022d818e 100644 --- a/tests/unit/test_deepnote_streamlit_client.py +++ b/tests/unit/test_deepnote_streamlit_client.py @@ -520,3 +520,38 @@ def test_worker_thread_never_falls_back_to_environment_token( DeepnoteCloudRunner("notebook-1", opener=opener).info() opener.assert_not_called() + + +def test_cloud_run_retries_a_transient_token_exchange_failure() -> None: + responses = iter( + [ + {"run": {"runId": "run-1", "status": "pending"}}, + {"run": {"runId": "run-1", "status": "success", "snapshotBlocks": []}}, + ] + ) + credentials = CurrentUserApiCredentials( + token="viewer-token", + api_origin="https://api.deepnote.com", + expires_at_seconds=1_800_000_000, + ) + with ( + patch( + "deepnote_toolkit.streamlit.client._has_hosted_streamlit_context", + return_value=True, + ), + patch( + "deepnote_toolkit.streamlit.client.current_user_api_credentials", + side_effect=[ + credentials, + CurrentUserApiTokenError("exchange timed out", transient=True), + credentials, + ], + ), + ): + result = DeepnoteCloudRunner( + "notebook-1", + opener=lambda _request, *, timeout: FakeResponse(next(responses)), + sleep=lambda _delay: None, + ).run({}) + + assert result.success is True From 0fe8fc73476b8ce033b96666da73a494f2ff9aad Mon Sep 17 00:00:00 2001 From: Wojciech Apanowicz Date: Fri, 18 Sep 2026 13:41:22 +0200 Subject: [PATCH 12/43] chore(streamlit): trim comments --- deepnote_toolkit/streamlit/auth.py | 6 +++--- deepnote_toolkit/streamlit/client.py | 4 +--- deepnote_toolkit/streamlit/document.py | 5 ++--- deepnote_toolkit/streamlit/widgets.py | 4 ++-- 4 files changed, 8 insertions(+), 11 deletions(-) diff --git a/deepnote_toolkit/streamlit/auth.py b/deepnote_toolkit/streamlit/auth.py index 72340ec9..98cffb95 100644 --- a/deepnote_toolkit/streamlit/auth.py +++ b/deepnote_toolkit/streamlit/auth.py @@ -68,9 +68,9 @@ def current_user_api_credentials( ) -> CurrentUserApiCredentials: """Exchange the active viewer cookie for public API credentials. - The returned API origin must be used with the returned bearer. Credentials are - reused within the current Streamlit session until shortly before they expire, - and never shared between sessions. + The bearer is only valid at the returned API origin. Credentials are reused + within the current Streamlit session until shortly before they expire, and + never shared between sessions. """ resolved_app_id = app_id or _read_streamlit_app_id_from_context() diff --git a/deepnote_toolkit/streamlit/client.py b/deepnote_toolkit/streamlit/client.py index 8fc5113e..72a9b5c4 100644 --- a/deepnote_toolkit/streamlit/client.py +++ b/deepnote_toolkit/streamlit/client.py @@ -315,9 +315,7 @@ def _authentication(self) -> tuple[str, str]: if self._static_token is not None: return self._required_token(self._static_token), self.base_url - # Hosted apps always authenticate as the current viewer. In particular, - # never fall back to a process-wide environment token when this request - # has a hosted Streamlit app hostname. + # A hosted request authenticates as the viewer and never uses DEEPNOTE_TOKEN. if _has_hosted_streamlit_context(): try: credentials = current_user_api_credentials( diff --git a/deepnote_toolkit/streamlit/document.py b/deepnote_toolkit/streamlit/document.py index 3e4504c2..b7415093 100644 --- a/deepnote_toolkit/streamlit/document.py +++ b/deepnote_toolkit/streamlit/document.py @@ -1,4 +1,4 @@ -"""Typed, deliberately small views over `.deepnote` YAML and run responses.""" +"""Typed views over `.deepnote` YAML and run responses.""" from __future__ import annotations @@ -202,8 +202,7 @@ def agent_text(self) -> str: class DeepnoteDocument(OutputCollection): """A parsed source or snapshot `.deepnote` file. - Pass `notebook_id` to read one notebook of a multi-notebook project, so the - inputs match what a runner for that notebook accepts. + `notebook_id` limits the inputs and outputs to one notebook of the project. """ def __init__(self, raw: Mapping[str, Any], *, notebook_id: str | None = None): diff --git a/deepnote_toolkit/streamlit/widgets.py b/deepnote_toolkit/streamlit/widgets.py index 34e41b97..c95113a3 100644 --- a/deepnote_toolkit/streamlit/widgets.py +++ b/deepnote_toolkit/streamlit/widgets.py @@ -127,7 +127,7 @@ def _as_number(value: Any, fallback: float | int) -> float | int: def _as_date(value: Any) -> date | None: - """Read a date or the date part of a timestamp; None leaves the widget empty.""" + """Read a date or the date part of a timestamp. None leaves the widget empty.""" if isinstance(value, date): return value @@ -138,7 +138,7 @@ def _as_date(value: Any) -> date | None: def _as_date_range(value: Any) -> tuple[date, ...]: - """Resolve an absolute or relative Deepnote range; () leaves the widget empty.""" + """Resolve an absolute or relative Deepnote range. () leaves the widget empty.""" if isinstance(value, list): dates = tuple(_as_date(item) for item in value[:2]) From b312e3e15b2f11a0df3836bda103f1a1f15dc846 Mon Sep 17 00:00:00 2001 From: Wojciech Apanowicz Date: Fri, 18 Sep 2026 14:44:07 +0200 Subject: [PATCH 13/43] refactor(streamlit): move notebook models and runners to deepnote_toolkit.notebooks Reading `.deepnote` files and running notebooks does not depend on Streamlit, so it moves to its own package, split by concern: models, output queries, document, run result, and one module per runner. - Both runners implement a `Runner` protocol and share one JSON request helper in place of two copies. - `deepnote_toolkit.streamlit` keeps the widgets and viewer credentials. `StreamlitCloudRunner` extends the cloud runner with viewer auth. --- deepnote_toolkit/streamlit/__init__.py | 31 +- deepnote_toolkit/streamlit/client.py | 386 ------------ deepnote_toolkit/streamlit/document.py | 362 ------------ deepnote_toolkit/streamlit/widgets.py | 2 +- tests/unit/test_deepnote_streamlit_client.py | 557 ------------------ tests/unit/test_deepnote_streamlit_widgets.py | 3 +- ...document.py => test_notebooks_document.py} | 2 +- 7 files changed, 9 insertions(+), 1334 deletions(-) delete mode 100644 deepnote_toolkit/streamlit/client.py delete mode 100644 deepnote_toolkit/streamlit/document.py delete mode 100644 tests/unit/test_deepnote_streamlit_client.py rename tests/unit/{test_deepnote_streamlit_document.py => test_notebooks_document.py} (99%) diff --git a/deepnote_toolkit/streamlit/__init__.py b/deepnote_toolkit/streamlit/__init__.py index 87a67393..96bc72a1 100644 --- a/deepnote_toolkit/streamlit/__init__.py +++ b/deepnote_toolkit/streamlit/__init__.py @@ -1,4 +1,4 @@ -"""Helpers for building Streamlit apps over local Deepnote files.""" +"""Helpers for Streamlit apps built on Deepnote notebooks.""" from .auth import ( CurrentUserApiCredentials, @@ -6,35 +6,14 @@ current_user_api_credentials, current_user_api_token, ) -from .client import DeepnoteCloudRunner, DeepnoteRunner, RunnerError, RunnerInfo -from .document import ( - DATAFRAME_MIME, - INDEX_COLUMN, - DeepnoteDataframe, - DeepnoteDocument, - InputBlock, - NotebookOutput, - RunResult, - join_text, -) +from .cloud_runner import StreamlitCloudRunner from .widgets import render_inputs __all__ = [ - "DATAFRAME_MIME", - "INDEX_COLUMN", - "CurrentUserApiTokenError", "CurrentUserApiCredentials", - "DeepnoteDataframe", - "DeepnoteCloudRunner", - "DeepnoteDocument", - "DeepnoteRunner", - "InputBlock", - "NotebookOutput", - "RunResult", - "RunnerError", - "RunnerInfo", - "current_user_api_token", + "CurrentUserApiTokenError", + "StreamlitCloudRunner", "current_user_api_credentials", - "join_text", + "current_user_api_token", "render_inputs", ] diff --git a/deepnote_toolkit/streamlit/client.py b/deepnote_toolkit/streamlit/client.py deleted file mode 100644 index 72a9b5c4..00000000 --- a/deepnote_toolkit/streamlit/client.py +++ /dev/null @@ -1,386 +0,0 @@ -"""HTTP client for the unified Deepnote app runner API.""" - -from __future__ import annotations - -import json -import os -import time -from collections.abc import Callable, Iterable, Mapping -from dataclasses import dataclass -from typing import Any -from urllib.error import HTTPError, URLError -from urllib.request import Request, urlopen - -from .auth import ( - CurrentUserApiTokenError, - _has_hosted_streamlit_context, - _is_streamlit_thread_without_request, - current_user_api_credentials, -) -from .document import InputBlock, RunResult - -OpenUrl = Callable[..., Any] -TokenProvider = Callable[[], str] -Sleep = Callable[[float], None] - -TERMINAL_RUN_STATUSES = frozenset({"success", "error", "internal_error", "stopped"}) -DEFAULT_API_ORIGIN = "https://api.deepnote.com" -MAX_TRANSIENT_POLL_FAILURES = 5 -SNAPSHOT_SETTLE_ATTEMPTS = 3 - - -class RunnerError(RuntimeError): - """The Deepnote runner was unavailable or rejected a request.""" - - def __init__(self, message: str, *, transient: bool = False): - super().__init__(message) - self.transient = transient - - -@dataclass(frozen=True) -class RunnerInfo: - """The target and input contract exposed by a Deepnote runner.""" - - notebook: str - inputs: tuple[InputBlock, ...] - run_target: str - - def accepts_inputs(self, inputs: Iterable[InputBlock]) -> bool: - """Return whether input variable names and block types match this runner.""" - - return _input_contract(inputs) == _input_contract(self.inputs) - - -class DeepnoteRunner: - """One client for a runner configured for Deepnote Cloud or a local kernel.""" - - def __init__( - self, - base_url: str = "http://127.0.0.1:8787", - *, - timeout: float = 600, - opener: OpenUrl = urlopen, - ): - self.base_url = base_url.rstrip("/") - self.timeout = timeout - self._open = opener - - def info(self) -> RunnerInfo: - payload = self._request("GET", "/api/info") - values = payload.get("inputs") - inputs = ( - tuple( - InputBlock.from_api(value) - for value in values - if isinstance(value, Mapping) - ) - if isinstance(values, list) - else () - ) - return RunnerInfo( - notebook=str(payload.get("notebook", "Untitled project")), - inputs=inputs, - run_target=str(payload.get("runTarget", "")), - ) - - def run(self, inputs: Mapping[str, Any]) -> RunResult: - return RunResult(self._request("POST", "/api/run", {"inputs": dict(inputs)})) - - def _request( - self, method: str, path: str, body: Mapping[str, Any] | None = None - ) -> Mapping[str, Any]: - encoded = json.dumps(body).encode() if body is not None else None - request = Request( - f"{self.base_url}{path}", - data=encoded, - method=method, - headers={"Content-Type": "application/json", "Accept": "application/json"}, - ) - try: - with self._open(request, timeout=self.timeout) as response: - payload = json.loads(response.read()) - except HTTPError as error: - detail = error.read().decode(errors="replace") - try: - parsed_detail = json.loads(detail) - message = ( - parsed_detail.get("error", detail) - if isinstance(parsed_detail, Mapping) - else detail - ) - except json.JSONDecodeError: - message = detail - raise RunnerError( - f"Deepnote runner returned HTTP {error.code}: {message}" - ) from error - except URLError as error: - raise RunnerError( - f"Could not reach Deepnote runner at {self.base_url}: {error.reason}" - ) from error - except TimeoutError as error: - raise RunnerError( - f"Deepnote runner at {self.base_url} timed out after {self.timeout:g} seconds" - ) from error - except (json.JSONDecodeError, UnicodeDecodeError) as error: - raise RunnerError( - "Deepnote runner returned an invalid JSON response" - ) from error - if not isinstance(payload, Mapping): - raise RunnerError("Deepnote runner returned a non-object response") - return payload - - -class DeepnoteCloudRunner: - """Run an existing notebook directly through the Deepnote public API. - - A token provider is called for every request, which lets long-lived Streamlit - sessions use short-lived credentials. Hosted viewer credentials are reused - within the viewer's own Streamlit session until shortly before they expire. - """ - - def __init__( - self, - notebook_id: str, - *, - token: str | None = None, - token_provider: TokenProvider | None = None, - base_url: str = DEFAULT_API_ORIGIN, - timeout: float = 600, - poll_interval: float = 2, - opener: OpenUrl = urlopen, - sleep: Sleep = time.sleep, - ): - if not notebook_id: - raise ValueError("notebook_id is required") - if token is not None and token_provider is not None: - raise ValueError("Pass token or token_provider, not both") - self.notebook_id = notebook_id - self.base_url = base_url.rstrip("/") - self.timeout = timeout - self.poll_interval = poll_interval - self._static_token = token - self._token_provider = token_provider - self._open = opener - self._sleep = sleep - - def info(self) -> RunnerInfo: - payload = self._request("GET", f"/v2/notebooks/{self.notebook_id}") - notebook = payload.get("notebook") - if not isinstance(notebook, Mapping): - raise RunnerError("Deepnote API response did not include a notebook") - raw_inputs = notebook.get("inputs") - inputs = tuple( - InputBlock.from_api( - { - "variableName": value.get("name"), - "type": value.get("type"), - "value": value.get("value"), - "label": value.get("label"), - } - ) - for value in raw_inputs or [] - if isinstance(value, Mapping) and isinstance(value.get("name"), str) - ) - return RunnerInfo( - notebook=str(notebook.get("name", "Untitled notebook")), - inputs=inputs, - run_target="cloud", - ) - - def run(self, inputs: Mapping[str, Any]) -> RunResult: - started = self._run_payload( - self._request( - "POST", - "/v2/runs", - { - "notebookId": self.notebook_id, - "detached": True, - "inputs": _normalize_cloud_inputs(inputs), - }, - ) - ) - run_id = _required_run_id(started) - deadline = time.monotonic() + self.timeout - current = started - transient_failures = 0 - while str(current.get("status", "")) not in TERMINAL_RUN_STATUSES: - if time.monotonic() >= deadline: - raise RunnerError( - f"Deepnote run {run_id} did not finish within {self.timeout:g} seconds" - ) - self._sleep(self.poll_interval) - try: - current = self._get_run(run_id) - transient_failures = 0 - except RunnerError as error: - transient_failures += 1 - if ( - not error.transient - or transient_failures > MAX_TRANSIENT_POLL_FAILURES - ): - raise - - # The snapshot can attach shortly after the status turns terminal. - for _ in range(SNAPSHOT_SETTLE_ATTEMPTS): - if _has_snapshot(current): - break - self._sleep(self.poll_interval) - try: - current = self._get_run(run_id) - except RunnerError as error: - if not error.transient: - raise - - status = str(current.get("status", "")) - snapshot = current.get("snapshot") - snapshot_yaml = current.get("snapshotContent") - if snapshot_yaml is None and isinstance(snapshot, Mapping): - snapshot_yaml = snapshot.get("snapshotContent") - error = current.get("error") - if isinstance(error, Mapping): - error = error.get("message") or json.dumps(error) - return RunResult( - { - "target": "cloud", - "success": status == "success", - "runId": run_id, - "status": status, - "error": str(error) if error is not None else None, - "snapshotYaml": snapshot_yaml, - "snapshotBlocks": current.get("snapshotBlocks"), - "viewUrl": current.get("viewUrl"), - } - ) - - def _get_run(self, run_id: str) -> Mapping[str, Any]: - return self._run_payload( - self._request("GET", f"/v2/runs/{run_id}?snapshotDelivery=inline") - ) - - def _request( - self, method: str, path: str, body: Mapping[str, Any] | None = None - ) -> Mapping[str, Any]: - encoded = json.dumps(body).encode() if body is not None else None - token, api_origin = self._authentication() - request = Request( - f"{api_origin}{path}", - data=encoded, - method=method, - headers={ - "Authorization": f"Bearer {token}", - "Content-Type": "application/json", - "Accept": "application/json", - }, - ) - try: - with self._open(request, timeout=min(self.timeout, 30)) as response: - payload = json.loads(response.read()) - except HTTPError as error: - detail = error.read().decode(errors="replace") - try: - parsed = json.loads(detail) - message = ( - parsed.get("message") or parsed.get("error") or detail - if isinstance(parsed, Mapping) - else detail - ) - except json.JSONDecodeError: - message = detail - raise RunnerError( - f"Deepnote API returned HTTP {error.code}: {message}", - transient=error.code == 429 or error.code >= 500, - ) from error - except URLError as error: - raise RunnerError( - f"Could not reach the Deepnote API at {api_origin}: {error.reason}", - transient=True, - ) from error - except TimeoutError as error: - raise RunnerError( - "Deepnote API request timed out", transient=True - ) from error - except (json.JSONDecodeError, UnicodeDecodeError) as error: - raise RunnerError( - "Deepnote API returned an invalid JSON response" - ) from error - if not isinstance(payload, Mapping): - raise RunnerError("Deepnote API returned a non-object response") - return payload - - def _authentication(self) -> tuple[str, str]: - if self._token_provider is not None: - token = self._token_provider() - return self._required_token(token), self.base_url - - if self._static_token is not None: - return self._required_token(self._static_token), self.base_url - - # A hosted request authenticates as the viewer and never uses DEEPNOTE_TOKEN. - if _has_hosted_streamlit_context(): - try: - credentials = current_user_api_credentials( - timeout=min(self.timeout, 30), opener=self._open - ) - except CurrentUserApiTokenError as error: - raise RunnerError(str(error), transient=error.transient) from error - api_origin = ( - credentials.api_origin - if self.base_url == DEFAULT_API_ORIGIN - else self.base_url - ) - return credentials.token, api_origin - - if _is_streamlit_thread_without_request(): - raise RunnerError( - "No viewer request is available on this thread. Call the runner from " - "the Streamlit script thread, or pass token= or token_provider=." - ) - - return self._required_token(os.environ.get("DEEPNOTE_TOKEN")), self.base_url - - @staticmethod - def _required_token(token: str | None) -> str: - if not token: - raise RunnerError("A Deepnote API token is required") - return token - - @staticmethod - def _run_payload(payload: Mapping[str, Any]) -> Mapping[str, Any]: - run = payload.get("run") - return run if isinstance(run, Mapping) else payload - - -def _has_snapshot(run: Mapping[str, Any]) -> bool: - snapshot = run.get("snapshot") - return bool( - run.get("snapshotContent") - or isinstance(run.get("snapshotBlocks"), list) - or (isinstance(snapshot, Mapping) and snapshot.get("snapshotContent")) - ) - - -def _required_run_id(run: Mapping[str, Any]) -> str: - run_id = run.get("runId") or run.get("id") - if not isinstance(run_id, str) or not run_id: - raise RunnerError("Deepnote API response did not include a run id") - return run_id - - -def _input_contract(inputs: Iterable[InputBlock]) -> tuple[tuple[str, str], ...]: - return tuple( - sorted((input_block.variable_name, input_block.type) for input_block in inputs) - ) - - -def _normalize_cloud_inputs( - inputs: Mapping[str, Any], -) -> dict[str, str | bool | list[str]]: - normalized: dict[str, str | bool | list[str]] = {} - for name, value in inputs.items(): - if isinstance(value, bool): - normalized[name] = value - elif isinstance(value, list): - normalized[name] = [str(item) for item in value] - else: - normalized[name] = str(value) - return normalized diff --git a/deepnote_toolkit/streamlit/document.py b/deepnote_toolkit/streamlit/document.py deleted file mode 100644 index b7415093..00000000 --- a/deepnote_toolkit/streamlit/document.py +++ /dev/null @@ -1,362 +0,0 @@ -"""Typed views over `.deepnote` YAML and run responses.""" - -from __future__ import annotations - -import base64 -import binascii -from collections.abc import Mapping, Sequence -from dataclasses import dataclass -from pathlib import Path -from typing import Any - -import yaml - -DATAFRAME_MIME = "application/vnd.deepnote.dataframe.v3+json" -INDEX_COLUMN = "_deepnote_index_column" - - -def join_text(value: Any) -> str: - """Normalize nbformat's string-or-list text values to one string.""" - - if isinstance(value, list): - return "".join(str(part) for part in value) - return "" if value is None else str(value) - - -@dataclass(frozen=True) -class InputBlock: - """The metadata a UI needs to render one Deepnote input block.""" - - variable_name: str - type: str - value: Any - label: str | None = None - options: tuple[str, ...] = () - multiple: bool = False - min: float | int | None = None - max: float | int | None = None - step: float | int | None = None - - @classmethod - def from_block(cls, block: Mapping[str, Any]) -> InputBlock | None: - block_type = str(block.get("type", "")) - metadata = block.get("metadata") - if not block_type.startswith("input-") or not isinstance(metadata, Mapping): - return None - variable_name = metadata.get("deepnote_variable_name") - if not isinstance(variable_name, str) or not variable_name: - return None - options = metadata.get("deepnote_variable_options") - return cls( - variable_name=variable_name, - type=block_type, - label=_optional_string(metadata.get("deepnote_input_label")), - value=metadata.get("deepnote_variable_value"), - options=( - tuple(str(option) for option in options) - if isinstance(options, list) - else () - ), - multiple=metadata.get("deepnote_allow_multiple_values") is True, - min=_optional_number(metadata.get("deepnote_slider_min_value")), - max=_optional_number(metadata.get("deepnote_slider_max_value")), - step=_optional_number(metadata.get("deepnote_slider_step")), - ) - - @classmethod - def from_api(cls, value: Mapping[str, Any]) -> InputBlock: - """Read the camelCase shape returned by `GET /api/info`.""" - - options = value.get("options") - return cls( - variable_name=str(value["variableName"]), - type=str(value["type"]), - label=_optional_string(value.get("label")), - value=value.get("value"), - options=( - tuple(str(option) for option in options) - if isinstance(options, list) - else () - ), - multiple=value.get("multiple") is True, - min=_optional_number(value.get("min")), - max=_optional_number(value.get("max")), - step=_optional_number(value.get("step")), - ) - - -@dataclass(frozen=True) -class DeepnoteDataframe: - """A structured Deepnote dataframe output, independent of pandas.""" - - columns: tuple[Mapping[str, Any], ...] - rows: tuple[Mapping[str, Any], ...] - raw: Mapping[str, Any] - - @classmethod - def from_value(cls, value: Any) -> DeepnoteDataframe | None: - if not isinstance(value, Mapping): - return None - columns = value.get("columns") - rows = value.get("rows") - if not isinstance(columns, list) or not isinstance(rows, list): - return None - if not all(isinstance(column, Mapping) for column in columns): - return None - if not all(isinstance(row, Mapping) for row in rows): - return None - return cls(columns=tuple(columns), rows=tuple(rows), raw=value) - - @property - def data_columns(self) -> tuple[str, ...]: - return tuple( - str(column.get("name")) - for column in self.columns - if column.get("name") not in (None, INDEX_COLUMN) - ) - - def records(self, *, include_index: bool = True) -> list[dict[str, Any]]: - """Return rows ready for `st.dataframe`, optionally omitting Deepnote's index column.""" - - if include_index: - return [dict(row) for row in self.rows] - return [ - {key: value for key, value in row.items() if key != INDEX_COLUMN} - for row in self.rows - ] - - -@dataclass(frozen=True) -class NotebookOutput: - """One nbformat-compatible output emitted by a Deepnote block.""" - - block_id: str - block_type: str | None - raw: Mapping[str, Any] - - @property - def output_type(self) -> str: - return str(self.raw.get("output_type", "")) - - @property - def data(self) -> Mapping[str, Any]: - value = self.raw.get("data") - return value if isinstance(value, Mapping) else {} - - def text(self, mime: str = "text/plain") -> str: - if self.output_type == "stream" and mime == "text/plain": - return join_text(self.raw.get("text")) - return join_text(self.data.get(mime)) - - def image_bytes(self, mime: str = "image/png") -> bytes | None: - value = self.data.get(mime) - if value is None: - return None - encoded = "".join(join_text(value).split()) - try: - return base64.b64decode(encoded, validate=True) - except (ValueError, binascii.Error): - return None - - @property - def dataframe(self) -> DeepnoteDataframe | None: - return DeepnoteDataframe.from_value(self.data.get(DATAFRAME_MIME)) - - -class OutputCollection: - """Shared output queries for a loaded document and a live run result.""" - - outputs: tuple[NotebookOutput, ...] - - def outputs_for_mime(self, mime: str) -> list[NotebookOutput]: - return [output for output in self.outputs if mime in output.data] - - def first_dataframe(self) -> DeepnoteDataframe | None: - for output in self.outputs: - if dataframe := output.dataframe: - return dataframe - return None - - def images(self, mime: str = "image/png") -> list[bytes]: - return [ - image - for output in self.outputs - if (image := output.image_bytes(mime)) is not None - ] - - def text(self, mime: str = "text/plain") -> str: - return "".join(output.text(mime) for output in self.outputs).strip() - - def agent_text(self) -> str: - chunks: list[str] = [] - for output in self.outputs: - if output.block_type != "agent": - continue - if output.output_type == "stream": - chunks.append(output.text()) - else: - chunks.append(output.text("text/markdown") or output.text()) - return "".join(chunks).strip() - - -class DeepnoteDocument(OutputCollection): - """A parsed source or snapshot `.deepnote` file. - - `notebook_id` limits the inputs and outputs to one notebook of the project. - """ - - def __init__(self, raw: Mapping[str, Any], *, notebook_id: str | None = None): - project = raw.get("project") - if not isinstance(project, Mapping) or not isinstance( - project.get("notebooks"), list - ): - raise ValueError("Expected a .deepnote document with project.notebooks") - notebooks = project["notebooks"] - if notebook_id is not None: - notebooks = [ - notebook - for notebook in notebooks - if isinstance(notebook, Mapping) and notebook.get("id") == notebook_id - ] - if not notebooks: - raise ValueError(f"Notebook {notebook_id} is not in this document") - self.raw = raw - self.project_name = str(project.get("name", "Untitled project")) - self.inputs, self.outputs = _read_blocks(notebooks) - - @classmethod - def load( - cls, path: str | Path, *, notebook_id: str | None = None - ) -> DeepnoteDocument: - source = Path(path) - try: - raw = yaml.safe_load(source.read_text(encoding="utf-8")) - except yaml.YAMLError as error: - raise ValueError(f"Could not parse {source}: {error}") from error - if not isinstance(raw, Mapping): - raise ValueError(f"Expected {source} to contain a YAML object") - return cls(raw, notebook_id=notebook_id) - - @classmethod - def parse(cls, content: str, *, notebook_id: str | None = None) -> DeepnoteDocument: - try: - raw = yaml.safe_load(content) - except yaml.YAMLError as error: - raise ValueError(f"Could not parse .deepnote YAML: {error}") from error - if not isinstance(raw, Mapping): - raise ValueError("Expected .deepnote YAML to contain an object") - return cls(raw, notebook_id=notebook_id) - - -class RunResult(OutputCollection): - """The normalized result of `POST /api/run`, for either cloud or local execution.""" - - def __init__(self, raw: Mapping[str, Any]): - self.raw = raw - self.target = str(raw.get("target", "")) - self.success = raw.get("success") is True - self.run_id = _optional_string(raw.get("runId")) - self.status = _optional_string(raw.get("status")) - self.created = raw.get("created") is True - self.view_url = _optional_string(raw.get("viewUrl")) - self.error = _optional_string(raw.get("error")) - self.snapshot_yaml = _optional_string(raw.get("snapshotYaml")) - self.snapshot = None - if self.snapshot_yaml: - try: - self.snapshot = DeepnoteDocument.parse(self.snapshot_yaml) - except ValueError: - pass - if self.snapshot: - self.outputs = self.snapshot.outputs - else: - snapshot_blocks = raw.get("snapshotBlocks") - self.outputs = ( - _outputs_from_snapshot_blocks(snapshot_blocks) - if isinstance(snapshot_blocks, list) - else _outputs_from_run(raw.get("outputs")) - ) - - -def _read_blocks( - notebooks: Sequence[Any], -) -> tuple[tuple[InputBlock, ...], tuple[NotebookOutput, ...]]: - inputs: list[InputBlock] = [] - outputs: list[NotebookOutput] = [] - for notebook in notebooks: - if not isinstance(notebook, Mapping): - continue - blocks = notebook.get("blocks") - if not isinstance(blocks, list): - continue - for block in blocks: - if not isinstance(block, Mapping): - continue - if input_block := InputBlock.from_block(block): - inputs.append(input_block) - block_outputs = block.get("outputs") - if not isinstance(block_outputs, list): - continue - block_id = str(block.get("id", "")) - block_type = _optional_string(block.get("type")) - outputs.extend( - NotebookOutput(block_id=block_id, block_type=block_type, raw=output) - for output in block_outputs - if isinstance(output, Mapping) - ) - return tuple(inputs), tuple(outputs) - - -def _outputs_from_run(value: Any) -> tuple[NotebookOutput, ...]: - if not isinstance(value, list): - return () - outputs: list[NotebookOutput] = [] - for block in value: - if not isinstance(block, Mapping): - continue - block_id = str(block.get("blockId", "")) - raw_outputs = block.get("outputs") - if not isinstance(raw_outputs, list): - continue - outputs.extend( - NotebookOutput(block_id=block_id, block_type=None, raw=output) - for output in raw_outputs - if isinstance(output, Mapping) - ) - return tuple(outputs) - - -def _outputs_from_snapshot_blocks(value: Any) -> tuple[NotebookOutput, ...]: - if not isinstance(value, list): - return () - outputs: list[NotebookOutput] = [] - for block in value: - if not isinstance(block, Mapping): - continue - block_id = str(block.get("id", "")) - block_type = _optional_string(block.get("type")) - raw_outputs = block.get("outputs") - if not isinstance(raw_outputs, list): - continue - outputs.extend( - NotebookOutput( - block_id=block_id, - block_type=block_type, - raw=output, - ) - for output in raw_outputs - if isinstance(output, Mapping) - ) - return tuple(outputs) - - -def _optional_string(value: Any) -> str | None: - return value if isinstance(value, str) else None - - -def _optional_number(value: Any) -> float | int | None: - return ( - value - if isinstance(value, (float, int)) and not isinstance(value, bool) - else None - ) diff --git a/deepnote_toolkit/streamlit/widgets.py b/deepnote_toolkit/streamlit/widgets.py index c95113a3..11cbc526 100644 --- a/deepnote_toolkit/streamlit/widgets.py +++ b/deepnote_toolkit/streamlit/widgets.py @@ -8,7 +8,7 @@ from datetime import date, timedelta from typing import Any -from .document import InputBlock +from deepnote_toolkit.notebooks.models import InputBlock _RELATIVE_RANGE_MONTHS = { "pastMonth": 1, diff --git a/tests/unit/test_deepnote_streamlit_client.py b/tests/unit/test_deepnote_streamlit_client.py deleted file mode 100644 index 022d818e..00000000 --- a/tests/unit/test_deepnote_streamlit_client.py +++ /dev/null @@ -1,557 +0,0 @@ -import io -import json -from typing import Any -from unittest.mock import MagicMock, patch -from urllib.error import HTTPError, URLError - -import pytest - -from deepnote_toolkit.streamlit import ( - CurrentUserApiCredentials, - CurrentUserApiTokenError, - DeepnoteCloudRunner, - DeepnoteRunner, - InputBlock, - RunnerError, - RunnerInfo, -) - - -class FakeResponse: - def __init__(self, payload: Any): - self.payload = payload - - def __enter__(self) -> "FakeResponse": - return self - - def __exit__(self, *_: object) -> None: - return None - - def read(self) -> bytes: - return json.dumps(self.payload).encode() - - -def test_info_parses_runner_contract() -> None: - calls = [] - - def open_request(request: Any, *, timeout: float) -> FakeResponse: - calls.append((request.full_url, request.method, timeout)) - return FakeResponse( - { - "notebook": "Revenue", - "runTarget": "cloud", - "inputs": [ - {"variableName": "region", "type": "input-select", "value": "All"} - ], - } - ) - - info = DeepnoteRunner("http://runner/", timeout=12, opener=open_request).info() - - assert calls == [("http://runner/api/info", "GET", 12)] - assert info.notebook == "Revenue" - assert info.run_target == "cloud" - assert info.inputs[0].variable_name == "region" - - -def test_runner_info_requires_matching_input_names_and_types() -> None: - info = RunnerInfo( - notebook="Revenue", - inputs=(InputBlock("region", "input-select", "All"),), - run_target="cloud", - ) - - assert info.accepts_inputs([InputBlock("region", "input-select", "Europe")]) - assert not info.accepts_inputs([InputBlock("market", "input-select", "Europe")]) - assert not info.accepts_inputs([InputBlock("region", "input-text", "Europe")]) - - -def test_run_posts_inputs_and_parses_one_result_shape() -> None: - def open_request(request: Any, *, timeout: float) -> FakeResponse: - assert timeout == 600 - assert request.method == "POST" - assert json.loads(request.data) == {"inputs": {"limit": 20}} - return FakeResponse({"target": "local", "success": True, "outputs": []}) - - result = DeepnoteRunner(opener=open_request).run({"limit": 20}) - - assert result.target == "local" - assert result.success is True - - -def test_http_error_surfaces_runner_message() -> None: - def open_request(*_: Any, **__: Any) -> FakeResponse: - raise HTTPError( - "http://runner/api/run", - 500, - "Server error", - {}, - io.BytesIO(b'{"error":"DEEPNOTE_TOKEN is required"}'), - ) - - with pytest.raises(RunnerError, match="DEEPNOTE_TOKEN is required"): - DeepnoteRunner("http://runner", opener=open_request).run({}) - - -def test_connection_error_names_runner_url() -> None: - def open_request(*_: Any, **__: Any) -> FakeResponse: - raise URLError("connection refused") - - with pytest.raises(RunnerError, match="http://runner"): - DeepnoteRunner("http://runner", opener=open_request).info() - - -def test_timeout_names_runner_url_and_duration() -> None: - def open_request(*_: Any, **__: Any) -> FakeResponse: - raise TimeoutError - - with pytest.raises(RunnerError, match="http://runner.*12 seconds"): - DeepnoteRunner("http://runner", timeout=12, opener=open_request).info() - - -def test_cloud_info_reads_public_notebook_contract() -> None: - def open_request(request: Any, *, timeout: float) -> FakeResponse: - assert request.full_url == "https://api.deepnote.com/v2/notebooks/notebook-1" - assert request.headers["Authorization"] == "Bearer token-1" - assert timeout == 30 - return FakeResponse( - { - "notebook": { - "name": "Revenue", - "inputs": [ - { - "name": "region", - "type": "input-select", - "value": "All", - "label": "Region", - } - ], - } - } - ) - - info = DeepnoteCloudRunner( - "notebook-1", token="token-1", opener=open_request - ).info() - - assert info.notebook == "Revenue" - assert info.run_target == "cloud" - assert info.inputs[0].variable_name == "region" - - -def test_cloud_run_posts_inputs_polls_and_parses_inline_snapshot() -> None: - calls = [] - responses = iter( - [ - {"run": {"runId": "run-1", "status": "pending"}}, - {"run": {"runId": "run-1", "status": "running"}}, - { - "run": { - "runId": "run-1", - "status": "success", - "snapshot": { - "snapshotContent": "project:\n name: Result\n notebooks:\n - blocks: []\n" - }, - } - }, - ] - ) - - def open_request(request: Any, *, timeout: float) -> FakeResponse: - calls.append( - ( - request.full_url, - request.method, - request.headers["Authorization"], - request.data, - timeout, - ) - ) - return FakeResponse(next(responses)) - - tokens = iter(["token-1", "token-2", "token-3"]) - sleeps = [] - result = DeepnoteCloudRunner( - "notebook-1", - token_provider=lambda: next(tokens), - opener=open_request, - sleep=sleeps.append, - poll_interval=0.25, - ).run({"limit": 20, "enabled": True, "regions": ["EU"]}) - - assert json.loads(calls[0][3]) == { - "notebookId": "notebook-1", - "detached": True, - "inputs": {"limit": "20", "enabled": True, "regions": ["EU"]}, - } - assert calls[1][0].endswith("/v2/runs/run-1?snapshotDelivery=inline") - assert [call[2] for call in calls] == [ - "Bearer token-1", - "Bearer token-2", - "Bearer token-3", - ] - assert sleeps == [0.25, 0.25] - assert result.success is True - assert result.snapshot is not None - assert result.snapshot.project_name == "Result" - - -def test_cloud_run_reads_sanitized_snapshot_blocks_without_raw_snapshot() -> None: - responses = iter( - [ - {"run": {"runId": "run-1", "status": "pending"}}, - { - "run": { - "runId": "run-1", - "status": "success", - "snapshotBlocks": [ - { - "id": "code-1", - "type": "code", - "outputs": [ - { - "output_type": "execute_result", - "data": { - "application/vnd.deepnote.dataframe.v3+json": { - "columns": [{"name": "revenue"}], - "rows": [{"revenue": 42}], - } - }, - } - ], - "metadata": {"deepnote_table_state": {}}, - }, - { - "id": "agent-1", - "type": "agent", - "outputs": [ - { - "output_type": "display_data", - "data": {"text/markdown": "**Done**"}, - } - ], - "metadata": {}, - }, - ], - } - }, - ] - ) - - def open_request(request: Any, *, timeout: float) -> FakeResponse: - assert timeout == 30 - if request.method == "POST": - assert json.loads(request.data) == { - "notebookId": "notebook-1", - "detached": True, - "inputs": {"region": "EU"}, - } - return FakeResponse(next(responses)) - - result = DeepnoteCloudRunner( - "notebook-1", - token="token", - opener=open_request, - sleep=lambda _delay: None, - ).run({"region": "EU"}) - - assert result.snapshot is None - assert result.snapshot_yaml is None - assert [output.block_id for output in result.outputs] == ["code-1", "agent-1"] - assert [output.block_type for output in result.outputs] == ["code", "agent"] - dataframe = result.first_dataframe() - assert dataframe is not None - assert dataframe.records() == [{"revenue": 42}] - assert result.agent_text() == "**Done**" - - -def test_cloud_run_surfaces_terminal_error() -> None: - def open_request(_request: Any, *, timeout: float) -> FakeResponse: - assert timeout == 30 - return FakeResponse( - { - "run": { - "id": "run-1", - "status": "error", - "error": {"message": "bad input"}, - } - } - ) - - result = DeepnoteCloudRunner( - "notebook-1", token="token", opener=open_request, sleep=lambda _delay: None - ).run({}) - - assert result.success is False - assert result.error == "bad input" - - -def test_hosted_cloud_runner_exchanges_per_request_and_uses_api_origin( - monkeypatch: pytest.MonkeyPatch, -) -> None: - monkeypatch.setenv("DEEPNOTE_TOKEN", "must-not-be-used") - api_calls = [] - responses = iter( - [ - {"run": {"runId": "run-1", "status": "pending"}}, - {"run": {"runId": "run-1", "status": "success", "snapshotBlocks": []}}, - ] - ) - - def open_request(request: Any, *, timeout: float) -> FakeResponse: - api_calls.append( - ( - request.full_url, - request.headers["Authorization"], - timeout, - ) - ) - return FakeResponse(next(responses)) - - credentials = [ - CurrentUserApiCredentials( - token="viewer-token-1", - api_origin="https://api.deepnote-staging.com", - expires_at_seconds=1_800_000_000, - ), - CurrentUserApiCredentials( - token="viewer-token-2", - api_origin="https://api.deepnote-staging.com", - expires_at_seconds=1_800_000_001, - ), - ] - with ( - patch( - "deepnote_toolkit.streamlit.client._has_hosted_streamlit_context", - return_value=True, - ), - patch( - "deepnote_toolkit.streamlit.client.current_user_api_credentials", - side_effect=credentials, - ) as exchange, - ): - result = DeepnoteCloudRunner( - "notebook-1", - opener=open_request, - sleep=lambda _delay: None, - ).run({}) - - assert result.success is True - assert exchange.call_count == 2 - assert api_calls == [ - ( - "https://api.deepnote-staging.com/v2/runs", - "Bearer viewer-token-1", - 30, - ), - ( - "https://api.deepnote-staging.com/v2/runs/run-1" "?snapshotDelivery=inline", - "Bearer viewer-token-2", - 30, - ), - ] - - -def test_hosted_runner_never_falls_back_to_environment_token( - monkeypatch: pytest.MonkeyPatch, -) -> None: - monkeypatch.setenv("DEEPNOTE_TOKEN", "shared-token") - opener = MagicMock() - with ( - patch( - "deepnote_toolkit.streamlit.client._has_hosted_streamlit_context", - return_value=True, - ), - patch( - "deepnote_toolkit.streamlit.client.current_user_api_credentials", - side_effect=CurrentUserApiTokenError("viewer token unavailable"), - ), - pytest.raises(RunnerError, match="viewer token unavailable"), - ): - DeepnoteCloudRunner("notebook-1", opener=opener).info() - - opener.assert_not_called() - - -def test_local_cloud_runner_uses_environment_token( - monkeypatch: pytest.MonkeyPatch, -) -> None: - monkeypatch.setenv("DEEPNOTE_TOKEN", "local-token") - - def open_request(request: Any, *, timeout: float) -> FakeResponse: - assert request.headers["Authorization"] == "Bearer local-token" - assert timeout == 30 - return FakeResponse({"notebook": {"name": "Revenue", "inputs": []}}) - - with patch( - "deepnote_toolkit.streamlit.client._has_hosted_streamlit_context", - return_value=False, - ): - info = DeepnoteCloudRunner("notebook-1", opener=open_request).info() - - assert info.notebook == "Revenue" - - -def test_cloud_runner_requires_one_token_source( - monkeypatch: pytest.MonkeyPatch, -) -> None: - with pytest.raises(ValueError, match="not both"): - DeepnoteCloudRunner("notebook-1", token="token", token_provider=lambda: "other") - - monkeypatch.delenv("DEEPNOTE_TOKEN", raising=False) - with pytest.raises(RunnerError, match="token is required"): - DeepnoteCloudRunner( - "notebook-1", - token="", - opener=lambda *_args, **_kwargs: FakeResponse({}), - ).info() - - -def test_cloud_run_retries_transient_poll_failures() -> None: - responses = iter( - [ - {"run": {"runId": "run-1", "status": "pending"}}, - HTTPError("http://api", 503, "Unavailable", {}, io.BytesIO(b"{}")), - URLError("connection reset"), - {"run": {"runId": "run-1", "status": "success", "snapshotBlocks": []}}, - ] - ) - - def open_request(_request: Any, *, timeout: float) -> FakeResponse: - response = next(responses) - if isinstance(response, Exception): - raise response - return FakeResponse(response) - - result = DeepnoteCloudRunner( - "notebook-1", token="token", opener=open_request, sleep=lambda _delay: None - ).run({}) - - assert result.success is True - - -def test_cloud_run_raises_poll_failures_that_are_not_transient() -> None: - responses = iter( - [ - {"run": {"runId": "run-1", "status": "pending"}}, - HTTPError("http://api", 403, "Forbidden", {}, io.BytesIO(b"{}")), - ] - ) - - def open_request(_request: Any, *, timeout: float) -> FakeResponse: - response = next(responses) - if isinstance(response, Exception): - raise response - return FakeResponse(response) - - with pytest.raises(RunnerError, match="HTTP 403"): - DeepnoteCloudRunner( - "notebook-1", token="token", opener=open_request, sleep=lambda _delay: None - ).run({}) - - -def test_cloud_run_stops_retrying_after_repeated_transient_failures() -> None: - calls = [] - - def open_request(request: Any, *, timeout: float) -> FakeResponse: - calls.append(request.method) - if request.method == "POST": - return FakeResponse({"run": {"runId": "run-1", "status": "pending"}}) - raise URLError("connection reset") - - with pytest.raises(RunnerError, match="connection reset"): - DeepnoteCloudRunner( - "notebook-1", token="token", opener=open_request, sleep=lambda _delay: None - ).run({}) - - assert calls == ["POST"] + ["GET"] * 6 - - -def test_cloud_run_waits_for_a_snapshot_that_lags_the_terminal_status() -> None: - responses = iter( - [ - {"run": {"runId": "run-1", "status": "success"}}, - {"run": {"runId": "run-1", "status": "success"}}, - { - "run": { - "runId": "run-1", - "status": "success", - "snapshotBlocks": [ - { - "id": "code-1", - "type": "code", - "outputs": [{"output_type": "stream", "text": "done"}], - } - ], - } - }, - ] - ) - sleeps = [] - - result = DeepnoteCloudRunner( - "notebook-1", - token="token", - opener=lambda _request, *, timeout: FakeResponse(next(responses)), - sleep=sleeps.append, - poll_interval=0.5, - ).run({}) - - assert sleeps == [0.5, 0.5] - assert result.text() == "done" - - -def test_worker_thread_never_falls_back_to_environment_token( - monkeypatch: pytest.MonkeyPatch, -) -> None: - monkeypatch.setenv("DEEPNOTE_TOKEN", "shared-token") - opener = MagicMock() - with ( - patch( - "deepnote_toolkit.streamlit.client._has_hosted_streamlit_context", - return_value=False, - ), - patch( - "deepnote_toolkit.streamlit.client._is_streamlit_thread_without_request", - return_value=True, - ), - pytest.raises(RunnerError, match="No viewer request"), - ): - DeepnoteCloudRunner("notebook-1", opener=opener).info() - - opener.assert_not_called() - - -def test_cloud_run_retries_a_transient_token_exchange_failure() -> None: - responses = iter( - [ - {"run": {"runId": "run-1", "status": "pending"}}, - {"run": {"runId": "run-1", "status": "success", "snapshotBlocks": []}}, - ] - ) - credentials = CurrentUserApiCredentials( - token="viewer-token", - api_origin="https://api.deepnote.com", - expires_at_seconds=1_800_000_000, - ) - with ( - patch( - "deepnote_toolkit.streamlit.client._has_hosted_streamlit_context", - return_value=True, - ), - patch( - "deepnote_toolkit.streamlit.client.current_user_api_credentials", - side_effect=[ - credentials, - CurrentUserApiTokenError("exchange timed out", transient=True), - credentials, - ], - ), - ): - result = DeepnoteCloudRunner( - "notebook-1", - opener=lambda _request, *, timeout: FakeResponse(next(responses)), - sleep=lambda _delay: None, - ).run({}) - - assert result.success is True diff --git a/tests/unit/test_deepnote_streamlit_widgets.py b/tests/unit/test_deepnote_streamlit_widgets.py index 349c37e9..22ddb49e 100644 --- a/tests/unit/test_deepnote_streamlit_widgets.py +++ b/tests/unit/test_deepnote_streamlit_widgets.py @@ -1,7 +1,8 @@ from datetime import date, timedelta from typing import Any -from deepnote_toolkit.streamlit import InputBlock, render_inputs +from deepnote_toolkit.notebooks import InputBlock +from deepnote_toolkit.streamlit import render_inputs class FakeContainer: diff --git a/tests/unit/test_deepnote_streamlit_document.py b/tests/unit/test_notebooks_document.py similarity index 99% rename from tests/unit/test_deepnote_streamlit_document.py rename to tests/unit/test_notebooks_document.py index 78b9be1f..00e4cbcf 100644 --- a/tests/unit/test_deepnote_streamlit_document.py +++ b/tests/unit/test_notebooks_document.py @@ -2,7 +2,7 @@ import pytest -from deepnote_toolkit.streamlit import ( +from deepnote_toolkit.notebooks import ( DATAFRAME_MIME, DeepnoteDocument, InputBlock, From d2d3b8c0cf52df9469fe088ae15ae2b4c4e4576b Mon Sep 17 00:00:00 2001 From: Wojciech Apanowicz Date: Fri, 18 Sep 2026 14:44:07 +0200 Subject: [PATCH 14/43] docs(streamlit): drop the CLI steps and the auth internals from the app guide --- docs/streamlit-apps.md | 112 ++++++++++++++++++----------------------- 1 file changed, 49 insertions(+), 63 deletions(-) diff --git a/docs/streamlit-apps.md b/docs/streamlit-apps.md index f21a81f8..f92ecdce 100644 --- a/docs/streamlit-apps.md +++ b/docs/streamlit-apps.md @@ -7,87 +7,73 @@ Deepnote Toolkit provides a small typed layer for custom Streamlit apps backed b from pathlib import Path import streamlit as st -from deepnote_toolkit.streamlit import ( - DeepnoteCloudRunner, - DeepnoteDocument, - render_inputs, -) +from deepnote_toolkit.notebooks import DeepnoteDocument +from deepnote_toolkit.streamlit import StreamlitCloudRunner, render_inputs document = DeepnoteDocument.load(Path("report.deepnote")) values = render_inputs(document.inputs, st.sidebar) if st.button("Run"): - result = DeepnoteCloudRunner("your-notebook-id").run(values) + result = StreamlitCloudRunner("your-notebook-id").run(values) st.dataframe(result.first_dataframe().records()) ``` -`DeepnoteDocument` reads typed input definitions and structured notebook outputs. -In a project with several notebooks, pass the notebook you run so the inputs match -it: `DeepnoteDocument.load(path, notebook_id="your-notebook-id")`. -`render_inputs` maps Deepnote input blocks to native Streamlit widgets. -`DeepnoteCloudRunner` calls the same public notebooks and runs API used by the -Deepnote CLI. `DeepnoteRunner` is available for the local-runner sidecar. - -## Authentication modes - -A hosted Deepnote Streamlit app needs no token configuration. The cloud runner: - -1. reads the current viewer's opaque `streamlit-token` cookie; -2. resolves the app ID from `x-original-host`, falling back to `host`; -3. exchanges the cookie through the project's userpod API at - `POST /userpod-api/streamlit-apps/{appId}/api-token`; and -4. calls the returned `apiOrigin` with the short-lived token as a bearer. - -Cloud runs explicitly request `detached: true`, keeping viewer-triggered work out -of the shared project session. Hosted app tokens receive sanitized -`snapshotBlocks` containing the executed notebook's outputs, not the raw -project snapshot. API-key clients remain compatible with inline -`snapshotContent` responses. - -The opaque cookie is never sent to the public API. The exchanged credentials are kept -in the viewer's own Streamlit session state and reused until a minute before they -expire. They are never kept in process globals or shared between sessions, and a -hosted request never falls back to a shared environment token. Call the runner -from the Streamlit script thread: a worker thread has no viewer request, so the -runner raises instead of using `DEEPNOTE_TOKEN`. Deepnote rechecks -the viewer's access on every API request, so a reused bearer stops working as soon -as access is revoked. - -The exchange endpoint must return `token`, `apiOrigin`, and -`expiresAtSeconds`. The request goes through the same userpod API route as the -Toolkit's other webapp calls, so a hosted app needs no extra configuration. - -For another public API client, use both values returned by -`current_user_api_credentials()`. `current_user_api_token()` is a token-provider -convenience for clients whose API origin is configured separately. - -For local development, pass a user's API token explicitly or set -`DEEPNOTE_TOKEN`: +## Two packages -```python -runner = DeepnoteCloudRunner("your-notebook-id", token="your-api-token") -``` +`deepnote_toolkit.notebooks` has no Streamlit dependency and works in any Python +program: -A callable `token_provider=` can supply a renewable token. It is invoked for every -request. `DeepnoteRunner` can instead call a local `@deepnote/local-runner` -sidecar at `http://127.0.0.1:8787`. +- `DeepnoteDocument` reads typed input definitions and structured outputs from a + `.deepnote` source or snapshot file. In a project with several notebooks, pass + the notebook you run so the inputs match it: + `DeepnoteDocument.load(path, notebook_id="your-notebook-id")`. +- `DeepnoteCloudRunner` runs an existing notebook in Deepnote Cloud and returns + its outputs as a `RunResult`. +- `DeepnoteRunner` does the same through a local `@deepnote/local-runner` sidecar + at `http://127.0.0.1:8787`. +- `Runner` is the interface both runners implement, for code that accepts either. + +`deepnote_toolkit.streamlit` holds the Streamlit-specific parts: -Static apps only load a committed snapshot with `DeepnoteDocument`; they require +- `render_inputs` maps Deepnote input blocks to native Streamlit widgets and + returns values ready to submit to a runner. +- `StreamlitCloudRunner` is a `DeepnoteCloudRunner` that runs notebooks as the + person viewing the app when Deepnote hosts it. + +A static app only loads a committed snapshot with `DeepnoteDocument`. It requires no token or network access. -## Synchronize at deployment +## Authentication + +A Streamlit app hosted by Deepnote needs no token configuration. +`StreamlitCloudRunner` runs the notebook as the current viewer, with that viewer's +access, and never as the app's owner. A viewer who loses access to the project +can no longer run it. -Runtime requests only read and run the existing cloud notebook. Synchronize source -in an explicit deployment step: +Call the runner from the Streamlit script thread. A worker thread has no viewer +request, so the runner raises there instead of using `DEEPNOTE_TOKEN`. -```bash -deepnote run report.deepnote --cloud --notebook-id "$DEEPNOTE_NOTEBOOK_ID" --push --dry-run -deepnote run report.deepnote --cloud --notebook-id "$DEEPNOTE_NOTEBOOK_ID" --push --yes +For local development, pass an API token explicitly or set `DEEPNOTE_TOKEN`: + +```python +runner = StreamlitCloudRunner("your-notebook-id", token="your-api-token") ``` -Use `RunnerInfo.accepts_inputs(document.inputs)` before submitting values to -verify that the deployed notebook still has matching input names and block types. +A callable `token_provider=` can supply a renewable token. It is invoked for every +request. + +For another Deepnote API client inside a hosted app, +`current_user_api_credentials()` returns a short-lived token for the current +viewer together with the API origin to send it to. + +## Runs + +Cloud runs are detached, which keeps viewer-triggered work out of the shared +project session. The cloud runner retries a poll that fails with a timeout, a network error, HTTP 429 or a 5xx, up to five times in a row. After the run finishes it waits briefly for the outputs, which can arrive after the final status. + +Use `runner.info().accepts_inputs(document.inputs)` before submitting values to +verify that the deployed notebook still has matching input names and block types. From 400f844c636933f83c0063f75dc002ed69b5ab90 Mon Sep 17 00:00:00 2001 From: Wojciech Apanowicz Date: Fri, 18 Sep 2026 14:44:25 +0200 Subject: [PATCH 15/43] refactor(streamlit): add the notebooks package files missing from the previous commit --- deepnote_toolkit/notebooks/__init__.py | 32 ++ deepnote_toolkit/notebooks/cloud_runner.py | 223 ++++++++++ deepnote_toolkit/notebooks/document.py | 90 ++++ deepnote_toolkit/notebooks/http.py | 54 +++ deepnote_toolkit/notebooks/local_runner.py | 65 +++ deepnote_toolkit/notebooks/models.py | 192 +++++++++ deepnote_toolkit/notebooks/outputs.py | 41 ++ deepnote_toolkit/notebooks/run_result.py | 83 ++++ deepnote_toolkit/notebooks/runner.py | 27 ++ deepnote_toolkit/streamlit/cloud_runner.py | 50 +++ .../test_deepnote_streamlit_cloud_runner.py | 193 +++++++++ tests/unit/test_notebooks_runners.py | 407 ++++++++++++++++++ 12 files changed, 1457 insertions(+) create mode 100644 deepnote_toolkit/notebooks/__init__.py create mode 100644 deepnote_toolkit/notebooks/cloud_runner.py create mode 100644 deepnote_toolkit/notebooks/document.py create mode 100644 deepnote_toolkit/notebooks/http.py create mode 100644 deepnote_toolkit/notebooks/local_runner.py create mode 100644 deepnote_toolkit/notebooks/models.py create mode 100644 deepnote_toolkit/notebooks/outputs.py create mode 100644 deepnote_toolkit/notebooks/run_result.py create mode 100644 deepnote_toolkit/notebooks/runner.py create mode 100644 deepnote_toolkit/streamlit/cloud_runner.py create mode 100644 tests/unit/test_deepnote_streamlit_cloud_runner.py create mode 100644 tests/unit/test_notebooks_runners.py diff --git a/deepnote_toolkit/notebooks/__init__.py b/deepnote_toolkit/notebooks/__init__.py new file mode 100644 index 00000000..b206508a --- /dev/null +++ b/deepnote_toolkit/notebooks/__init__.py @@ -0,0 +1,32 @@ +"""Read `.deepnote` files and run notebooks, independent of any UI framework.""" + +from .cloud_runner import DeepnoteCloudRunner +from .document import DeepnoteDocument +from .local_runner import DeepnoteRunner +from .models import ( + DATAFRAME_MIME, + INDEX_COLUMN, + DeepnoteDataframe, + InputBlock, + NotebookOutput, + RunnerInfo, + join_text, +) +from .run_result import RunResult +from .runner import Runner, RunnerError + +__all__ = [ + "DATAFRAME_MIME", + "INDEX_COLUMN", + "DeepnoteCloudRunner", + "DeepnoteDataframe", + "DeepnoteDocument", + "DeepnoteRunner", + "InputBlock", + "NotebookOutput", + "RunResult", + "Runner", + "RunnerError", + "RunnerInfo", + "join_text", +] diff --git a/deepnote_toolkit/notebooks/cloud_runner.py b/deepnote_toolkit/notebooks/cloud_runner.py new file mode 100644 index 00000000..3cda1364 --- /dev/null +++ b/deepnote_toolkit/notebooks/cloud_runner.py @@ -0,0 +1,223 @@ +"""Run a notebook in Deepnote Cloud through the public API.""" + +from __future__ import annotations + +import json +import os +import time +from collections.abc import Callable, Mapping +from typing import Any +from urllib.request import Request, urlopen + +from .http import OpenUrl, request_json +from .models import InputBlock, RunnerInfo +from .run_result import RunResult +from .runner import RunnerError + +TokenProvider = Callable[[], str] +Sleep = Callable[[float], None] + +TERMINAL_RUN_STATUSES = frozenset({"success", "error", "internal_error", "stopped"}) +DEFAULT_API_ORIGIN = "https://api.deepnote.com" +MAX_TRANSIENT_POLL_FAILURES = 5 +SNAPSHOT_SETTLE_ATTEMPTS = 3 + + +class DeepnoteCloudRunner: + """Run an existing notebook directly through the Deepnote public API. + + The token comes from `token`, `token_provider` or the `DEEPNOTE_TOKEN` + environment variable. A token provider is called for every request, which lets + a long-lived process use short-lived credentials. + """ + + def __init__( + self, + notebook_id: str, + *, + token: str | None = None, + token_provider: TokenProvider | None = None, + base_url: str = DEFAULT_API_ORIGIN, + timeout: float = 600, + poll_interval: float = 2, + opener: OpenUrl = urlopen, + sleep: Sleep = time.sleep, + ): + if not notebook_id: + raise ValueError("notebook_id is required") + if token is not None and token_provider is not None: + raise ValueError("Pass token or token_provider, not both") + self.notebook_id = notebook_id + self.base_url = base_url.rstrip("/") + self.timeout = timeout + self.poll_interval = poll_interval + self._static_token = token + self._token_provider = token_provider + self._open = opener + self._sleep = sleep + + def info(self) -> RunnerInfo: + payload = self._request("GET", f"/v2/notebooks/{self.notebook_id}") + notebook = payload.get("notebook") + if not isinstance(notebook, Mapping): + raise RunnerError("Deepnote API response did not include a notebook") + raw_inputs = notebook.get("inputs") + inputs = tuple( + InputBlock.from_api( + { + "variableName": value.get("name"), + "type": value.get("type"), + "value": value.get("value"), + "label": value.get("label"), + } + ) + for value in raw_inputs or [] + if isinstance(value, Mapping) and isinstance(value.get("name"), str) + ) + return RunnerInfo( + notebook=str(notebook.get("name", "Untitled notebook")), + inputs=inputs, + run_target="cloud", + ) + + def run(self, inputs: Mapping[str, Any]) -> RunResult: + started = self._run_payload( + self._request( + "POST", + "/v2/runs", + { + "notebookId": self.notebook_id, + "detached": True, + "inputs": _normalize_cloud_inputs(inputs), + }, + ) + ) + run_id = _required_run_id(started) + deadline = time.monotonic() + self.timeout + current = started + transient_failures = 0 + while str(current.get("status", "")) not in TERMINAL_RUN_STATUSES: + if time.monotonic() >= deadline: + raise RunnerError( + f"Deepnote run {run_id} did not finish within {self.timeout:g} seconds" + ) + self._sleep(self.poll_interval) + try: + current = self._get_run(run_id) + transient_failures = 0 + except RunnerError as error: + transient_failures += 1 + if ( + not error.transient + or transient_failures > MAX_TRANSIENT_POLL_FAILURES + ): + raise + + # The snapshot can attach shortly after the status turns terminal. + for _ in range(SNAPSHOT_SETTLE_ATTEMPTS): + if _has_snapshot(current): + break + self._sleep(self.poll_interval) + try: + current = self._get_run(run_id) + except RunnerError as error: + if not error.transient: + raise + + status = str(current.get("status", "")) + snapshot = current.get("snapshot") + snapshot_yaml = current.get("snapshotContent") + if snapshot_yaml is None and isinstance(snapshot, Mapping): + snapshot_yaml = snapshot.get("snapshotContent") + error = current.get("error") + if isinstance(error, Mapping): + error = error.get("message") or json.dumps(error) + return RunResult( + { + "target": "cloud", + "success": status == "success", + "runId": run_id, + "status": status, + "error": str(error) if error is not None else None, + "snapshotYaml": snapshot_yaml, + "snapshotBlocks": current.get("snapshotBlocks"), + "viewUrl": current.get("viewUrl"), + } + ) + + def _get_run(self, run_id: str) -> Mapping[str, Any]: + return self._run_payload( + self._request("GET", f"/v2/runs/{run_id}?snapshotDelivery=inline") + ) + + def _request( + self, method: str, path: str, body: Mapping[str, Any] | None = None + ) -> Mapping[str, Any]: + token, api_origin = self._credentials() + request = Request( + f"{api_origin}{path}", + data=json.dumps(body).encode() if body is not None else None, + method=method, + headers={ + "Authorization": f"Bearer {token}", + "Content-Type": "application/json", + "Accept": "application/json", + }, + ) + return request_json( + self._open, + request, + timeout=min(self.timeout, 30), + service="Deepnote API", + origin=api_origin, + ) + + def _credentials(self) -> tuple[str, str]: + """Return the bearer token and the API origin to send it to.""" + + if self._token_provider is not None: + return self._required_token(self._token_provider()), self.base_url + if self._static_token is not None: + return self._required_token(self._static_token), self.base_url + return self._required_token(os.environ.get("DEEPNOTE_TOKEN")), self.base_url + + @staticmethod + def _required_token(token: str | None) -> str: + if not token: + raise RunnerError("A Deepnote API token is required") + return token + + @staticmethod + def _run_payload(payload: Mapping[str, Any]) -> Mapping[str, Any]: + run = payload.get("run") + return run if isinstance(run, Mapping) else payload + + +def _has_snapshot(run: Mapping[str, Any]) -> bool: + snapshot = run.get("snapshot") + return bool( + run.get("snapshotContent") + or isinstance(run.get("snapshotBlocks"), list) + or (isinstance(snapshot, Mapping) and snapshot.get("snapshotContent")) + ) + + +def _required_run_id(run: Mapping[str, Any]) -> str: + run_id = run.get("runId") or run.get("id") + if not isinstance(run_id, str) or not run_id: + raise RunnerError("Deepnote API response did not include a run id") + return run_id + + +def _normalize_cloud_inputs( + inputs: Mapping[str, Any], +) -> dict[str, str | bool | list[str]]: + normalized: dict[str, str | bool | list[str]] = {} + for name, value in inputs.items(): + if isinstance(value, bool): + normalized[name] = value + elif isinstance(value, list): + normalized[name] = [str(item) for item in value] + else: + normalized[name] = str(value) + return normalized diff --git a/deepnote_toolkit/notebooks/document.py b/deepnote_toolkit/notebooks/document.py new file mode 100644 index 00000000..3d0c6b5e --- /dev/null +++ b/deepnote_toolkit/notebooks/document.py @@ -0,0 +1,90 @@ +"""Read `.deepnote` source and snapshot files.""" + +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from pathlib import Path +from typing import Any + +import yaml + +from .models import InputBlock, NotebookOutput, optional_string +from .outputs import OutputCollection + + +class DeepnoteDocument(OutputCollection): + """A parsed source or snapshot `.deepnote` file. + + `notebook_id` limits the inputs and outputs to one notebook of the project. + """ + + def __init__(self, raw: Mapping[str, Any], *, notebook_id: str | None = None): + project = raw.get("project") + if not isinstance(project, Mapping) or not isinstance( + project.get("notebooks"), list + ): + raise ValueError("Expected a .deepnote document with project.notebooks") + notebooks = project["notebooks"] + if notebook_id is not None: + notebooks = [ + notebook + for notebook in notebooks + if isinstance(notebook, Mapping) and notebook.get("id") == notebook_id + ] + if not notebooks: + raise ValueError(f"Notebook {notebook_id} is not in this document") + self.raw = raw + self.project_name = str(project.get("name", "Untitled project")) + self.inputs, self.outputs = _read_blocks(notebooks) + + @classmethod + def load( + cls, path: str | Path, *, notebook_id: str | None = None + ) -> DeepnoteDocument: + source = Path(path) + try: + raw = yaml.safe_load(source.read_text(encoding="utf-8")) + except yaml.YAMLError as error: + raise ValueError(f"Could not parse {source}: {error}") from error + if not isinstance(raw, Mapping): + raise ValueError(f"Expected {source} to contain a YAML object") + return cls(raw, notebook_id=notebook_id) + + @classmethod + def parse(cls, content: str, *, notebook_id: str | None = None) -> DeepnoteDocument: + try: + raw = yaml.safe_load(content) + except yaml.YAMLError as error: + raise ValueError(f"Could not parse .deepnote YAML: {error}") from error + if not isinstance(raw, Mapping): + raise ValueError("Expected .deepnote YAML to contain an object") + return cls(raw, notebook_id=notebook_id) + + +def _read_blocks( + notebooks: Sequence[Any], +) -> tuple[tuple[InputBlock, ...], tuple[NotebookOutput, ...]]: + inputs: list[InputBlock] = [] + outputs: list[NotebookOutput] = [] + for notebook in notebooks: + if not isinstance(notebook, Mapping): + continue + blocks = notebook.get("blocks") + if not isinstance(blocks, list): + continue + for block in blocks: + if not isinstance(block, Mapping): + continue + if input_block := InputBlock.from_block(block): + inputs.append(input_block) + block_outputs = block.get("outputs") + if not isinstance(block_outputs, list): + continue + block_id = str(block.get("id", "")) + block_type = optional_string(block.get("type")) + outputs.extend( + NotebookOutput(block_id=block_id, block_type=block_type, raw=output) + for output in block_outputs + if isinstance(output, Mapping) + ) + return tuple(inputs), tuple(outputs) diff --git a/deepnote_toolkit/notebooks/http.py b/deepnote_toolkit/notebooks/http.py new file mode 100644 index 00000000..c96f8908 --- /dev/null +++ b/deepnote_toolkit/notebooks/http.py @@ -0,0 +1,54 @@ +"""JSON requests shared by the notebook runners.""" + +from __future__ import annotations + +import json +from collections.abc import Callable, Mapping +from typing import Any +from urllib.error import HTTPError, URLError +from urllib.request import Request + +from .runner import RunnerError + +OpenUrl = Callable[..., Any] + + +def request_json( + opener: OpenUrl, request: Request, *, timeout: float, service: str, origin: str +) -> Mapping[str, Any]: + """Send a request and return its JSON object, raising `RunnerError` otherwise.""" + + try: + with opener(request, timeout=timeout) as response: + payload = json.loads(response.read()) + except HTTPError as error: + raise RunnerError( + f"The {service} returned HTTP {error.code}: {_error_message(error)}", + transient=error.code == 429 or error.code >= 500, + ) from error + except URLError as error: + raise RunnerError( + f"Could not reach the {service} at {origin}: {error.reason}", + transient=True, + ) from error + except TimeoutError as error: + raise RunnerError( + f"The {service} at {origin} timed out after {timeout:g} seconds", + transient=True, + ) from error + except (json.JSONDecodeError, UnicodeDecodeError) as error: + raise RunnerError(f"The {service} returned an invalid JSON response") from error + if not isinstance(payload, Mapping): + raise RunnerError(f"The {service} returned a non-object response") + return payload + + +def _error_message(error: HTTPError) -> str: + detail = error.read().decode(errors="replace") + try: + parsed = json.loads(detail) + except json.JSONDecodeError: + return detail + if not isinstance(parsed, Mapping): + return detail + return str(parsed.get("message") or parsed.get("error") or detail) diff --git a/deepnote_toolkit/notebooks/local_runner.py b/deepnote_toolkit/notebooks/local_runner.py new file mode 100644 index 00000000..add290ac --- /dev/null +++ b/deepnote_toolkit/notebooks/local_runner.py @@ -0,0 +1,65 @@ +"""Run a notebook through a local `@deepnote/local-runner` sidecar.""" + +from __future__ import annotations + +import json +from collections.abc import Mapping +from typing import Any +from urllib.request import Request, urlopen + +from .http import OpenUrl, request_json +from .models import InputBlock, RunnerInfo +from .run_result import RunResult + + +class DeepnoteRunner: + """One client for a runner configured for Deepnote Cloud or a local kernel.""" + + def __init__( + self, + base_url: str = "http://127.0.0.1:8787", + *, + timeout: float = 600, + opener: OpenUrl = urlopen, + ): + self.base_url = base_url.rstrip("/") + self.timeout = timeout + self._open = opener + + def info(self) -> RunnerInfo: + payload = self._request("GET", "/api/info") + values = payload.get("inputs") + inputs = ( + tuple( + InputBlock.from_api(value) + for value in values + if isinstance(value, Mapping) + ) + if isinstance(values, list) + else () + ) + return RunnerInfo( + notebook=str(payload.get("notebook", "Untitled project")), + inputs=inputs, + run_target=str(payload.get("runTarget", "")), + ) + + def run(self, inputs: Mapping[str, Any]) -> RunResult: + return RunResult(self._request("POST", "/api/run", {"inputs": dict(inputs)})) + + def _request( + self, method: str, path: str, body: Mapping[str, Any] | None = None + ) -> Mapping[str, Any]: + request = Request( + f"{self.base_url}{path}", + data=json.dumps(body).encode() if body is not None else None, + method=method, + headers={"Content-Type": "application/json", "Accept": "application/json"}, + ) + return request_json( + self._open, + request, + timeout=self.timeout, + service="Deepnote runner", + origin=self.base_url, + ) diff --git a/deepnote_toolkit/notebooks/models.py b/deepnote_toolkit/notebooks/models.py new file mode 100644 index 00000000..bf70a942 --- /dev/null +++ b/deepnote_toolkit/notebooks/models.py @@ -0,0 +1,192 @@ +"""Typed models for Deepnote input blocks, outputs and runner metadata.""" + +from __future__ import annotations + +import base64 +import binascii +from collections.abc import Iterable, Mapping +from dataclasses import dataclass +from typing import Any + +DATAFRAME_MIME = "application/vnd.deepnote.dataframe.v3+json" +INDEX_COLUMN = "_deepnote_index_column" + + +def join_text(value: Any) -> str: + """Normalize nbformat's string-or-list text values to one string.""" + + if isinstance(value, list): + return "".join(str(part) for part in value) + return "" if value is None else str(value) + + +@dataclass(frozen=True) +class InputBlock: + """The metadata a UI needs to render one Deepnote input block.""" + + variable_name: str + type: str + value: Any + label: str | None = None + options: tuple[str, ...] = () + multiple: bool = False + min: float | int | None = None + max: float | int | None = None + step: float | int | None = None + + @classmethod + def from_block(cls, block: Mapping[str, Any]) -> InputBlock | None: + block_type = str(block.get("type", "")) + metadata = block.get("metadata") + if not block_type.startswith("input-") or not isinstance(metadata, Mapping): + return None + variable_name = metadata.get("deepnote_variable_name") + if not isinstance(variable_name, str) or not variable_name: + return None + options = metadata.get("deepnote_variable_options") + return cls( + variable_name=variable_name, + type=block_type, + label=optional_string(metadata.get("deepnote_input_label")), + value=metadata.get("deepnote_variable_value"), + options=( + tuple(str(option) for option in options) + if isinstance(options, list) + else () + ), + multiple=metadata.get("deepnote_allow_multiple_values") is True, + min=optional_number(metadata.get("deepnote_slider_min_value")), + max=optional_number(metadata.get("deepnote_slider_max_value")), + step=optional_number(metadata.get("deepnote_slider_step")), + ) + + @classmethod + def from_api(cls, value: Mapping[str, Any]) -> InputBlock: + """Read the camelCase shape returned by `GET /api/info`.""" + + options = value.get("options") + return cls( + variable_name=str(value["variableName"]), + type=str(value["type"]), + label=optional_string(value.get("label")), + value=value.get("value"), + options=( + tuple(str(option) for option in options) + if isinstance(options, list) + else () + ), + multiple=value.get("multiple") is True, + min=optional_number(value.get("min")), + max=optional_number(value.get("max")), + step=optional_number(value.get("step")), + ) + + +@dataclass(frozen=True) +class DeepnoteDataframe: + """A structured Deepnote dataframe output, independent of pandas.""" + + columns: tuple[Mapping[str, Any], ...] + rows: tuple[Mapping[str, Any], ...] + raw: Mapping[str, Any] + + @classmethod + def from_value(cls, value: Any) -> DeepnoteDataframe | None: + if not isinstance(value, Mapping): + return None + columns = value.get("columns") + rows = value.get("rows") + if not isinstance(columns, list) or not isinstance(rows, list): + return None + if not all(isinstance(column, Mapping) for column in columns): + return None + if not all(isinstance(row, Mapping) for row in rows): + return None + return cls(columns=tuple(columns), rows=tuple(rows), raw=value) + + @property + def data_columns(self) -> tuple[str, ...]: + return tuple( + str(column.get("name")) + for column in self.columns + if column.get("name") not in (None, INDEX_COLUMN) + ) + + def records(self, *, include_index: bool = True) -> list[dict[str, Any]]: + """Return rows ready for `st.dataframe`, optionally omitting Deepnote's index column.""" + + if include_index: + return [dict(row) for row in self.rows] + return [ + {key: value for key, value in row.items() if key != INDEX_COLUMN} + for row in self.rows + ] + + +@dataclass(frozen=True) +class NotebookOutput: + """One nbformat-compatible output emitted by a Deepnote block.""" + + block_id: str + block_type: str | None + raw: Mapping[str, Any] + + @property + def output_type(self) -> str: + return str(self.raw.get("output_type", "")) + + @property + def data(self) -> Mapping[str, Any]: + value = self.raw.get("data") + return value if isinstance(value, Mapping) else {} + + def text(self, mime: str = "text/plain") -> str: + if self.output_type == "stream" and mime == "text/plain": + return join_text(self.raw.get("text")) + return join_text(self.data.get(mime)) + + def image_bytes(self, mime: str = "image/png") -> bytes | None: + value = self.data.get(mime) + if value is None: + return None + encoded = "".join(join_text(value).split()) + try: + return base64.b64decode(encoded, validate=True) + except (ValueError, binascii.Error): + return None + + @property + def dataframe(self) -> DeepnoteDataframe | None: + return DeepnoteDataframe.from_value(self.data.get(DATAFRAME_MIME)) + + +@dataclass(frozen=True) +class RunnerInfo: + """The target and input contract exposed by a Deepnote runner.""" + + notebook: str + inputs: tuple[InputBlock, ...] + run_target: str + + def accepts_inputs(self, inputs: Iterable[InputBlock]) -> bool: + """Return whether input variable names and block types match this runner.""" + + return _input_contract(inputs) == _input_contract(self.inputs) + + +def _input_contract(inputs: Iterable[InputBlock]) -> tuple[tuple[str, str], ...]: + return tuple( + sorted((input_block.variable_name, input_block.type) for input_block in inputs) + ) + + +def optional_string(value: Any) -> str | None: + return value if isinstance(value, str) else None + + +def optional_number(value: Any) -> float | int | None: + return ( + value + if isinstance(value, (float, int)) and not isinstance(value, bool) + else None + ) diff --git a/deepnote_toolkit/notebooks/outputs.py b/deepnote_toolkit/notebooks/outputs.py new file mode 100644 index 00000000..65a29b3f --- /dev/null +++ b/deepnote_toolkit/notebooks/outputs.py @@ -0,0 +1,41 @@ +"""Queries shared by every source of notebook outputs.""" + +from __future__ import annotations + +from .models import DeepnoteDataframe, NotebookOutput + + +class OutputCollection: + """Shared output queries for a loaded document and a live run result.""" + + outputs: tuple[NotebookOutput, ...] + + def outputs_for_mime(self, mime: str) -> list[NotebookOutput]: + return [output for output in self.outputs if mime in output.data] + + def first_dataframe(self) -> DeepnoteDataframe | None: + for output in self.outputs: + if dataframe := output.dataframe: + return dataframe + return None + + def images(self, mime: str = "image/png") -> list[bytes]: + return [ + image + for output in self.outputs + if (image := output.image_bytes(mime)) is not None + ] + + def text(self, mime: str = "text/plain") -> str: + return "".join(output.text(mime) for output in self.outputs).strip() + + def agent_text(self) -> str: + chunks: list[str] = [] + for output in self.outputs: + if output.block_type != "agent": + continue + if output.output_type == "stream": + chunks.append(output.text()) + else: + chunks.append(output.text("text/markdown") or output.text()) + return "".join(chunks).strip() diff --git a/deepnote_toolkit/notebooks/run_result.py b/deepnote_toolkit/notebooks/run_result.py new file mode 100644 index 00000000..ce09fd3b --- /dev/null +++ b/deepnote_toolkit/notebooks/run_result.py @@ -0,0 +1,83 @@ +"""The result of one notebook run, from either runner.""" + +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any + +from .document import DeepnoteDocument +from .models import NotebookOutput, optional_string +from .outputs import OutputCollection + + +class RunResult(OutputCollection): + """The normalized result of `POST /api/run`, for either cloud or local execution.""" + + def __init__(self, raw: Mapping[str, Any]): + self.raw = raw + self.target = str(raw.get("target", "")) + self.success = raw.get("success") is True + self.run_id = optional_string(raw.get("runId")) + self.status = optional_string(raw.get("status")) + self.created = raw.get("created") is True + self.view_url = optional_string(raw.get("viewUrl")) + self.error = optional_string(raw.get("error")) + self.snapshot_yaml = optional_string(raw.get("snapshotYaml")) + self.snapshot = None + if self.snapshot_yaml: + try: + self.snapshot = DeepnoteDocument.parse(self.snapshot_yaml) + except ValueError: + pass + if self.snapshot: + self.outputs = self.snapshot.outputs + else: + snapshot_blocks = raw.get("snapshotBlocks") + self.outputs = ( + _outputs_from_snapshot_blocks(snapshot_blocks) + if isinstance(snapshot_blocks, list) + else _outputs_from_run(raw.get("outputs")) + ) + + +def _outputs_from_run(value: Any) -> tuple[NotebookOutput, ...]: + if not isinstance(value, list): + return () + outputs: list[NotebookOutput] = [] + for block in value: + if not isinstance(block, Mapping): + continue + block_id = str(block.get("blockId", "")) + raw_outputs = block.get("outputs") + if not isinstance(raw_outputs, list): + continue + outputs.extend( + NotebookOutput(block_id=block_id, block_type=None, raw=output) + for output in raw_outputs + if isinstance(output, Mapping) + ) + return tuple(outputs) + + +def _outputs_from_snapshot_blocks(value: Any) -> tuple[NotebookOutput, ...]: + if not isinstance(value, list): + return () + outputs: list[NotebookOutput] = [] + for block in value: + if not isinstance(block, Mapping): + continue + block_id = str(block.get("id", "")) + block_type = optional_string(block.get("type")) + raw_outputs = block.get("outputs") + if not isinstance(raw_outputs, list): + continue + outputs.extend( + NotebookOutput( + block_id=block_id, + block_type=block_type, + raw=output, + ) + for output in raw_outputs + if isinstance(output, Mapping) + ) + return tuple(outputs) diff --git a/deepnote_toolkit/notebooks/runner.py b/deepnote_toolkit/notebooks/runner.py new file mode 100644 index 00000000..6ceb1407 --- /dev/null +++ b/deepnote_toolkit/notebooks/runner.py @@ -0,0 +1,27 @@ +"""The interface every notebook runner implements.""" + +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any, Protocol + +from .models import RunnerInfo +from .run_result import RunResult + + +class RunnerError(RuntimeError): + """The Deepnote runner was unavailable or rejected a request.""" + + def __init__(self, message: str, *, transient: bool = False): + super().__init__(message) + self.transient = transient + + +class Runner(Protocol): + """Runs one notebook and reports the inputs it accepts.""" + + def info(self) -> RunnerInfo: + """Return the notebook's name and the inputs it accepts.""" + + def run(self, inputs: Mapping[str, Any]) -> RunResult: + """Run the notebook with the given input values and wait for the result.""" diff --git a/deepnote_toolkit/streamlit/cloud_runner.py b/deepnote_toolkit/streamlit/cloud_runner.py new file mode 100644 index 00000000..9ed88ba9 --- /dev/null +++ b/deepnote_toolkit/streamlit/cloud_runner.py @@ -0,0 +1,50 @@ +"""The cloud runner for Streamlit apps hosted by Deepnote.""" + +from __future__ import annotations + +from deepnote_toolkit.notebooks.cloud_runner import ( + DEFAULT_API_ORIGIN, + DeepnoteCloudRunner, +) +from deepnote_toolkit.notebooks.runner import RunnerError + +from .auth import ( + CurrentUserApiTokenError, + _has_hosted_streamlit_context, + _is_streamlit_thread_without_request, + current_user_api_credentials, +) + + +class StreamlitCloudRunner(DeepnoteCloudRunner): + """Run a notebook from a Streamlit app, as the current viewer when Deepnote hosts it. + + A hosted app needs no token. Elsewhere this behaves like `DeepnoteCloudRunner`. + """ + + def _credentials(self) -> tuple[str, str]: + if self._token_provider is not None or self._static_token is not None: + return super()._credentials() + + # A hosted request authenticates as the viewer and never uses DEEPNOTE_TOKEN. + if _has_hosted_streamlit_context(): + try: + credentials = current_user_api_credentials( + timeout=min(self.timeout, 30), opener=self._open + ) + except CurrentUserApiTokenError as error: + raise RunnerError(str(error), transient=error.transient) from error + api_origin = ( + credentials.api_origin + if self.base_url == DEFAULT_API_ORIGIN + else self.base_url + ) + return credentials.token, api_origin + + if _is_streamlit_thread_without_request(): + raise RunnerError( + "No viewer request is available on this thread. Call the runner from " + "the Streamlit script thread, or pass token= or token_provider=." + ) + + return super()._credentials() diff --git a/tests/unit/test_deepnote_streamlit_cloud_runner.py b/tests/unit/test_deepnote_streamlit_cloud_runner.py new file mode 100644 index 00000000..7b98602a --- /dev/null +++ b/tests/unit/test_deepnote_streamlit_cloud_runner.py @@ -0,0 +1,193 @@ +import json +from typing import Any +from unittest.mock import MagicMock, patch + +import pytest + +from deepnote_toolkit.notebooks import RunnerError +from deepnote_toolkit.streamlit import ( + CurrentUserApiCredentials, + CurrentUserApiTokenError, + StreamlitCloudRunner, +) + + +class FakeResponse: + def __init__(self, payload: Any): + self.payload = payload + + def __enter__(self) -> "FakeResponse": + return self + + def __exit__(self, *_: object) -> None: + return None + + def read(self) -> bytes: + return json.dumps(self.payload).encode() + + +def test_hosted_cloud_runner_exchanges_per_request_and_uses_api_origin( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("DEEPNOTE_TOKEN", "must-not-be-used") + api_calls = [] + responses = iter( + [ + {"run": {"runId": "run-1", "status": "pending"}}, + {"run": {"runId": "run-1", "status": "success", "snapshotBlocks": []}}, + ] + ) + + def open_request(request: Any, *, timeout: float) -> FakeResponse: + api_calls.append( + ( + request.full_url, + request.headers["Authorization"], + timeout, + ) + ) + return FakeResponse(next(responses)) + + credentials = [ + CurrentUserApiCredentials( + token="viewer-token-1", + api_origin="https://api.deepnote-staging.com", + expires_at_seconds=1_800_000_000, + ), + CurrentUserApiCredentials( + token="viewer-token-2", + api_origin="https://api.deepnote-staging.com", + expires_at_seconds=1_800_000_001, + ), + ] + with ( + patch( + "deepnote_toolkit.streamlit.cloud_runner._has_hosted_streamlit_context", + return_value=True, + ), + patch( + "deepnote_toolkit.streamlit.cloud_runner.current_user_api_credentials", + side_effect=credentials, + ) as exchange, + ): + result = StreamlitCloudRunner( + "notebook-1", + opener=open_request, + sleep=lambda _delay: None, + ).run({}) + + assert result.success is True + assert exchange.call_count == 2 + assert api_calls == [ + ( + "https://api.deepnote-staging.com/v2/runs", + "Bearer viewer-token-1", + 30, + ), + ( + "https://api.deepnote-staging.com/v2/runs/run-1" "?snapshotDelivery=inline", + "Bearer viewer-token-2", + 30, + ), + ] + + +def test_hosted_runner_never_falls_back_to_environment_token( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("DEEPNOTE_TOKEN", "shared-token") + opener = MagicMock() + with ( + patch( + "deepnote_toolkit.streamlit.cloud_runner._has_hosted_streamlit_context", + return_value=True, + ), + patch( + "deepnote_toolkit.streamlit.cloud_runner.current_user_api_credentials", + side_effect=CurrentUserApiTokenError("viewer token unavailable"), + ), + pytest.raises(RunnerError, match="viewer token unavailable"), + ): + StreamlitCloudRunner("notebook-1", opener=opener).info() + + opener.assert_not_called() + + +def test_worker_thread_never_falls_back_to_environment_token( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("DEEPNOTE_TOKEN", "shared-token") + opener = MagicMock() + with ( + patch( + "deepnote_toolkit.streamlit.cloud_runner._has_hosted_streamlit_context", + return_value=False, + ), + patch( + "deepnote_toolkit.streamlit.cloud_runner._is_streamlit_thread_without_request", + return_value=True, + ), + pytest.raises(RunnerError, match="No viewer request"), + ): + StreamlitCloudRunner("notebook-1", opener=opener).info() + + opener.assert_not_called() + + +def test_cloud_run_retries_a_transient_token_exchange_failure() -> None: + responses = iter( + [ + {"run": {"runId": "run-1", "status": "pending"}}, + {"run": {"runId": "run-1", "status": "success", "snapshotBlocks": []}}, + ] + ) + credentials = CurrentUserApiCredentials( + token="viewer-token", + api_origin="https://api.deepnote.com", + expires_at_seconds=1_800_000_000, + ) + with ( + patch( + "deepnote_toolkit.streamlit.cloud_runner._has_hosted_streamlit_context", + return_value=True, + ), + patch( + "deepnote_toolkit.streamlit.cloud_runner.current_user_api_credentials", + side_effect=[ + credentials, + CurrentUserApiTokenError("exchange timed out", transient=True), + credentials, + ], + ), + ): + result = StreamlitCloudRunner( + "notebook-1", + opener=lambda _request, *, timeout: FakeResponse(next(responses)), + sleep=lambda _delay: None, + ).run({}) + + assert result.success is True + + +def test_local_streamlit_runner_uses_environment_token( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("DEEPNOTE_TOKEN", "local-token") + + def open_request(request: Any, *, timeout: float) -> FakeResponse: + assert request.headers["Authorization"] == "Bearer local-token" + return FakeResponse({"notebook": {"name": "Revenue", "inputs": []}}) + + with ( + patch( + "deepnote_toolkit.streamlit.cloud_runner._has_hosted_streamlit_context", + return_value=False, + ), + patch( + "deepnote_toolkit.streamlit.cloud_runner._is_streamlit_thread_without_request", + return_value=False, + ), + ): + info = StreamlitCloudRunner("notebook-1", opener=open_request).info() + + assert info.notebook == "Revenue" diff --git a/tests/unit/test_notebooks_runners.py b/tests/unit/test_notebooks_runners.py new file mode 100644 index 00000000..ad3dd55b --- /dev/null +++ b/tests/unit/test_notebooks_runners.py @@ -0,0 +1,407 @@ +import io +import json +from typing import Any +from urllib.error import HTTPError, URLError + +import pytest + +from deepnote_toolkit.notebooks import ( + DeepnoteCloudRunner, + DeepnoteRunner, + InputBlock, + RunnerError, + RunnerInfo, +) + + +class FakeResponse: + def __init__(self, payload: Any): + self.payload = payload + + def __enter__(self) -> "FakeResponse": + return self + + def __exit__(self, *_: object) -> None: + return None + + def read(self) -> bytes: + return json.dumps(self.payload).encode() + + +def test_info_parses_runner_contract() -> None: + calls = [] + + def open_request(request: Any, *, timeout: float) -> FakeResponse: + calls.append((request.full_url, request.method, timeout)) + return FakeResponse( + { + "notebook": "Revenue", + "runTarget": "cloud", + "inputs": [ + {"variableName": "region", "type": "input-select", "value": "All"} + ], + } + ) + + info = DeepnoteRunner("http://runner/", timeout=12, opener=open_request).info() + + assert calls == [("http://runner/api/info", "GET", 12)] + assert info.notebook == "Revenue" + assert info.run_target == "cloud" + assert info.inputs[0].variable_name == "region" + + +def test_runner_info_requires_matching_input_names_and_types() -> None: + info = RunnerInfo( + notebook="Revenue", + inputs=(InputBlock("region", "input-select", "All"),), + run_target="cloud", + ) + + assert info.accepts_inputs([InputBlock("region", "input-select", "Europe")]) + assert not info.accepts_inputs([InputBlock("market", "input-select", "Europe")]) + assert not info.accepts_inputs([InputBlock("region", "input-text", "Europe")]) + + +def test_run_posts_inputs_and_parses_one_result_shape() -> None: + def open_request(request: Any, *, timeout: float) -> FakeResponse: + assert timeout == 600 + assert request.method == "POST" + assert json.loads(request.data) == {"inputs": {"limit": 20}} + return FakeResponse({"target": "local", "success": True, "outputs": []}) + + result = DeepnoteRunner(opener=open_request).run({"limit": 20}) + + assert result.target == "local" + assert result.success is True + + +def test_http_error_surfaces_runner_message() -> None: + def open_request(*_: Any, **__: Any) -> FakeResponse: + raise HTTPError( + "http://runner/api/run", + 500, + "Server error", + {}, + io.BytesIO(b'{"error":"DEEPNOTE_TOKEN is required"}'), + ) + + with pytest.raises(RunnerError, match="DEEPNOTE_TOKEN is required"): + DeepnoteRunner("http://runner", opener=open_request).run({}) + + +def test_connection_error_names_runner_url() -> None: + def open_request(*_: Any, **__: Any) -> FakeResponse: + raise URLError("connection refused") + + with pytest.raises(RunnerError, match="http://runner"): + DeepnoteRunner("http://runner", opener=open_request).info() + + +def test_timeout_names_runner_url_and_duration() -> None: + def open_request(*_: Any, **__: Any) -> FakeResponse: + raise TimeoutError + + with pytest.raises(RunnerError, match="http://runner.*12 seconds"): + DeepnoteRunner("http://runner", timeout=12, opener=open_request).info() + + +def test_cloud_info_reads_public_notebook_contract() -> None: + def open_request(request: Any, *, timeout: float) -> FakeResponse: + assert request.full_url == "https://api.deepnote.com/v2/notebooks/notebook-1" + assert request.headers["Authorization"] == "Bearer token-1" + assert timeout == 30 + return FakeResponse( + { + "notebook": { + "name": "Revenue", + "inputs": [ + { + "name": "region", + "type": "input-select", + "value": "All", + "label": "Region", + } + ], + } + } + ) + + info = DeepnoteCloudRunner( + "notebook-1", token="token-1", opener=open_request + ).info() + + assert info.notebook == "Revenue" + assert info.run_target == "cloud" + assert info.inputs[0].variable_name == "region" + + +def test_cloud_run_posts_inputs_polls_and_parses_inline_snapshot() -> None: + calls = [] + responses = iter( + [ + {"run": {"runId": "run-1", "status": "pending"}}, + {"run": {"runId": "run-1", "status": "running"}}, + { + "run": { + "runId": "run-1", + "status": "success", + "snapshot": { + "snapshotContent": "project:\n name: Result\n notebooks:\n - blocks: []\n" + }, + } + }, + ] + ) + + def open_request(request: Any, *, timeout: float) -> FakeResponse: + calls.append( + ( + request.full_url, + request.method, + request.headers["Authorization"], + request.data, + timeout, + ) + ) + return FakeResponse(next(responses)) + + tokens = iter(["token-1", "token-2", "token-3"]) + sleeps = [] + result = DeepnoteCloudRunner( + "notebook-1", + token_provider=lambda: next(tokens), + opener=open_request, + sleep=sleeps.append, + poll_interval=0.25, + ).run({"limit": 20, "enabled": True, "regions": ["EU"]}) + + assert json.loads(calls[0][3]) == { + "notebookId": "notebook-1", + "detached": True, + "inputs": {"limit": "20", "enabled": True, "regions": ["EU"]}, + } + assert calls[1][0].endswith("/v2/runs/run-1?snapshotDelivery=inline") + assert [call[2] for call in calls] == [ + "Bearer token-1", + "Bearer token-2", + "Bearer token-3", + ] + assert sleeps == [0.25, 0.25] + assert result.success is True + assert result.snapshot is not None + assert result.snapshot.project_name == "Result" + + +def test_cloud_run_reads_sanitized_snapshot_blocks_without_raw_snapshot() -> None: + responses = iter( + [ + {"run": {"runId": "run-1", "status": "pending"}}, + { + "run": { + "runId": "run-1", + "status": "success", + "snapshotBlocks": [ + { + "id": "code-1", + "type": "code", + "outputs": [ + { + "output_type": "execute_result", + "data": { + "application/vnd.deepnote.dataframe.v3+json": { + "columns": [{"name": "revenue"}], + "rows": [{"revenue": 42}], + } + }, + } + ], + "metadata": {"deepnote_table_state": {}}, + }, + { + "id": "agent-1", + "type": "agent", + "outputs": [ + { + "output_type": "display_data", + "data": {"text/markdown": "**Done**"}, + } + ], + "metadata": {}, + }, + ], + } + }, + ] + ) + + def open_request(request: Any, *, timeout: float) -> FakeResponse: + assert timeout == 30 + if request.method == "POST": + assert json.loads(request.data) == { + "notebookId": "notebook-1", + "detached": True, + "inputs": {"region": "EU"}, + } + return FakeResponse(next(responses)) + + result = DeepnoteCloudRunner( + "notebook-1", + token="token", + opener=open_request, + sleep=lambda _delay: None, + ).run({"region": "EU"}) + + assert result.snapshot is None + assert result.snapshot_yaml is None + assert [output.block_id for output in result.outputs] == ["code-1", "agent-1"] + assert [output.block_type for output in result.outputs] == ["code", "agent"] + dataframe = result.first_dataframe() + assert dataframe is not None + assert dataframe.records() == [{"revenue": 42}] + assert result.agent_text() == "**Done**" + + +def test_cloud_run_surfaces_terminal_error() -> None: + def open_request(_request: Any, *, timeout: float) -> FakeResponse: + assert timeout == 30 + return FakeResponse( + { + "run": { + "id": "run-1", + "status": "error", + "error": {"message": "bad input"}, + } + } + ) + + result = DeepnoteCloudRunner( + "notebook-1", token="token", opener=open_request, sleep=lambda _delay: None + ).run({}) + + assert result.success is False + assert result.error == "bad input" + + +def test_cloud_runner_uses_environment_token( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("DEEPNOTE_TOKEN", "local-token") + + def open_request(request: Any, *, timeout: float) -> FakeResponse: + assert request.headers["Authorization"] == "Bearer local-token" + assert timeout == 30 + return FakeResponse({"notebook": {"name": "Revenue", "inputs": []}}) + + info = DeepnoteCloudRunner("notebook-1", opener=open_request).info() + + assert info.notebook == "Revenue" + + +def test_cloud_runner_requires_one_token_source( + monkeypatch: pytest.MonkeyPatch, +) -> None: + with pytest.raises(ValueError, match="not both"): + DeepnoteCloudRunner("notebook-1", token="token", token_provider=lambda: "other") + + monkeypatch.delenv("DEEPNOTE_TOKEN", raising=False) + with pytest.raises(RunnerError, match="token is required"): + DeepnoteCloudRunner( + "notebook-1", + token="", + opener=lambda *_args, **_kwargs: FakeResponse({}), + ).info() + + +def test_cloud_run_retries_transient_poll_failures() -> None: + responses = iter( + [ + {"run": {"runId": "run-1", "status": "pending"}}, + HTTPError("http://api", 503, "Unavailable", {}, io.BytesIO(b"{}")), + URLError("connection reset"), + {"run": {"runId": "run-1", "status": "success", "snapshotBlocks": []}}, + ] + ) + + def open_request(_request: Any, *, timeout: float) -> FakeResponse: + response = next(responses) + if isinstance(response, Exception): + raise response + return FakeResponse(response) + + result = DeepnoteCloudRunner( + "notebook-1", token="token", opener=open_request, sleep=lambda _delay: None + ).run({}) + + assert result.success is True + + +def test_cloud_run_raises_poll_failures_that_are_not_transient() -> None: + responses = iter( + [ + {"run": {"runId": "run-1", "status": "pending"}}, + HTTPError("http://api", 403, "Forbidden", {}, io.BytesIO(b"{}")), + ] + ) + + def open_request(_request: Any, *, timeout: float) -> FakeResponse: + response = next(responses) + if isinstance(response, Exception): + raise response + return FakeResponse(response) + + with pytest.raises(RunnerError, match="HTTP 403"): + DeepnoteCloudRunner( + "notebook-1", token="token", opener=open_request, sleep=lambda _delay: None + ).run({}) + + +def test_cloud_run_stops_retrying_after_repeated_transient_failures() -> None: + calls = [] + + def open_request(request: Any, *, timeout: float) -> FakeResponse: + calls.append(request.method) + if request.method == "POST": + return FakeResponse({"run": {"runId": "run-1", "status": "pending"}}) + raise URLError("connection reset") + + with pytest.raises(RunnerError, match="connection reset"): + DeepnoteCloudRunner( + "notebook-1", token="token", opener=open_request, sleep=lambda _delay: None + ).run({}) + + assert calls == ["POST"] + ["GET"] * 6 + + +def test_cloud_run_waits_for_a_snapshot_that_lags_the_terminal_status() -> None: + responses = iter( + [ + {"run": {"runId": "run-1", "status": "success"}}, + {"run": {"runId": "run-1", "status": "success"}}, + { + "run": { + "runId": "run-1", + "status": "success", + "snapshotBlocks": [ + { + "id": "code-1", + "type": "code", + "outputs": [{"output_type": "stream", "text": "done"}], + } + ], + } + }, + ] + ) + sleeps = [] + + result = DeepnoteCloudRunner( + "notebook-1", + token="token", + opener=lambda _request, *, timeout: FakeResponse(next(responses)), + sleep=sleeps.append, + poll_interval=0.5, + ).run({}) + + assert sleeps == [0.5, 0.5] + assert result.text() == "done" From d3396786c97df094c4cb696d3810979faea3a07a Mon Sep 17 00:00:00 2001 From: Wojciech Apanowicz Date: Fri, 18 Sep 2026 15:21:22 +0200 Subject: [PATCH 16/43] fix(streamlit): send the viewer token only to its own API origin - A hosted run always uses the origin returned with the viewer credentials. A custom base_url no longer receives the viewer token. - The local runner skips input entries without a name or type, matching the cloud runner, so a malformed entry no longer raises KeyError. --- deepnote_toolkit/notebooks/local_runner.py | 2 ++ deepnote_toolkit/streamlit/cloud_runner.py | 12 ++------ .../test_deepnote_streamlit_cloud_runner.py | 28 +++++++++++++++++++ tests/unit/test_notebooks_runners.py | 17 +++++++++++ 4 files changed, 49 insertions(+), 10 deletions(-) diff --git a/deepnote_toolkit/notebooks/local_runner.py b/deepnote_toolkit/notebooks/local_runner.py index add290ac..e973cff2 100644 --- a/deepnote_toolkit/notebooks/local_runner.py +++ b/deepnote_toolkit/notebooks/local_runner.py @@ -34,6 +34,8 @@ def info(self) -> RunnerInfo: InputBlock.from_api(value) for value in values if isinstance(value, Mapping) + and isinstance(value.get("variableName"), str) + and isinstance(value.get("type"), str) ) if isinstance(values, list) else () diff --git a/deepnote_toolkit/streamlit/cloud_runner.py b/deepnote_toolkit/streamlit/cloud_runner.py index 9ed88ba9..729575c6 100644 --- a/deepnote_toolkit/streamlit/cloud_runner.py +++ b/deepnote_toolkit/streamlit/cloud_runner.py @@ -2,10 +2,7 @@ from __future__ import annotations -from deepnote_toolkit.notebooks.cloud_runner import ( - DEFAULT_API_ORIGIN, - DeepnoteCloudRunner, -) +from deepnote_toolkit.notebooks.cloud_runner import DeepnoteCloudRunner from deepnote_toolkit.notebooks.runner import RunnerError from .auth import ( @@ -34,12 +31,7 @@ def _credentials(self) -> tuple[str, str]: ) except CurrentUserApiTokenError as error: raise RunnerError(str(error), transient=error.transient) from error - api_origin = ( - credentials.api_origin - if self.base_url == DEFAULT_API_ORIGIN - else self.base_url - ) - return credentials.token, api_origin + return credentials.token, credentials.api_origin if _is_streamlit_thread_without_request(): raise RunnerError( diff --git a/tests/unit/test_deepnote_streamlit_cloud_runner.py b/tests/unit/test_deepnote_streamlit_cloud_runner.py index 7b98602a..e45f3fed 100644 --- a/tests/unit/test_deepnote_streamlit_cloud_runner.py +++ b/tests/unit/test_deepnote_streamlit_cloud_runner.py @@ -191,3 +191,31 @@ def open_request(request: Any, *, timeout: float) -> FakeResponse: info = StreamlitCloudRunner("notebook-1", opener=open_request).info() assert info.notebook == "Revenue" + + +def test_hosted_runner_sends_the_viewer_token_only_to_the_returned_origin() -> None: + urls = [] + + def open_request(request: Any, *, timeout: float) -> FakeResponse: + urls.append(request.full_url) + return FakeResponse({"notebook": {"name": "Revenue", "inputs": []}}) + + with ( + patch( + "deepnote_toolkit.streamlit.cloud_runner._has_hosted_streamlit_context", + return_value=True, + ), + patch( + "deepnote_toolkit.streamlit.cloud_runner.current_user_api_credentials", + return_value=CurrentUserApiCredentials( + token="viewer-token", + api_origin="https://api.deepnote.com", + expires_at_seconds=1_800_000_000, + ), + ), + ): + StreamlitCloudRunner( + "notebook-1", base_url="https://elsewhere.example", opener=open_request + ).info() + + assert urls == ["https://api.deepnote.com/v2/notebooks/notebook-1"] diff --git a/tests/unit/test_notebooks_runners.py b/tests/unit/test_notebooks_runners.py index ad3dd55b..75ed75ba 100644 --- a/tests/unit/test_notebooks_runners.py +++ b/tests/unit/test_notebooks_runners.py @@ -405,3 +405,20 @@ def test_cloud_run_waits_for_a_snapshot_that_lags_the_terminal_status() -> None: assert sleeps == [0.5, 0.5] assert result.text() == "done" + + +def test_info_skips_inputs_without_a_name_or_type() -> None: + def open_request(_request: Any, *, timeout: float) -> FakeResponse: + return FakeResponse( + { + "inputs": [ + {"type": "input-text"}, + {"variableName": "orphan"}, + {"variableName": "region", "type": "input-text"}, + ] + } + ) + + info = DeepnoteRunner(opener=open_request).info() + + assert info.inputs == (InputBlock("region", "input-text", None),) From 9ac0c27e3319664a929d1632694c2fcfd643985b Mon Sep 17 00:00:00 2001 From: Wojciech Apanowicz Date: Fri, 18 Sep 2026 15:36:01 +0200 Subject: [PATCH 17/43] fix(streamlit): read .deepnote YAML as written and harden the runners - Load `.deepnote` files by the YAML 1.2 core schema their writer uses. PyYAML's YAML 1.1 rules turned unquoted `Yes`/`No` into booleans, `12:30` into a number and timestamps into datetime objects, which broke select options and legacy date inputs. The C loader is used when available and parses a 3.6 MB snapshot in 2 s, down from 9 s. - A dropped connection is a transient failure for both the runs API and the viewer token exchange. - The settle wait follows the run's `snapshotStatus`, so a run whose snapshot will never be stored returns at once. `RunResult` exposes it. - The cloud runner's `info()` keeps select options and slider bounds. - `DeepnoteDataframe` reports `row_count` and `is_truncated`, since `rows` holds only the first page. - `accepts_inputs` compares names and types as a set. - The viewer token is hidden from the credentials repr. - The quick-start handles a failed run and a run without a table. --- deepnote_toolkit/notebooks/cloud_runner.py | 17 ++-- deepnote_toolkit/notebooks/document.py | 5 +- deepnote_toolkit/notebooks/http.py | 6 ++ deepnote_toolkit/notebooks/models.py | 17 +++- deepnote_toolkit/notebooks/run_result.py | 1 + deepnote_toolkit/notebooks/yaml_loader.py | 47 +++++++++++ deepnote_toolkit/streamlit/auth.py | 10 ++- docs/streamlit-apps.md | 11 ++- tests/unit/test_deepnote_streamlit_auth.py | 30 ++++++- tests/unit/test_notebooks_document.py | 48 +++++++++++ tests/unit/test_notebooks_runners.py | 94 ++++++++++++++++++++++ 11 files changed, 264 insertions(+), 22 deletions(-) create mode 100644 deepnote_toolkit/notebooks/yaml_loader.py diff --git a/deepnote_toolkit/notebooks/cloud_runner.py b/deepnote_toolkit/notebooks/cloud_runner.py index 3cda1364..245de845 100644 --- a/deepnote_toolkit/notebooks/cloud_runner.py +++ b/deepnote_toolkit/notebooks/cloud_runner.py @@ -63,16 +63,11 @@ def info(self) -> RunnerInfo: raise RunnerError("Deepnote API response did not include a notebook") raw_inputs = notebook.get("inputs") inputs = tuple( - InputBlock.from_api( - { - "variableName": value.get("name"), - "type": value.get("type"), - "value": value.get("value"), - "label": value.get("label"), - } - ) + InputBlock.from_api({**value, "variableName": value["name"]}) for value in raw_inputs or [] - if isinstance(value, Mapping) and isinstance(value.get("name"), str) + if isinstance(value, Mapping) + and isinstance(value.get("name"), str) + and isinstance(value.get("type"), str) ) return RunnerInfo( notebook=str(notebook.get("name", "Untitled notebook")), @@ -115,7 +110,8 @@ def run(self, inputs: Mapping[str, Any]) -> RunResult: # The snapshot can attach shortly after the status turns terminal. for _ in range(SNAPSHOT_SETTLE_ATTEMPTS): - if _has_snapshot(current): + is_pending = current.get("snapshotStatus", "pending") == "pending" + if not is_pending or _has_snapshot(current): break self._sleep(self.poll_interval) try: @@ -139,6 +135,7 @@ def run(self, inputs: Mapping[str, Any]) -> RunResult: "runId": run_id, "status": status, "error": str(error) if error is not None else None, + "snapshotStatus": current.get("snapshotStatus"), "snapshotYaml": snapshot_yaml, "snapshotBlocks": current.get("snapshotBlocks"), "viewUrl": current.get("viewUrl"), diff --git a/deepnote_toolkit/notebooks/document.py b/deepnote_toolkit/notebooks/document.py index 3d0c6b5e..a83782a2 100644 --- a/deepnote_toolkit/notebooks/document.py +++ b/deepnote_toolkit/notebooks/document.py @@ -10,6 +10,7 @@ from .models import InputBlock, NotebookOutput, optional_string from .outputs import OutputCollection +from .yaml_loader import load_yaml class DeepnoteDocument(OutputCollection): @@ -43,7 +44,7 @@ def load( ) -> DeepnoteDocument: source = Path(path) try: - raw = yaml.safe_load(source.read_text(encoding="utf-8")) + raw = load_yaml(source.read_text(encoding="utf-8")) except yaml.YAMLError as error: raise ValueError(f"Could not parse {source}: {error}") from error if not isinstance(raw, Mapping): @@ -53,7 +54,7 @@ def load( @classmethod def parse(cls, content: str, *, notebook_id: str | None = None) -> DeepnoteDocument: try: - raw = yaml.safe_load(content) + raw = load_yaml(content) except yaml.YAMLError as error: raise ValueError(f"Could not parse .deepnote YAML: {error}") from error if not isinstance(raw, Mapping): diff --git a/deepnote_toolkit/notebooks/http.py b/deepnote_toolkit/notebooks/http.py index c96f8908..5b850bbd 100644 --- a/deepnote_toolkit/notebooks/http.py +++ b/deepnote_toolkit/notebooks/http.py @@ -4,6 +4,7 @@ import json from collections.abc import Callable, Mapping +from http.client import HTTPException from typing import Any from urllib.error import HTTPError, URLError from urllib.request import Request @@ -36,6 +37,11 @@ def request_json( f"The {service} at {origin} timed out after {timeout:g} seconds", transient=True, ) from error + except (OSError, HTTPException) as error: + raise RunnerError( + f"The connection to the {service} at {origin} dropped: {error}", + transient=True, + ) from error except (json.JSONDecodeError, UnicodeDecodeError) as error: raise RunnerError(f"The {service} returned an invalid JSON response") from error if not isinstance(payload, Mapping): diff --git a/deepnote_toolkit/notebooks/models.py b/deepnote_toolkit/notebooks/models.py index bf70a942..4bc9dc69 100644 --- a/deepnote_toolkit/notebooks/models.py +++ b/deepnote_toolkit/notebooks/models.py @@ -104,6 +104,17 @@ def from_value(cls, value: Any) -> DeepnoteDataframe | None: return None return cls(columns=tuple(columns), rows=tuple(rows), raw=value) + @property + def row_count(self) -> int: + """Rows in the full dataframe. `rows` holds only the first page of them.""" + + count = self.raw.get("row_count") + return count if isinstance(count, int) else len(self.rows) + + @property + def is_truncated(self) -> bool: + return self.row_count > len(self.rows) + @property def data_columns(self) -> tuple[str, ...]: return tuple( @@ -174,9 +185,9 @@ def accepts_inputs(self, inputs: Iterable[InputBlock]) -> bool: return _input_contract(inputs) == _input_contract(self.inputs) -def _input_contract(inputs: Iterable[InputBlock]) -> tuple[tuple[str, str], ...]: - return tuple( - sorted((input_block.variable_name, input_block.type) for input_block in inputs) +def _input_contract(inputs: Iterable[InputBlock]) -> frozenset[tuple[str, str]]: + return frozenset( + (input_block.variable_name, input_block.type) for input_block in inputs ) diff --git a/deepnote_toolkit/notebooks/run_result.py b/deepnote_toolkit/notebooks/run_result.py index ce09fd3b..7816e7db 100644 --- a/deepnote_toolkit/notebooks/run_result.py +++ b/deepnote_toolkit/notebooks/run_result.py @@ -22,6 +22,7 @@ def __init__(self, raw: Mapping[str, Any]): self.created = raw.get("created") is True self.view_url = optional_string(raw.get("viewUrl")) self.error = optional_string(raw.get("error")) + self.snapshot_status = optional_string(raw.get("snapshotStatus")) self.snapshot_yaml = optional_string(raw.get("snapshotYaml")) self.snapshot = None if self.snapshot_yaml: diff --git a/deepnote_toolkit/notebooks/yaml_loader.py b/deepnote_toolkit/notebooks/yaml_loader.py new file mode 100644 index 00000000..e5b060be --- /dev/null +++ b/deepnote_toolkit/notebooks/yaml_loader.py @@ -0,0 +1,47 @@ +"""Load `.deepnote` YAML with the rules its writer uses.""" + +from __future__ import annotations + +import re +from typing import Any + +import yaml + +_BaseLoader: type = getattr(yaml, "CSafeLoader", yaml.SafeLoader) + + +class _CoreSchemaLoader(_BaseLoader): # type: ignore[misc,valid-type] + """A safe loader that resolves plain scalars by the YAML 1.2 core schema. + + `.deepnote` files are written as YAML 1.2, where `No`, `on`, `12:30` and + `2026-08-17` are strings. PyYAML's YAML 1.1 rules read them as booleans, + numbers and dates. + """ + + yaml_implicit_resolvers: dict[str, Any] = {} + + +for _tag, _pattern, _first in ( + ("null", r"^(?:~|null|Null|NULL|)$", ["~", "n", "N", ""]), + ("bool", r"^(?:true|True|TRUE|false|False|FALSE)$", list("tTfF")), + ("int", r"^(?:[-+]?[0-9]+|0o[0-7]+|0x[0-9a-fA-F]+)$", list("-+0123456789")), + ( + "float", + r"^(?:[-+]?(?:\.[0-9]+|[0-9]+(?:\.[0-9]*)?)(?:[eE][-+]?[0-9]+)?" + r"|[-+]?\.(?:inf|Inf|INF)|\.(?:nan|NaN|NAN))$", + list("-+0123456789."), + ), +): + _CoreSchemaLoader.add_implicit_resolver( + f"tag:yaml.org,2002:{_tag}", re.compile(_pattern), _first + ) + + +def load_yaml(content: str) -> Any: + """Parse one YAML document. Raises `yaml.YAMLError` when it is malformed.""" + + loader = _CoreSchemaLoader(content) + try: + return loader.get_single_data() + finally: + loader.dispose() diff --git a/deepnote_toolkit/streamlit/auth.py b/deepnote_toolkit/streamlit/auth.py index 98cffb95..8fc7d5d8 100644 --- a/deepnote_toolkit/streamlit/auth.py +++ b/deepnote_toolkit/streamlit/auth.py @@ -7,7 +7,8 @@ import re import time from collections.abc import Callable, Mapping -from dataclasses import dataclass +from dataclasses import dataclass, field +from http.client import HTTPException from typing import Any from urllib.error import HTTPError, URLError from urllib.parse import urlparse @@ -44,7 +45,7 @@ def __init__(self, message: str, *, transient: bool = False): class CurrentUserApiCredentials: """A short-lived viewer-scoped public API credential.""" - token: str + token: str = field(repr=False) api_origin: str expires_at_seconds: float @@ -127,6 +128,11 @@ def current_user_api_credentials( raise CurrentUserApiTokenError( "Current viewer API-token exchange timed out.", transient=True ) from error + except (OSError, HTTPException) as error: + raise CurrentUserApiTokenError( + "The connection dropped during the current viewer API-token exchange.", + transient=True, + ) from error except (json.JSONDecodeError, UnicodeDecodeError) as error: raise CurrentUserApiTokenError( "Current viewer API-token exchange returned invalid JSON." diff --git a/docs/streamlit-apps.md b/docs/streamlit-apps.md index f92ecdce..ba40aa6b 100644 --- a/docs/streamlit-apps.md +++ b/docs/streamlit-apps.md @@ -15,7 +15,13 @@ values = render_inputs(document.inputs, st.sidebar) if st.button("Run"): result = StreamlitCloudRunner("your-notebook-id").run(values) - st.dataframe(result.first_dataframe().records()) + dataframe = result.first_dataframe() + if not result.success: + st.error(result.error or "The run failed.") + elif dataframe is None: + st.info("The run produced no table.") + else: + st.dataframe(dataframe.records()) ``` ## Two packages @@ -28,7 +34,8 @@ program: the notebook you run so the inputs match it: `DeepnoteDocument.load(path, notebook_id="your-notebook-id")`. - `DeepnoteCloudRunner` runs an existing notebook in Deepnote Cloud and returns - its outputs as a `RunResult`. + its outputs as a `RunResult`. A dataframe output holds the first page of rows. + `row_count` is the full size and `is_truncated` tells whether rows are missing. - `DeepnoteRunner` does the same through a local `@deepnote/local-runner` sidecar at `http://127.0.0.1:8787`. - `Runner` is the interface both runners implement, for code that accepts either. diff --git a/tests/unit/test_deepnote_streamlit_auth.py b/tests/unit/test_deepnote_streamlit_auth.py index 9078a7d5..616084e4 100644 --- a/tests/unit/test_deepnote_streamlit_auth.py +++ b/tests/unit/test_deepnote_streamlit_auth.py @@ -4,6 +4,7 @@ import json import sys import time +from http.client import RemoteDisconnected from types import SimpleNamespace from typing import Any from unittest.mock import patch @@ -12,13 +13,12 @@ import pytest from deepnote_toolkit.streamlit import ( + CurrentUserApiCredentials, CurrentUserApiTokenError, current_user_api_credentials, current_user_api_token, ) -from deepnote_toolkit.streamlit.auth import ( - _read_streamlit_app_id_from_context, -) +from deepnote_toolkit.streamlit.auth import _read_streamlit_app_id_from_context APP_ID = "3853c7f5-2048-4b57-946d-6c5592c3317e" @@ -277,3 +277,27 @@ def open_request(*_args: Any, **_kwargs: Any) -> FakeResponse: assert "HTTP 401" in str(exc_info.value) assert secret_response not in str(exc_info.value) + + +def test_credentials_repr_hides_the_token() -> None: + credentials = CurrentUserApiCredentials( + token="secret-token", + api_origin="https://api.deepnote.com", + expires_at_seconds=1_800_000_000, + ) + + assert "secret-token" not in repr(credentials) + + +def test_dropped_connection_during_exchange_is_transient() -> None: + def open_request(_request: Any, *, timeout: float) -> Any: + raise RemoteDisconnected("Remote end closed connection without response") + + with pytest.raises(CurrentUserApiTokenError) as exc_info: + current_user_api_credentials( + app_id="11111111-2222-3333-4444-555555555555", + streamlit_token="cookie", + opener=open_request, + ) + + assert exc_info.value.transient is True diff --git a/tests/unit/test_notebooks_document.py b/tests/unit/test_notebooks_document.py index 00e4cbcf..c0e9bcb8 100644 --- a/tests/unit/test_notebooks_document.py +++ b/tests/unit/test_notebooks_document.py @@ -4,6 +4,7 @@ from deepnote_toolkit.notebooks import ( DATAFRAME_MIME, + DeepnoteDataframe, DeepnoteDocument, InputBlock, RunResult, @@ -235,3 +236,50 @@ def test_notebook_id_scopes_inputs_to_one_notebook() -> None: def test_unknown_notebook_id_is_rejected() -> None: with pytest.raises(ValueError, match="notebook-c is not in this document"): DeepnoteDocument.parse(MULTI_NOTEBOOK_YAML, notebook_id="notebook-c") + + +WRITER_STYLE_YAML = """ +project: + name: Survey + notebooks: + - id: notebook-a + blocks: + - type: input-select + metadata: + deepnote_variable_name: answer + deepnote_variable_value: No + deepnote_variable_options: + - Yes + - No + - type: input-date + metadata: + deepnote_variable_name: as_of + deepnote_variable_value: 2026-08-17T00:00:00.000Z + - type: input-text + metadata: + deepnote_variable_name: time + deepnote_variable_value: 12:30 +""" + + +def test_plain_scalars_the_deepnote_writer_leaves_unquoted_stay_strings() -> None: + document = DeepnoteDocument.parse(WRITER_STYLE_YAML) + + assert document.inputs == ( + InputBlock("answer", "input-select", "No", options=("Yes", "No")), + InputBlock("as_of", "input-date", "2026-08-17T00:00:00.000Z"), + InputBlock("time", "input-text", "12:30"), + ) + + +def test_dataframe_reports_rows_beyond_the_first_page() -> None: + dataframe = DeepnoteDataframe.from_value( + {"columns": [{"name": "a"}], "rows": [{"a": 1}], "row_count": 250} + ) + whole = DeepnoteDataframe.from_value( + {"columns": [{"name": "a"}], "rows": [{"a": 1}]} + ) + + assert dataframe is not None and whole is not None + assert (dataframe.row_count, dataframe.is_truncated) == (250, True) + assert (whole.row_count, whole.is_truncated) == (1, False) diff --git a/tests/unit/test_notebooks_runners.py b/tests/unit/test_notebooks_runners.py index 75ed75ba..2e67bf79 100644 --- a/tests/unit/test_notebooks_runners.py +++ b/tests/unit/test_notebooks_runners.py @@ -1,5 +1,6 @@ import io import json +from http.client import RemoteDisconnected from typing import Any from urllib.error import HTTPError, URLError @@ -422,3 +423,96 @@ def open_request(_request: Any, *, timeout: float) -> FakeResponse: info = DeepnoteRunner(opener=open_request).info() assert info.inputs == (InputBlock("region", "input-text", None),) + + +def test_cloud_run_retries_a_dropped_connection() -> None: + responses = iter( + [ + {"run": {"runId": "run-1", "status": "pending"}}, + RemoteDisconnected("Remote end closed connection without response"), + {"run": {"runId": "run-1", "status": "success", "snapshotBlocks": []}}, + ] + ) + + def open_request(_request: Any, *, timeout: float) -> FakeResponse: + response = next(responses) + if isinstance(response, Exception): + raise response + return FakeResponse(response) + + result = DeepnoteCloudRunner( + "notebook-1", token="token", opener=open_request, sleep=lambda _delay: None + ).run({}) + + assert result.success is True + + +def test_cloud_run_does_not_wait_for_a_snapshot_that_will_not_come() -> None: + sleeps = [] + result = DeepnoteCloudRunner( + "notebook-1", + token="token", + opener=lambda _request, *, timeout: FakeResponse( + { + "run": { + "runId": "run-1", + "status": "error", + "snapshotStatus": "unavailable", + } + } + ), + sleep=sleeps.append, + ).run({}) + + assert sleeps == [] + assert result.snapshot_status == "unavailable" + + +def test_cloud_info_keeps_select_options_and_slider_bounds() -> None: + def open_request(_request: Any, *, timeout: float) -> FakeResponse: + return FakeResponse( + { + "notebook": { + "name": "Revenue", + "inputs": [ + { + "name": "region", + "type": "input-select", + "value": "EU", + "options": ["EU", "US"], + "multiple": True, + }, + { + "name": "limit", + "type": "input-slider", + "value": "5", + "min": 1, + "max": 9, + "step": 2, + }, + ], + } + } + ) + + info = DeepnoteCloudRunner("notebook-1", token="token", opener=open_request).info() + + assert info.inputs == ( + InputBlock("region", "input-select", "EU", options=("EU", "US"), multiple=True), + InputBlock("limit", "input-slider", "5", min=1, max=9, step=2), + ) + + +def test_runner_info_ignores_repeated_input_names() -> None: + info = RunnerInfo( + notebook="Revenue", + inputs=(InputBlock("region", "input-text", "EU"),), + run_target="cloud", + ) + + assert info.accepts_inputs( + [ + InputBlock("region", "input-text", "EU"), + InputBlock("region", "input-text", "US"), + ] + ) From 817eca09398980bf82d67da60c8fc6a2af8435e4 Mon Sep 17 00:00:00 2001 From: Wojciech Apanowicz Date: Fri, 18 Sep 2026 16:12:45 +0200 Subject: [PATCH 18/43] fix(streamlit): return only the executed notebook's outputs from cloud runs - Poll runs with `snapshotDelivery=blocks`. The inline delivery returns a snapshot of the whole project, so an API-key run collected outputs from every notebook in it. Found in a live test with a second notebook. - Skip the Streamlit request lookups when no script is running. Outside Streamlit each request logged "missing ScriptRunContext" twice. - Document that non-numeric dataframe cells arrive as text. --- deepnote_toolkit/notebooks/cloud_runner.py | 19 ++------ deepnote_toolkit/streamlit/auth.py | 18 ++++++-- deepnote_toolkit/streamlit/cloud_runner.py | 3 +- docs/streamlit-apps.md | 6 ++- .../test_deepnote_streamlit_cloud_runner.py | 45 ++++++++++++++++++- tests/unit/test_notebooks_runners.py | 17 ++++--- 6 files changed, 77 insertions(+), 31 deletions(-) diff --git a/deepnote_toolkit/notebooks/cloud_runner.py b/deepnote_toolkit/notebooks/cloud_runner.py index 245de845..52f2232e 100644 --- a/deepnote_toolkit/notebooks/cloud_runner.py +++ b/deepnote_toolkit/notebooks/cloud_runner.py @@ -111,7 +111,7 @@ def run(self, inputs: Mapping[str, Any]) -> RunResult: # The snapshot can attach shortly after the status turns terminal. for _ in range(SNAPSHOT_SETTLE_ATTEMPTS): is_pending = current.get("snapshotStatus", "pending") == "pending" - if not is_pending or _has_snapshot(current): + if not is_pending or isinstance(current.get("snapshotBlocks"), list): break self._sleep(self.poll_interval) try: @@ -121,10 +121,6 @@ def run(self, inputs: Mapping[str, Any]) -> RunResult: raise status = str(current.get("status", "")) - snapshot = current.get("snapshot") - snapshot_yaml = current.get("snapshotContent") - if snapshot_yaml is None and isinstance(snapshot, Mapping): - snapshot_yaml = snapshot.get("snapshotContent") error = current.get("error") if isinstance(error, Mapping): error = error.get("message") or json.dumps(error) @@ -136,15 +132,15 @@ def run(self, inputs: Mapping[str, Any]) -> RunResult: "status": status, "error": str(error) if error is not None else None, "snapshotStatus": current.get("snapshotStatus"), - "snapshotYaml": snapshot_yaml, "snapshotBlocks": current.get("snapshotBlocks"), "viewUrl": current.get("viewUrl"), } ) def _get_run(self, run_id: str) -> Mapping[str, Any]: + # The blocks delivery holds the executed notebook alone, not the whole project. return self._run_payload( - self._request("GET", f"/v2/runs/{run_id}?snapshotDelivery=inline") + self._request("GET", f"/v2/runs/{run_id}?snapshotDelivery=blocks") ) def _request( @@ -190,15 +186,6 @@ def _run_payload(payload: Mapping[str, Any]) -> Mapping[str, Any]: return run if isinstance(run, Mapping) else payload -def _has_snapshot(run: Mapping[str, Any]) -> bool: - snapshot = run.get("snapshot") - return bool( - run.get("snapshotContent") - or isinstance(run.get("snapshotBlocks"), list) - or (isinstance(snapshot, Mapping) and snapshot.get("snapshotContent")) - ) - - def _required_run_id(run: Mapping[str, Any]) -> str: run_id = run.get("runId") or run.get("id") if not isinstance(run_id, str) or not run_id: diff --git a/deepnote_toolkit/streamlit/auth.py b/deepnote_toolkit/streamlit/auth.py index 8fc7d5d8..1c4bc77a 100644 --- a/deepnote_toolkit/streamlit/auth.py +++ b/deepnote_toolkit/streamlit/auth.py @@ -217,6 +217,19 @@ def _has_hosted_streamlit_context() -> bool: ) +def _has_script_run_context() -> bool: + """Return whether this thread is running a Streamlit script for a viewer.""" + + try: + from streamlit.runtime.scriptrunner import ( # type: ignore[import-not-found] + get_script_run_ctx, + ) + except ImportError: + return False + + return get_script_run_ctx(suppress_warning=True) is not None + + def _is_streamlit_thread_without_request() -> bool: """Return whether a Streamlit server is running but this thread has no viewer request. @@ -225,13 +238,10 @@ def _is_streamlit_thread_without_request() -> bool: try: from streamlit import runtime # type: ignore[import-not-found] - from streamlit.runtime.scriptrunner import ( # type: ignore[import-not-found] - get_script_run_ctx, - ) except ImportError: return False - return runtime.exists() and get_script_run_ctx(suppress_warning=True) is None + return runtime.exists() and not _has_script_run_context() def _validated_origin(value: str, *, name: str) -> str: diff --git a/deepnote_toolkit/streamlit/cloud_runner.py b/deepnote_toolkit/streamlit/cloud_runner.py index 729575c6..6c49576d 100644 --- a/deepnote_toolkit/streamlit/cloud_runner.py +++ b/deepnote_toolkit/streamlit/cloud_runner.py @@ -8,6 +8,7 @@ from .auth import ( CurrentUserApiTokenError, _has_hosted_streamlit_context, + _has_script_run_context, _is_streamlit_thread_without_request, current_user_api_credentials, ) @@ -24,7 +25,7 @@ def _credentials(self) -> tuple[str, str]: return super()._credentials() # A hosted request authenticates as the viewer and never uses DEEPNOTE_TOKEN. - if _has_hosted_streamlit_context(): + if _has_script_run_context() and _has_hosted_streamlit_context(): try: credentials = current_user_api_credentials( timeout=min(self.timeout, 30), opener=self._open diff --git a/docs/streamlit-apps.md b/docs/streamlit-apps.md index ba40aa6b..bbcce425 100644 --- a/docs/streamlit-apps.md +++ b/docs/streamlit-apps.md @@ -34,8 +34,10 @@ program: the notebook you run so the inputs match it: `DeepnoteDocument.load(path, notebook_id="your-notebook-id")`. - `DeepnoteCloudRunner` runs an existing notebook in Deepnote Cloud and returns - its outputs as a `RunResult`. A dataframe output holds the first page of rows. - `row_count` is the full size and `is_truncated` tells whether rows are missing. + its outputs as a `RunResult`, which holds the outputs of that notebook alone. A + dataframe output holds the first page of rows. `row_count` is the full size and + `is_truncated` tells whether rows are missing. Deepnote sends every non-numeric + cell as text, so a boolean column arrives as `"True"` and `"False"`. - `DeepnoteRunner` does the same through a local `@deepnote/local-runner` sidecar at `http://127.0.0.1:8787`. - `Runner` is the interface both runners implement, for code that accepts either. diff --git a/tests/unit/test_deepnote_streamlit_cloud_runner.py b/tests/unit/test_deepnote_streamlit_cloud_runner.py index e45f3fed..de182656 100644 --- a/tests/unit/test_deepnote_streamlit_cloud_runner.py +++ b/tests/unit/test_deepnote_streamlit_cloud_runner.py @@ -61,6 +61,10 @@ def open_request(request: Any, *, timeout: float) -> FakeResponse: ), ] with ( + patch( + "deepnote_toolkit.streamlit.cloud_runner._has_script_run_context", + return_value=True, + ), patch( "deepnote_toolkit.streamlit.cloud_runner._has_hosted_streamlit_context", return_value=True, @@ -85,7 +89,7 @@ def open_request(request: Any, *, timeout: float) -> FakeResponse: 30, ), ( - "https://api.deepnote-staging.com/v2/runs/run-1" "?snapshotDelivery=inline", + "https://api.deepnote-staging.com/v2/runs/run-1" "?snapshotDelivery=blocks", "Bearer viewer-token-2", 30, ), @@ -98,6 +102,10 @@ def test_hosted_runner_never_falls_back_to_environment_token( monkeypatch.setenv("DEEPNOTE_TOKEN", "shared-token") opener = MagicMock() with ( + patch( + "deepnote_toolkit.streamlit.cloud_runner._has_script_run_context", + return_value=True, + ), patch( "deepnote_toolkit.streamlit.cloud_runner._has_hosted_streamlit_context", return_value=True, @@ -147,6 +155,10 @@ def test_cloud_run_retries_a_transient_token_exchange_failure() -> None: expires_at_seconds=1_800_000_000, ) with ( + patch( + "deepnote_toolkit.streamlit.cloud_runner._has_script_run_context", + return_value=True, + ), patch( "deepnote_toolkit.streamlit.cloud_runner._has_hosted_streamlit_context", return_value=True, @@ -201,6 +213,10 @@ def open_request(request: Any, *, timeout: float) -> FakeResponse: return FakeResponse({"notebook": {"name": "Revenue", "inputs": []}}) with ( + patch( + "deepnote_toolkit.streamlit.cloud_runner._has_script_run_context", + return_value=True, + ), patch( "deepnote_toolkit.streamlit.cloud_runner._has_hosted_streamlit_context", return_value=True, @@ -219,3 +235,30 @@ def open_request(request: Any, *, timeout: float) -> FakeResponse: ).info() assert urls == ["https://api.deepnote.com/v2/notebooks/notebook-1"] + + +def test_runner_skips_streamlit_lookups_outside_a_script_run( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("DEEPNOTE_TOKEN", "local-token") + + def open_request(request: Any, *, timeout: float) -> FakeResponse: + assert request.headers["Authorization"] == "Bearer local-token" + return FakeResponse({"notebook": {"name": "Revenue", "inputs": []}}) + + with ( + patch( + "deepnote_toolkit.streamlit.cloud_runner._has_script_run_context", + return_value=False, + ), + patch( + "deepnote_toolkit.streamlit.cloud_runner._has_hosted_streamlit_context" + ) as hosted_lookup, + patch( + "deepnote_toolkit.streamlit.cloud_runner._is_streamlit_thread_without_request", + return_value=False, + ), + ): + StreamlitCloudRunner("notebook-1", opener=open_request).info() + + hosted_lookup.assert_not_called() diff --git a/tests/unit/test_notebooks_runners.py b/tests/unit/test_notebooks_runners.py index 2e67bf79..f76237a7 100644 --- a/tests/unit/test_notebooks_runners.py +++ b/tests/unit/test_notebooks_runners.py @@ -137,7 +137,7 @@ def open_request(request: Any, *, timeout: float) -> FakeResponse: assert info.inputs[0].variable_name == "region" -def test_cloud_run_posts_inputs_polls_and_parses_inline_snapshot() -> None: +def test_cloud_run_posts_inputs_polls_and_reads_the_executed_blocks() -> None: calls = [] responses = iter( [ @@ -147,9 +147,13 @@ def test_cloud_run_posts_inputs_polls_and_parses_inline_snapshot() -> None: "run": { "runId": "run-1", "status": "success", - "snapshot": { - "snapshotContent": "project:\n name: Result\n notebooks:\n - blocks: []\n" - }, + "snapshotBlocks": [ + { + "id": "code-1", + "type": "code", + "outputs": [{"output_type": "stream", "text": "done"}], + } + ], } }, ] @@ -182,7 +186,7 @@ def open_request(request: Any, *, timeout: float) -> FakeResponse: "detached": True, "inputs": {"limit": "20", "enabled": True, "regions": ["EU"]}, } - assert calls[1][0].endswith("/v2/runs/run-1?snapshotDelivery=inline") + assert calls[1][0].endswith("/v2/runs/run-1?snapshotDelivery=blocks") assert [call[2] for call in calls] == [ "Bearer token-1", "Bearer token-2", @@ -190,8 +194,7 @@ def open_request(request: Any, *, timeout: float) -> FakeResponse: ] assert sleeps == [0.25, 0.25] assert result.success is True - assert result.snapshot is not None - assert result.snapshot.project_name == "Result" + assert result.text() == "done" def test_cloud_run_reads_sanitized_snapshot_blocks_without_raw_snapshot() -> None: From 2c6f2522c0fd05009550df1b52adf45a7b4ba216 Mon Sep 17 00:00:00 2001 From: Wojciech Apanowicz Date: Fri, 18 Sep 2026 16:34:16 +0200 Subject: [PATCH 19/43] fix(streamlit): keep leading-zero scalars as strings in the YAML loader The YAML 1.2 int pattern accepted `08540` and handed it to PyYAML's YAML 1.1 constructor, which reads a leading zero as octal: `012` loaded as 10 and `08540` raised ValueError. PyYAML writes the string "08540" unquoted, so a file written by Python tooling stopped loading. - Numbers with a leading zero resolve as strings. No YAML writer emits a number that way, so this also keeps a postal code intact. - A scalar the constructor rejects raises `yaml.YAMLError`, which keeps the "Could not parse" context in `DeepnoteDocument`. - A parametrized scalar test runs against both the libyaml and the pure-Python loader. - Document `RunResult.snapshot_status` and that `base_url` does not apply to a hosted app. --- deepnote_toolkit/notebooks/yaml_loader.py | 11 +++- deepnote_toolkit/streamlit/cloud_runner.py | 3 +- docs/streamlit-apps.md | 4 +- tests/unit/test_notebooks_yaml_loader.py | 69 ++++++++++++++++++++++ 4 files changed, 83 insertions(+), 4 deletions(-) create mode 100644 tests/unit/test_notebooks_yaml_loader.py diff --git a/deepnote_toolkit/notebooks/yaml_loader.py b/deepnote_toolkit/notebooks/yaml_loader.py index e5b060be..580bf0a1 100644 --- a/deepnote_toolkit/notebooks/yaml_loader.py +++ b/deepnote_toolkit/notebooks/yaml_loader.py @@ -24,10 +24,15 @@ class _CoreSchemaLoader(_BaseLoader): # type: ignore[misc,valid-type] for _tag, _pattern, _first in ( ("null", r"^(?:~|null|Null|NULL|)$", ["~", "n", "N", ""]), ("bool", r"^(?:true|True|TRUE|false|False|FALSE)$", list("tTfF")), - ("int", r"^(?:[-+]?[0-9]+|0o[0-7]+|0x[0-9a-fA-F]+)$", list("-+0123456789")), + # A leading zero marks a string such as a postal code. No writer emits numbers so. + ( + "int", + r"^(?:[-+]?(?:0|[1-9][0-9]*)|0o[0-7]+|0x[0-9a-fA-F]+)$", + list("-+0123456789"), + ), ( "float", - r"^(?:[-+]?(?:\.[0-9]+|[0-9]+(?:\.[0-9]*)?)(?:[eE][-+]?[0-9]+)?" + r"^(?:[-+]?(?:\.[0-9]+|(?:0|[1-9][0-9]*)(?:\.[0-9]*)?)(?:[eE][-+]?[0-9]+)?" r"|[-+]?\.(?:inf|Inf|INF)|\.(?:nan|NaN|NAN))$", list("-+0123456789."), ), @@ -43,5 +48,7 @@ def load_yaml(content: str) -> Any: loader = _CoreSchemaLoader(content) try: return loader.get_single_data() + except ValueError as error: + raise yaml.YAMLError(str(error)) from error finally: loader.dispose() diff --git a/deepnote_toolkit/streamlit/cloud_runner.py b/deepnote_toolkit/streamlit/cloud_runner.py index 6c49576d..843e72bb 100644 --- a/deepnote_toolkit/streamlit/cloud_runner.py +++ b/deepnote_toolkit/streamlit/cloud_runner.py @@ -17,7 +17,8 @@ class StreamlitCloudRunner(DeepnoteCloudRunner): """Run a notebook from a Streamlit app, as the current viewer when Deepnote hosts it. - A hosted app needs no token. Elsewhere this behaves like `DeepnoteCloudRunner`. + A hosted app needs no token, and `base_url` does not apply to it. Elsewhere + this behaves like `DeepnoteCloudRunner`. """ def _credentials(self) -> tuple[str, str]: diff --git a/docs/streamlit-apps.md b/docs/streamlit-apps.md index bbcce425..04e216df 100644 --- a/docs/streamlit-apps.md +++ b/docs/streamlit-apps.md @@ -82,7 +82,9 @@ project session. The cloud runner retries a poll that fails with a timeout, a network error, HTTP 429 or a 5xx, up to five times in a row. After the run finishes it waits briefly -for the outputs, which can arrive after the final status. +for the outputs, which can arrive after the final status. When +`result.snapshot_status` is still `pending`, the outputs had not arrived by the end +of that wait. Use `runner.info().accepts_inputs(document.inputs)` before submitting values to verify that the deployed notebook still has matching input names and block types. diff --git a/tests/unit/test_notebooks_yaml_loader.py b/tests/unit/test_notebooks_yaml_loader.py new file mode 100644 index 00000000..11be7377 --- /dev/null +++ b/tests/unit/test_notebooks_yaml_loader.py @@ -0,0 +1,69 @@ +import importlib +from collections.abc import Iterator +from typing import Any + +import pytest +import yaml + +from deepnote_toolkit.notebooks import yaml_loader + + +@pytest.fixture(params=["libyaml", "pure_python"]) +def load_yaml( + request: pytest.FixtureRequest, monkeypatch: pytest.MonkeyPatch +) -> Iterator[Any]: + if request.param == "libyaml" and not hasattr(yaml, "CSafeLoader"): + pytest.skip("PyYAML was built without libyaml") + if request.param == "pure_python": + monkeypatch.delattr(yaml, "CSafeLoader", raising=False) + yield importlib.reload(yaml_loader).load_yaml + monkeypatch.undo() + importlib.reload(yaml_loader) + + +@pytest.mark.parametrize( + ("scalar", "expected"), + [ + ("No", "No"), + ("yes", "yes"), + ("on", "on"), + ("true", True), + ("FALSE", False), + ("12:30", "12:30"), + ("1_000", "1_000"), + ("2026-08-17", "2026-08-17"), + ("2026-08-17T00:00:00.000Z", "2026-08-17T00:00:00.000Z"), + ("08540", "08540"), + ("012", "012"), + ("08540.5", "08540.5"), + ("0.5", 0.5), + ("10", 10), + ("0", 0), + ("-7", -7), + ("+12", 12), + ("0o17", 15), + ("0x1F", 31), + ("2.50", 2.5), + (".5", 0.5), + ("1e3", 1000.0), + ("~", None), + ("null", None), + ("", None), + ("'08540'", "08540"), + ('"true"', "true"), + ], +) +def test_plain_scalars_resolve_by_the_yaml_1_2_core_schema( + load_yaml: Any, scalar: str, expected: object +) -> None: + assert load_yaml(f"value: {scalar}\n") == {"value": expected} + + +def test_python_object_tags_are_rejected(load_yaml: Any) -> None: + with pytest.raises(yaml.YAMLError): + load_yaml("value: !!python/object/apply:os.getcwd []\n") + + +def test_a_scalar_the_constructor_rejects_raises_a_yaml_error(load_yaml: Any) -> None: + with pytest.raises(yaml.YAMLError): + load_yaml("value: !!int twelve\n") From 372c4fd799b9194c5fda43a9dc34beb2fdcc2129 Mon Sep 17 00:00:00 2001 From: Wojciech Apanowicz Date: Fri, 18 Sep 2026 16:56:37 +0200 Subject: [PATCH 20/43] fix(streamlit): reject duplicate YAML keys and keep a scalar multi-select default - A `.deepnote` file that repeats a mapping key fails to load. PyYAML kept the last value, which could drop inputs or outputs silently. - A multi-select whose stored value is a single string renders with that option selected, the way Deepnote itself reads it. - Add docstrings to the public functions of `deepnote_toolkit.notebooks`. --- deepnote_toolkit/notebooks/cloud_runner.py | 4 ++++ deepnote_toolkit/notebooks/document.py | 4 ++++ deepnote_toolkit/notebooks/local_runner.py | 4 ++++ deepnote_toolkit/notebooks/models.py | 22 +++++++++++++++++++ deepnote_toolkit/notebooks/outputs.py | 10 +++++++++ deepnote_toolkit/notebooks/yaml_loader.py | 20 +++++++++++++++++ deepnote_toolkit/streamlit/widgets.py | 10 ++++----- tests/unit/test_deepnote_streamlit_widgets.py | 16 ++++++++++++++ tests/unit/test_notebooks_yaml_loader.py | 13 +++++++++++ 9 files changed, 97 insertions(+), 6 deletions(-) diff --git a/deepnote_toolkit/notebooks/cloud_runner.py b/deepnote_toolkit/notebooks/cloud_runner.py index 52f2232e..b44049f5 100644 --- a/deepnote_toolkit/notebooks/cloud_runner.py +++ b/deepnote_toolkit/notebooks/cloud_runner.py @@ -57,6 +57,8 @@ def __init__( self._sleep = sleep def info(self) -> RunnerInfo: + """Read the notebook's name and input blocks from the public API.""" + payload = self._request("GET", f"/v2/notebooks/{self.notebook_id}") notebook = payload.get("notebook") if not isinstance(notebook, Mapping): @@ -76,6 +78,8 @@ def info(self) -> RunnerInfo: ) def run(self, inputs: Mapping[str, Any]) -> RunResult: + """Start a detached run with the given input values and wait for its result.""" + started = self._run_payload( self._request( "POST", diff --git a/deepnote_toolkit/notebooks/document.py b/deepnote_toolkit/notebooks/document.py index a83782a2..e6a7da14 100644 --- a/deepnote_toolkit/notebooks/document.py +++ b/deepnote_toolkit/notebooks/document.py @@ -42,6 +42,8 @@ def __init__(self, raw: Mapping[str, Any], *, notebook_id: str | None = None): def load( cls, path: str | Path, *, notebook_id: str | None = None ) -> DeepnoteDocument: + """Read a `.deepnote` file from disk. Raises `ValueError` when it cannot be parsed.""" + source = Path(path) try: raw = load_yaml(source.read_text(encoding="utf-8")) @@ -53,6 +55,8 @@ def load( @classmethod def parse(cls, content: str, *, notebook_id: str | None = None) -> DeepnoteDocument: + """Read `.deepnote` YAML from a string. Raises `ValueError` when it cannot be parsed.""" + try: raw = load_yaml(content) except yaml.YAMLError as error: diff --git a/deepnote_toolkit/notebooks/local_runner.py b/deepnote_toolkit/notebooks/local_runner.py index e973cff2..73391ed8 100644 --- a/deepnote_toolkit/notebooks/local_runner.py +++ b/deepnote_toolkit/notebooks/local_runner.py @@ -27,6 +27,8 @@ def __init__( self._open = opener def info(self) -> RunnerInfo: + """Read the notebook's name and input blocks from the sidecar.""" + payload = self._request("GET", "/api/info") values = payload.get("inputs") inputs = ( @@ -47,6 +49,8 @@ def info(self) -> RunnerInfo: ) def run(self, inputs: Mapping[str, Any]) -> RunResult: + """Run the notebook in the sidecar with the given input values.""" + return RunResult(self._request("POST", "/api/run", {"inputs": dict(inputs)})) def _request( diff --git a/deepnote_toolkit/notebooks/models.py b/deepnote_toolkit/notebooks/models.py index 4bc9dc69..899ea010 100644 --- a/deepnote_toolkit/notebooks/models.py +++ b/deepnote_toolkit/notebooks/models.py @@ -36,6 +36,8 @@ class InputBlock: @classmethod def from_block(cls, block: Mapping[str, Any]) -> InputBlock | None: + """Read an input block from a `.deepnote` file. Returns None for other blocks.""" + block_type = str(block.get("type", "")) metadata = block.get("metadata") if not block_type.startswith("input-") or not isinstance(metadata, Mapping): @@ -92,6 +94,8 @@ class DeepnoteDataframe: @classmethod def from_value(cls, value: Any) -> DeepnoteDataframe | None: + """Read a dataframe output payload. Returns None when the value is not one.""" + if not isinstance(value, Mapping): return None columns = value.get("columns") @@ -113,10 +117,14 @@ def row_count(self) -> int: @property def is_truncated(self) -> bool: + """Whether the full dataframe has more rows than `rows` holds.""" + return self.row_count > len(self.rows) @property def data_columns(self) -> tuple[str, ...]: + """Column names without Deepnote's index column.""" + return tuple( str(column.get("name")) for column in self.columns @@ -144,19 +152,27 @@ class NotebookOutput: @property def output_type(self) -> str: + """The nbformat output type, such as `stream` or `execute_result`.""" + return str(self.raw.get("output_type", "")) @property def data(self) -> Mapping[str, Any]: + """The output's MIME bundle, empty for outputs that have none.""" + value = self.raw.get("data") return value if isinstance(value, Mapping) else {} def text(self, mime: str = "text/plain") -> str: + """The output's text for a MIME type. Stream outputs count as `text/plain`.""" + if self.output_type == "stream" and mime == "text/plain": return join_text(self.raw.get("text")) return join_text(self.data.get(mime)) def image_bytes(self, mime: str = "image/png") -> bytes | None: + """The decoded image for a MIME type, or None when absent or not valid base64.""" + value = self.data.get(mime) if value is None: return None @@ -168,6 +184,8 @@ def image_bytes(self, mime: str = "image/png") -> bytes | None: @property def dataframe(self) -> DeepnoteDataframe | None: + """The output as a Deepnote dataframe, or None when it is not one.""" + return DeepnoteDataframe.from_value(self.data.get(DATAFRAME_MIME)) @@ -192,10 +210,14 @@ def _input_contract(inputs: Iterable[InputBlock]) -> frozenset[tuple[str, str]]: def optional_string(value: Any) -> str | None: + """Return the value when it is a string, otherwise None.""" + return value if isinstance(value, str) else None def optional_number(value: Any) -> float | int | None: + """Return the value when it is a number other than a boolean, otherwise None.""" + return ( value if isinstance(value, (float, int)) and not isinstance(value, bool) diff --git a/deepnote_toolkit/notebooks/outputs.py b/deepnote_toolkit/notebooks/outputs.py index 65a29b3f..23c27e58 100644 --- a/deepnote_toolkit/notebooks/outputs.py +++ b/deepnote_toolkit/notebooks/outputs.py @@ -11,15 +11,21 @@ class OutputCollection: outputs: tuple[NotebookOutput, ...] def outputs_for_mime(self, mime: str) -> list[NotebookOutput]: + """Outputs that carry the given MIME type.""" + return [output for output in self.outputs if mime in output.data] def first_dataframe(self) -> DeepnoteDataframe | None: + """The first dataframe output, or None when there is none.""" + for output in self.outputs: if dataframe := output.dataframe: return dataframe return None def images(self, mime: str = "image/png") -> list[bytes]: + """Every image of the given MIME type, decoded.""" + return [ image for output in self.outputs @@ -27,9 +33,13 @@ def images(self, mime: str = "image/png") -> list[bytes]: ] def text(self, mime: str = "text/plain") -> str: + """The text of all outputs for a MIME type, joined.""" + return "".join(output.text(mime) for output in self.outputs).strip() def agent_text(self) -> str: + """The text written by agent blocks, preferring Markdown over plain text.""" + chunks: list[str] = [] for output in self.outputs: if output.block_type != "agent": diff --git a/deepnote_toolkit/notebooks/yaml_loader.py b/deepnote_toolkit/notebooks/yaml_loader.py index 580bf0a1..838cc57e 100644 --- a/deepnote_toolkit/notebooks/yaml_loader.py +++ b/deepnote_toolkit/notebooks/yaml_loader.py @@ -20,6 +20,26 @@ class _CoreSchemaLoader(_BaseLoader): # type: ignore[misc,valid-type] yaml_implicit_resolvers: dict[str, Any] = {} + def construct_mapping( + self, node: yaml.MappingNode, deep: bool = False + ) -> dict[Any, Any]: + """Build a mapping, rejecting a repeated key. PyYAML keeps the last one silently.""" + + seen: set[tuple[str, str]] = set() + for key_node, _value_node in node.value: + if not isinstance(key_node, yaml.ScalarNode): + continue + key = (key_node.tag, key_node.value) + if key in seen: + raise yaml.constructor.ConstructorError( + None, + None, + f"found duplicate key {key_node.value!r}", + key_node.start_mark, + ) + seen.add(key) + return super().construct_mapping(node, deep=deep) + for _tag, _pattern, _first in ( ("null", r"^(?:~|null|Null|NULL|)$", ["~", "n", "N", ""]), diff --git a/deepnote_toolkit/streamlit/widgets.py b/deepnote_toolkit/streamlit/widgets.py index 11cbc526..02281491 100644 --- a/deepnote_toolkit/streamlit/widgets.py +++ b/deepnote_toolkit/streamlit/widgets.py @@ -52,13 +52,11 @@ def _render_one(container: Any, input_block: InputBlock, label: str, key: str) - if input_block.type == "input-select": options = list(input_block.options) if input_block.multiple: - raw_defaults = ( - input_block.value if isinstance(input_block.value, list) else [] - ) + value = input_block.value + if not isinstance(value, list): + value = [] if value is None else [value] defaults = [ - normalized - for value in raw_defaults - if (normalized := str(value)) in options + normalized for item in value if (normalized := str(item)) in options ] return container.multiselect(label, options, default=defaults, key=key) index = ( diff --git a/tests/unit/test_deepnote_streamlit_widgets.py b/tests/unit/test_deepnote_streamlit_widgets.py index 22ddb49e..e6675729 100644 --- a/tests/unit/test_deepnote_streamlit_widgets.py +++ b/tests/unit/test_deepnote_streamlit_widgets.py @@ -181,3 +181,19 @@ def test_inputs_sharing_a_variable_name_render_once() -> None: ) assert values == {"region": "EU"} + + +def test_multiselect_treats_a_scalar_default_as_one_selection() -> None: + values = render_inputs( + [ + InputBlock( + "regions", "input-select", "EU", options=("EU", "US"), multiple=True + ), + InputBlock( + "empty", "input-select", None, options=("EU", "US"), multiple=True + ), + ], + FakeContainer(), + ) + + assert values == {"regions": ["EU"], "empty": []} diff --git a/tests/unit/test_notebooks_yaml_loader.py b/tests/unit/test_notebooks_yaml_loader.py index 11be7377..b7b47f67 100644 --- a/tests/unit/test_notebooks_yaml_loader.py +++ b/tests/unit/test_notebooks_yaml_loader.py @@ -67,3 +67,16 @@ def test_python_object_tags_are_rejected(load_yaml: Any) -> None: def test_a_scalar_the_constructor_rejects_raises_a_yaml_error(load_yaml: Any) -> None: with pytest.raises(yaml.YAMLError): load_yaml("value: !!int twelve\n") + + +def test_a_repeated_mapping_key_is_rejected(load_yaml: Any) -> None: + with pytest.raises(yaml.YAMLError, match="duplicate key 'notebooks'"): + load_yaml("project:\n notebooks: [a]\n notebooks: [b]\n") + + +def test_the_same_key_may_repeat_in_separate_mappings(load_yaml: Any) -> None: + assert load_yaml("- id: a\n- id: b\n- 1: x\n '1': y\n") == [ + {"id": "a"}, + {"id": "b"}, + {1: "x", "1": "y"}, + ] From d5d2798167aee0eb1e1af5d573ac48a9d600a345 Mon Sep 17 00:00:00 2001 From: Wojciech Apanowicz Date: Fri, 18 Sep 2026 17:10:41 +0200 Subject: [PATCH 21/43] fix(streamlit): show the server's reason when viewer credentials are refused A refused credential request reported only its HTTP status, so an app author saw "HTTP 403" with no hint that API access is turned off for the project. The error now includes the message from a JSON error response. A response body of any other shape, such as a proxy error page, stays out of the error. --- deepnote_toolkit/streamlit/auth.py | 17 +++++++++- tests/unit/test_deepnote_streamlit_auth.py | 38 ++++++++++++++++++---- 2 files changed, 47 insertions(+), 8 deletions(-) diff --git a/deepnote_toolkit/streamlit/auth.py b/deepnote_toolkit/streamlit/auth.py index 1c4bc77a..2bf675e7 100644 --- a/deepnote_toolkit/streamlit/auth.py +++ b/deepnote_toolkit/streamlit/auth.py @@ -115,8 +115,10 @@ def current_user_api_credentials( with opener(request, timeout=timeout) as response: payload = json.loads(response.read()) except HTTPError as error: + message = _server_message(error) raise CurrentUserApiTokenError( - f"Current viewer API-token exchange returned HTTP {error.code}.", + f"Current viewer API-token exchange returned HTTP {error.code}" + + (f": {message}" if message else "."), transient=error.code == 429 or error.code >= 500, ) from error except URLError as error: @@ -244,6 +246,19 @@ def _is_streamlit_thread_without_request() -> bool: return runtime.exists() and not _has_script_run_context() +def _server_message(error: HTTPError) -> str | None: + """Return the message of a JSON error response. A body of any other shape is not shown.""" + + try: + payload = json.loads(error.read()) + except (json.JSONDecodeError, UnicodeDecodeError): + return None + if not isinstance(payload, Mapping): + return None + message = payload.get("error") or payload.get("message") + return message if isinstance(message, str) else None + + def _validated_origin(value: str, *, name: str) -> str: parsed = urlparse(value) if ( diff --git a/tests/unit/test_deepnote_streamlit_auth.py b/tests/unit/test_deepnote_streamlit_auth.py index 616084e4..1baf441c 100644 --- a/tests/unit/test_deepnote_streamlit_auth.py +++ b/tests/unit/test_deepnote_streamlit_auth.py @@ -256,16 +256,41 @@ def test_exchange_rejects_invalid_response(payload: dict[str, Any]) -> None: ) -def test_exchange_error_does_not_expose_response_body() -> None: - secret_response = "must-not-leak" +def test_exchange_error_includes_the_server_message() -> None: + def open_request(*_args: Any, **_kwargs: Any) -> FakeResponse: + raise HTTPError( + "http://localhost:19456/userpod-api/streamlit-apps/id/api-token", + 403, + "Forbidden", + {}, + io.BytesIO( + json.dumps( + {"error": "API access is not available for this app"} + ).encode() + ), + ) + + with pytest.raises(CurrentUserApiTokenError) as exc_info: + current_user_api_credentials( + app_id=APP_ID, + streamlit_token="opaque-cookie", + opener=open_request, + ) + + assert str(exc_info.value) == ( + "Current viewer API-token exchange returned HTTP 403: " + "API access is not available for this app" + ) + +def test_exchange_error_does_not_expose_a_raw_response_body() -> None: def open_request(*_args: Any, **_kwargs: Any) -> FakeResponse: raise HTTPError( "http://localhost:19456/userpod-api/streamlit-apps/id/api-token", - 401, - "Unauthorized", + 502, + "Bad Gateway", {}, - io.BytesIO(json.dumps({"error": secret_response}).encode()), + io.BytesIO(b"must-not-leak"), ) with pytest.raises(CurrentUserApiTokenError) as exc_info: @@ -275,8 +300,7 @@ def open_request(*_args: Any, **_kwargs: Any) -> FakeResponse: opener=open_request, ) - assert "HTTP 401" in str(exc_info.value) - assert secret_response not in str(exc_info.value) + assert str(exc_info.value) == "Current viewer API-token exchange returned HTTP 502." def test_credentials_repr_hides_the_token() -> None: From 6bfbe9fdae2a8c461438e8a1c18152df5178662d Mon Sep 17 00:00:00 2001 From: Wojciech Apanowicz Date: Fri, 18 Sep 2026 18:08:44 +0200 Subject: [PATCH 22/43] style(streamlit): keep source lines within 88 characters Black does not wrap docstrings or strings, so eleven docstring lines and one message ran past the 88 characters the contributor guidelines ask for. --- deepnote_toolkit/notebooks/cloud_runner.py | 2 +- deepnote_toolkit/notebooks/document.py | 4 ++-- deepnote_toolkit/notebooks/models.py | 6 +++--- deepnote_toolkit/notebooks/yaml_loader.py | 2 +- deepnote_toolkit/streamlit/auth.py | 4 ++-- deepnote_toolkit/streamlit/cloud_runner.py | 2 +- deepnote_toolkit/streamlit/widgets.py | 5 +++-- 7 files changed, 13 insertions(+), 12 deletions(-) diff --git a/deepnote_toolkit/notebooks/cloud_runner.py b/deepnote_toolkit/notebooks/cloud_runner.py index b44049f5..18c2f6bd 100644 --- a/deepnote_toolkit/notebooks/cloud_runner.py +++ b/deepnote_toolkit/notebooks/cloud_runner.py @@ -98,7 +98,7 @@ def run(self, inputs: Mapping[str, Any]) -> RunResult: while str(current.get("status", "")) not in TERMINAL_RUN_STATUSES: if time.monotonic() >= deadline: raise RunnerError( - f"Deepnote run {run_id} did not finish within {self.timeout:g} seconds" + f"Deepnote run {run_id} did not finish in {self.timeout:g} seconds" ) self._sleep(self.poll_interval) try: diff --git a/deepnote_toolkit/notebooks/document.py b/deepnote_toolkit/notebooks/document.py index e6a7da14..52578490 100644 --- a/deepnote_toolkit/notebooks/document.py +++ b/deepnote_toolkit/notebooks/document.py @@ -42,7 +42,7 @@ def __init__(self, raw: Mapping[str, Any], *, notebook_id: str | None = None): def load( cls, path: str | Path, *, notebook_id: str | None = None ) -> DeepnoteDocument: - """Read a `.deepnote` file from disk. Raises `ValueError` when it cannot be parsed.""" + """Read a `.deepnote` file from disk. Raises `ValueError` on invalid content.""" source = Path(path) try: @@ -55,7 +55,7 @@ def load( @classmethod def parse(cls, content: str, *, notebook_id: str | None = None) -> DeepnoteDocument: - """Read `.deepnote` YAML from a string. Raises `ValueError` when it cannot be parsed.""" + """Read `.deepnote` YAML from a string. Raises `ValueError` when invalid.""" try: raw = load_yaml(content) diff --git a/deepnote_toolkit/notebooks/models.py b/deepnote_toolkit/notebooks/models.py index 899ea010..8b287e25 100644 --- a/deepnote_toolkit/notebooks/models.py +++ b/deepnote_toolkit/notebooks/models.py @@ -36,7 +36,7 @@ class InputBlock: @classmethod def from_block(cls, block: Mapping[str, Any]) -> InputBlock | None: - """Read an input block from a `.deepnote` file. Returns None for other blocks.""" + """Read an input block from a `.deepnote` file, or None for another block.""" block_type = str(block.get("type", "")) metadata = block.get("metadata") @@ -132,7 +132,7 @@ def data_columns(self) -> tuple[str, ...]: ) def records(self, *, include_index: bool = True) -> list[dict[str, Any]]: - """Return rows ready for `st.dataframe`, optionally omitting Deepnote's index column.""" + """Return rows ready for `st.dataframe`, optionally without the index column.""" if include_index: return [dict(row) for row in self.rows] @@ -171,7 +171,7 @@ def text(self, mime: str = "text/plain") -> str: return join_text(self.data.get(mime)) def image_bytes(self, mime: str = "image/png") -> bytes | None: - """The decoded image for a MIME type, or None when absent or not valid base64.""" + """The decoded image for a MIME type, or None when absent or not base64.""" value = self.data.get(mime) if value is None: diff --git a/deepnote_toolkit/notebooks/yaml_loader.py b/deepnote_toolkit/notebooks/yaml_loader.py index 838cc57e..f540939b 100644 --- a/deepnote_toolkit/notebooks/yaml_loader.py +++ b/deepnote_toolkit/notebooks/yaml_loader.py @@ -23,7 +23,7 @@ class _CoreSchemaLoader(_BaseLoader): # type: ignore[misc,valid-type] def construct_mapping( self, node: yaml.MappingNode, deep: bool = False ) -> dict[Any, Any]: - """Build a mapping, rejecting a repeated key. PyYAML keeps the last one silently.""" + """Build a mapping, rejecting a repeated key. PyYAML would keep the last.""" seen: set[tuple[str, str]] = set() for key_node, _value_node in node.value: diff --git a/deepnote_toolkit/streamlit/auth.py b/deepnote_toolkit/streamlit/auth.py index 2bf675e7..cf360de3 100644 --- a/deepnote_toolkit/streamlit/auth.py +++ b/deepnote_toolkit/streamlit/auth.py @@ -233,7 +233,7 @@ def _has_script_run_context() -> bool: def _is_streamlit_thread_without_request() -> bool: - """Return whether a Streamlit server is running but this thread has no viewer request. + """Return whether Streamlit is running but this thread has no viewer request. Worker threads see no headers or cookies, so they look identical to a local script. """ @@ -247,7 +247,7 @@ def _is_streamlit_thread_without_request() -> bool: def _server_message(error: HTTPError) -> str | None: - """Return the message of a JSON error response. A body of any other shape is not shown.""" + """Return the message of a JSON error response, or None for any other body.""" try: payload = json.loads(error.read()) diff --git a/deepnote_toolkit/streamlit/cloud_runner.py b/deepnote_toolkit/streamlit/cloud_runner.py index 843e72bb..02e1976e 100644 --- a/deepnote_toolkit/streamlit/cloud_runner.py +++ b/deepnote_toolkit/streamlit/cloud_runner.py @@ -15,7 +15,7 @@ class StreamlitCloudRunner(DeepnoteCloudRunner): - """Run a notebook from a Streamlit app, as the current viewer when Deepnote hosts it. + """Run a notebook from a Streamlit app, as the viewer when Deepnote hosts it. A hosted app needs no token, and `base_url` does not apply to it. Elsewhere this behaves like `DeepnoteCloudRunner`. diff --git a/deepnote_toolkit/streamlit/widgets.py b/deepnote_toolkit/streamlit/widgets.py index 02281491..5295de32 100644 --- a/deepnote_toolkit/streamlit/widgets.py +++ b/deepnote_toolkit/streamlit/widgets.py @@ -23,8 +23,9 @@ def render_inputs( ) -> dict[str, Any]: """Render input blocks and return API-ready values keyed by variable name. - `container` may be `st`, `st.sidebar`, or a fake with the same widget methods for tests. When it - is omitted, Streamlit is imported lazily so parsing and API clients work without the app extra. + `container` may be `st`, `st.sidebar`, or a fake with the same widget methods + for tests. When it is omitted, Streamlit is imported lazily so parsing and API + clients work without the app extra. """ if container is None: From 125232043cacc84b0622d509d7073c6ccf60cd77 Mon Sep 17 00:00:00 2001 From: Wojciech Apanowicz Date: Fri, 18 Sep 2026 18:58:15 +0200 Subject: [PATCH 23/43] fix(streamlit): run hosted apps as the viewer with read-only storage A hosted app now uses the viewer's credentials even when the script passes token= or token_provider=, so a script developed with a token does not run every viewer's notebook as its author once deployed. StreamlitCloudRunner starts runs with storage_mode="readonly". The new storage_mode option on DeepnoteCloudRunner defaults to the API's choice. --- deepnote_toolkit/notebooks/cloud_runner.py | 27 ++++---- deepnote_toolkit/streamlit/cloud_runner.py | 51 +++++++++++--- docs/streamlit-apps.md | 16 +++-- .../test_deepnote_streamlit_cloud_runner.py | 66 +++++++++++++++++++ tests/unit/test_notebooks_runners.py | 23 +++++++ 5 files changed, 158 insertions(+), 25 deletions(-) diff --git a/deepnote_toolkit/notebooks/cloud_runner.py b/deepnote_toolkit/notebooks/cloud_runner.py index 18c2f6bd..729e1f44 100644 --- a/deepnote_toolkit/notebooks/cloud_runner.py +++ b/deepnote_toolkit/notebooks/cloud_runner.py @@ -6,7 +6,7 @@ import os import time from collections.abc import Callable, Mapping -from typing import Any +from typing import Any, Literal from urllib.request import Request, urlopen from .http import OpenUrl, request_json @@ -15,6 +15,7 @@ from .runner import RunnerError TokenProvider = Callable[[], str] +StorageMode = Literal["read_write", "readonly"] Sleep = Callable[[float], None] TERMINAL_RUN_STATUSES = frozenset({"success", "error", "internal_error", "stopped"}) @@ -29,6 +30,9 @@ class DeepnoteCloudRunner: The token comes from `token`, `token_provider` or the `DEEPNOTE_TOKEN` environment variable. A token provider is called for every request, which lets a long-lived process use short-lived credentials. + + `storage_mode="readonly"` keeps the run from changing the project's stored + files. None leaves the choice to the API, which allows writes. """ def __init__( @@ -38,6 +42,7 @@ def __init__( token: str | None = None, token_provider: TokenProvider | None = None, base_url: str = DEFAULT_API_ORIGIN, + storage_mode: StorageMode | None = None, timeout: float = 600, poll_interval: float = 2, opener: OpenUrl = urlopen, @@ -49,6 +54,7 @@ def __init__( raise ValueError("Pass token or token_provider, not both") self.notebook_id = notebook_id self.base_url = base_url.rstrip("/") + self.storage_mode = storage_mode self.timeout = timeout self.poll_interval = poll_interval self._static_token = token @@ -80,17 +86,14 @@ def info(self) -> RunnerInfo: def run(self, inputs: Mapping[str, Any]) -> RunResult: """Start a detached run with the given input values and wait for its result.""" - started = self._run_payload( - self._request( - "POST", - "/v2/runs", - { - "notebookId": self.notebook_id, - "detached": True, - "inputs": _normalize_cloud_inputs(inputs), - }, - ) - ) + body: dict[str, Any] = { + "notebookId": self.notebook_id, + "detached": True, + "inputs": _normalize_cloud_inputs(inputs), + } + if self.storage_mode is not None: + body["detachedRunStorageMode"] = self.storage_mode + started = self._run_payload(self._request("POST", "/v2/runs", body)) run_id = _required_run_id(started) deadline = time.monotonic() + self.timeout current = started diff --git a/deepnote_toolkit/streamlit/cloud_runner.py b/deepnote_toolkit/streamlit/cloud_runner.py index 02e1976e..1201766a 100644 --- a/deepnote_toolkit/streamlit/cloud_runner.py +++ b/deepnote_toolkit/streamlit/cloud_runner.py @@ -2,7 +2,17 @@ from __future__ import annotations -from deepnote_toolkit.notebooks.cloud_runner import DeepnoteCloudRunner +import time +from urllib.request import urlopen + +from deepnote_toolkit.notebooks.cloud_runner import ( + DEFAULT_API_ORIGIN, + DeepnoteCloudRunner, + Sleep, + StorageMode, + TokenProvider, +) +from deepnote_toolkit.notebooks.http import OpenUrl from deepnote_toolkit.notebooks.runner import RunnerError from .auth import ( @@ -17,15 +27,37 @@ class StreamlitCloudRunner(DeepnoteCloudRunner): """Run a notebook from a Streamlit app, as the viewer when Deepnote hosts it. - A hosted app needs no token, and `base_url` does not apply to it. Elsewhere - this behaves like `DeepnoteCloudRunner`. + A hosted app always runs as the viewer. `token`, `token_provider` and + `base_url` apply only outside Deepnote hosting. Runs cannot change the + project's stored files unless `storage_mode` says otherwise. """ - def _credentials(self) -> tuple[str, str]: - if self._token_provider is not None or self._static_token is not None: - return super()._credentials() + def __init__( + self, + notebook_id: str, + *, + token: str | None = None, + token_provider: TokenProvider | None = None, + base_url: str = DEFAULT_API_ORIGIN, + storage_mode: StorageMode | None = "readonly", + timeout: float = 600, + poll_interval: float = 2, + opener: OpenUrl = urlopen, + sleep: Sleep = time.sleep, + ): + super().__init__( + notebook_id, + token=token, + token_provider=token_provider, + base_url=base_url, + storage_mode=storage_mode, + timeout=timeout, + poll_interval=poll_interval, + opener=opener, + sleep=sleep, + ) - # A hosted request authenticates as the viewer and never uses DEEPNOTE_TOKEN. + def _credentials(self) -> tuple[str, str]: if _has_script_run_context() and _has_hosted_streamlit_context(): try: credentials = current_user_api_credentials( @@ -35,7 +67,10 @@ def _credentials(self) -> tuple[str, str]: raise RunnerError(str(error), transient=error.transient) from error return credentials.token, credentials.api_origin - if _is_streamlit_thread_without_request(): + is_token_explicit = ( + self._token_provider is not None or self._static_token is not None + ) + if not is_token_explicit and _is_streamlit_thread_without_request(): raise RunnerError( "No viewer request is available on this thread. Call the runner from " "the Streamlit script thread, or pass token= or token_provider=." diff --git a/docs/streamlit-apps.md b/docs/streamlit-apps.md index 04e216df..423f95a4 100644 --- a/docs/streamlit-apps.md +++ b/docs/streamlit-apps.md @@ -59,9 +59,6 @@ A Streamlit app hosted by Deepnote needs no token configuration. access, and never as the app's owner. A viewer who loses access to the project can no longer run it. -Call the runner from the Streamlit script thread. A worker thread has no viewer -request, so the runner raises there instead of using `DEEPNOTE_TOKEN`. - For local development, pass an API token explicitly or set `DEEPNOTE_TOKEN`: ```python @@ -69,7 +66,13 @@ runner = StreamlitCloudRunner("your-notebook-id", token="your-api-token") ``` A callable `token_provider=` can supply a renewable token. It is invoked for every -request. +request. A hosted app ignores both and still runs as the viewer, so the same +script works locally and deployed. To run notebooks with one fixed token for every +viewer, use `DeepnoteCloudRunner` with that token. + +Call the runner from the Streamlit script thread. A worker thread has no viewer +request, so the runner raises there instead of using `DEEPNOTE_TOKEN`. With an +explicit token it uses that token there, even in a hosted app. For another Deepnote API client inside a hosted app, `current_user_api_credentials()` returns a short-lived token for the current @@ -78,7 +81,10 @@ viewer together with the API origin to send it to. ## Runs Cloud runs are detached, which keeps viewer-triggered work out of the shared -project session. +project session. `StreamlitCloudRunner` also starts them with +`storage_mode="readonly"`, so a run can read the project's stored files but not +change them. Pass `storage_mode="read_write"` for a notebook that must write them. +`DeepnoteCloudRunner` leaves the mode to the API, which allows writes. The cloud runner retries a poll that fails with a timeout, a network error, HTTP 429 or a 5xx, up to five times in a row. After the run finishes it waits briefly diff --git a/tests/unit/test_deepnote_streamlit_cloud_runner.py b/tests/unit/test_deepnote_streamlit_cloud_runner.py index de182656..52bb3e90 100644 --- a/tests/unit/test_deepnote_streamlit_cloud_runner.py +++ b/tests/unit/test_deepnote_streamlit_cloud_runner.py @@ -262,3 +262,69 @@ def open_request(request: Any, *, timeout: float) -> FakeResponse: StreamlitCloudRunner("notebook-1", opener=open_request).info() hosted_lookup.assert_not_called() + + +def test_hosted_runner_ignores_an_explicit_token() -> None: + authorizations = [] + + def open_request(request: Any, *, timeout: float) -> FakeResponse: + authorizations.append(request.headers["Authorization"]) + return FakeResponse({"notebook": {"name": "Revenue", "inputs": []}}) + + with ( + patch( + "deepnote_toolkit.streamlit.cloud_runner._has_script_run_context", + return_value=True, + ), + patch( + "deepnote_toolkit.streamlit.cloud_runner._has_hosted_streamlit_context", + return_value=True, + ), + patch( + "deepnote_toolkit.streamlit.cloud_runner.current_user_api_credentials", + return_value=CurrentUserApiCredentials( + token="viewer-token", + api_origin="https://api.deepnote.com", + expires_at_seconds=1_800_000_000, + ), + ), + ): + StreamlitCloudRunner( + "notebook-1", token="owner-token", opener=open_request + ).info() + + assert authorizations == ["Bearer viewer-token"] + + +def test_streamlit_runs_are_readonly_by_default( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("DEEPNOTE_TOKEN", "local-token") + bodies = [] + + def open_request(request: Any, *, timeout: float) -> FakeResponse: + bodies.append(json.loads(request.data)) + return FakeResponse( + {"run": {"runId": "run-1", "status": "success", "snapshotBlocks": []}} + ) + + with ( + patch( + "deepnote_toolkit.streamlit.cloud_runner._has_script_run_context", + return_value=False, + ), + patch( + "deepnote_toolkit.streamlit.cloud_runner._is_streamlit_thread_without_request", + return_value=False, + ), + ): + StreamlitCloudRunner("notebook-1", opener=open_request).run({}) + + assert bodies == [ + { + "notebookId": "notebook-1", + "detached": True, + "inputs": {}, + "detachedRunStorageMode": "readonly", + } + ] diff --git a/tests/unit/test_notebooks_runners.py b/tests/unit/test_notebooks_runners.py index f76237a7..09108f2d 100644 --- a/tests/unit/test_notebooks_runners.py +++ b/tests/unit/test_notebooks_runners.py @@ -519,3 +519,26 @@ def test_runner_info_ignores_repeated_input_names() -> None: InputBlock("region", "input-text", "US"), ] ) + + +def test_cloud_run_sends_the_requested_storage_mode() -> None: + bodies = [] + + def open_request(request: Any, *, timeout: float) -> FakeResponse: + bodies.append(json.loads(request.data)) + return FakeResponse( + {"run": {"runId": "run-1", "status": "success", "snapshotBlocks": []}} + ) + + DeepnoteCloudRunner( + "notebook-1", token="token", storage_mode="readonly", opener=open_request + ).run({}) + + assert bodies == [ + { + "notebookId": "notebook-1", + "detached": True, + "inputs": {}, + "detachedRunStorageMode": "readonly", + } + ] From fa8c501c570a0ea6fc4b1b8ab851682c72c3b7bb Mon Sep 17 00:00:00 2001 From: Wojciech Apanowicz Date: Fri, 18 Sep 2026 19:06:40 +0200 Subject: [PATCH 24/43] refactor(notebooks): compose the cloud runner from a client, credentials and a transport DeepnoteCloudRunner now only starts a run, polls it and waits for its outputs. DeepnoteApiClient sends the API requests and validates the responses. A CredentialsProvider returns the token with the origin it is valid at. A Transport sends one JSON request, with UrllibTransport as the default, and replaces the opener argument. StreamlitCloudRunner no longer overrides runner internals. It passes ViewerCredentials, which is also exported for direct use. The models hold data only. Reading .deepnote blocks moved to the document module and API decoding to the client. RunResult is a dataclass built by each runner. Run statuses, snapshot statuses, storage modes, input block types and input values have narrow types. --- deepnote_toolkit/notebooks/__init__.py | 23 +++ deepnote_toolkit/notebooks/api_client.py | 146 +++++++++++++ deepnote_toolkit/notebooks/api_types.py | 29 +++ deepnote_toolkit/notebooks/cloud_runner.py | 191 +++++------------- deepnote_toolkit/notebooks/credentials.py | 54 +++++ deepnote_toolkit/notebooks/document.py | 48 +++-- deepnote_toolkit/notebooks/http.py | 60 ------ deepnote_toolkit/notebooks/local_runner.py | 69 ++++--- deepnote_toolkit/notebooks/models.py | 71 +------ deepnote_toolkit/notebooks/run_result.py | 94 ++------- deepnote_toolkit/notebooks/transport.py | 101 +++++++++ deepnote_toolkit/notebooks/wire.py | 77 +++++++ deepnote_toolkit/streamlit/__init__.py | 2 + deepnote_toolkit/streamlit/cloud_runner.py | 53 +---- .../streamlit/viewer_credentials.py | 57 ++++++ docs/streamlit-apps.md | 12 ++ .../test_deepnote_streamlit_cloud_runner.py | 95 +++++---- tests/unit/test_notebooks_document.py | 80 +++++--- tests/unit/test_notebooks_runners.py | 125 +++++++++--- 19 files changed, 875 insertions(+), 512 deletions(-) create mode 100644 deepnote_toolkit/notebooks/api_client.py create mode 100644 deepnote_toolkit/notebooks/api_types.py create mode 100644 deepnote_toolkit/notebooks/credentials.py delete mode 100644 deepnote_toolkit/notebooks/http.py create mode 100644 deepnote_toolkit/notebooks/transport.py create mode 100644 deepnote_toolkit/notebooks/wire.py create mode 100644 deepnote_toolkit/streamlit/viewer_credentials.py diff --git a/deepnote_toolkit/notebooks/__init__.py b/deepnote_toolkit/notebooks/__init__.py index b206508a..e088ca58 100644 --- a/deepnote_toolkit/notebooks/__init__.py +++ b/deepnote_toolkit/notebooks/__init__.py @@ -1,6 +1,15 @@ """Read `.deepnote` files and run notebooks, independent of any UI framework.""" +from .api_client import CloudNotebook, CloudRun, DeepnoteApiClient +from .api_types import ( + InputBlockType, + InputValue, + RunStatus, + SnapshotStatus, + StorageMode, +) from .cloud_runner import DeepnoteCloudRunner +from .credentials import ApiCredentials, CredentialsProvider, token_credentials from .document import DeepnoteDocument from .local_runner import DeepnoteRunner from .models import ( @@ -14,19 +23,33 @@ ) from .run_result import RunResult from .runner import Runner, RunnerError +from .transport import Transport, UrllibTransport __all__ = [ "DATAFRAME_MIME", "INDEX_COLUMN", + "ApiCredentials", + "CloudNotebook", + "CloudRun", + "CredentialsProvider", + "DeepnoteApiClient", "DeepnoteCloudRunner", "DeepnoteDataframe", "DeepnoteDocument", "DeepnoteRunner", "InputBlock", + "InputBlockType", + "InputValue", "NotebookOutput", "RunResult", + "RunStatus", "Runner", "RunnerError", "RunnerInfo", + "SnapshotStatus", + "StorageMode", + "Transport", + "UrllibTransport", "join_text", + "token_credentials", ] diff --git a/deepnote_toolkit/notebooks/api_client.py b/deepnote_toolkit/notebooks/api_client.py new file mode 100644 index 00000000..973301ee --- /dev/null +++ b/deepnote_toolkit/notebooks/api_client.py @@ -0,0 +1,146 @@ +"""The Deepnote public API operations a runner needs.""" + +from __future__ import annotations + +import json +from collections.abc import Mapping +from dataclasses import dataclass +from typing import Any, cast + +from .api_types import ( + SNAPSHOT_STATUSES, + TERMINAL_RUN_STATUSES, + InputValue, + RunStatus, + SnapshotStatus, + StorageMode, +) +from .credentials import CredentialsProvider +from .models import InputBlock, NotebookOutput +from .runner import RunnerError +from .transport import Transport, UrllibTransport +from .wire import decode_block_outputs, decode_inputs, optional_string + + +@dataclass(frozen=True) +class CloudNotebook: + """A notebook's name and input blocks.""" + + name: str + inputs: tuple[InputBlock, ...] + + +@dataclass(frozen=True) +class CloudRun: + """The state of one run. `outputs` is None until the run's snapshot is stored.""" + + run_id: str + status: RunStatus + snapshot_status: SnapshotStatus | None + outputs: tuple[NotebookOutput, ...] | None + error: str | None + view_url: str | None + + @property + def is_finished(self) -> bool: + """Whether the run has reached a final status.""" + + return self.status in TERMINAL_RUN_STATUSES + + +class DeepnoteApiClient: + """Sends requests to the Deepnote public API and validates what comes back.""" + + def __init__( + self, + credentials: CredentialsProvider, + *, + transport: Transport | None = None, + request_timeout: float = 30, + ): + self._credentials = credentials + self._transport = transport or UrllibTransport() + self._request_timeout = request_timeout + + def get_notebook(self, notebook_id: str) -> CloudNotebook: + """Read a notebook's name and input blocks.""" + + notebook = self._request("GET", f"/v2/notebooks/{notebook_id}").get("notebook") + if not isinstance(notebook, Mapping): + raise RunnerError("Deepnote API response did not include a notebook") + return CloudNotebook( + name=str(notebook.get("name", "Untitled notebook")), + inputs=decode_inputs(notebook.get("inputs"), name_key="name"), + ) + + def create_run( + self, + notebook_id: str, + inputs: Mapping[str, Any], + *, + storage_mode: StorageMode | None = None, + ) -> CloudRun: + """Start a detached run of the whole notebook.""" + + body: dict[str, Any] = { + "notebookId": notebook_id, + "detached": True, + "inputs": {name: _encode_input(value) for name, value in inputs.items()}, + } + if storage_mode is not None: + body["detachedRunStorageMode"] = storage_mode + return _decode_run(self._request("POST", "/v2/runs", body)) + + def get_run(self, run_id: str) -> CloudRun: + """Read a run with the outputs of the notebook it executed.""" + + # The blocks delivery holds the executed notebook alone, not the whole project. + payload = self._request("GET", f"/v2/runs/{run_id}?snapshotDelivery=blocks") + return _decode_run(payload, run_id=run_id) + + def _request( + self, method: str, path: str, body: Mapping[str, Any] | None = None + ) -> Mapping[str, Any]: + credentials = self._credentials() + return self._transport.request_json( + method, + f"{credentials.api_origin}{path}", + headers={"Authorization": f"Bearer {credentials.token}"}, + body=body, + timeout=self._request_timeout, + ) + + +def _encode_input(value: Any) -> InputValue: + if isinstance(value, bool): + return value + if isinstance(value, list): + return [str(item) for item in value] + return str(value) + + +def _decode_run(payload: Mapping[str, Any], *, run_id: str | None = None) -> CloudRun: + nested = payload.get("run") + run = nested if isinstance(nested, Mapping) else payload + run_id = run.get("runId") or run.get("id") or run_id + if not isinstance(run_id, str) or not run_id: + raise RunnerError("Deepnote API response did not include a run id") + snapshot_status = run.get("snapshotStatus") + blocks = run.get("snapshotBlocks") + error = run.get("error") + if isinstance(error, Mapping): + error = error.get("message") or json.dumps(error) + return CloudRun( + run_id=run_id, + status=cast(RunStatus, str(run.get("status", ""))), + snapshot_status=( + snapshot_status if snapshot_status in SNAPSHOT_STATUSES else None + ), + outputs=( + decode_block_outputs(blocks, id_key="id") + if isinstance(blocks, list) + else None + ), + error=str(error) if error is not None else None, + view_url=optional_string(run.get("viewUrl")), + ) diff --git a/deepnote_toolkit/notebooks/api_types.py b/deepnote_toolkit/notebooks/api_types.py new file mode 100644 index 00000000..79c1a694 --- /dev/null +++ b/deepnote_toolkit/notebooks/api_types.py @@ -0,0 +1,29 @@ +"""Value types of the Deepnote public API.""" + +from __future__ import annotations + +from typing import Literal, Union + +InputBlockType = Literal[ + "input-checkbox", + "input-date", + "input-date-range", + "input-file", + "input-select", + "input-slider", + "input-text", + "input-textarea", +] +RunStatus = Literal[ + "pending", "running", "success", "error", "internal_error", "stopped" +] +SnapshotStatus = Literal["pending", "available", "unavailable"] +StorageMode = Literal["read_write", "readonly"] +InputValue = Union[str, bool, list[str]] + +TERMINAL_RUN_STATUSES: frozenset[RunStatus] = frozenset( + {"success", "error", "internal_error", "stopped"} +) +SNAPSHOT_STATUSES: frozenset[SnapshotStatus] = frozenset( + {"pending", "available", "unavailable"} +) diff --git a/deepnote_toolkit/notebooks/cloud_runner.py b/deepnote_toolkit/notebooks/cloud_runner.py index 729e1f44..137ae49f 100644 --- a/deepnote_toolkit/notebooks/cloud_runner.py +++ b/deepnote_toolkit/notebooks/cloud_runner.py @@ -2,24 +2,25 @@ from __future__ import annotations -import json -import os import time from collections.abc import Callable, Mapping -from typing import Any, Literal -from urllib.request import Request, urlopen - -from .http import OpenUrl, request_json -from .models import InputBlock, RunnerInfo +from typing import Any + +from .api_client import CloudRun, DeepnoteApiClient +from .api_types import StorageMode +from .credentials import ( + DEFAULT_API_ORIGIN, + CredentialsProvider, + TokenProvider, + token_credentials, +) +from .models import RunnerInfo from .run_result import RunResult from .runner import RunnerError +from .transport import Transport -TokenProvider = Callable[[], str] -StorageMode = Literal["read_write", "readonly"] Sleep = Callable[[float], None] -TERMINAL_RUN_STATUSES = frozenset({"success", "error", "internal_error", "stopped"}) -DEFAULT_API_ORIGIN = "https://api.deepnote.com" MAX_TRANSIENT_POLL_FAILURES = 5 SNAPSHOT_SETTLE_ATTEMPTS = 3 @@ -28,8 +29,9 @@ class DeepnoteCloudRunner: """Run an existing notebook directly through the Deepnote public API. The token comes from `token`, `token_provider` or the `DEEPNOTE_TOKEN` - environment variable. A token provider is called for every request, which lets - a long-lived process use short-lived credentials. + environment variable, and is sent to `base_url`. A token provider is called for + every request, which lets a long-lived process use short-lived credentials. + `credentials` replaces all three with one provider of the token and its origin. `storage_mode="readonly"` keeps the run from changing the project's stored files. None leaves the choice to the API, which allows writes. @@ -42,70 +44,68 @@ def __init__( token: str | None = None, token_provider: TokenProvider | None = None, base_url: str = DEFAULT_API_ORIGIN, + credentials: CredentialsProvider | None = None, storage_mode: StorageMode | None = None, timeout: float = 600, poll_interval: float = 2, - opener: OpenUrl = urlopen, + transport: Transport | None = None, sleep: Sleep = time.sleep, ): if not notebook_id: raise ValueError("notebook_id is required") - if token is not None and token_provider is not None: - raise ValueError("Pass token or token_provider, not both") + if credentials is not None and ( + token is not None or token_provider is not None + ): + raise ValueError("Pass credentials or a token, not both") self.notebook_id = notebook_id - self.base_url = base_url.rstrip("/") self.storage_mode = storage_mode self.timeout = timeout self.poll_interval = poll_interval - self._static_token = token - self._token_provider = token_provider - self._open = opener + self._client = DeepnoteApiClient( + credentials or token_credentials(token, token_provider, base_url=base_url), + transport=transport, + request_timeout=min(timeout, 30), + ) self._sleep = sleep def info(self) -> RunnerInfo: """Read the notebook's name and input blocks from the public API.""" - payload = self._request("GET", f"/v2/notebooks/{self.notebook_id}") - notebook = payload.get("notebook") - if not isinstance(notebook, Mapping): - raise RunnerError("Deepnote API response did not include a notebook") - raw_inputs = notebook.get("inputs") - inputs = tuple( - InputBlock.from_api({**value, "variableName": value["name"]}) - for value in raw_inputs or [] - if isinstance(value, Mapping) - and isinstance(value.get("name"), str) - and isinstance(value.get("type"), str) - ) + notebook = self._client.get_notebook(self.notebook_id) return RunnerInfo( - notebook=str(notebook.get("name", "Untitled notebook")), - inputs=inputs, - run_target="cloud", + notebook=notebook.name, inputs=notebook.inputs, run_target="cloud" ) def run(self, inputs: Mapping[str, Any]) -> RunResult: """Start a detached run with the given input values and wait for its result.""" - body: dict[str, Any] = { - "notebookId": self.notebook_id, - "detached": True, - "inputs": _normalize_cloud_inputs(inputs), - } - if self.storage_mode is not None: - body["detachedRunStorageMode"] = self.storage_mode - started = self._run_payload(self._request("POST", "/v2/runs", body)) - run_id = _required_run_id(started) + run = self._client.create_run( + self.notebook_id, inputs, storage_mode=self.storage_mode + ) + run = self._settle_snapshot(self._wait_until_finished(run)) + return RunResult( + target="cloud", + success=run.status == "success", + outputs=run.outputs or (), + run_id=run.run_id, + status=run.status, + error=run.error, + snapshot_status=run.snapshot_status, + view_url=run.view_url, + ) + + def _wait_until_finished(self, run: CloudRun) -> CloudRun: deadline = time.monotonic() + self.timeout - current = started transient_failures = 0 - while str(current.get("status", "")) not in TERMINAL_RUN_STATUSES: + while not run.is_finished: if time.monotonic() >= deadline: raise RunnerError( - f"Deepnote run {run_id} did not finish in {self.timeout:g} seconds" + f"Deepnote run {run.run_id} did not finish in " + f"{self.timeout:g} seconds" ) self._sleep(self.poll_interval) try: - current = self._get_run(run_id) + run = self._client.get_run(run.run_id) transient_failures = 0 except RunnerError as error: transient_failures += 1 @@ -114,101 +114,18 @@ def run(self, inputs: Mapping[str, Any]) -> RunResult: or transient_failures > MAX_TRANSIENT_POLL_FAILURES ): raise + return run + def _settle_snapshot(self, run: CloudRun) -> CloudRun: # The snapshot can attach shortly after the status turns terminal. for _ in range(SNAPSHOT_SETTLE_ATTEMPTS): - is_pending = current.get("snapshotStatus", "pending") == "pending" - if not is_pending or isinstance(current.get("snapshotBlocks"), list): + is_pending = run.snapshot_status in (None, "pending") + if not is_pending or run.outputs is not None: break self._sleep(self.poll_interval) try: - current = self._get_run(run_id) + run = self._client.get_run(run.run_id) except RunnerError as error: if not error.transient: raise - - status = str(current.get("status", "")) - error = current.get("error") - if isinstance(error, Mapping): - error = error.get("message") or json.dumps(error) - return RunResult( - { - "target": "cloud", - "success": status == "success", - "runId": run_id, - "status": status, - "error": str(error) if error is not None else None, - "snapshotStatus": current.get("snapshotStatus"), - "snapshotBlocks": current.get("snapshotBlocks"), - "viewUrl": current.get("viewUrl"), - } - ) - - def _get_run(self, run_id: str) -> Mapping[str, Any]: - # The blocks delivery holds the executed notebook alone, not the whole project. - return self._run_payload( - self._request("GET", f"/v2/runs/{run_id}?snapshotDelivery=blocks") - ) - - def _request( - self, method: str, path: str, body: Mapping[str, Any] | None = None - ) -> Mapping[str, Any]: - token, api_origin = self._credentials() - request = Request( - f"{api_origin}{path}", - data=json.dumps(body).encode() if body is not None else None, - method=method, - headers={ - "Authorization": f"Bearer {token}", - "Content-Type": "application/json", - "Accept": "application/json", - }, - ) - return request_json( - self._open, - request, - timeout=min(self.timeout, 30), - service="Deepnote API", - origin=api_origin, - ) - - def _credentials(self) -> tuple[str, str]: - """Return the bearer token and the API origin to send it to.""" - - if self._token_provider is not None: - return self._required_token(self._token_provider()), self.base_url - if self._static_token is not None: - return self._required_token(self._static_token), self.base_url - return self._required_token(os.environ.get("DEEPNOTE_TOKEN")), self.base_url - - @staticmethod - def _required_token(token: str | None) -> str: - if not token: - raise RunnerError("A Deepnote API token is required") - return token - - @staticmethod - def _run_payload(payload: Mapping[str, Any]) -> Mapping[str, Any]: - run = payload.get("run") - return run if isinstance(run, Mapping) else payload - - -def _required_run_id(run: Mapping[str, Any]) -> str: - run_id = run.get("runId") or run.get("id") - if not isinstance(run_id, str) or not run_id: - raise RunnerError("Deepnote API response did not include a run id") - return run_id - - -def _normalize_cloud_inputs( - inputs: Mapping[str, Any], -) -> dict[str, str | bool | list[str]]: - normalized: dict[str, str | bool | list[str]] = {} - for name, value in inputs.items(): - if isinstance(value, bool): - normalized[name] = value - elif isinstance(value, list): - normalized[name] = [str(item) for item in value] - else: - normalized[name] = str(value) - return normalized + return run diff --git a/deepnote_toolkit/notebooks/credentials.py b/deepnote_toolkit/notebooks/credentials.py new file mode 100644 index 00000000..e335c844 --- /dev/null +++ b/deepnote_toolkit/notebooks/credentials.py @@ -0,0 +1,54 @@ +"""Where a runner gets its API token and the origin to send it to.""" + +from __future__ import annotations + +import os +from collections.abc import Callable +from dataclasses import dataclass, field +from typing import Protocol + +from .runner import RunnerError + +DEFAULT_API_ORIGIN = "https://api.deepnote.com" +TokenProvider = Callable[[], str] + + +@dataclass(frozen=True) +class ApiCredentials: + """A bearer token and the API origin it is valid at.""" + + token: str = field(repr=False) + api_origin: str = DEFAULT_API_ORIGIN + + +class CredentialsProvider(Protocol): + """Returns the credentials for one request. Called before every request.""" + + def __call__(self) -> ApiCredentials: + """Return the credentials, or raise `RunnerError` when there are none.""" + + +def token_credentials( + token: str | None = None, + token_provider: TokenProvider | None = None, + *, + base_url: str = DEFAULT_API_ORIGIN, +) -> CredentialsProvider: + """Credentials from `token`, `token_provider` or `DEEPNOTE_TOKEN`, in that order.""" + + if token is not None and token_provider is not None: + raise ValueError("Pass token or token_provider, not both") + api_origin = base_url.rstrip("/") + + def provide() -> ApiCredentials: + if token_provider is not None: + value: str | None = token_provider() + elif token is not None: + value = token + else: + value = os.environ.get("DEEPNOTE_TOKEN") + if not value: + raise RunnerError("A Deepnote API token is required") + return ApiCredentials(token=value, api_origin=api_origin) + + return provide diff --git a/deepnote_toolkit/notebooks/document.py b/deepnote_toolkit/notebooks/document.py index 52578490..0e37b183 100644 --- a/deepnote_toolkit/notebooks/document.py +++ b/deepnote_toolkit/notebooks/document.py @@ -4,12 +4,14 @@ from collections.abc import Mapping, Sequence from pathlib import Path -from typing import Any +from typing import Any, cast import yaml -from .models import InputBlock, NotebookOutput, optional_string +from .api_types import InputBlockType +from .models import InputBlock, NotebookOutput from .outputs import OutputCollection +from .wire import decode_block_outputs, optional_number, optional_string, string_tuple from .yaml_loader import load_yaml @@ -77,19 +79,31 @@ def _read_blocks( blocks = notebook.get("blocks") if not isinstance(blocks, list): continue - for block in blocks: - if not isinstance(block, Mapping): - continue - if input_block := InputBlock.from_block(block): - inputs.append(input_block) - block_outputs = block.get("outputs") - if not isinstance(block_outputs, list): - continue - block_id = str(block.get("id", "")) - block_type = optional_string(block.get("type")) - outputs.extend( - NotebookOutput(block_id=block_id, block_type=block_type, raw=output) - for output in block_outputs - if isinstance(output, Mapping) - ) + inputs.extend( + input_block + for block in blocks + if isinstance(block, Mapping) and (input_block := _read_input_block(block)) + ) + outputs.extend(decode_block_outputs(blocks, id_key="id")) return tuple(inputs), tuple(outputs) + + +def _read_input_block(block: Mapping[str, Any]) -> InputBlock | None: + block_type = str(block.get("type", "")) + metadata = block.get("metadata") + if not block_type.startswith("input-") or not isinstance(metadata, Mapping): + return None + variable_name = metadata.get("deepnote_variable_name") + if not isinstance(variable_name, str) or not variable_name: + return None + return InputBlock( + variable_name=variable_name, + type=cast(InputBlockType, block_type), + label=optional_string(metadata.get("deepnote_input_label")), + value=metadata.get("deepnote_variable_value"), + options=string_tuple(metadata.get("deepnote_variable_options")), + multiple=metadata.get("deepnote_allow_multiple_values") is True, + min=optional_number(metadata.get("deepnote_slider_min_value")), + max=optional_number(metadata.get("deepnote_slider_max_value")), + step=optional_number(metadata.get("deepnote_slider_step")), + ) diff --git a/deepnote_toolkit/notebooks/http.py b/deepnote_toolkit/notebooks/http.py deleted file mode 100644 index 5b850bbd..00000000 --- a/deepnote_toolkit/notebooks/http.py +++ /dev/null @@ -1,60 +0,0 @@ -"""JSON requests shared by the notebook runners.""" - -from __future__ import annotations - -import json -from collections.abc import Callable, Mapping -from http.client import HTTPException -from typing import Any -from urllib.error import HTTPError, URLError -from urllib.request import Request - -from .runner import RunnerError - -OpenUrl = Callable[..., Any] - - -def request_json( - opener: OpenUrl, request: Request, *, timeout: float, service: str, origin: str -) -> Mapping[str, Any]: - """Send a request and return its JSON object, raising `RunnerError` otherwise.""" - - try: - with opener(request, timeout=timeout) as response: - payload = json.loads(response.read()) - except HTTPError as error: - raise RunnerError( - f"The {service} returned HTTP {error.code}: {_error_message(error)}", - transient=error.code == 429 or error.code >= 500, - ) from error - except URLError as error: - raise RunnerError( - f"Could not reach the {service} at {origin}: {error.reason}", - transient=True, - ) from error - except TimeoutError as error: - raise RunnerError( - f"The {service} at {origin} timed out after {timeout:g} seconds", - transient=True, - ) from error - except (OSError, HTTPException) as error: - raise RunnerError( - f"The connection to the {service} at {origin} dropped: {error}", - transient=True, - ) from error - except (json.JSONDecodeError, UnicodeDecodeError) as error: - raise RunnerError(f"The {service} returned an invalid JSON response") from error - if not isinstance(payload, Mapping): - raise RunnerError(f"The {service} returned a non-object response") - return payload - - -def _error_message(error: HTTPError) -> str: - detail = error.read().decode(errors="replace") - try: - parsed = json.loads(detail) - except json.JSONDecodeError: - return detail - if not isinstance(parsed, Mapping): - return detail - return str(parsed.get("message") or parsed.get("error") or detail) diff --git a/deepnote_toolkit/notebooks/local_runner.py b/deepnote_toolkit/notebooks/local_runner.py index 73391ed8..ac116307 100644 --- a/deepnote_toolkit/notebooks/local_runner.py +++ b/deepnote_toolkit/notebooks/local_runner.py @@ -2,14 +2,14 @@ from __future__ import annotations -import json from collections.abc import Mapping from typing import Any -from urllib.request import Request, urlopen -from .http import OpenUrl, request_json -from .models import InputBlock, RunnerInfo +from .document import DeepnoteDocument +from .models import RunnerInfo from .run_result import RunResult +from .transport import Transport, UrllibTransport +from .wire import decode_block_outputs, decode_inputs, optional_string class DeepnoteRunner: @@ -20,52 +20,61 @@ def __init__( base_url: str = "http://127.0.0.1:8787", *, timeout: float = 600, - opener: OpenUrl = urlopen, + transport: Transport | None = None, ): self.base_url = base_url.rstrip("/") self.timeout = timeout - self._open = opener + self._transport = transport or UrllibTransport() def info(self) -> RunnerInfo: """Read the notebook's name and input blocks from the sidecar.""" payload = self._request("GET", "/api/info") - values = payload.get("inputs") - inputs = ( - tuple( - InputBlock.from_api(value) - for value in values - if isinstance(value, Mapping) - and isinstance(value.get("variableName"), str) - and isinstance(value.get("type"), str) - ) - if isinstance(values, list) - else () - ) return RunnerInfo( notebook=str(payload.get("notebook", "Untitled project")), - inputs=inputs, + inputs=decode_inputs(payload.get("inputs"), name_key="variableName"), run_target=str(payload.get("runTarget", "")), ) def run(self, inputs: Mapping[str, Any]) -> RunResult: """Run the notebook in the sidecar with the given input values.""" - return RunResult(self._request("POST", "/api/run", {"inputs": dict(inputs)})) + return _decode_run_result( + self._request("POST", "/api/run", {"inputs": dict(inputs)}) + ) def _request( self, method: str, path: str, body: Mapping[str, Any] | None = None ) -> Mapping[str, Any]: - request = Request( + return self._transport.request_json( + method, f"{self.base_url}{path}", - data=json.dumps(body).encode() if body is not None else None, - method=method, - headers={"Content-Type": "application/json", "Accept": "application/json"}, - ) - return request_json( - self._open, - request, + headers={}, + body=body, timeout=self.timeout, - service="Deepnote runner", - origin=self.base_url, ) + + +def _decode_run_result(payload: Mapping[str, Any]) -> RunResult: + snapshot = None + snapshot_yaml = payload.get("snapshotYaml") + if isinstance(snapshot_yaml, str) and snapshot_yaml: + try: + snapshot = DeepnoteDocument.parse(snapshot_yaml) + except ValueError: + pass + return RunResult( + target=str(payload.get("target", "")), + success=payload.get("success") is True, + outputs=( + snapshot.outputs + if snapshot + else decode_block_outputs(payload.get("outputs"), id_key="blockId") + ), + run_id=optional_string(payload.get("runId")), + status=optional_string(payload.get("status")), + error=optional_string(payload.get("error")), + view_url=optional_string(payload.get("viewUrl")), + snapshot=snapshot, + created=payload.get("created") is True, + ) diff --git a/deepnote_toolkit/notebooks/models.py b/deepnote_toolkit/notebooks/models.py index 8b287e25..2584b732 100644 --- a/deepnote_toolkit/notebooks/models.py +++ b/deepnote_toolkit/notebooks/models.py @@ -8,6 +8,8 @@ from dataclasses import dataclass from typing import Any +from .api_types import InputBlockType + DATAFRAME_MIME = "application/vnd.deepnote.dataframe.v3+json" INDEX_COLUMN = "_deepnote_index_column" @@ -25,7 +27,7 @@ class InputBlock: """The metadata a UI needs to render one Deepnote input block.""" variable_name: str - type: str + type: InputBlockType value: Any label: str | None = None options: tuple[str, ...] = () @@ -34,55 +36,6 @@ class InputBlock: max: float | int | None = None step: float | int | None = None - @classmethod - def from_block(cls, block: Mapping[str, Any]) -> InputBlock | None: - """Read an input block from a `.deepnote` file, or None for another block.""" - - block_type = str(block.get("type", "")) - metadata = block.get("metadata") - if not block_type.startswith("input-") or not isinstance(metadata, Mapping): - return None - variable_name = metadata.get("deepnote_variable_name") - if not isinstance(variable_name, str) or not variable_name: - return None - options = metadata.get("deepnote_variable_options") - return cls( - variable_name=variable_name, - type=block_type, - label=optional_string(metadata.get("deepnote_input_label")), - value=metadata.get("deepnote_variable_value"), - options=( - tuple(str(option) for option in options) - if isinstance(options, list) - else () - ), - multiple=metadata.get("deepnote_allow_multiple_values") is True, - min=optional_number(metadata.get("deepnote_slider_min_value")), - max=optional_number(metadata.get("deepnote_slider_max_value")), - step=optional_number(metadata.get("deepnote_slider_step")), - ) - - @classmethod - def from_api(cls, value: Mapping[str, Any]) -> InputBlock: - """Read the camelCase shape returned by `GET /api/info`.""" - - options = value.get("options") - return cls( - variable_name=str(value["variableName"]), - type=str(value["type"]), - label=optional_string(value.get("label")), - value=value.get("value"), - options=( - tuple(str(option) for option in options) - if isinstance(options, list) - else () - ), - multiple=value.get("multiple") is True, - min=optional_number(value.get("min")), - max=optional_number(value.get("max")), - step=optional_number(value.get("step")), - ) - @dataclass(frozen=True) class DeepnoteDataframe: @@ -132,7 +85,7 @@ def data_columns(self) -> tuple[str, ...]: ) def records(self, *, include_index: bool = True) -> list[dict[str, Any]]: - """Return rows ready for `st.dataframe`, optionally without the index column.""" + """Return rows as plain dicts, optionally without the index column.""" if include_index: return [dict(row) for row in self.rows] @@ -207,19 +160,3 @@ def _input_contract(inputs: Iterable[InputBlock]) -> frozenset[tuple[str, str]]: return frozenset( (input_block.variable_name, input_block.type) for input_block in inputs ) - - -def optional_string(value: Any) -> str | None: - """Return the value when it is a string, otherwise None.""" - - return value if isinstance(value, str) else None - - -def optional_number(value: Any) -> float | int | None: - """Return the value when it is a number other than a boolean, otherwise None.""" - - return ( - value - if isinstance(value, (float, int)) and not isinstance(value, bool) - else None - ) diff --git a/deepnote_toolkit/notebooks/run_result.py b/deepnote_toolkit/notebooks/run_result.py index 7816e7db..d818ff49 100644 --- a/deepnote_toolkit/notebooks/run_result.py +++ b/deepnote_toolkit/notebooks/run_result.py @@ -2,83 +2,29 @@ from __future__ import annotations -from collections.abc import Mapping -from typing import Any +from dataclasses import dataclass +from .api_types import SnapshotStatus from .document import DeepnoteDocument -from .models import NotebookOutput, optional_string +from .models import NotebookOutput from .outputs import OutputCollection +@dataclass(frozen=True) class RunResult(OutputCollection): - """The normalized result of `POST /api/run`, for either cloud or local execution.""" - - def __init__(self, raw: Mapping[str, Any]): - self.raw = raw - self.target = str(raw.get("target", "")) - self.success = raw.get("success") is True - self.run_id = optional_string(raw.get("runId")) - self.status = optional_string(raw.get("status")) - self.created = raw.get("created") is True - self.view_url = optional_string(raw.get("viewUrl")) - self.error = optional_string(raw.get("error")) - self.snapshot_status = optional_string(raw.get("snapshotStatus")) - self.snapshot_yaml = optional_string(raw.get("snapshotYaml")) - self.snapshot = None - if self.snapshot_yaml: - try: - self.snapshot = DeepnoteDocument.parse(self.snapshot_yaml) - except ValueError: - pass - if self.snapshot: - self.outputs = self.snapshot.outputs - else: - snapshot_blocks = raw.get("snapshotBlocks") - self.outputs = ( - _outputs_from_snapshot_blocks(snapshot_blocks) - if isinstance(snapshot_blocks, list) - else _outputs_from_run(raw.get("outputs")) - ) - - -def _outputs_from_run(value: Any) -> tuple[NotebookOutput, ...]: - if not isinstance(value, list): - return () - outputs: list[NotebookOutput] = [] - for block in value: - if not isinstance(block, Mapping): - continue - block_id = str(block.get("blockId", "")) - raw_outputs = block.get("outputs") - if not isinstance(raw_outputs, list): - continue - outputs.extend( - NotebookOutput(block_id=block_id, block_type=None, raw=output) - for output in raw_outputs - if isinstance(output, Mapping) - ) - return tuple(outputs) - - -def _outputs_from_snapshot_blocks(value: Any) -> tuple[NotebookOutput, ...]: - if not isinstance(value, list): - return () - outputs: list[NotebookOutput] = [] - for block in value: - if not isinstance(block, Mapping): - continue - block_id = str(block.get("id", "")) - block_type = optional_string(block.get("type")) - raw_outputs = block.get("outputs") - if not isinstance(raw_outputs, list): - continue - outputs.extend( - NotebookOutput( - block_id=block_id, - block_type=block_type, - raw=output, - ) - for output in raw_outputs - if isinstance(output, Mapping) - ) - return tuple(outputs) + """What one run produced, whether it ran in Deepnote Cloud or locally. + + `snapshot_status` is set for cloud runs. `snapshot` and `created` are set by + the local runner. + """ + + target: str + success: bool + outputs: tuple[NotebookOutput, ...] = () + run_id: str | None = None + status: str | None = None + error: str | None = None + snapshot_status: SnapshotStatus | None = None + view_url: str | None = None + snapshot: DeepnoteDocument | None = None + created: bool = False diff --git a/deepnote_toolkit/notebooks/transport.py b/deepnote_toolkit/notebooks/transport.py new file mode 100644 index 00000000..c87f6463 --- /dev/null +++ b/deepnote_toolkit/notebooks/transport.py @@ -0,0 +1,101 @@ +"""The HTTP layer under the notebook runners.""" + +from __future__ import annotations + +import json +from collections.abc import Callable, Mapping +from http.client import HTTPException +from typing import Any, Protocol +from urllib.error import HTTPError, URLError +from urllib.parse import urlsplit +from urllib.request import Request, urlopen + +from .runner import RunnerError + +OpenUrl = Callable[..., Any] + + +class Transport(Protocol): + """Sends one JSON request. Implement it to use another HTTP library.""" + + def request_json( + self, + method: str, + url: str, + *, + headers: Mapping[str, str], + body: Mapping[str, Any] | None, + timeout: float, + ) -> Mapping[str, Any]: + """Return the response's JSON object, or raise `RunnerError`. + + The error is `transient` when a retry can succeed: HTTP 429 or 5xx, a + timeout, or a network failure. + """ + + +class UrllibTransport: + """The default transport, on the standard library.""" + + def __init__(self, opener: OpenUrl = urlopen): + self._open = opener + + def request_json( + self, + method: str, + url: str, + *, + headers: Mapping[str, str], + body: Mapping[str, Any] | None, + timeout: float, + ) -> Mapping[str, Any]: + """Send the request with `urllib` and return its JSON object.""" + + parts = urlsplit(url) + origin = f"{parts.scheme}://{parts.netloc}" + request = Request( + url, + data=json.dumps(body).encode() if body is not None else None, + method=method, + headers={ + "Content-Type": "application/json", + "Accept": "application/json", + **headers, + }, + ) + try: + with self._open(request, timeout=timeout) as response: + payload = json.loads(response.read()) + except HTTPError as error: + raise RunnerError( + f"{origin} returned HTTP {error.code}: {_error_message(error)}", + transient=error.code == 429 or error.code >= 500, + ) from error + except URLError as error: + raise RunnerError( + f"Could not reach {origin}: {error.reason}", transient=True + ) from error + except TimeoutError as error: + raise RunnerError( + f"{origin} timed out after {timeout:g} seconds", transient=True + ) from error + except (OSError, HTTPException) as error: + raise RunnerError( + f"The connection to {origin} dropped: {error}", transient=True + ) from error + except (json.JSONDecodeError, UnicodeDecodeError) as error: + raise RunnerError(f"{origin} returned an invalid JSON response") from error + if not isinstance(payload, Mapping): + raise RunnerError(f"{origin} returned a non-object response") + return payload + + +def _error_message(error: HTTPError) -> str: + detail = error.read().decode(errors="replace") + try: + parsed = json.loads(detail) + except json.JSONDecodeError: + return detail + if not isinstance(parsed, Mapping): + return detail + return str(parsed.get("message") or parsed.get("error") or detail) diff --git a/deepnote_toolkit/notebooks/wire.py b/deepnote_toolkit/notebooks/wire.py new file mode 100644 index 00000000..dc9dd549 --- /dev/null +++ b/deepnote_toolkit/notebooks/wire.py @@ -0,0 +1,77 @@ +"""Decode the JSON shapes shared by the API, the sidecar and `.deepnote` files.""" + +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any, cast + +from .api_types import InputBlockType +from .models import InputBlock, NotebookOutput + + +def optional_string(value: Any) -> str | None: + """Return the value when it is a string, otherwise None.""" + + return value if isinstance(value, str) else None + + +def optional_number(value: Any) -> float | int | None: + """Return the value when it is a number other than a boolean, otherwise None.""" + + return ( + value + if isinstance(value, (float, int)) and not isinstance(value, bool) + else None + ) + + +def string_tuple(value: Any) -> tuple[str, ...]: + """Return a list's items as strings, or an empty tuple for any other value.""" + + return tuple(str(item) for item in value) if isinstance(value, list) else () + + +def decode_inputs(values: Any, *, name_key: str) -> tuple[InputBlock, ...]: + """Read an API's camelCase input list, skipping entries without a name or type.""" + + if not isinstance(values, list): + return () + return tuple( + InputBlock( + variable_name=value[name_key], + type=cast(InputBlockType, value["type"]), + label=optional_string(value.get("label")), + value=value.get("value"), + options=string_tuple(value.get("options")), + multiple=value.get("multiple") is True, + min=optional_number(value.get("min")), + max=optional_number(value.get("max")), + step=optional_number(value.get("step")), + ) + for value in values + if isinstance(value, Mapping) + and isinstance(value.get(name_key), str) + and isinstance(value.get("type"), str) + ) + + +def decode_block_outputs(blocks: Any, *, id_key: str) -> tuple[NotebookOutput, ...]: + """Read the outputs of a list of blocks, in block order.""" + + if not isinstance(blocks, list): + return () + outputs: list[NotebookOutput] = [] + for block in blocks: + if not isinstance(block, Mapping): + continue + block_outputs = block.get("outputs") + if not isinstance(block_outputs, list): + continue + block_id = str(block.get(id_key, "")) + block_type = optional_string(block.get("type")) + outputs.extend( + NotebookOutput(block_id=block_id, block_type=block_type, raw=output) + for output in block_outputs + if isinstance(output, Mapping) + ) + return tuple(outputs) diff --git a/deepnote_toolkit/streamlit/__init__.py b/deepnote_toolkit/streamlit/__init__.py index 96bc72a1..c981118a 100644 --- a/deepnote_toolkit/streamlit/__init__.py +++ b/deepnote_toolkit/streamlit/__init__.py @@ -7,12 +7,14 @@ current_user_api_token, ) from .cloud_runner import StreamlitCloudRunner +from .viewer_credentials import ViewerCredentials from .widgets import render_inputs __all__ = [ "CurrentUserApiCredentials", "CurrentUserApiTokenError", "StreamlitCloudRunner", + "ViewerCredentials", "current_user_api_credentials", "current_user_api_token", "render_inputs", diff --git a/deepnote_toolkit/streamlit/cloud_runner.py b/deepnote_toolkit/streamlit/cloud_runner.py index 1201766a..df6103f8 100644 --- a/deepnote_toolkit/streamlit/cloud_runner.py +++ b/deepnote_toolkit/streamlit/cloud_runner.py @@ -3,25 +3,13 @@ from __future__ import annotations import time -from urllib.request import urlopen -from deepnote_toolkit.notebooks.cloud_runner import ( - DEFAULT_API_ORIGIN, - DeepnoteCloudRunner, - Sleep, - StorageMode, - TokenProvider, -) -from deepnote_toolkit.notebooks.http import OpenUrl -from deepnote_toolkit.notebooks.runner import RunnerError +from deepnote_toolkit.notebooks.api_types import StorageMode +from deepnote_toolkit.notebooks.cloud_runner import DeepnoteCloudRunner, Sleep +from deepnote_toolkit.notebooks.credentials import DEFAULT_API_ORIGIN, TokenProvider +from deepnote_toolkit.notebooks.transport import Transport -from .auth import ( - CurrentUserApiTokenError, - _has_hosted_streamlit_context, - _has_script_run_context, - _is_streamlit_thread_without_request, - current_user_api_credentials, -) +from .viewer_credentials import ViewerCredentials class StreamlitCloudRunner(DeepnoteCloudRunner): @@ -42,38 +30,17 @@ def __init__( storage_mode: StorageMode | None = "readonly", timeout: float = 600, poll_interval: float = 2, - opener: OpenUrl = urlopen, + transport: Transport | None = None, sleep: Sleep = time.sleep, ): super().__init__( notebook_id, - token=token, - token_provider=token_provider, - base_url=base_url, + credentials=ViewerCredentials( + token, token_provider, base_url=base_url, timeout=min(timeout, 30) + ), storage_mode=storage_mode, timeout=timeout, poll_interval=poll_interval, - opener=opener, + transport=transport, sleep=sleep, ) - - def _credentials(self) -> tuple[str, str]: - if _has_script_run_context() and _has_hosted_streamlit_context(): - try: - credentials = current_user_api_credentials( - timeout=min(self.timeout, 30), opener=self._open - ) - except CurrentUserApiTokenError as error: - raise RunnerError(str(error), transient=error.transient) from error - return credentials.token, credentials.api_origin - - is_token_explicit = ( - self._token_provider is not None or self._static_token is not None - ) - if not is_token_explicit and _is_streamlit_thread_without_request(): - raise RunnerError( - "No viewer request is available on this thread. Call the runner from " - "the Streamlit script thread, or pass token= or token_provider=." - ) - - return super()._credentials() diff --git a/deepnote_toolkit/streamlit/viewer_credentials.py b/deepnote_toolkit/streamlit/viewer_credentials.py new file mode 100644 index 00000000..d0e4aebe --- /dev/null +++ b/deepnote_toolkit/streamlit/viewer_credentials.py @@ -0,0 +1,57 @@ +"""API credentials of the person viewing a hosted Streamlit app.""" + +from __future__ import annotations + +from deepnote_toolkit.notebooks.credentials import ( + DEFAULT_API_ORIGIN, + ApiCredentials, + TokenProvider, + token_credentials, +) +from deepnote_toolkit.notebooks.runner import RunnerError + +from .auth import ( + CurrentUserApiTokenError, + _has_hosted_streamlit_context, + _has_script_run_context, + _is_streamlit_thread_without_request, + current_user_api_credentials, +) + + +class ViewerCredentials: + """Credentials of the current viewer when Deepnote hosts the app. + + `token`, `token_provider`, `base_url` and the `DEEPNOTE_TOKEN` environment + variable apply only outside Deepnote hosting. + """ + + def __init__( + self, + token: str | None = None, + token_provider: TokenProvider | None = None, + *, + base_url: str = DEFAULT_API_ORIGIN, + timeout: float = 10, + ): + self._is_token_explicit = token is not None or token_provider is not None + self._local = token_credentials(token, token_provider, base_url=base_url) + self._timeout = timeout + + def __call__(self) -> ApiCredentials: + """Return the viewer's credentials, or the local ones outside hosting.""" + + if _has_script_run_context() and _has_hosted_streamlit_context(): + try: + viewer = current_user_api_credentials(timeout=self._timeout) + except CurrentUserApiTokenError as error: + raise RunnerError(str(error), transient=error.transient) from error + return ApiCredentials(token=viewer.token, api_origin=viewer.api_origin) + + if not self._is_token_explicit and _is_streamlit_thread_without_request(): + raise RunnerError( + "No viewer request is available on this thread. Call the runner from " + "the Streamlit script thread, or pass token= or token_provider=." + ) + + return self._local() diff --git a/docs/streamlit-apps.md b/docs/streamlit-apps.md index 423f95a4..63e41b9f 100644 --- a/docs/streamlit-apps.md +++ b/docs/streamlit-apps.md @@ -42,12 +42,24 @@ program: at `http://127.0.0.1:8787`. - `Runner` is the interface both runners implement, for code that accepts either. +The cloud runner only starts a run and waits for it. The parts under it can be +used or replaced on their own: + +- `DeepnoteApiClient` sends the API requests and validates the responses. +- A `CredentialsProvider` is any callable that returns `ApiCredentials`, a token + with the API origin it is valid at. It is called before every request. Pass one + as `credentials=` in place of `token`, `token_provider` and `base_url`. +- A `Transport` sends one JSON request. `UrllibTransport` is the default. Pass + your own as `transport=` to use another HTTP library. + `deepnote_toolkit.streamlit` holds the Streamlit-specific parts: - `render_inputs` maps Deepnote input blocks to native Streamlit widgets and returns values ready to submit to a runner. - `StreamlitCloudRunner` is a `DeepnoteCloudRunner` that runs notebooks as the person viewing the app when Deepnote hosts it. +- `ViewerCredentials` is the `CredentialsProvider` behind it, for use with + `DeepnoteCloudRunner` or `DeepnoteApiClient` directly. A static app only loads a committed snapshot with `DeepnoteDocument`. It requires no token or network access. diff --git a/tests/unit/test_deepnote_streamlit_cloud_runner.py b/tests/unit/test_deepnote_streamlit_cloud_runner.py index 52bb3e90..0cc70dce 100644 --- a/tests/unit/test_deepnote_streamlit_cloud_runner.py +++ b/tests/unit/test_deepnote_streamlit_cloud_runner.py @@ -4,11 +4,12 @@ import pytest -from deepnote_toolkit.notebooks import RunnerError +from deepnote_toolkit.notebooks import ApiCredentials, RunnerError, UrllibTransport from deepnote_toolkit.streamlit import ( CurrentUserApiCredentials, CurrentUserApiTokenError, StreamlitCloudRunner, + ViewerCredentials, ) @@ -62,21 +63,21 @@ def open_request(request: Any, *, timeout: float) -> FakeResponse: ] with ( patch( - "deepnote_toolkit.streamlit.cloud_runner._has_script_run_context", + "deepnote_toolkit.streamlit.viewer_credentials._has_script_run_context", return_value=True, ), patch( - "deepnote_toolkit.streamlit.cloud_runner._has_hosted_streamlit_context", + "deepnote_toolkit.streamlit.viewer_credentials._has_hosted_streamlit_context", return_value=True, ), patch( - "deepnote_toolkit.streamlit.cloud_runner.current_user_api_credentials", + "deepnote_toolkit.streamlit.viewer_credentials.current_user_api_credentials", side_effect=credentials, ) as exchange, ): result = StreamlitCloudRunner( "notebook-1", - opener=open_request, + transport=UrllibTransport(open_request), sleep=lambda _delay: None, ).run({}) @@ -103,20 +104,20 @@ def test_hosted_runner_never_falls_back_to_environment_token( opener = MagicMock() with ( patch( - "deepnote_toolkit.streamlit.cloud_runner._has_script_run_context", + "deepnote_toolkit.streamlit.viewer_credentials._has_script_run_context", return_value=True, ), patch( - "deepnote_toolkit.streamlit.cloud_runner._has_hosted_streamlit_context", + "deepnote_toolkit.streamlit.viewer_credentials._has_hosted_streamlit_context", return_value=True, ), patch( - "deepnote_toolkit.streamlit.cloud_runner.current_user_api_credentials", + "deepnote_toolkit.streamlit.viewer_credentials.current_user_api_credentials", side_effect=CurrentUserApiTokenError("viewer token unavailable"), ), pytest.raises(RunnerError, match="viewer token unavailable"), ): - StreamlitCloudRunner("notebook-1", opener=opener).info() + StreamlitCloudRunner("notebook-1", transport=UrllibTransport(opener)).info() opener.assert_not_called() @@ -128,16 +129,16 @@ def test_worker_thread_never_falls_back_to_environment_token( opener = MagicMock() with ( patch( - "deepnote_toolkit.streamlit.cloud_runner._has_hosted_streamlit_context", + "deepnote_toolkit.streamlit.viewer_credentials._has_hosted_streamlit_context", return_value=False, ), patch( - "deepnote_toolkit.streamlit.cloud_runner._is_streamlit_thread_without_request", + "deepnote_toolkit.streamlit.viewer_credentials._is_streamlit_thread_without_request", return_value=True, ), pytest.raises(RunnerError, match="No viewer request"), ): - StreamlitCloudRunner("notebook-1", opener=opener).info() + StreamlitCloudRunner("notebook-1", transport=UrllibTransport(opener)).info() opener.assert_not_called() @@ -156,15 +157,15 @@ def test_cloud_run_retries_a_transient_token_exchange_failure() -> None: ) with ( patch( - "deepnote_toolkit.streamlit.cloud_runner._has_script_run_context", + "deepnote_toolkit.streamlit.viewer_credentials._has_script_run_context", return_value=True, ), patch( - "deepnote_toolkit.streamlit.cloud_runner._has_hosted_streamlit_context", + "deepnote_toolkit.streamlit.viewer_credentials._has_hosted_streamlit_context", return_value=True, ), patch( - "deepnote_toolkit.streamlit.cloud_runner.current_user_api_credentials", + "deepnote_toolkit.streamlit.viewer_credentials.current_user_api_credentials", side_effect=[ credentials, CurrentUserApiTokenError("exchange timed out", transient=True), @@ -174,7 +175,9 @@ def test_cloud_run_retries_a_transient_token_exchange_failure() -> None: ): result = StreamlitCloudRunner( "notebook-1", - opener=lambda _request, *, timeout: FakeResponse(next(responses)), + transport=UrllibTransport( + lambda _request, *, timeout: FakeResponse(next(responses)) + ), sleep=lambda _delay: None, ).run({}) @@ -192,15 +195,17 @@ def open_request(request: Any, *, timeout: float) -> FakeResponse: with ( patch( - "deepnote_toolkit.streamlit.cloud_runner._has_hosted_streamlit_context", + "deepnote_toolkit.streamlit.viewer_credentials._has_hosted_streamlit_context", return_value=False, ), patch( - "deepnote_toolkit.streamlit.cloud_runner._is_streamlit_thread_without_request", + "deepnote_toolkit.streamlit.viewer_credentials._is_streamlit_thread_without_request", return_value=False, ), ): - info = StreamlitCloudRunner("notebook-1", opener=open_request).info() + info = StreamlitCloudRunner( + "notebook-1", transport=UrllibTransport(open_request) + ).info() assert info.notebook == "Revenue" @@ -214,15 +219,15 @@ def open_request(request: Any, *, timeout: float) -> FakeResponse: with ( patch( - "deepnote_toolkit.streamlit.cloud_runner._has_script_run_context", + "deepnote_toolkit.streamlit.viewer_credentials._has_script_run_context", return_value=True, ), patch( - "deepnote_toolkit.streamlit.cloud_runner._has_hosted_streamlit_context", + "deepnote_toolkit.streamlit.viewer_credentials._has_hosted_streamlit_context", return_value=True, ), patch( - "deepnote_toolkit.streamlit.cloud_runner.current_user_api_credentials", + "deepnote_toolkit.streamlit.viewer_credentials.current_user_api_credentials", return_value=CurrentUserApiCredentials( token="viewer-token", api_origin="https://api.deepnote.com", @@ -231,7 +236,9 @@ def open_request(request: Any, *, timeout: float) -> FakeResponse: ), ): StreamlitCloudRunner( - "notebook-1", base_url="https://elsewhere.example", opener=open_request + "notebook-1", + base_url="https://elsewhere.example", + transport=UrllibTransport(open_request), ).info() assert urls == ["https://api.deepnote.com/v2/notebooks/notebook-1"] @@ -248,18 +255,20 @@ def open_request(request: Any, *, timeout: float) -> FakeResponse: with ( patch( - "deepnote_toolkit.streamlit.cloud_runner._has_script_run_context", + "deepnote_toolkit.streamlit.viewer_credentials._has_script_run_context", return_value=False, ), patch( - "deepnote_toolkit.streamlit.cloud_runner._has_hosted_streamlit_context" + "deepnote_toolkit.streamlit.viewer_credentials._has_hosted_streamlit_context" ) as hosted_lookup, patch( - "deepnote_toolkit.streamlit.cloud_runner._is_streamlit_thread_without_request", + "deepnote_toolkit.streamlit.viewer_credentials._is_streamlit_thread_without_request", return_value=False, ), ): - StreamlitCloudRunner("notebook-1", opener=open_request).info() + StreamlitCloudRunner( + "notebook-1", transport=UrllibTransport(open_request) + ).info() hosted_lookup.assert_not_called() @@ -273,15 +282,15 @@ def open_request(request: Any, *, timeout: float) -> FakeResponse: with ( patch( - "deepnote_toolkit.streamlit.cloud_runner._has_script_run_context", + "deepnote_toolkit.streamlit.viewer_credentials._has_script_run_context", return_value=True, ), patch( - "deepnote_toolkit.streamlit.cloud_runner._has_hosted_streamlit_context", + "deepnote_toolkit.streamlit.viewer_credentials._has_hosted_streamlit_context", return_value=True, ), patch( - "deepnote_toolkit.streamlit.cloud_runner.current_user_api_credentials", + "deepnote_toolkit.streamlit.viewer_credentials.current_user_api_credentials", return_value=CurrentUserApiCredentials( token="viewer-token", api_origin="https://api.deepnote.com", @@ -290,7 +299,7 @@ def open_request(request: Any, *, timeout: float) -> FakeResponse: ), ): StreamlitCloudRunner( - "notebook-1", token="owner-token", opener=open_request + "notebook-1", token="owner-token", transport=UrllibTransport(open_request) ).info() assert authorizations == ["Bearer viewer-token"] @@ -310,15 +319,17 @@ def open_request(request: Any, *, timeout: float) -> FakeResponse: with ( patch( - "deepnote_toolkit.streamlit.cloud_runner._has_script_run_context", + "deepnote_toolkit.streamlit.viewer_credentials._has_script_run_context", return_value=False, ), patch( - "deepnote_toolkit.streamlit.cloud_runner._is_streamlit_thread_without_request", + "deepnote_toolkit.streamlit.viewer_credentials._is_streamlit_thread_without_request", return_value=False, ), ): - StreamlitCloudRunner("notebook-1", opener=open_request).run({}) + StreamlitCloudRunner("notebook-1", transport=UrllibTransport(open_request)).run( + {} + ) assert bodies == [ { @@ -328,3 +339,19 @@ def open_request(request: Any, *, timeout: float) -> FakeResponse: "detachedRunStorageMode": "readonly", } ] + + +def test_viewer_credentials_use_an_explicit_token_on_a_worker_thread() -> None: + with ( + patch( + "deepnote_toolkit.streamlit.viewer_credentials._has_script_run_context", + return_value=False, + ), + patch( + "deepnote_toolkit.streamlit.viewer_credentials._is_streamlit_thread_without_request", + return_value=True, + ), + ): + credentials = ViewerCredentials(token="local-token")() + + assert credentials == ApiCredentials("local-token", "https://api.deepnote.com") diff --git a/tests/unit/test_notebooks_document.py b/tests/unit/test_notebooks_document.py index c0e9bcb8..43bee029 100644 --- a/tests/unit/test_notebooks_document.py +++ b/tests/unit/test_notebooks_document.py @@ -1,4 +1,6 @@ +from collections.abc import Mapping from pathlib import Path +from typing import Any import pytest @@ -6,11 +8,25 @@ DATAFRAME_MIME, DeepnoteDataframe, DeepnoteDocument, + DeepnoteRunner, InputBlock, RunResult, join_text, ) + +class FakeTransport: + def __init__(self, payload: Mapping[str, Any]): + self.payload = payload + + def request_json(self, *_args: Any, **_kwargs: Any) -> Mapping[str, Any]: + return self.payload + + +def run_locally(payload: Mapping[str, Any]) -> RunResult: + return DeepnoteRunner(transport=FakeTransport(payload)).run({}) + + SNAPSHOT_YAML = """ project: name: Sales performance @@ -87,29 +103,47 @@ def test_dataframe_ignores_columns_without_names() -> None: def test_reads_input_metadata_from_file_and_api_shapes() -> None: - file_input = InputBlock.from_block( - { - "type": "input-slider", - "metadata": { - "deepnote_variable_name": "limit", - "deepnote_input_label": "Row limit", - "deepnote_variable_value": "20", - "deepnote_slider_min_value": 10, - "deepnote_slider_max_value": 100, - "deepnote_slider_step": 10, - }, - } - ) - api_input = InputBlock.from_api( + document = DeepnoteDocument( { - "variableName": "countries", - "type": "input-select", - "label": "Countries", - "value": ["Panama"], - "options": ["Panama", "Colombia"], - "multiple": True, + "project": { + "notebooks": [ + { + "blocks": [ + { + "type": "input-slider", + "metadata": { + "deepnote_variable_name": "limit", + "deepnote_input_label": "Row limit", + "deepnote_variable_value": "20", + "deepnote_slider_min_value": 10, + "deepnote_slider_max_value": 100, + "deepnote_slider_step": 10, + }, + } + ] + } + ] + } } ) + info = DeepnoteRunner( + transport=FakeTransport( + { + "inputs": [ + { + "variableName": "countries", + "type": "input-select", + "label": "Countries", + "value": ["Panama"], + "options": ["Panama", "Colombia"], + "multiple": True, + } + ] + } + ) + ).info() + (file_input,) = document.inputs + (api_input,) = info.inputs assert file_input == InputBlock( variable_name="limit", @@ -125,7 +159,7 @@ def test_reads_input_metadata_from_file_and_api_shapes() -> None: def test_run_result_prefers_snapshot_outputs_and_preserves_cloud_fields() -> None: - result = RunResult( + result = run_locally( { "target": "cloud", "success": True, @@ -144,7 +178,7 @@ def test_run_result_prefers_snapshot_outputs_and_preserves_cloud_fields() -> Non def test_run_result_falls_back_to_inline_outputs_without_snapshot() -> None: - result = RunResult( + result = run_locally( { "target": "local", "success": True, @@ -173,7 +207,7 @@ def test_run_result_falls_back_to_inline_outputs_without_snapshot() -> None: def test_run_result_falls_back_to_inline_outputs_for_malformed_snapshot() -> None: - result = RunResult( + result = run_locally( { "target": "cloud", "success": True, diff --git a/tests/unit/test_notebooks_runners.py b/tests/unit/test_notebooks_runners.py index 09108f2d..a9e3d8b0 100644 --- a/tests/unit/test_notebooks_runners.py +++ b/tests/unit/test_notebooks_runners.py @@ -7,11 +7,13 @@ import pytest from deepnote_toolkit.notebooks import ( + ApiCredentials, DeepnoteCloudRunner, DeepnoteRunner, InputBlock, RunnerError, RunnerInfo, + UrllibTransport, ) @@ -44,7 +46,9 @@ def open_request(request: Any, *, timeout: float) -> FakeResponse: } ) - info = DeepnoteRunner("http://runner/", timeout=12, opener=open_request).info() + info = DeepnoteRunner( + "http://runner/", timeout=12, transport=UrllibTransport(open_request) + ).info() assert calls == [("http://runner/api/info", "GET", 12)] assert info.notebook == "Revenue" @@ -71,7 +75,7 @@ def open_request(request: Any, *, timeout: float) -> FakeResponse: assert json.loads(request.data) == {"inputs": {"limit": 20}} return FakeResponse({"target": "local", "success": True, "outputs": []}) - result = DeepnoteRunner(opener=open_request).run({"limit": 20}) + result = DeepnoteRunner(transport=UrllibTransport(open_request)).run({"limit": 20}) assert result.target == "local" assert result.success is True @@ -88,7 +92,7 @@ def open_request(*_: Any, **__: Any) -> FakeResponse: ) with pytest.raises(RunnerError, match="DEEPNOTE_TOKEN is required"): - DeepnoteRunner("http://runner", opener=open_request).run({}) + DeepnoteRunner("http://runner", transport=UrllibTransport(open_request)).run({}) def test_connection_error_names_runner_url() -> None: @@ -96,7 +100,7 @@ def open_request(*_: Any, **__: Any) -> FakeResponse: raise URLError("connection refused") with pytest.raises(RunnerError, match="http://runner"): - DeepnoteRunner("http://runner", opener=open_request).info() + DeepnoteRunner("http://runner", transport=UrllibTransport(open_request)).info() def test_timeout_names_runner_url_and_duration() -> None: @@ -104,7 +108,9 @@ def open_request(*_: Any, **__: Any) -> FakeResponse: raise TimeoutError with pytest.raises(RunnerError, match="http://runner.*12 seconds"): - DeepnoteRunner("http://runner", timeout=12, opener=open_request).info() + DeepnoteRunner( + "http://runner", timeout=12, transport=UrllibTransport(open_request) + ).info() def test_cloud_info_reads_public_notebook_contract() -> None: @@ -129,7 +135,7 @@ def open_request(request: Any, *, timeout: float) -> FakeResponse: ) info = DeepnoteCloudRunner( - "notebook-1", token="token-1", opener=open_request + "notebook-1", token="token-1", transport=UrllibTransport(open_request) ).info() assert info.notebook == "Revenue" @@ -176,7 +182,7 @@ def open_request(request: Any, *, timeout: float) -> FakeResponse: result = DeepnoteCloudRunner( "notebook-1", token_provider=lambda: next(tokens), - opener=open_request, + transport=UrllibTransport(open_request), sleep=sleeps.append, poll_interval=0.25, ).run({"limit": 20, "enabled": True, "regions": ["EU"]}) @@ -252,12 +258,11 @@ def open_request(request: Any, *, timeout: float) -> FakeResponse: result = DeepnoteCloudRunner( "notebook-1", token="token", - opener=open_request, + transport=UrllibTransport(open_request), sleep=lambda _delay: None, ).run({"region": "EU"}) assert result.snapshot is None - assert result.snapshot_yaml is None assert [output.block_id for output in result.outputs] == ["code-1", "agent-1"] assert [output.block_type for output in result.outputs] == ["code", "agent"] dataframe = result.first_dataframe() @@ -280,7 +285,10 @@ def open_request(_request: Any, *, timeout: float) -> FakeResponse: ) result = DeepnoteCloudRunner( - "notebook-1", token="token", opener=open_request, sleep=lambda _delay: None + "notebook-1", + token="token", + transport=UrllibTransport(open_request), + sleep=lambda _delay: None, ).run({}) assert result.success is False @@ -297,7 +305,9 @@ def open_request(request: Any, *, timeout: float) -> FakeResponse: assert timeout == 30 return FakeResponse({"notebook": {"name": "Revenue", "inputs": []}}) - info = DeepnoteCloudRunner("notebook-1", opener=open_request).info() + info = DeepnoteCloudRunner( + "notebook-1", transport=UrllibTransport(open_request) + ).info() assert info.notebook == "Revenue" @@ -313,7 +323,7 @@ def test_cloud_runner_requires_one_token_source( DeepnoteCloudRunner( "notebook-1", token="", - opener=lambda *_args, **_kwargs: FakeResponse({}), + transport=UrllibTransport(lambda *_args, **_kwargs: FakeResponse({})), ).info() @@ -334,7 +344,10 @@ def open_request(_request: Any, *, timeout: float) -> FakeResponse: return FakeResponse(response) result = DeepnoteCloudRunner( - "notebook-1", token="token", opener=open_request, sleep=lambda _delay: None + "notebook-1", + token="token", + transport=UrllibTransport(open_request), + sleep=lambda _delay: None, ).run({}) assert result.success is True @@ -356,7 +369,10 @@ def open_request(_request: Any, *, timeout: float) -> FakeResponse: with pytest.raises(RunnerError, match="HTTP 403"): DeepnoteCloudRunner( - "notebook-1", token="token", opener=open_request, sleep=lambda _delay: None + "notebook-1", + token="token", + transport=UrllibTransport(open_request), + sleep=lambda _delay: None, ).run({}) @@ -371,7 +387,10 @@ def open_request(request: Any, *, timeout: float) -> FakeResponse: with pytest.raises(RunnerError, match="connection reset"): DeepnoteCloudRunner( - "notebook-1", token="token", opener=open_request, sleep=lambda _delay: None + "notebook-1", + token="token", + transport=UrllibTransport(open_request), + sleep=lambda _delay: None, ).run({}) assert calls == ["POST"] + ["GET"] * 6 @@ -402,7 +421,9 @@ def test_cloud_run_waits_for_a_snapshot_that_lags_the_terminal_status() -> None: result = DeepnoteCloudRunner( "notebook-1", token="token", - opener=lambda _request, *, timeout: FakeResponse(next(responses)), + transport=UrllibTransport( + lambda _request, *, timeout: FakeResponse(next(responses)) + ), sleep=sleeps.append, poll_interval=0.5, ).run({}) @@ -423,7 +444,7 @@ def open_request(_request: Any, *, timeout: float) -> FakeResponse: } ) - info = DeepnoteRunner(opener=open_request).info() + info = DeepnoteRunner(transport=UrllibTransport(open_request)).info() assert info.inputs == (InputBlock("region", "input-text", None),) @@ -444,7 +465,10 @@ def open_request(_request: Any, *, timeout: float) -> FakeResponse: return FakeResponse(response) result = DeepnoteCloudRunner( - "notebook-1", token="token", opener=open_request, sleep=lambda _delay: None + "notebook-1", + token="token", + transport=UrllibTransport(open_request), + sleep=lambda _delay: None, ).run({}) assert result.success is True @@ -455,14 +479,16 @@ def test_cloud_run_does_not_wait_for_a_snapshot_that_will_not_come() -> None: result = DeepnoteCloudRunner( "notebook-1", token="token", - opener=lambda _request, *, timeout: FakeResponse( - { - "run": { - "runId": "run-1", - "status": "error", - "snapshotStatus": "unavailable", + transport=UrllibTransport( + lambda _request, *, timeout: FakeResponse( + { + "run": { + "runId": "run-1", + "status": "error", + "snapshotStatus": "unavailable", + } } - } + ) ), sleep=sleeps.append, ).run({}) @@ -498,7 +524,9 @@ def open_request(_request: Any, *, timeout: float) -> FakeResponse: } ) - info = DeepnoteCloudRunner("notebook-1", token="token", opener=open_request).info() + info = DeepnoteCloudRunner( + "notebook-1", token="token", transport=UrllibTransport(open_request) + ).info() assert info.inputs == ( InputBlock("region", "input-select", "EU", options=("EU", "US"), multiple=True), @@ -531,7 +559,10 @@ def open_request(request: Any, *, timeout: float) -> FakeResponse: ) DeepnoteCloudRunner( - "notebook-1", token="token", storage_mode="readonly", opener=open_request + "notebook-1", + token="token", + storage_mode="readonly", + transport=UrllibTransport(open_request), ).run({}) assert bodies == [ @@ -542,3 +573,43 @@ def open_request(request: Any, *, timeout: float) -> FakeResponse: "detachedRunStorageMode": "readonly", } ] + + +class FakeTransport: + def __init__(self, payload: Any): + self.payload = payload + self.calls: list[tuple[str, str, Any]] = [] + + def request_json( + self, method: str, url: str, *, headers: Any, body: Any, timeout: float + ) -> Any: + self.calls.append((method, url, headers)) + return self.payload + + +def test_cloud_runner_uses_injected_credentials_and_transport() -> None: + transport = FakeTransport({"notebook": {"name": "Revenue", "inputs": []}}) + + info = DeepnoteCloudRunner( + "notebook-1", + credentials=lambda: ApiCredentials("token", "https://api.example"), + transport=transport, + ).info() + + assert info.notebook == "Revenue" + assert transport.calls == [ + ( + "GET", + "https://api.example/v2/notebooks/notebook-1", + {"Authorization": "Bearer token"}, + ) + ] + + +def test_cloud_runner_rejects_credentials_together_with_a_token() -> None: + with pytest.raises(ValueError, match="not both"): + DeepnoteCloudRunner( + "notebook-1", + token="token", + credentials=lambda: ApiCredentials("other-token"), + ) From 254f696e2d1969fd2f1c81bfff304944200b26b7 Mon Sep 17 00:00:00 2001 From: Wojciech Apanowicz Date: Fri, 18 Sep 2026 19:07:01 +0200 Subject: [PATCH 25/43] feat(notebooks): compare value constraints in RunnerInfo.accepts_inputs The check compared input names and block types only. It now also compares what decides whether a submitted value is valid: single or multiple selection, select options, and slider bounds with the block defaults filled in. Options that a select fills from a variable change between runs, so they are skipped. InputBlock.options_from_variable marks them. --- deepnote_toolkit/notebooks/document.py | 3 ++ deepnote_toolkit/notebooks/models.py | 48 ++++++++++++++++++++--- docs/streamlit-apps.md | 6 ++- tests/unit/test_notebooks_runners.py | 54 ++++++++++++++++++++++++++ 4 files changed, 105 insertions(+), 6 deletions(-) diff --git a/deepnote_toolkit/notebooks/document.py b/deepnote_toolkit/notebooks/document.py index 0e37b183..50cf16af 100644 --- a/deepnote_toolkit/notebooks/document.py +++ b/deepnote_toolkit/notebooks/document.py @@ -102,6 +102,9 @@ def _read_input_block(block: Mapping[str, Any]) -> InputBlock | None: label=optional_string(metadata.get("deepnote_input_label")), value=metadata.get("deepnote_variable_value"), options=string_tuple(metadata.get("deepnote_variable_options")), + options_from_variable=( + metadata.get("deepnote_variable_select_type") == "from-variable" + ), multiple=metadata.get("deepnote_allow_multiple_values") is True, min=optional_number(metadata.get("deepnote_slider_min_value")), max=optional_number(metadata.get("deepnote_slider_max_value")), diff --git a/deepnote_toolkit/notebooks/models.py b/deepnote_toolkit/notebooks/models.py index 2584b732..9ba0d097 100644 --- a/deepnote_toolkit/notebooks/models.py +++ b/deepnote_toolkit/notebooks/models.py @@ -35,6 +35,7 @@ class InputBlock: min: float | int | None = None max: float | int | None = None step: float | int | None = None + options_from_variable: bool = False @dataclass(frozen=True) @@ -151,12 +152,49 @@ class RunnerInfo: run_target: str def accepts_inputs(self, inputs: Iterable[InputBlock]) -> bool: - """Return whether input variable names and block types match this runner.""" - - return _input_contract(inputs) == _input_contract(self.inputs) + """Return whether values made for `inputs` fit this runner's notebook. + + Names, block types, single or multiple selection, slider bounds and select + options must match. Options filled from a variable change between runs, so + they are not compared. + """ + + expected = tuple(inputs) + dynamic = frozenset( + input_block.variable_name + for input_block in expected + if input_block.options_from_variable + ) + return _input_contract(expected, dynamic) == _input_contract( + self.inputs, dynamic + ) -def _input_contract(inputs: Iterable[InputBlock]) -> frozenset[tuple[str, str]]: +def _input_contract( + inputs: Iterable[InputBlock], dynamic_options: frozenset[str] +) -> frozenset[tuple[Any, ...]]: return frozenset( - (input_block.variable_name, input_block.type) for input_block in inputs + ( + input_block.variable_name, + input_block.type, + *_value_constraints(input_block, dynamic_options), + ) + for input_block in inputs ) + + +def _value_constraints( + input_block: InputBlock, dynamic_options: frozenset[str] +) -> tuple[Any, ...]: + if input_block.type == "input-slider": + return ( + input_block.min if input_block.min is not None else 0, + input_block.max if input_block.max is not None else 100, + ) + if input_block.type == "input-select": + is_dynamic = input_block.variable_name in dynamic_options + return ( + input_block.multiple, + None if is_dynamic else frozenset(input_block.options), + ) + return () diff --git a/docs/streamlit-apps.md b/docs/streamlit-apps.md index 63e41b9f..ed39ce2f 100644 --- a/docs/streamlit-apps.md +++ b/docs/streamlit-apps.md @@ -105,4 +105,8 @@ for the outputs, which can arrive after the final status. When of that wait. Use `runner.info().accepts_inputs(document.inputs)` before submitting values to -verify that the deployed notebook still has matching input names and block types. +verify that the deployed notebook still matches the file the app was built from. +It compares input names, block types, single or multiple selection, slider bounds +and select options. Options that a select fills from a variable change between +runs, so they are not compared. The API also rejects a value that does not fit its +input block. diff --git a/tests/unit/test_notebooks_runners.py b/tests/unit/test_notebooks_runners.py index a9e3d8b0..3ee72d77 100644 --- a/tests/unit/test_notebooks_runners.py +++ b/tests/unit/test_notebooks_runners.py @@ -575,6 +575,60 @@ def open_request(request: Any, *, timeout: float) -> FakeResponse: ] +@pytest.mark.parametrize( + "changed", + [ + InputBlock("region", "input-select", "EU", options=("EU", "US"), multiple=True), + InputBlock("region", "input-select", "EU", options=("EU", "APAC")), + ], + ids=["multiple", "options"], +) +def test_runner_info_rejects_a_select_that_takes_other_values( + changed: InputBlock, +) -> None: + info = RunnerInfo( + notebook="Revenue", + inputs=(InputBlock("region", "input-select", "EU", options=("US", "EU")),), + run_target="cloud", + ) + + assert info.accepts_inputs( + [InputBlock("region", "input-select", "US", options=("EU", "US"))] + ) + assert not info.accepts_inputs([changed]) + + +def test_runner_info_ignores_select_options_filled_from_a_variable() -> None: + info = RunnerInfo( + notebook="Revenue", + inputs=(InputBlock("region", "input-select", "EU", options=("EU", "US")),), + run_target="cloud", + ) + + assert info.accepts_inputs( + [ + InputBlock( + "region", + "input-select", + "EU", + options=("EU",), + options_from_variable=True, + ) + ] + ) + + +def test_runner_info_compares_slider_bounds_with_the_defaults_filled_in() -> None: + info = RunnerInfo( + notebook="Revenue", + inputs=(InputBlock("limit", "input-slider", "20", min=0, max=100, step=1),), + run_target="cloud", + ) + + assert info.accepts_inputs([InputBlock("limit", "input-slider", "20")]) + assert not info.accepts_inputs([InputBlock("limit", "input-slider", "20", max=50)]) + + class FakeTransport: def __init__(self, payload: Any): self.payload = payload From 1687ee86c5926e4fe1ccbdd8eb75223c93e92931 Mon Sep 17 00:00:00 2001 From: Wojciech Apanowicz Date: Fri, 18 Sep 2026 22:11:30 +0200 Subject: [PATCH 26/43] fix(streamlit): harden the hosted path and the transport after review The default transport refuses a redirect to another origin, which urllib would follow with the bearer token attached. The credential exchange uses the same opener. A response body that is not JSON stays out of error messages. The launcher exports each app's ID to its process as DEEPNOTE_STREAMLIT_APP_ID. The toolkit treats such a process as hosted without needing request headers, takes the app ID from there first, and raises off the script thread whatever token was passed. The wait for a run's outputs is set in seconds with snapshot_timeout. A tuple input is sent as a list, and None or a mapping is rejected instead of being sent as its repr. A mapping tag on another YAML node is a YAML error. Tests cover the poll timeout, giving up on the snapshot, the image helpers, and render_inputs on real Streamlit widgets. The relative date range test runs on a fixed date. --- deepnote_toolkit/notebooks/api_client.py | 13 ++- deepnote_toolkit/notebooks/cloud_runner.py | 21 ++-- deepnote_toolkit/notebooks/transport.py | 47 +++++--- deepnote_toolkit/notebooks/yaml_loader.py | 2 + deepnote_toolkit/streamlit/auth.py | 30 ++++-- deepnote_toolkit/streamlit/cloud_runner.py | 2 + .../streamlit/viewer_credentials.py | 22 ++-- docs/streamlit-apps.md | 15 +-- installer/module/streamlit.py | 15 ++- tests/unit/test_deepnote_streamlit_auth.py | 29 +++++ .../test_deepnote_streamlit_cloud_runner.py | 16 +++ tests/unit/test_deepnote_streamlit_widgets.py | 77 +++++++++++-- tests/unit/test_notebooks_document.py | 26 +++++ tests/unit/test_notebooks_runners.py | 102 ++++++++++++++++++ tests/unit/test_notebooks_yaml_loader.py | 5 + tests/unit/test_streamlit.py | 25 +++++ 16 files changed, 393 insertions(+), 54 deletions(-) diff --git a/deepnote_toolkit/notebooks/api_client.py b/deepnote_toolkit/notebooks/api_client.py index 973301ee..14064723 100644 --- a/deepnote_toolkit/notebooks/api_client.py +++ b/deepnote_toolkit/notebooks/api_client.py @@ -85,7 +85,9 @@ def create_run( body: dict[str, Any] = { "notebookId": notebook_id, "detached": True, - "inputs": {name: _encode_input(value) for name, value in inputs.items()}, + "inputs": { + name: _encode_input(name, value) for name, value in inputs.items() + }, } if storage_mode is not None: body["detachedRunStorageMode"] = storage_mode @@ -111,11 +113,16 @@ def _request( ) -def _encode_input(value: Any) -> InputValue: +def _encode_input(name: str, value: Any) -> InputValue: if isinstance(value, bool): return value - if isinstance(value, list): + if isinstance(value, (list, tuple)): return [str(item) for item in value] + if value is None or isinstance(value, (Mapping, set, frozenset, bytes)): + raise ValueError( + f'Input "{name}" has a {type(value).__name__} value. ' + "Pass text, a number, a boolean or a list of texts." + ) return str(value) diff --git a/deepnote_toolkit/notebooks/cloud_runner.py b/deepnote_toolkit/notebooks/cloud_runner.py index 137ae49f..5cb3a899 100644 --- a/deepnote_toolkit/notebooks/cloud_runner.py +++ b/deepnote_toolkit/notebooks/cloud_runner.py @@ -22,7 +22,6 @@ Sleep = Callable[[float], None] MAX_TRANSIENT_POLL_FAILURES = 5 -SNAPSHOT_SETTLE_ATTEMPTS = 3 class DeepnoteCloudRunner: @@ -35,6 +34,10 @@ class DeepnoteCloudRunner: `storage_mode="readonly"` keeps the run from changing the project's stored files. None leaves the choice to the API, which allows writes. + + The outputs can arrive after the run finishes. `snapshot_timeout` is how many + seconds to wait for them. A result whose `snapshot_status` is still `pending` + has none because that wait ran out. """ def __init__( @@ -47,12 +50,15 @@ def __init__( credentials: CredentialsProvider | None = None, storage_mode: StorageMode | None = None, timeout: float = 600, + snapshot_timeout: float = 10, poll_interval: float = 2, transport: Transport | None = None, sleep: Sleep = time.sleep, ): if not notebook_id: raise ValueError("notebook_id is required") + if poll_interval <= 0: + raise ValueError("poll_interval must be positive") if credentials is not None and ( token is not None or token_provider is not None ): @@ -60,6 +66,7 @@ def __init__( self.notebook_id = notebook_id self.storage_mode = storage_mode self.timeout = timeout + self.snapshot_timeout = snapshot_timeout self.poll_interval = poll_interval self._client = DeepnoteApiClient( credentials or token_credentials(token, token_provider, base_url=base_url), @@ -117,12 +124,14 @@ def _wait_until_finished(self, run: CloudRun) -> CloudRun: return run def _settle_snapshot(self, run: CloudRun) -> CloudRun: - # The snapshot can attach shortly after the status turns terminal. - for _ in range(SNAPSHOT_SETTLE_ATTEMPTS): - is_pending = run.snapshot_status in (None, "pending") - if not is_pending or run.outputs is not None: - break + waited = 0.0 + while ( + run.outputs is None + and run.snapshot_status in (None, "pending") + and waited < self.snapshot_timeout + ): self._sleep(self.poll_interval) + waited += self.poll_interval try: run = self._client.get_run(run.run_id) except RunnerError as error: diff --git a/deepnote_toolkit/notebooks/transport.py b/deepnote_toolkit/notebooks/transport.py index c87f6463..b1a60f22 100644 --- a/deepnote_toolkit/notebooks/transport.py +++ b/deepnote_toolkit/notebooks/transport.py @@ -8,13 +8,29 @@ from typing import Any, Protocol from urllib.error import HTTPError, URLError from urllib.parse import urlsplit -from urllib.request import Request, urlopen +from urllib.request import HTTPRedirectHandler, Request, build_opener from .runner import RunnerError OpenUrl = Callable[..., Any] +class _SameOriginRedirectHandler(HTTPRedirectHandler): + """Refuses a redirect to another origin, which would receive the bearer token.""" + + def redirect_request( + self, req: Request, fp: Any, code: int, msg: str, headers: Any, newurl: str + ) -> Request | None: + if _origin(newurl) != _origin(req.full_url): + raise HTTPError( + req.full_url, code, "Refused a redirect to another origin", headers, fp + ) + return super().redirect_request(req, fp, code, msg, headers, newurl) + + +open_url: OpenUrl = build_opener(_SameOriginRedirectHandler).open + + class Transport(Protocol): """Sends one JSON request. Implement it to use another HTTP library.""" @@ -37,7 +53,7 @@ def request_json( class UrllibTransport: """The default transport, on the standard library.""" - def __init__(self, opener: OpenUrl = urlopen): + def __init__(self, opener: OpenUrl = open_url): self._open = opener def request_json( @@ -51,8 +67,7 @@ def request_json( ) -> Mapping[str, Any]: """Send the request with `urllib` and return its JSON object.""" - parts = urlsplit(url) - origin = f"{parts.scheme}://{parts.netloc}" + origin = _origin(url) request = Request( url, data=json.dumps(body).encode() if body is not None else None, @@ -67,8 +82,9 @@ def request_json( with self._open(request, timeout=timeout) as response: payload = json.loads(response.read()) except HTTPError as error: + message = _error_message(error) or error.reason raise RunnerError( - f"{origin} returned HTTP {error.code}: {_error_message(error)}", + f"{origin} returned HTTP {error.code}: {message}", transient=error.code == 429 or error.code >= 500, ) from error except URLError as error: @@ -90,12 +106,19 @@ def request_json( return payload -def _error_message(error: HTTPError) -> str: - detail = error.read().decode(errors="replace") +def _origin(url: str) -> str: + parts = urlsplit(url) + return f"{parts.scheme}://{parts.netloc}" + + +def _error_message(error: HTTPError) -> str | None: + """Return the message of a JSON error response, or None for any other body.""" + try: - parsed = json.loads(detail) - except json.JSONDecodeError: - return detail + parsed = json.loads(error.read()) + except (json.JSONDecodeError, UnicodeDecodeError): + return None if not isinstance(parsed, Mapping): - return detail - return str(parsed.get("message") or parsed.get("error") or detail) + return None + message = parsed.get("message") or parsed.get("error") + return message if isinstance(message, str) else None diff --git a/deepnote_toolkit/notebooks/yaml_loader.py b/deepnote_toolkit/notebooks/yaml_loader.py index f540939b..4cb1b62d 100644 --- a/deepnote_toolkit/notebooks/yaml_loader.py +++ b/deepnote_toolkit/notebooks/yaml_loader.py @@ -25,6 +25,8 @@ def construct_mapping( ) -> dict[Any, Any]: """Build a mapping, rejecting a repeated key. PyYAML would keep the last.""" + if not isinstance(node, yaml.MappingNode): + return super().construct_mapping(node, deep=deep) seen: set[tuple[str, str]] = set() for key_node, _value_node in node.value: if not isinstance(key_node, yaml.ScalarNode): diff --git a/deepnote_toolkit/streamlit/auth.py b/deepnote_toolkit/streamlit/auth.py index cf360de3..bba390de 100644 --- a/deepnote_toolkit/streamlit/auth.py +++ b/deepnote_toolkit/streamlit/auth.py @@ -4,6 +4,7 @@ import hashlib import json +import os import re import time from collections.abc import Callable, Mapping @@ -12,21 +13,21 @@ from typing import Any from urllib.error import HTTPError, URLError from urllib.parse import urlparse -from urllib.request import Request, urlopen +from urllib.request import Request from deepnote_toolkit.get_webapp_url import ( get_absolute_userpod_api_url, get_project_auth_headers, ) +from deepnote_toolkit.notebooks.transport import open_url from deepnote_toolkit.streamlit_data_apps import ( read_streamlit_token_from_context, ) OpenUrl = Callable[..., Any] -STREAMLIT_APP_HOST_PATTERN = re.compile( - r"^streamlit-([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})\.", - re.IGNORECASE, -) +_APP_ID = r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}" +STREAMLIT_APP_HOST_PATTERN = re.compile(rf"^streamlit-({_APP_ID})\.", re.IGNORECASE) +STREAMLIT_APP_ID_ENV = "DEEPNOTE_STREAMLIT_APP_ID" _SESSION_STATE_KEY = "_deepnote_current_user_api_credentials" @@ -65,7 +66,7 @@ def current_user_api_credentials( app_id: str | None = None, streamlit_token: str | None = None, timeout: float = 10, - opener: OpenUrl = urlopen, + opener: OpenUrl = open_url, ) -> CurrentUserApiCredentials: """Exchange the active viewer cookie for public API credentials. @@ -74,7 +75,9 @@ def current_user_api_credentials( never shared between sessions. """ - resolved_app_id = app_id or _read_streamlit_app_id_from_context() + resolved_app_id = ( + app_id or _read_hosted_app_id() or _read_streamlit_app_id_from_context() + ) if not resolved_app_id: raise CurrentUserApiTokenError( "Could not resolve a Deepnote Streamlit app ID from the request host." @@ -185,8 +188,19 @@ def _read_streamlit_session_state() -> Any | None: return st.session_state +def _read_hosted_app_id() -> str | None: + """Return the app ID that Deepnote's launcher exports to a hosted app's process.""" + + app_id = os.environ.get(STREAMLIT_APP_ID_ENV, "") + return app_id.lower() if re.fullmatch(_APP_ID, app_id, re.IGNORECASE) else None + + def _read_streamlit_app_id_from_context() -> str | None: - """Resolve the app UUID from the external Streamlit request hostname.""" + """Resolve the app UUID from the external Streamlit request hostname. + + The exchange checks the viewer's token against this app, so a forged host + gains nothing. + """ try: import streamlit as st # type: ignore[import-not-found] diff --git a/deepnote_toolkit/streamlit/cloud_runner.py b/deepnote_toolkit/streamlit/cloud_runner.py index df6103f8..12746e71 100644 --- a/deepnote_toolkit/streamlit/cloud_runner.py +++ b/deepnote_toolkit/streamlit/cloud_runner.py @@ -29,6 +29,7 @@ def __init__( base_url: str = DEFAULT_API_ORIGIN, storage_mode: StorageMode | None = "readonly", timeout: float = 600, + snapshot_timeout: float = 10, poll_interval: float = 2, transport: Transport | None = None, sleep: Sleep = time.sleep, @@ -40,6 +41,7 @@ def __init__( ), storage_mode=storage_mode, timeout=timeout, + snapshot_timeout=snapshot_timeout, poll_interval=poll_interval, transport=transport, sleep=sleep, diff --git a/deepnote_toolkit/streamlit/viewer_credentials.py b/deepnote_toolkit/streamlit/viewer_credentials.py index d0e4aebe..568fd27f 100644 --- a/deepnote_toolkit/streamlit/viewer_credentials.py +++ b/deepnote_toolkit/streamlit/viewer_credentials.py @@ -15,15 +15,22 @@ _has_hosted_streamlit_context, _has_script_run_context, _is_streamlit_thread_without_request, + _read_hosted_app_id, current_user_api_credentials, ) +_NO_REQUEST = ( + "No viewer request is available on this thread. Call the runner from the " + "Streamlit script thread" +) + class ViewerCredentials: """Credentials of the current viewer when Deepnote hosts the app. `token`, `token_provider`, `base_url` and the `DEEPNOTE_TOKEN` environment - variable apply only outside Deepnote hosting. + variable apply only outside Deepnote hosting. A hosted app has a viewer only on + the Streamlit script thread, and the call raises on any other thread. """ def __init__( @@ -41,7 +48,13 @@ def __init__( def __call__(self) -> ApiCredentials: """Return the viewer's credentials, or the local ones outside hosting.""" - if _has_script_run_context() and _has_hosted_streamlit_context(): + has_request = _has_script_run_context() + is_hosted = _read_hosted_app_id() is not None or ( + has_request and _has_hosted_streamlit_context() + ) + if is_hosted: + if not has_request: + raise RunnerError(_NO_REQUEST + ".") try: viewer = current_user_api_credentials(timeout=self._timeout) except CurrentUserApiTokenError as error: @@ -49,9 +62,6 @@ def __call__(self) -> ApiCredentials: return ApiCredentials(token=viewer.token, api_origin=viewer.api_origin) if not self._is_token_explicit and _is_streamlit_thread_without_request(): - raise RunnerError( - "No viewer request is available on this thread. Call the runner from " - "the Streamlit script thread, or pass token= or token_provider=." - ) + raise RunnerError(_NO_REQUEST + ", or pass token= or token_provider=.") return self._local() diff --git a/docs/streamlit-apps.md b/docs/streamlit-apps.md index ed39ce2f..d48ca052 100644 --- a/docs/streamlit-apps.md +++ b/docs/streamlit-apps.md @@ -39,7 +39,8 @@ program: `is_truncated` tells whether rows are missing. Deepnote sends every non-numeric cell as text, so a boolean column arrives as `"True"` and `"False"`. - `DeepnoteRunner` does the same through a local `@deepnote/local-runner` sidecar - at `http://127.0.0.1:8787`. + at `http://127.0.0.1:8787`. The sidecar does not say which notebook it ran, so + for a file with several notebooks the result holds the outputs of all of them. - `Runner` is the interface both runners implement, for code that accepts either. The cloud runner only starts a run and waits for it. The parts under it can be @@ -83,8 +84,8 @@ script works locally and deployed. To run notebooks with one fixed token for eve viewer, use `DeepnoteCloudRunner` with that token. Call the runner from the Streamlit script thread. A worker thread has no viewer -request, so the runner raises there instead of using `DEEPNOTE_TOKEN`. With an -explicit token it uses that token there, even in a hosted app. +request. In a hosted app the runner raises there, whatever token it was given. +Elsewhere it raises instead of using `DEEPNOTE_TOKEN`, and uses an explicit token. For another Deepnote API client inside a hosted app, `current_user_api_credentials()` returns a short-lived token for the current @@ -99,10 +100,10 @@ change them. Pass `storage_mode="read_write"` for a notebook that must write the `DeepnoteCloudRunner` leaves the mode to the API, which allows writes. The cloud runner retries a poll that fails with a timeout, a network error, HTTP -429 or a 5xx, up to five times in a row. After the run finishes it waits briefly -for the outputs, which can arrive after the final status. When -`result.snapshot_status` is still `pending`, the outputs had not arrived by the end -of that wait. +429 or a 5xx, up to five times in a row. After the run finishes it waits up to +`snapshot_timeout` seconds, 10 by default, for the outputs, which can arrive after +the final status. When `result.snapshot_status` is still `pending`, the outputs +had not arrived by the end of that wait, so a successful run can have none. Use `runner.info().accepts_inputs(document.inputs)` before submitting values to verify that the deployed notebook still matches the file the app was built from. diff --git a/installer/module/streamlit.py b/installer/module/streamlit.py index c20c1aee..1fd91e81 100644 --- a/installer/module/streamlit.py +++ b/installer/module/streamlit.py @@ -3,12 +3,17 @@ import json import logging import os +import re import urllib.request from typing import List from .helper import request_with_retries from .virtual_environment import VirtualEnvironment +_APP_ID_PATTERN = re.compile( + r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}", re.IGNORECASE +) + def get_webapp_url() -> str: """Returns the WebApp URL.""" @@ -118,9 +123,17 @@ def start_streamlit_servers( arg_str = " ".join(args) + # The toolkit reads the app ID to run notebooks as the app's viewer. + app_id = app.get("id") + is_app_id_valid = isinstance(app_id, str) and _APP_ID_PATTERN.fullmatch( + app_id + ) + env = f"DEEPNOTE_STREAMLIT_APP_ID={app_id} " if is_app_id_valid else "" + processes.append( venv.start_server( - f"streamlit run '{entrypoint_path}' {arg_str}", cwd=directory_path + f"{env}streamlit run '{entrypoint_path}' {arg_str}", + cwd=directory_path, ) ) except Exception as e: diff --git a/tests/unit/test_deepnote_streamlit_auth.py b/tests/unit/test_deepnote_streamlit_auth.py index 1baf441c..12c7d996 100644 --- a/tests/unit/test_deepnote_streamlit_auth.py +++ b/tests/unit/test_deepnote_streamlit_auth.py @@ -325,3 +325,32 @@ def open_request(_request: Any, *, timeout: float) -> Any: ) assert exc_info.value.transient is True + + +def test_exchange_prefers_the_app_id_exported_by_the_launcher( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("DEEPNOTE_STREAMLIT_APP_ID", APP_ID.upper()) + urls = [] + + def open_request(request: Any, *, timeout: float) -> FakeResponse: + urls.append(request.full_url) + return FakeResponse( + { + "token": "viewer-api-token", + "apiOrigin": "https://api.deepnote.com", + "expiresAtSeconds": 1_800_000_000, + } + ) + + with patch( + "deepnote_toolkit.streamlit.auth._read_streamlit_app_id_from_context", + return_value="00000000-0000-0000-0000-000000000000", + ): + current_user_api_credentials( + streamlit_token="opaque-cookie", opener=open_request + ) + + assert urls == [ + f"http://localhost:19456/userpod-api/streamlit-apps/{APP_ID}/api-token" + ] diff --git a/tests/unit/test_deepnote_streamlit_cloud_runner.py b/tests/unit/test_deepnote_streamlit_cloud_runner.py index 0cc70dce..e1224585 100644 --- a/tests/unit/test_deepnote_streamlit_cloud_runner.py +++ b/tests/unit/test_deepnote_streamlit_cloud_runner.py @@ -355,3 +355,19 @@ def test_viewer_credentials_use_an_explicit_token_on_a_worker_thread() -> None: credentials = ViewerCredentials(token="local-token")() assert credentials == ApiCredentials("local-token", "https://api.deepnote.com") + + +def test_hosted_process_raises_off_the_script_thread_even_with_a_token( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv( + "DEEPNOTE_STREAMLIT_APP_ID", "11111111-2222-3333-4444-555555555555" + ) + with ( + patch( + "deepnote_toolkit.streamlit.viewer_credentials._has_script_run_context", + return_value=False, + ), + pytest.raises(RunnerError, match="No viewer request"), + ): + ViewerCredentials(token="owner-token")() diff --git a/tests/unit/test_deepnote_streamlit_widgets.py b/tests/unit/test_deepnote_streamlit_widgets.py index e6675729..0066dedc 100644 --- a/tests/unit/test_deepnote_streamlit_widgets.py +++ b/tests/unit/test_deepnote_streamlit_widgets.py @@ -1,6 +1,8 @@ -from datetime import date, timedelta +from datetime import date from typing import Any +import pytest + from deepnote_toolkit.notebooks import InputBlock from deepnote_toolkit.streamlit import render_inputs @@ -148,26 +150,79 @@ def test_empty_dates_stay_empty_instead_of_becoming_today() -> None: assert values == {"as_of": "", "period": ["", ""]} -def test_relative_date_ranges_resolve_to_concrete_dates() -> None: +class FrozenDate(date): + @classmethod + def today(cls) -> "FrozenDate": + return cls(2024, 3, 31) + + +def test_relative_date_ranges_resolve_to_concrete_dates( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr("deepnote_toolkit.streamlit.widgets.date", FrozenDate) + values = render_inputs( [ InputBlock("week", "input-date-range", "past7days"), InputBlock("custom", "input-date-range", "customDays3"), + InputBlock("month", "input-date-range", "pastMonth"), InputBlock("year", "input-date-range", "pastYear"), ], FakeContainer(), ) - today = date.today() - # Feb 29 has no counterpart a year earlier and clamps to Feb 28. - year_ago_day = 28 if (today.month, today.day) == (2, 29) else today.day + # Mar 31 has no counterpart a month earlier and clamps to Feb 29. assert values == { - "week": [(today - timedelta(days=7)).isoformat(), today.isoformat()], - "custom": [(today - timedelta(days=3)).isoformat(), today.isoformat()], - "year": [ - today.replace(year=today.year - 1, day=year_ago_day).isoformat(), - today.isoformat(), - ], + "week": ["2024-03-24", "2024-03-31"], + "custom": ["2024-03-28", "2024-03-31"], + "month": ["2024-02-29", "2024-03-31"], + "year": ["2023-03-31", "2024-03-31"], + } + + +def test_render_inputs_runs_on_real_streamlit_widgets() -> None: + pytest.importorskip("streamlit") + from streamlit.testing.v1 import AppTest + + def app() -> None: + import streamlit as st + + from deepnote_toolkit.notebooks import InputBlock + from deepnote_toolkit.streamlit import render_inputs + + st.session_state["values"] = render_inputs( + [ + InputBlock("name", "input-text", "Ada"), + InputBlock("enabled", "input-checkbox", True), + InputBlock("region", "input-select", "EU", options=("US", "EU")), + InputBlock( + "regions", + "input-select", + ["EU"], + options=("US", "EU"), + multiple=True, + ), + InputBlock("limit", "input-slider", "20", min=0, max=100, step=5), + InputBlock("day", "input-date", "2026-08-17"), + InputBlock("no_day", "input-date", ""), + InputBlock("span", "input-date-range", ["2026-08-01", "2026-08-17"]), + InputBlock("no_span", "input-date-range", ["", ""]), + ] + ) + + at = AppTest.from_function(app).run() + + assert not at.exception + assert at.session_state["values"] == { + "name": "Ada", + "enabled": True, + "region": "EU", + "regions": ["EU"], + "limit": 20, + "day": "2026-08-17", + "no_day": "", + "span": ["2026-08-01", "2026-08-17"], + "no_span": ["", ""], } diff --git a/tests/unit/test_notebooks_document.py b/tests/unit/test_notebooks_document.py index 43bee029..6df241f6 100644 --- a/tests/unit/test_notebooks_document.py +++ b/tests/unit/test_notebooks_document.py @@ -317,3 +317,29 @@ def test_dataframe_reports_rows_beyond_the_first_page() -> None: assert dataframe is not None and whole is not None assert (dataframe.row_count, dataframe.is_truncated) == (250, True) assert (whole.row_count, whole.is_truncated) == (1, False) + + +def test_images_decode_wrapped_base64_and_skip_invalid_data() -> None: + result = run_locally( + { + "outputs": [ + { + "blockId": "code-1", + "outputs": [ + { + "output_type": "display_data", + "data": {"image/png": "aGVs\nbG8="}, + }, + { + "output_type": "display_data", + "data": {"image/png": "not base64!"}, + }, + {"output_type": "display_data", "data": {"image/jpeg": "aGk="}}, + ], + } + ] + } + ) + + assert result.images() == [b"hello"] + assert result.images("image/jpeg") == [b"hi"] diff --git a/tests/unit/test_notebooks_runners.py b/tests/unit/test_notebooks_runners.py index 3ee72d77..1c2d4e45 100644 --- a/tests/unit/test_notebooks_runners.py +++ b/tests/unit/test_notebooks_runners.py @@ -1,6 +1,8 @@ import io import json +import threading from http.client import RemoteDisconnected +from http.server import BaseHTTPRequestHandler, HTTPServer from typing import Any from urllib.error import HTTPError, URLError @@ -667,3 +669,103 @@ def test_cloud_runner_rejects_credentials_together_with_a_token() -> None: token="token", credentials=lambda: ApiCredentials("other-token"), ) + + +def test_default_transport_refuses_a_redirect_to_another_origin() -> None: + received = [] + + class Handler(BaseHTTPRequestHandler): + def do_GET(self) -> None: + received.append((self.server.server_port, self.headers["Authorization"])) + self.send_response(302) + self.send_header("Location", f"http://127.0.0.1:{other.server_port}/") + self.end_headers() + + def log_message(self, *_args: Any) -> None: + return None + + api = HTTPServer(("127.0.0.1", 0), Handler) + other = HTTPServer(("127.0.0.1", 0), Handler) + for server in (api, other): + threading.Thread(target=server.serve_forever, daemon=True).start() + try: + with pytest.raises(RunnerError, match="HTTP 302: Refused a redirect"): + DeepnoteCloudRunner( + "notebook-1", + token="token", + base_url=f"http://127.0.0.1:{api.server_port}", + ).info() + finally: + for server in (api, other): + server.shutdown() + server.server_close() + + assert received == [(api.server_port, "Bearer token")] + + +def test_http_error_keeps_a_body_that_is_not_json_out_of_the_message() -> None: + def open_request(*_: Any, **__: Any) -> FakeResponse: + raise HTTPError( + "http://runner/api/run", + 502, + "Bad Gateway", + {}, + io.BytesIO(b"proxy internals"), + ) + + with pytest.raises(RunnerError) as raised: + DeepnoteRunner("http://runner", transport=UrllibTransport(open_request)).run({}) + + assert str(raised.value) == "http://runner returned HTTP 502: Bad Gateway" + + +def test_cloud_run_raises_when_the_run_outlasts_the_timeout() -> None: + transport = FakeTransport({"run": {"runId": "run-1", "status": "running"}}) + + with pytest.raises(RunnerError, match="run-1 did not finish in 0 seconds"): + DeepnoteCloudRunner( + "notebook-1", token="token", timeout=0, transport=transport + ).run({}) + + +def test_cloud_run_stops_waiting_for_a_snapshot_after_the_snapshot_timeout() -> None: + transport = FakeTransport( + {"run": {"runId": "run-1", "status": "success", "snapshotStatus": "pending"}} + ) + sleeps: list[float] = [] + + result = DeepnoteCloudRunner( + "notebook-1", + token="token", + snapshot_timeout=1, + poll_interval=0.5, + transport=transport, + sleep=sleeps.append, + ).run({}) + + assert sleeps == [0.5, 0.5] + assert result.success is True + assert result.snapshot_status == "pending" + assert result.outputs == () + + +def test_cloud_run_sends_a_tuple_as_a_list_and_rejects_a_missing_value() -> None: + bodies = [] + + class RecordingTransport(FakeTransport): + def request_json(self, method: str, url: str, **kwargs: Any) -> Any: + bodies.append(kwargs["body"]) + return self.payload + + runner = DeepnoteCloudRunner( + "notebook-1", + token="token", + transport=RecordingTransport( + {"run": {"runId": "run-1", "status": "success", "snapshotBlocks": []}} + ), + ) + runner.run({"regions": ("EU", "US")}) + + assert bodies[0]["inputs"] == {"regions": ["EU", "US"]} + with pytest.raises(ValueError, match='Input "region" has a NoneType value'): + runner.run({"region": None}) diff --git a/tests/unit/test_notebooks_yaml_loader.py b/tests/unit/test_notebooks_yaml_loader.py index b7b47f67..cd67fbee 100644 --- a/tests/unit/test_notebooks_yaml_loader.py +++ b/tests/unit/test_notebooks_yaml_loader.py @@ -80,3 +80,8 @@ def test_the_same_key_may_repeat_in_separate_mappings(load_yaml: Any) -> None: {"id": "b"}, {1: "x", "1": "y"}, ] + + +def test_mapping_tag_on_another_node_is_a_yaml_error(load_yaml: Any) -> None: + with pytest.raises(yaml.YAMLError, match="expected a mapping node"): + load_yaml("!!map [1, 2]") diff --git a/tests/unit/test_streamlit.py b/tests/unit/test_streamlit.py index e3dfb7ec..819bc74f 100644 --- a/tests/unit/test_streamlit.py +++ b/tests/unit/test_streamlit.py @@ -97,3 +97,28 @@ def exists_side_effect(path: str) -> bool: assert mock_logger.warning.call_count == 2 assert mock_venv.start_server.call_count == 1 + + def test_exports_a_valid_app_id_to_the_app_process(self): + """The app ID reaches the process only when it is a UUID, since it enters a shell command.""" + apps = [ + { + "id": "11111111-2222-3333-4444-555555555555", + "entrypoint": "a/app.py", + "port": "8501", + }, + {"id": "x; rm -rf /", "entrypoint": "b/app.py", "port": "8502"}, + ] + mock_venv = MagicMock() + + with ( + patch("installer.module.streamlit.fetch_streamlit_apps", return_value=apps), + patch("installer.module.streamlit.os.path.exists", return_value=True), + ): + start_streamlit_servers(mock_venv, MagicMock(spec=logging.Logger)) + + commands = [call[0][0] for call in mock_venv.start_server.call_args_list] + assert commands[0].startswith( + "DEEPNOTE_STREAMLIT_APP_ID=11111111-2222-3333-4444-555555555555 " + "streamlit run '/work/a/app.py' " + ) + assert commands[1].startswith("streamlit run '/work/b/app.py' ") From 1728ad397f8400feca6bab1b1be8b206ec5ab873 Mon Sep 17 00:00:00 2001 From: Wojciech Apanowicz Date: Mon, 21 Sep 2026 15:39:07 +0200 Subject: [PATCH 27/43] fix(notebooks): reject unknown run statuses and input types, cap the snapshot wait A run response whose status is missing or not one of the API's six run statuses raises a RunnerError. The runner used to treat it as unfinished and poll until the timeout. An input block of a type the toolkit does not support is skipped, in a .deepnote file and in an API or sidecar response. The check used to accept any type starting with "input-". The wait for a run's outputs sleeps no longer than what is left of snapshot_timeout, so a timeout shorter than poll_interval is honored. --- deepnote_toolkit/notebooks/api_client.py | 8 ++++- deepnote_toolkit/notebooks/api_types.py | 8 ++--- deepnote_toolkit/notebooks/cloud_runner.py | 5 +-- deepnote_toolkit/notebooks/document.py | 4 +-- deepnote_toolkit/notebooks/transport.py | 2 ++ deepnote_toolkit/notebooks/wire.py | 5 +-- tests/unit/test_notebooks_document.py | 14 ++++++++ tests/unit/test_notebooks_runners.py | 40 +++++++++++++++++++++- 8 files changed, 74 insertions(+), 12 deletions(-) diff --git a/deepnote_toolkit/notebooks/api_client.py b/deepnote_toolkit/notebooks/api_client.py index 14064723..43165f18 100644 --- a/deepnote_toolkit/notebooks/api_client.py +++ b/deepnote_toolkit/notebooks/api_client.py @@ -8,6 +8,7 @@ from typing import Any, cast from .api_types import ( + RUN_STATUSES, SNAPSHOT_STATUSES, TERMINAL_RUN_STATUSES, InputValue, @@ -114,6 +115,8 @@ def _request( def _encode_input(name: str, value: Any) -> InputValue: + """Convert a value to the form the runs API accepts, or raise `ValueError`.""" + if isinstance(value, bool): return value if isinstance(value, (list, tuple)): @@ -132,6 +135,9 @@ def _decode_run(payload: Mapping[str, Any], *, run_id: str | None = None) -> Clo run_id = run.get("runId") or run.get("id") or run_id if not isinstance(run_id, str) or not run_id: raise RunnerError("Deepnote API response did not include a run id") + status = run.get("status") + if not isinstance(status, str) or status not in RUN_STATUSES: + raise RunnerError(f"Deepnote run {run_id} has an unknown status: {status!r}") snapshot_status = run.get("snapshotStatus") blocks = run.get("snapshotBlocks") error = run.get("error") @@ -139,7 +145,7 @@ def _decode_run(payload: Mapping[str, Any], *, run_id: str | None = None) -> Clo error = error.get("message") or json.dumps(error) return CloudRun( run_id=run_id, - status=cast(RunStatus, str(run.get("status", ""))), + status=cast(RunStatus, status), snapshot_status=( snapshot_status if snapshot_status in SNAPSHOT_STATUSES else None ), diff --git a/deepnote_toolkit/notebooks/api_types.py b/deepnote_toolkit/notebooks/api_types.py index 79c1a694..15a9d648 100644 --- a/deepnote_toolkit/notebooks/api_types.py +++ b/deepnote_toolkit/notebooks/api_types.py @@ -2,7 +2,7 @@ from __future__ import annotations -from typing import Literal, Union +from typing import Literal, Union, get_args InputBlockType = Literal[ "input-checkbox", @@ -21,9 +21,9 @@ StorageMode = Literal["read_write", "readonly"] InputValue = Union[str, bool, list[str]] +INPUT_BLOCK_TYPES: frozenset[InputBlockType] = frozenset(get_args(InputBlockType)) +RUN_STATUSES: frozenset[RunStatus] = frozenset(get_args(RunStatus)) TERMINAL_RUN_STATUSES: frozenset[RunStatus] = frozenset( {"success", "error", "internal_error", "stopped"} ) -SNAPSHOT_STATUSES: frozenset[SnapshotStatus] = frozenset( - {"pending", "available", "unavailable"} -) +SNAPSHOT_STATUSES: frozenset[SnapshotStatus] = frozenset(get_args(SnapshotStatus)) diff --git a/deepnote_toolkit/notebooks/cloud_runner.py b/deepnote_toolkit/notebooks/cloud_runner.py index 5cb3a899..0fc1cff2 100644 --- a/deepnote_toolkit/notebooks/cloud_runner.py +++ b/deepnote_toolkit/notebooks/cloud_runner.py @@ -130,8 +130,9 @@ def _settle_snapshot(self, run: CloudRun) -> CloudRun: and run.snapshot_status in (None, "pending") and waited < self.snapshot_timeout ): - self._sleep(self.poll_interval) - waited += self.poll_interval + delay = min(self.poll_interval, self.snapshot_timeout - waited) + self._sleep(delay) + waited += delay try: run = self._client.get_run(run.run_id) except RunnerError as error: diff --git a/deepnote_toolkit/notebooks/document.py b/deepnote_toolkit/notebooks/document.py index 50cf16af..e42a405d 100644 --- a/deepnote_toolkit/notebooks/document.py +++ b/deepnote_toolkit/notebooks/document.py @@ -8,7 +8,7 @@ import yaml -from .api_types import InputBlockType +from .api_types import INPUT_BLOCK_TYPES, InputBlockType from .models import InputBlock, NotebookOutput from .outputs import OutputCollection from .wire import decode_block_outputs, optional_number, optional_string, string_tuple @@ -91,7 +91,7 @@ def _read_blocks( def _read_input_block(block: Mapping[str, Any]) -> InputBlock | None: block_type = str(block.get("type", "")) metadata = block.get("metadata") - if not block_type.startswith("input-") or not isinstance(metadata, Mapping): + if block_type not in INPUT_BLOCK_TYPES or not isinstance(metadata, Mapping): return None variable_name = metadata.get("deepnote_variable_name") if not isinstance(variable_name, str) or not variable_name: diff --git a/deepnote_toolkit/notebooks/transport.py b/deepnote_toolkit/notebooks/transport.py index b1a60f22..354638c4 100644 --- a/deepnote_toolkit/notebooks/transport.py +++ b/deepnote_toolkit/notebooks/transport.py @@ -107,6 +107,8 @@ def request_json( def _origin(url: str) -> str: + """Return a URL's origin: its scheme, host and port.""" + parts = urlsplit(url) return f"{parts.scheme}://{parts.netloc}" diff --git a/deepnote_toolkit/notebooks/wire.py b/deepnote_toolkit/notebooks/wire.py index dc9dd549..28498939 100644 --- a/deepnote_toolkit/notebooks/wire.py +++ b/deepnote_toolkit/notebooks/wire.py @@ -5,7 +5,7 @@ from collections.abc import Mapping from typing import Any, cast -from .api_types import InputBlockType +from .api_types import INPUT_BLOCK_TYPES, InputBlockType from .models import InputBlock, NotebookOutput @@ -32,7 +32,7 @@ def string_tuple(value: Any) -> tuple[str, ...]: def decode_inputs(values: Any, *, name_key: str) -> tuple[InputBlock, ...]: - """Read an API's camelCase input list, skipping entries without a name or type.""" + """Read an API's camelCase inputs, skipping any without a name or a known type.""" if not isinstance(values, list): return () @@ -52,6 +52,7 @@ def decode_inputs(values: Any, *, name_key: str) -> tuple[InputBlock, ...]: if isinstance(value, Mapping) and isinstance(value.get(name_key), str) and isinstance(value.get("type"), str) + and value["type"] in INPUT_BLOCK_TYPES ) diff --git a/tests/unit/test_notebooks_document.py b/tests/unit/test_notebooks_document.py index 6df241f6..0883b4d2 100644 --- a/tests/unit/test_notebooks_document.py +++ b/tests/unit/test_notebooks_document.py @@ -272,6 +272,20 @@ def test_unknown_notebook_id_is_rejected() -> None: DeepnoteDocument.parse(MULTI_NOTEBOOK_YAML, notebook_id="notebook-c") +def test_skips_input_blocks_of_an_unknown_type() -> None: + document = DeepnoteDocument.parse(""" +project: + notebooks: + - blocks: + - type: input-unknown + metadata: {deepnote_variable_name: mystery} + - type: input-text + metadata: {deepnote_variable_name: region} +""") + + assert document.inputs == (InputBlock("region", "input-text", None),) + + WRITER_STYLE_YAML = """ project: name: Survey diff --git a/tests/unit/test_notebooks_runners.py b/tests/unit/test_notebooks_runners.py index 1c2d4e45..566796a5 100644 --- a/tests/unit/test_notebooks_runners.py +++ b/tests/unit/test_notebooks_runners.py @@ -434,13 +434,14 @@ def test_cloud_run_waits_for_a_snapshot_that_lags_the_terminal_status() -> None: assert result.text() == "done" -def test_info_skips_inputs_without_a_name_or_type() -> None: +def test_info_skips_inputs_without_a_name_or_a_known_type() -> None: def open_request(_request: Any, *, timeout: float) -> FakeResponse: return FakeResponse( { "inputs": [ {"type": "input-text"}, {"variableName": "orphan"}, + {"variableName": "mystery", "type": "input-unknown"}, {"variableName": "region", "type": "input-text"}, ] } @@ -749,6 +750,43 @@ def test_cloud_run_stops_waiting_for_a_snapshot_after_the_snapshot_timeout() -> assert result.outputs == () +def test_cloud_run_honors_a_snapshot_timeout_shorter_than_the_poll_interval() -> None: + transport = FakeTransport( + {"run": {"runId": "run-1", "status": "success", "snapshotStatus": "pending"}} + ) + sleeps: list[float] = [] + + DeepnoteCloudRunner( + "notebook-1", + token="token", + snapshot_timeout=1, + poll_interval=2, + transport=transport, + sleep=sleeps.append, + ).run({}) + + assert sleeps == [1] + + +@pytest.mark.parametrize( + "run", + [{"runId": "run-1"}, {"runId": "run-1", "status": "cancelled"}], + ids=["missing", "unknown"], +) +def test_cloud_run_rejects_a_run_without_a_known_status(run: dict[str, str]) -> None: + sleeps: list[float] = [] + + with pytest.raises(RunnerError, match="run-1 has an unknown status"): + DeepnoteCloudRunner( + "notebook-1", + token="token", + transport=FakeTransport({"run": run}), + sleep=sleeps.append, + ).run({}) + + assert sleeps == [] + + def test_cloud_run_sends_a_tuple_as_a_list_and_rejects_a_missing_value() -> None: bodies = [] From 2c10374c0098aaf5471366e95fdbaa39c603f7df Mon Sep 17 00:00:00 2001 From: Wojtek <6819634+voyti@users.noreply.github.com> Date: Mon, 21 Sep 2026 19:08:44 +0200 Subject: [PATCH 28/43] refactor(streamlit): finalize viewer execution and input contracts --- .github/workflows/ci.yml | 4 + deepnote_toolkit/notebooks/__init__.py | 45 +- deepnote_toolkit/notebooks/_schemas.py | 49 + deepnote_toolkit/notebooks/api_client.py | 101 +- deepnote_toolkit/notebooks/cloud_runner.py | 78 +- deepnote_toolkit/notebooks/credentials.py | 4 +- deepnote_toolkit/notebooks/document.py | 6 +- deepnote_toolkit/notebooks/local_runner.py | 15 +- deepnote_toolkit/notebooks/models.py | 9 +- deepnote_toolkit/notebooks/transport.py | 178 +-- deepnote_toolkit/notebooks/yaml_loader.py | 2 +- deepnote_toolkit/streamlit/__init__.py | 17 +- deepnote_toolkit/streamlit/auth.py | 136 +-- deepnote_toolkit/streamlit/cloud_runner.py | 41 +- .../streamlit/viewer_credentials.py | 41 +- deepnote_toolkit/streamlit/widgets.py | 55 +- docs/streamlit-apps.md | 182 ++- docs/testing/streamlit-hosted.md | 31 + installer/module/server_process.py | 11 +- installer/module/streamlit.py | 22 +- installer/module/virtual_environment.py | 10 +- tests/unit/helpers/notebook_api.py | 38 + tests/unit/test_deepnote_streamlit_auth.py | 441 +++---- .../test_deepnote_streamlit_cloud_runner.py | 524 +++------ tests/unit/test_deepnote_streamlit_widgets.py | 120 +- tests/unit/test_notebooks_document.py | 53 +- tests/unit/test_notebooks_runners.py | 1029 ++++++----------- tests/unit/test_notebooks_yaml_loader.py | 8 + tests/unit/test_streamlit.py | 31 +- tests/unit/test_virtual_environment.py | 30 + 30 files changed, 1476 insertions(+), 1835 deletions(-) create mode 100644 deepnote_toolkit/notebooks/_schemas.py create mode 100644 docs/testing/streamlit-hosted.md create mode 100644 tests/unit/helpers/notebook_api.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6d63ccda..b8034737 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -333,6 +333,10 @@ jobs: - name: Install project run: poetry install --no-interaction --only-root + - name: Install Streamlit for widget integration tests + if: matrix.python-version == '3.12' + run: poetry install --no-interaction --extras server + - name: Run unit tests env: TOOLKIT_VERSION: ${{ steps.version.outputs.VERSION }} diff --git a/deepnote_toolkit/notebooks/__init__.py b/deepnote_toolkit/notebooks/__init__.py index e088ca58..af7de61f 100644 --- a/deepnote_toolkit/notebooks/__init__.py +++ b/deepnote_toolkit/notebooks/__init__.py @@ -1,55 +1,28 @@ -"""Read `.deepnote` files and run notebooks, independent of any UI framework.""" +"""Read `.deepnote` files and run notebooks, independent of any UI framework. + +Only the names in __all__ are supported public API. Wire schemas and API clients +are implementation details. +""" -from .api_client import CloudNotebook, CloudRun, DeepnoteApiClient -from .api_types import ( - InputBlockType, - InputValue, - RunStatus, - SnapshotStatus, - StorageMode, -) from .cloud_runner import DeepnoteCloudRunner -from .credentials import ApiCredentials, CredentialsProvider, token_credentials +from .credentials import ApiCredentials, CredentialsProvider from .document import DeepnoteDocument -from .local_runner import DeepnoteRunner -from .models import ( - DATAFRAME_MIME, - INDEX_COLUMN, - DeepnoteDataframe, - InputBlock, - NotebookOutput, - RunnerInfo, - join_text, -) +from .local_runner import DeepnoteLocalRunner +from .models import DeepnoteDataframe, InputBlock, NotebookOutput, RunnerInfo from .run_result import RunResult from .runner import Runner, RunnerError -from .transport import Transport, UrllibTransport __all__ = [ - "DATAFRAME_MIME", - "INDEX_COLUMN", "ApiCredentials", - "CloudNotebook", - "CloudRun", "CredentialsProvider", - "DeepnoteApiClient", "DeepnoteCloudRunner", "DeepnoteDataframe", "DeepnoteDocument", - "DeepnoteRunner", + "DeepnoteLocalRunner", "InputBlock", - "InputBlockType", - "InputValue", "NotebookOutput", "RunResult", - "RunStatus", "Runner", "RunnerError", "RunnerInfo", - "SnapshotStatus", - "StorageMode", - "Transport", - "UrllibTransport", - "join_text", - "token_credentials", ] diff --git a/deepnote_toolkit/notebooks/_schemas.py b/deepnote_toolkit/notebooks/_schemas.py new file mode 100644 index 00000000..472d4a43 --- /dev/null +++ b/deepnote_toolkit/notebooks/_schemas.py @@ -0,0 +1,49 @@ +"""Consumed fields of the v2 API contracts (contracts/runs.ts and notebooks.ts). + +Extra fields are intentionally ignored. Status strings are extensible: only the +known terminal run statuses stop polling. Missing or ill-typed statuses are errors. +""" + +from __future__ import annotations + +from typing import Any + +from pydantic import BaseModel, Field, StrictBool, StrictFloat, StrictInt, StrictStr + + +class ApiInput(BaseModel): + name: StrictStr + type: StrictStr + value: StrictStr | StrictBool | list[StrictStr] | None = None + label: StrictStr | None = None + options: list[StrictStr] = Field(default_factory=list) + multiple: StrictBool = False + min: StrictInt | StrictFloat | None = None + max: StrictInt | StrictFloat | None = None + step: StrictInt | StrictFloat | None = None + + +class ApiNotebook(BaseModel): + name: StrictStr = "Untitled notebook" + inputs: list[ApiInput] = Field(default_factory=list) + + +class NotebookResponse(BaseModel): + notebook: ApiNotebook + + +class ApiRun(BaseModel): + run_id: StrictStr = Field(alias="runId", min_length=1) + status: StrictStr = Field(min_length=1) + snapshot_status: StrictStr | None = Field(default=None, alias="snapshotStatus") + snapshot_blocks: list[dict[str, Any]] | None = Field( + default=None, alias="snapshotBlocks" + ) + error: StrictStr | dict[str, Any] | None = None + view_url: StrictStr | None = Field(default=None, alias="viewUrl") + + +class ViewerTokenResponse(BaseModel): + token: StrictStr = Field(min_length=1) + api_origin: StrictStr = Field(alias="apiOrigin") + expires_at_seconds: StrictInt | StrictFloat = Field(alias="expiresAtSeconds") diff --git a/deepnote_toolkit/notebooks/api_client.py b/deepnote_toolkit/notebooks/api_client.py index 43165f18..71998e34 100644 --- a/deepnote_toolkit/notebooks/api_client.py +++ b/deepnote_toolkit/notebooks/api_client.py @@ -3,24 +3,28 @@ from __future__ import annotations import json -from collections.abc import Mapping +import time +from collections.abc import Callable, Mapping from dataclasses import dataclass -from typing import Any, cast +from typing import Any +from urllib.parse import quote +import requests +from pydantic import ValidationError + +from ._schemas import ApiRun, NotebookResponse from .api_types import ( - RUN_STATUSES, SNAPSHOT_STATUSES, TERMINAL_RUN_STATUSES, InputValue, - RunStatus, SnapshotStatus, StorageMode, ) from .credentials import CredentialsProvider from .models import InputBlock, NotebookOutput from .runner import RunnerError -from .transport import Transport, UrllibTransport -from .wire import decode_block_outputs, decode_inputs, optional_string +from .transport import request_json +from .wire import decode_block_outputs, decode_inputs @dataclass(frozen=True) @@ -36,7 +40,7 @@ class CloudRun: """The state of one run. `outputs` is None until the run's snapshot is stored.""" run_id: str - status: RunStatus + status: str snapshot_status: SnapshotStatus | None outputs: tuple[NotebookOutput, ...] | None error: str | None @@ -56,22 +60,28 @@ def __init__( self, credentials: CredentialsProvider, *, - transport: Transport | None = None, + session: requests.Session | None = None, request_timeout: float = 30, + clock: Callable[[], float] = time.monotonic, ): self._credentials = credentials - self._transport = transport or UrllibTransport() + self._clock = clock + self._session = session if session is not None else requests.Session() self._request_timeout = request_timeout def get_notebook(self, notebook_id: str) -> CloudNotebook: """Read a notebook's name and input blocks.""" - notebook = self._request("GET", f"/v2/notebooks/{notebook_id}").get("notebook") - if not isinstance(notebook, Mapping): - raise RunnerError("Deepnote API response did not include a notebook") + payload = self._request("GET", f"/v2/notebooks/{quote(notebook_id, safe='')}") + try: + notebook = NotebookResponse(**payload).notebook + except ValidationError as error: + raise RunnerError( + "Deepnote API returned an invalid notebook response" + ) from error return CloudNotebook( - name=str(notebook.get("name", "Untitled notebook")), - inputs=decode_inputs(notebook.get("inputs"), name_key="name"), + name=notebook.name, + inputs=decode_inputs(payload["notebook"].get("inputs"), name_key="name"), ) def create_run( @@ -80,6 +90,7 @@ def create_run( inputs: Mapping[str, Any], *, storage_mode: StorageMode | None = None, + timeout: float | None = None, ) -> CloudRun: """Start a detached run of the whole notebook.""" @@ -92,25 +103,46 @@ def create_run( } if storage_mode is not None: body["detachedRunStorageMode"] = storage_mode - return _decode_run(self._request("POST", "/v2/runs", body)) + return _decode_run(self._request("POST", "/v2/runs", body, timeout=timeout)) - def get_run(self, run_id: str) -> CloudRun: + def get_run(self, run_id: str, *, timeout: float | None = None) -> CloudRun: """Read a run with the outputs of the notebook it executed.""" # The blocks delivery holds the executed notebook alone, not the whole project. - payload = self._request("GET", f"/v2/runs/{run_id}?snapshotDelivery=blocks") + payload = self._request( + "GET", + f"/v2/runs/{quote(run_id, safe='')}?snapshotDelivery=blocks", + timeout=timeout, + ) return _decode_run(payload, run_id=run_id) def _request( - self, method: str, path: str, body: Mapping[str, Any] | None = None + self, + method: str, + path: str, + body: Mapping[str, Any] | None = None, + *, + timeout: float | None = None, ) -> Mapping[str, Any]: - credentials = self._credentials() - return self._transport.request_json( + budget = ( + self._request_timeout + if timeout is None + else min(timeout, self._request_timeout) + ) + deadline = self._clock() + budget + credentials = self._credentials(timeout=budget) + remaining = deadline - self._clock() + if remaining <= 0: + raise RunnerError( + "API credentials exhausted the request timeout", transient=True + ) + return request_json( + self._session, method, f"{credentials.api_origin}{path}", headers={"Authorization": f"Bearer {credentials.token}"}, body=body, - timeout=self._request_timeout, + timeout=remaining, ) @@ -135,25 +167,28 @@ def _decode_run(payload: Mapping[str, Any], *, run_id: str | None = None) -> Clo run_id = run.get("runId") or run.get("id") or run_id if not isinstance(run_id, str) or not run_id: raise RunnerError("Deepnote API response did not include a run id") - status = run.get("status") - if not isinstance(status, str) or status not in RUN_STATUSES: - raise RunnerError(f"Deepnote run {run_id} has an unknown status: {status!r}") - snapshot_status = run.get("snapshotStatus") - blocks = run.get("snapshotBlocks") - error = run.get("error") + try: + parsed = ApiRun(**{**run, "runId": run_id}) + except ValidationError as error: + raise RunnerError( + f"Deepnote API returned an invalid run response for {run_id}" + ) from error + error = parsed.error if isinstance(error, Mapping): error = error.get("message") or json.dumps(error) return CloudRun( - run_id=run_id, - status=cast(RunStatus, status), + run_id=parsed.run_id, + status=parsed.status, snapshot_status=( - snapshot_status if snapshot_status in SNAPSHOT_STATUSES else None + parsed.snapshot_status + if parsed.snapshot_status in SNAPSHOT_STATUSES + else None ), outputs=( - decode_block_outputs(blocks, id_key="id") - if isinstance(blocks, list) + decode_block_outputs(parsed.snapshot_blocks, id_key="id") + if parsed.snapshot_blocks is not None else None ), error=str(error) if error is not None else None, - view_url=optional_string(run.get("viewUrl")), + view_url=parsed.view_url, ) diff --git a/deepnote_toolkit/notebooks/cloud_runner.py b/deepnote_toolkit/notebooks/cloud_runner.py index 0fc1cff2..494beef5 100644 --- a/deepnote_toolkit/notebooks/cloud_runner.py +++ b/deepnote_toolkit/notebooks/cloud_runner.py @@ -2,10 +2,13 @@ from __future__ import annotations +import math import time from collections.abc import Callable, Mapping from typing import Any +import requests + from .api_client import CloudRun, DeepnoteApiClient from .api_types import StorageMode from .credentials import ( @@ -17,7 +20,6 @@ from .models import RunnerInfo from .run_result import RunResult from .runner import RunnerError -from .transport import Transport Sleep = Callable[[float], None] @@ -52,12 +54,17 @@ def __init__( timeout: float = 600, snapshot_timeout: float = 10, poll_interval: float = 2, - transport: Transport | None = None, + session: requests.Session | None = None, sleep: Sleep = time.sleep, + clock: Callable[[], float] = time.monotonic, ): if not notebook_id: raise ValueError("notebook_id is required") - if poll_interval <= 0: + if not math.isfinite(timeout) or timeout <= 0: + raise ValueError("timeout must be positive and finite") + if not math.isfinite(snapshot_timeout) or snapshot_timeout < 0: + raise ValueError("snapshot_timeout must be non-negative and finite") + if not math.isfinite(poll_interval) or poll_interval <= 0: raise ValueError("poll_interval must be positive") if credentials is not None and ( token is not None or token_provider is not None @@ -70,10 +77,12 @@ def __init__( self.poll_interval = poll_interval self._client = DeepnoteApiClient( credentials or token_credentials(token, token_provider, base_url=base_url), - transport=transport, + session=session, request_timeout=min(timeout, 30), + clock=clock, ) self._sleep = sleep + self._clock = clock def info(self) -> RunnerInfo: """Read the notebook's name and input blocks from the public API.""" @@ -86,10 +95,19 @@ def info(self) -> RunnerInfo: def run(self, inputs: Mapping[str, Any]) -> RunResult: """Start a detached run with the given input values and wait for its result.""" + deadline = self._clock() + self.timeout run = self._client.create_run( - self.notebook_id, inputs, storage_mode=self.storage_mode + self.notebook_id, + inputs, + storage_mode=self.storage_mode, + timeout=self.timeout, + ) + if self._clock() >= deadline: + self._timed_out(run) + run = self._wait_until_finished(run, deadline) + run = self._settle_snapshot( + run, min(deadline, self._clock() + self.snapshot_timeout) ) - run = self._settle_snapshot(self._wait_until_finished(run)) return RunResult( target="cloud", success=run.status == "success", @@ -101,18 +119,25 @@ def run(self, inputs: Mapping[str, Any]) -> RunResult: view_url=run.view_url, ) - def _wait_until_finished(self, run: CloudRun) -> CloudRun: - deadline = time.monotonic() + self.timeout + def _timed_out(self, run: CloudRun) -> None: + raise RunnerError( + f"Deepnote run {run.run_id} did not finish in {self.timeout:g} seconds" + ) + + def _pause(self, deadline: float) -> bool: + remaining = deadline - self._clock() + if remaining <= 0: + return False + self._sleep(min(self.poll_interval, remaining)) + return self._clock() < deadline + + def _wait_until_finished(self, run: CloudRun, deadline: float) -> CloudRun: transient_failures = 0 while not run.is_finished: - if time.monotonic() >= deadline: - raise RunnerError( - f"Deepnote run {run.run_id} did not finish in " - f"{self.timeout:g} seconds" - ) - self._sleep(self.poll_interval) + if not self._pause(deadline): + self._timed_out(run) try: - run = self._client.get_run(run.run_id) + run = self._client.get_run(run.run_id, timeout=deadline - self._clock()) transient_failures = 0 except RunnerError as error: transient_failures += 1 @@ -121,20 +146,21 @@ def _wait_until_finished(self, run: CloudRun) -> CloudRun: or transient_failures > MAX_TRANSIENT_POLL_FAILURES ): raise + if self._clock() >= deadline: + self._timed_out(run) return run - def _settle_snapshot(self, run: CloudRun) -> CloudRun: - waited = 0.0 - while ( - run.outputs is None - and run.snapshot_status in (None, "pending") - and waited < self.snapshot_timeout - ): - delay = min(self.poll_interval, self.snapshot_timeout - waited) - self._sleep(delay) - waited += delay + def _settle_snapshot(self, run: CloudRun, deadline: float) -> CloudRun: + while run.outputs is None and run.snapshot_status == "pending": + if not self._pause(deadline): + break try: - run = self._client.get_run(run.run_id) + updated = self._client.get_run( + run.run_id, timeout=deadline - self._clock() + ) + if self._clock() >= deadline: + break + run = updated except RunnerError as error: if not error.transient: raise diff --git a/deepnote_toolkit/notebooks/credentials.py b/deepnote_toolkit/notebooks/credentials.py index e335c844..3257bf7c 100644 --- a/deepnote_toolkit/notebooks/credentials.py +++ b/deepnote_toolkit/notebooks/credentials.py @@ -24,7 +24,7 @@ class ApiCredentials: class CredentialsProvider(Protocol): """Returns the credentials for one request. Called before every request.""" - def __call__(self) -> ApiCredentials: + def __call__(self, *, timeout: float = 30) -> ApiCredentials: """Return the credentials, or raise `RunnerError` when there are none.""" @@ -40,7 +40,7 @@ def token_credentials( raise ValueError("Pass token or token_provider, not both") api_origin = base_url.rstrip("/") - def provide() -> ApiCredentials: + def provide(*, timeout: float = 30) -> ApiCredentials: if token_provider is not None: value: str | None = token_provider() elif token is not None: diff --git a/deepnote_toolkit/notebooks/document.py b/deepnote_toolkit/notebooks/document.py index e42a405d..58fe6e51 100644 --- a/deepnote_toolkit/notebooks/document.py +++ b/deepnote_toolkit/notebooks/document.py @@ -4,11 +4,11 @@ from collections.abc import Mapping, Sequence from pathlib import Path -from typing import Any, cast +from typing import Any import yaml -from .api_types import INPUT_BLOCK_TYPES, InputBlockType +from .api_types import INPUT_BLOCK_TYPES from .models import InputBlock, NotebookOutput from .outputs import OutputCollection from .wire import decode_block_outputs, optional_number, optional_string, string_tuple @@ -98,7 +98,7 @@ def _read_input_block(block: Mapping[str, Any]) -> InputBlock | None: return None return InputBlock( variable_name=variable_name, - type=cast(InputBlockType, block_type), + type=block_type, label=optional_string(metadata.get("deepnote_input_label")), value=metadata.get("deepnote_variable_value"), options=string_tuple(metadata.get("deepnote_variable_options")), diff --git a/deepnote_toolkit/notebooks/local_runner.py b/deepnote_toolkit/notebooks/local_runner.py index ac116307..f2f13ad7 100644 --- a/deepnote_toolkit/notebooks/local_runner.py +++ b/deepnote_toolkit/notebooks/local_runner.py @@ -5,26 +5,28 @@ from collections.abc import Mapping from typing import Any +import requests + from .document import DeepnoteDocument from .models import RunnerInfo from .run_result import RunResult -from .transport import Transport, UrllibTransport +from .transport import request_json from .wire import decode_block_outputs, decode_inputs, optional_string -class DeepnoteRunner: - """One client for a runner configured for Deepnote Cloud or a local kernel.""" +class DeepnoteLocalRunner: + """Run a notebook through a local sidecar, configured for a local or cloud kernel.""" def __init__( self, base_url: str = "http://127.0.0.1:8787", *, timeout: float = 600, - transport: Transport | None = None, + session: requests.Session | None = None, ): self.base_url = base_url.rstrip("/") self.timeout = timeout - self._transport = transport or UrllibTransport() + self._session = session if session is not None else requests.Session() def info(self) -> RunnerInfo: """Read the notebook's name and input blocks from the sidecar.""" @@ -46,7 +48,8 @@ def run(self, inputs: Mapping[str, Any]) -> RunResult: def _request( self, method: str, path: str, body: Mapping[str, Any] | None = None ) -> Mapping[str, Any]: - return self._transport.request_json( + return request_json( + self._session, method, f"{self.base_url}{path}", headers={}, diff --git a/deepnote_toolkit/notebooks/models.py b/deepnote_toolkit/notebooks/models.py index 9ba0d097..cc9825b2 100644 --- a/deepnote_toolkit/notebooks/models.py +++ b/deepnote_toolkit/notebooks/models.py @@ -151,8 +151,8 @@ class RunnerInfo: inputs: tuple[InputBlock, ...] run_target: str - def accepts_inputs(self, inputs: Iterable[InputBlock]) -> bool: - """Return whether values made for `inputs` fit this runner's notebook. + def matches_inputs(self, inputs: Iterable[InputBlock]) -> bool: + """Return whether the static input definitions match this runner's notebook. Names, block types, single or multiple selection, slider bounds and select options must match. Options filled from a variable change between runs, so @@ -160,6 +160,10 @@ def accepts_inputs(self, inputs: Iterable[InputBlock]) -> bool: """ expected = tuple(inputs) + for blocks in (expected, self.inputs): + names = [block.variable_name for block in blocks] + if len(names) != len(set(names)): + return False dynamic = frozenset( input_block.variable_name for input_block in expected @@ -190,6 +194,7 @@ def _value_constraints( return ( input_block.min if input_block.min is not None else 0, input_block.max if input_block.max is not None else 100, + input_block.step if input_block.step is not None else 1, ) if input_block.type == "input-select": is_dynamic = input_block.variable_name in dynamic_options diff --git a/deepnote_toolkit/notebooks/transport.py b/deepnote_toolkit/notebooks/transport.py index 354638c4..de6db4f1 100644 --- a/deepnote_toolkit/notebooks/transport.py +++ b/deepnote_toolkit/notebooks/transport.py @@ -1,126 +1,70 @@ -"""The HTTP layer under the notebook runners.""" +"""Internal HTTP helpers shared by notebook execution and viewer authentication.""" from __future__ import annotations -import json -from collections.abc import Callable, Mapping -from http.client import HTTPException -from typing import Any, Protocol -from urllib.error import HTTPError, URLError +from collections.abc import Mapping +from typing import Any from urllib.parse import urlsplit -from urllib.request import HTTPRedirectHandler, Request, build_opener -from .runner import RunnerError - -OpenUrl = Callable[..., Any] - - -class _SameOriginRedirectHandler(HTTPRedirectHandler): - """Refuses a redirect to another origin, which would receive the bearer token.""" - - def redirect_request( - self, req: Request, fp: Any, code: int, msg: str, headers: Any, newurl: str - ) -> Request | None: - if _origin(newurl) != _origin(req.full_url): - raise HTTPError( - req.full_url, code, "Refused a redirect to another origin", headers, fp - ) - return super().redirect_request(req, fp, code, msg, headers, newurl) - - -open_url: OpenUrl = build_opener(_SameOriginRedirectHandler).open - - -class Transport(Protocol): - """Sends one JSON request. Implement it to use another HTTP library.""" - - def request_json( - self, - method: str, - url: str, - *, - headers: Mapping[str, str], - body: Mapping[str, Any] | None, - timeout: float, - ) -> Mapping[str, Any]: - """Return the response's JSON object, or raise `RunnerError`. - - The error is `transient` when a retry can succeed: HTTP 429 or 5xx, a - timeout, or a network failure. - """ - - -class UrllibTransport: - """The default transport, on the standard library.""" - - def __init__(self, opener: OpenUrl = open_url): - self._open = opener - - def request_json( - self, - method: str, - url: str, - *, - headers: Mapping[str, str], - body: Mapping[str, Any] | None, - timeout: float, - ) -> Mapping[str, Any]: - """Send the request with `urllib` and return its JSON object.""" - - origin = _origin(url) - request = Request( - url, - data=json.dumps(body).encode() if body is not None else None, - method=method, - headers={ - "Content-Type": "application/json", - "Accept": "application/json", - **headers, - }, - ) - try: - with self._open(request, timeout=timeout) as response: - payload = json.loads(response.read()) - except HTTPError as error: - message = _error_message(error) or error.reason - raise RunnerError( - f"{origin} returned HTTP {error.code}: {message}", - transient=error.code == 429 or error.code >= 500, - ) from error - except URLError as error: - raise RunnerError( - f"Could not reach {origin}: {error.reason}", transient=True - ) from error - except TimeoutError as error: - raise RunnerError( - f"{origin} timed out after {timeout:g} seconds", transient=True - ) from error - except (OSError, HTTPException) as error: - raise RunnerError( - f"The connection to {origin} dropped: {error}", transient=True - ) from error - except (json.JSONDecodeError, UnicodeDecodeError) as error: - raise RunnerError(f"{origin} returned an invalid JSON response") from error - if not isinstance(payload, Mapping): - raise RunnerError(f"{origin} returned a non-object response") - return payload - - -def _origin(url: str) -> str: - """Return a URL's origin: its scheme, host and port.""" - - parts = urlsplit(url) - return f"{parts.scheme}://{parts.netloc}" +import requests +from urllib3.util import Timeout +from .runner import RunnerError -def _error_message(error: HTTPError) -> str | None: - """Return the message of a JSON error response, or None for any other body.""" +def request_json( + session: requests.Session, + method: str, + url: str, + *, + headers: Mapping[str, str], + body: Mapping[str, Any] | None = None, + timeout: float, +) -> Mapping[str, Any]: + """Send one request, without replaying POSTs or forwarding credentials on redirects.""" + origin = urlsplit(url) + origin_name = f"{origin.scheme}://{origin.netloc}" try: - parsed = json.loads(error.read()) - except (json.JSONDecodeError, UnicodeDecodeError): - return None - if not isinstance(parsed, Mapping): - return None - message = parsed.get("message") or parsed.get("error") - return message if isinstance(message, str) else None + with session.request( + method, + url, + headers={"Accept": "application/json", **headers}, + json=body, + timeout=Timeout(total=timeout), + allow_redirects=False, + ) as response: + if 300 <= response.status_code < 400: + raise RunnerError( + f"{origin_name} returned HTTP {response.status_code}: " + "Refused a redirect" + ) + if response.status_code >= 400: + message = response.reason + try: + payload = response.json() + if isinstance(payload, Mapping): + reason = payload.get("message") or payload.get("error") + if isinstance(reason, str): + message = reason + except ValueError: + pass + raise RunnerError( + f"{origin_name} returned HTTP {response.status_code}: {message}", + transient=response.status_code == 429 + or response.status_code >= 500, + ) + try: + payload = response.json() + except ValueError as error: + raise RunnerError(f"{origin_name} returned invalid JSON") from error + except requests.Timeout as error: + raise RunnerError( + f"{origin_name} timed out after {timeout:g} seconds", transient=True + ) from error + except requests.RequestException as error: + raise RunnerError( + f"Could not reach {origin_name}: {error}", transient=True + ) from error + if not isinstance(payload, Mapping): + raise RunnerError(f"{origin_name} returned a non-object response") + return payload diff --git a/deepnote_toolkit/notebooks/yaml_loader.py b/deepnote_toolkit/notebooks/yaml_loader.py index 4cb1b62d..e5aed9ec 100644 --- a/deepnote_toolkit/notebooks/yaml_loader.py +++ b/deepnote_toolkit/notebooks/yaml_loader.py @@ -70,7 +70,7 @@ def load_yaml(content: str) -> Any: loader = _CoreSchemaLoader(content) try: return loader.get_single_data() - except ValueError as error: + except (ValueError, AttributeError, TypeError) as error: raise yaml.YAMLError(str(error)) from error finally: loader.dispose() diff --git a/deepnote_toolkit/streamlit/__init__.py b/deepnote_toolkit/streamlit/__init__.py index c981118a..9cd8fb83 100644 --- a/deepnote_toolkit/streamlit/__init__.py +++ b/deepnote_toolkit/streamlit/__init__.py @@ -1,21 +1,6 @@ """Helpers for Streamlit apps built on Deepnote notebooks.""" -from .auth import ( - CurrentUserApiCredentials, - CurrentUserApiTokenError, - current_user_api_credentials, - current_user_api_token, -) from .cloud_runner import StreamlitCloudRunner -from .viewer_credentials import ViewerCredentials from .widgets import render_inputs -__all__ = [ - "CurrentUserApiCredentials", - "CurrentUserApiTokenError", - "StreamlitCloudRunner", - "ViewerCredentials", - "current_user_api_credentials", - "current_user_api_token", - "render_inputs", -] +__all__ = ["StreamlitCloudRunner", "render_inputs"] diff --git a/deepnote_toolkit/streamlit/auth.py b/deepnote_toolkit/streamlit/auth.py index bba390de..6ffd5331 100644 --- a/deepnote_toolkit/streamlit/auth.py +++ b/deepnote_toolkit/streamlit/auth.py @@ -3,28 +3,26 @@ from __future__ import annotations import hashlib -import json +import math import os import re import time -from collections.abc import Callable, Mapping from dataclasses import dataclass, field -from http.client import HTTPException from typing import Any -from urllib.error import HTTPError, URLError from urllib.parse import urlparse -from urllib.request import Request + +import requests +from pydantic import ValidationError from deepnote_toolkit.get_webapp_url import ( get_absolute_userpod_api_url, get_project_auth_headers, ) -from deepnote_toolkit.notebooks.transport import open_url -from deepnote_toolkit.streamlit_data_apps import ( - read_streamlit_token_from_context, -) +from deepnote_toolkit.notebooks._schemas import ViewerTokenResponse +from deepnote_toolkit.notebooks.runner import RunnerError +from deepnote_toolkit.notebooks.transport import request_json +from deepnote_toolkit.streamlit_data_apps import read_streamlit_token_from_context -OpenUrl = Callable[..., Any] _APP_ID = r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}" STREAMLIT_APP_HOST_PATTERN = re.compile(rf"^streamlit-({_APP_ID})\.", re.IGNORECASE) STREAMLIT_APP_ID_ENV = "DEEPNOTE_STREAMLIT_APP_ID" @@ -66,7 +64,7 @@ def current_user_api_credentials( app_id: str | None = None, streamlit_token: str | None = None, timeout: float = 10, - opener: OpenUrl = open_url, + session: requests.Session | None = None, ) -> CurrentUserApiCredentials: """Exchange the active viewer cookie for public API credentials. @@ -76,13 +74,19 @@ def current_user_api_credentials( """ resolved_app_id = ( - app_id or _read_hosted_app_id() or _read_streamlit_app_id_from_context() + app_id + if app_id is not None + else (_read_hosted_app_id() or _read_streamlit_app_id_from_context()) ) if not resolved_app_id: raise CurrentUserApiTokenError( "Could not resolve a Deepnote Streamlit app ID from the request host." ) + if not re.fullmatch(_APP_ID, resolved_app_id, re.IGNORECASE): + raise CurrentUserApiTokenError("app_id must be a UUID.") + resolved_app_id = resolved_app_id.lower() + viewer_token = streamlit_token or read_streamlit_token_from_context() if not viewer_token: raise CurrentUserApiTokenError( @@ -103,70 +107,38 @@ def current_user_api_credentials( ): return cached[1] - request = Request( - get_absolute_userpod_api_url(f"streamlit-apps/{resolved_app_id}/api-token"), - data=b"", - method="POST", - headers={ - "Accept": "application/json", - "Content-Type": "application/json", - "StreamlitToken": viewer_token, - **get_project_auth_headers(), - }, - ) + owned_session = session is None + http = session if session is not None else requests.Session() try: - with opener(request, timeout=timeout) as response: - payload = json.loads(response.read()) - except HTTPError as error: - message = _server_message(error) - raise CurrentUserApiTokenError( - f"Current viewer API-token exchange returned HTTP {error.code}" - + (f": {message}" if message else "."), - transient=error.code == 429 or error.code >= 500, - ) from error - except URLError as error: - raise CurrentUserApiTokenError( - "Could not reach Deepnote to exchange the current viewer's API token.", - transient=True, - ) from error - except TimeoutError as error: - raise CurrentUserApiTokenError( - "Current viewer API-token exchange timed out.", transient=True - ) from error - except (OSError, HTTPException) as error: - raise CurrentUserApiTokenError( - "The connection dropped during the current viewer API-token exchange.", - transient=True, - ) from error - except (json.JSONDecodeError, UnicodeDecodeError) as error: - raise CurrentUserApiTokenError( - "Current viewer API-token exchange returned invalid JSON." - ) from error - - if not isinstance(payload, Mapping): - raise CurrentUserApiTokenError( - "Current viewer API-token exchange returned a non-object response." + payload = request_json( + http, + "POST", + get_absolute_userpod_api_url(f"streamlit-apps/{resolved_app_id}/api-token"), + headers={"StreamlitToken": viewer_token, **get_project_auth_headers()}, + timeout=timeout, ) - - token = payload.get("token") - api_origin = payload.get("apiOrigin") - expires_at_seconds = payload.get("expiresAtSeconds") - if ( - not isinstance(token, str) - or not token - or not isinstance(api_origin, str) - or not isinstance(expires_at_seconds, (int, float)) - or isinstance(expires_at_seconds, bool) - ): - raise CurrentUserApiTokenError( - "Current viewer API-token exchange response is missing required fields." + parsed = ViewerTokenResponse(**payload) + credentials = CurrentUserApiCredentials( + token=parsed.token, + api_origin=_validated_origin(parsed.api_origin, name="apiOrigin"), + expires_at_seconds=float(parsed.expires_at_seconds), ) - - credentials = CurrentUserApiCredentials( - token=token, - api_origin=_validated_origin(api_origin, name="apiOrigin"), - expires_at_seconds=float(expires_at_seconds), - ) + if ( + not math.isfinite(credentials.expires_at_seconds) + or credentials.expires_at_seconds <= time.time() + ): + raise CurrentUserApiTokenError( + "Viewer API credentials have already expired." + ) + except RunnerError as error: + raise CurrentUserApiTokenError(str(error), transient=error.transient) from error + except ValidationError as error: + raise CurrentUserApiTokenError( + "Viewer API-token response is missing or has invalid required fields." + ) from error + finally: + if owned_session: + http.close() if session_state is not None: session_state[_SESSION_STATE_KEY] = (cache_key, credentials) return credentials @@ -183,7 +155,7 @@ def _read_streamlit_session_state() -> Any | None: except ImportError: return None - if get_script_run_ctx() is None: + if get_script_run_ctx(suppress_warning=True) is None: return None return st.session_state @@ -191,8 +163,7 @@ def _read_streamlit_session_state() -> Any | None: def _read_hosted_app_id() -> str | None: """Return the app ID that Deepnote's launcher exports to a hosted app's process.""" - app_id = os.environ.get(STREAMLIT_APP_ID_ENV, "") - return app_id.lower() if re.fullmatch(_APP_ID, app_id, re.IGNORECASE) else None + return os.environ.get(STREAMLIT_APP_ID_ENV) def _read_streamlit_app_id_from_context() -> str | None: @@ -260,19 +231,6 @@ def _is_streamlit_thread_without_request() -> bool: return runtime.exists() and not _has_script_run_context() -def _server_message(error: HTTPError) -> str | None: - """Return the message of a JSON error response, or None for any other body.""" - - try: - payload = json.loads(error.read()) - except (json.JSONDecodeError, UnicodeDecodeError): - return None - if not isinstance(payload, Mapping): - return None - message = payload.get("error") or payload.get("message") - return message if isinstance(message, str) else None - - def _validated_origin(value: str, *, name: str) -> str: parsed = urlparse(value) if ( diff --git a/deepnote_toolkit/streamlit/cloud_runner.py b/deepnote_toolkit/streamlit/cloud_runner.py index 12746e71..1f70cd45 100644 --- a/deepnote_toolkit/streamlit/cloud_runner.py +++ b/deepnote_toolkit/streamlit/cloud_runner.py @@ -3,27 +3,34 @@ from __future__ import annotations import time +from collections.abc import Callable, Mapping +from typing import Any + +import requests from deepnote_toolkit.notebooks.api_types import StorageMode from deepnote_toolkit.notebooks.cloud_runner import DeepnoteCloudRunner, Sleep from deepnote_toolkit.notebooks.credentials import DEFAULT_API_ORIGIN, TokenProvider -from deepnote_toolkit.notebooks.transport import Transport +from deepnote_toolkit.notebooks.models import RunnerInfo +from deepnote_toolkit.notebooks.run_result import RunResult from .viewer_credentials import ViewerCredentials -class StreamlitCloudRunner(DeepnoteCloudRunner): +class StreamlitCloudRunner: """Run a notebook from a Streamlit app, as the viewer when Deepnote hosts it. - A hosted app always runs as the viewer. `token`, `token_provider` and - `base_url` apply only outside Deepnote hosting. Runs cannot change the - project's stored files unless `storage_mode` says otherwise. + The Streamlit script thread uses viewer authentication by default. Local + development requires `local=True` and an explicit token or token provider. + Deepnote hosting markers always override local credentials. Runs use read-only + project storage unless `storage_mode` says otherwise. """ def __init__( self, notebook_id: str, *, + local: bool = False, token: str | None = None, token_provider: TokenProvider | None = None, base_url: str = DEFAULT_API_ORIGIN, @@ -31,18 +38,34 @@ def __init__( timeout: float = 600, snapshot_timeout: float = 10, poll_interval: float = 2, - transport: Transport | None = None, + session: requests.Session | None = None, sleep: Sleep = time.sleep, + clock: Callable[[], float] = time.monotonic, ): - super().__init__( + session = session if session is not None else requests.Session() + self._runner = DeepnoteCloudRunner( notebook_id, credentials=ViewerCredentials( - token, token_provider, base_url=base_url, timeout=min(timeout, 30) + token, + token_provider, + base_url=base_url, + timeout=min(timeout, 10), + session=session, + local=local, ), storage_mode=storage_mode, timeout=timeout, snapshot_timeout=snapshot_timeout, poll_interval=poll_interval, - transport=transport, + session=session, sleep=sleep, + clock=clock, ) + + def info(self) -> RunnerInfo: + """Read the notebook's name and inputs using the current viewer.""" + return self._runner.info() + + def run(self, inputs: Mapping[str, Any]) -> RunResult: + """Run the notebook using the current viewer and return its outputs.""" + return self._runner.run(inputs) diff --git a/deepnote_toolkit/streamlit/viewer_credentials.py b/deepnote_toolkit/streamlit/viewer_credentials.py index 568fd27f..f6d47b99 100644 --- a/deepnote_toolkit/streamlit/viewer_credentials.py +++ b/deepnote_toolkit/streamlit/viewer_credentials.py @@ -2,6 +2,10 @@ from __future__ import annotations +import os + +import requests + from deepnote_toolkit.notebooks.credentials import ( DEFAULT_API_ORIGIN, ApiCredentials, @@ -28,9 +32,9 @@ class ViewerCredentials: """Credentials of the current viewer when Deepnote hosts the app. - `token`, `token_provider`, `base_url` and the `DEEPNOTE_TOKEN` environment - variable apply only outside Deepnote hosting. A hosted app has a viewer only on - the Streamlit script thread, and the call raises on any other thread. + Local Streamlit development requires `local=True` and explicit credentials. + Hosted processes and requests always use the viewer. Worker threads cannot + resolve a viewer and raise instead of using a shared token. """ def __init__( @@ -40,28 +44,41 @@ def __init__( *, base_url: str = DEFAULT_API_ORIGIN, timeout: float = 10, + session: requests.Session | None = None, + local: bool = False, ): - self._is_token_explicit = token is not None or token_provider is not None + self._local_mode = local + self._local_token_explicit = token is not None or token_provider is not None self._local = token_credentials(token, token_provider, base_url=base_url) self._timeout = timeout + self._session = session - def __call__(self) -> ApiCredentials: + def __call__(self, *, timeout: float = 30) -> ApiCredentials: """Return the viewer's credentials, or the local ones outside hosting.""" has_request = _has_script_run_context() - is_hosted = _read_hosted_app_id() is not None or ( - has_request and _has_hosted_streamlit_context() + is_hosted = ( + _read_hosted_app_id() is not None + or bool(os.environ.get("DEEPNOTE_PROJECT_ID")) + or (has_request and _has_hosted_streamlit_context()) ) - if is_hosted: + if is_hosted or (has_request and not self._local_mode): if not has_request: raise RunnerError(_NO_REQUEST + ".") try: - viewer = current_user_api_credentials(timeout=self._timeout) + viewer = current_user_api_credentials( + timeout=min(timeout, self._timeout), session=self._session + ) except CurrentUserApiTokenError as error: raise RunnerError(str(error), transient=error.transient) from error return ApiCredentials(token=viewer.token, api_origin=viewer.api_origin) - if not self._is_token_explicit and _is_streamlit_thread_without_request(): - raise RunnerError(_NO_REQUEST + ", or pass token= or token_provider=.") + if _is_streamlit_thread_without_request(): + raise RunnerError(_NO_REQUEST + ".") - return self._local() + if has_request and self._local_mode and not self._local_token_explicit: + raise RunnerError( + "Viewer identity is unavailable. For local development, " + "pass token= or token_provider= explicitly." + ) + return self._local(timeout=timeout) diff --git a/deepnote_toolkit/streamlit/widgets.py b/deepnote_toolkit/streamlit/widgets.py index 5295de32..028d8aaa 100644 --- a/deepnote_toolkit/streamlit/widgets.py +++ b/deepnote_toolkit/streamlit/widgets.py @@ -3,6 +3,7 @@ from __future__ import annotations import calendar +import math import re from collections.abc import Iterable from datetime import date, timedelta @@ -33,16 +34,17 @@ def render_inputs( container = st + inputs = tuple(inputs) + names = [block.variable_name for block in inputs] + if len(names) != len(set(names)): + raise ValueError("Input variable names must be unique") values: dict[str, Any] = {} for input_block in inputs: - # One submitted value applies to every block sharing a variable name. - if input_block.variable_name in values: - continue label = input_block.label or input_block.variable_name.replace("_", " ").title() key = f"{key_prefix}:{input_block.variable_name}" - values[input_block.variable_name] = _render_one( - container, input_block, label, key - ) + value = _render_one(container, input_block, label, key) + if value is not None: + values[input_block.variable_name] = value return values @@ -59,21 +61,32 @@ def _render_one(container: Any, input_block: InputBlock, label: str, key: str) - defaults = [ normalized for item in value if (normalized := str(item)) in options ] + if len(defaults) != len(value): + container.warning( + f"{label}: saved selections are no longer available. Review the selection before running." + ) return container.multiselect(label, options, default=defaults, key=key) index = ( options.index(str(input_block.value)) if str(input_block.value) in options - else 0 - ) - return ( - container.selectbox(label, options, index=index, key=key) if options else "" + else None ) + return container.selectbox(label, options, index=index, key=key) if input_block.type == "input-slider": minimum = input_block.min if input_block.min is not None else 0 maximum = input_block.max if input_block.max is not None else 100 step = input_block.step if input_block.step is not None else 1 value = _as_number(input_block.value, minimum) + if ( + not all(math.isfinite(n) for n in (minimum, maximum, step, value)) + or minimum >= maximum + or step <= 0 + or not minimum <= value <= maximum + ): + raise ValueError( + f"{label}: slider needs finite ordered bounds, a positive step, and a default within its bounds" + ) if any(isinstance(number, float) for number in (minimum, maximum, value, step)): minimum, maximum, value, step = ( float(number) for number in (minimum, maximum, value, step) @@ -95,16 +108,24 @@ def _render_one(container: Any, input_block: InputBlock, label: str, key: str) - label, value=_as_date_range(input_block.value), key=key ) if not isinstance(selected, (list, tuple)): - selected = (selected, selected) + return None serialized = [_serialize_date(value) for value in selected] if len(serialized) == 1: - return serialized * 2 + return None return serialized if len(serialized) == 2 else ["", ""] if input_block.type == "input-textarea": - return container.text_area(label, value=str(input_block.value or ""), key=key) + return container.text_area( + label, + value=str(input_block.value) if input_block.value is not None else "", + key=key, + ) - return container.text_input(label, value=str(input_block.value or ""), key=key) + return container.text_input( + label, + value=str(input_block.value) if input_block.value is not None else "", + key=key, + ) def _as_bool(value: Any) -> bool: @@ -114,6 +135,8 @@ def _as_bool(value: Any) -> bool: def _as_number(value: Any, fallback: float | int) -> float | int: + if value is None: + return fallback try: number = float(value) return ( @@ -121,8 +144,8 @@ def _as_number(value: Any, fallback: float | int) -> float | int: if isinstance(fallback, float) or not number.is_integer() else int(number) ) - except (TypeError, ValueError): - return fallback + except (TypeError, ValueError) as error: + raise ValueError("Slider default must be a number") from error def _as_date(value: Any) -> date | None: diff --git a/docs/streamlit-apps.md b/docs/streamlit-apps.md index d48ca052..2ab227b1 100644 --- a/docs/streamlit-apps.md +++ b/docs/streamlit-apps.md @@ -1,113 +1,103 @@ -# Build Streamlit apps from Deepnote notebooks +# Build a Streamlit app from a Deepnote notebook -Deepnote Toolkit provides a small typed layer for custom Streamlit apps backed by -`.deepnote` source files and snapshots. +Install `deepnote-toolkit` and `streamlit`, export your notebook as a `.deepnote` +file, and put it next to your app. Use the same notebook ID when reading inputs +and running the notebook, especially in projects with several notebooks. ```python -from pathlib import Path - import streamlit as st -from deepnote_toolkit.notebooks import DeepnoteDocument +from deepnote_toolkit.notebooks import DeepnoteDocument, RunnerError from deepnote_toolkit.streamlit import StreamlitCloudRunner, render_inputs -document = DeepnoteDocument.load(Path("report.deepnote")) +notebook_id = "your-notebook-id" +document = DeepnoteDocument.load("report.deepnote", notebook_id=notebook_id) +runner = StreamlitCloudRunner(notebook_id) values = render_inputs(document.inputs, st.sidebar) if st.button("Run"): - result = StreamlitCloudRunner("your-notebook-id").run(values) - dataframe = result.first_dataframe() - if not result.success: - st.error(result.error or "The run failed.") - elif dataframe is None: - st.info("The run produced no table.") - else: - st.dataframe(dataframe.records()) + try: + result = runner.run(values) + if not result.success: + st.error(result.error or "The run failed.") + elif result.snapshot_status == "pending": + st.info("The run finished, but its outputs are not available yet.") + elif (table := result.first_dataframe()) is not None: + st.dataframe(table.records()) + else: + st.write(result.text()) + except RunnerError as error: + st.error(str(error)) ``` -## Two packages - -`deepnote_toolkit.notebooks` has no Streamlit dependency and works in any Python -program: - -- `DeepnoteDocument` reads typed input definitions and structured outputs from a - `.deepnote` source or snapshot file. In a project with several notebooks, pass - the notebook you run so the inputs match it: - `DeepnoteDocument.load(path, notebook_id="your-notebook-id")`. -- `DeepnoteCloudRunner` runs an existing notebook in Deepnote Cloud and returns - its outputs as a `RunResult`, which holds the outputs of that notebook alone. A - dataframe output holds the first page of rows. `row_count` is the full size and - `is_truncated` tells whether rows are missing. Deepnote sends every non-numeric - cell as text, so a boolean column arrives as `"True"` and `"False"`. -- `DeepnoteRunner` does the same through a local `@deepnote/local-runner` sidecar - at `http://127.0.0.1:8787`. The sidecar does not say which notebook it ran, so - for a file with several notebooks the result holds the outputs of all of them. -- `Runner` is the interface both runners implement, for code that accepts either. - -The cloud runner only starts a run and waits for it. The parts under it can be -used or replaced on their own: - -- `DeepnoteApiClient` sends the API requests and validates the responses. -- A `CredentialsProvider` is any callable that returns `ApiCredentials`, a token - with the API origin it is valid at. It is called before every request. Pass one - as `credentials=` in place of `token`, `token_provider` and `base_url`. -- A `Transport` sends one JSON request. `UrllibTransport` is the default. Pass - your own as `transport=` to use another HTTP library. +## Authentication and local development -`deepnote_toolkit.streamlit` holds the Streamlit-specific parts: +On Deepnote, `StreamlitCloudRunner` uses the current viewer's permissions. +The hosting environment must support viewer-token exchange. If the app ID, +viewer cookie, or exchange is unavailable, the call fails; it does not fall back +to an owner token. Call it on the Streamlit script thread, not a worker thread. -- `render_inputs` maps Deepnote input blocks to native Streamlit widgets and - returns values ready to submit to a runner. -- `StreamlitCloudRunner` is a `DeepnoteCloudRunner` that runs notebooks as the - person viewing the app when Deepnote hosts it. -- `ViewerCredentials` is the `CredentialsProvider` behind it, for use with - `DeepnoteCloudRunner` or `DeepnoteApiClient` directly. - -A static app only loads a committed snapshot with `DeepnoteDocument`. It requires -no token or network access. - -## Authentication - -A Streamlit app hosted by Deepnote needs no token configuration. -`StreamlitCloudRunner` runs the notebook as the current viewer, with that viewer's -access, and never as the app's owner. A viewer who loses access to the project -can no longer run it. - -For local development, pass an API token explicitly or set `DEEPNOTE_TOKEN`: +For a locally hosted Streamlit app, opt into local credentials explicitly: ```python -runner = StreamlitCloudRunner("your-notebook-id", token="your-api-token") +import os + +runner = StreamlitCloudRunner( + notebook_id, local=True, token=os.environ["DEEPNOTE_TOKEN"] +) ``` -A callable `token_provider=` can supply a renewable token. It is invoked for every -request. A hosted app ignores both and still runs as the viewer, so the same -script works locally and deployed. To run notebooks with one fixed token for every -viewer, use `DeepnoteCloudRunner` with that token. - -Call the runner from the Streamlit script thread. A worker thread has no viewer -request. In a hosted app the runner raises there, whatever token it was given. -Elsewhere it raises instead of using `DEEPNOTE_TOKEN`, and uses an explicit token. - -For another Deepnote API client inside a hosted app, -`current_user_api_credentials()` returns a short-lived token for the current -viewer together with the API origin to send it to. - -## Runs - -Cloud runs are detached, which keeps viewer-triggered work out of the shared -project session. `StreamlitCloudRunner` also starts them with -`storage_mode="readonly"`, so a run can read the project's stored files but not -change them. Pass `storage_mode="read_write"` for a notebook that must write them. -`DeepnoteCloudRunner` leaves the mode to the API, which allows writes. - -The cloud runner retries a poll that fails with a timeout, a network error, HTTP -429 or a 5xx, up to five times in a row. After the run finishes it waits up to -`snapshot_timeout` seconds, 10 by default, for the outputs, which can arrive after -the final status. When `result.snapshot_status` is still `pending`, the outputs -had not arrived by the end of that wait, so a successful run can have none. - -Use `runner.info().accepts_inputs(document.inputs)` before submitting values to -verify that the deployed notebook still matches the file the app was built from. -It compares input names, block types, single or multiple selection, slider bounds -and select options. Options that a select fills from a variable change between -runs, so they are not compared. The API also rejects a value that does not fit its -input block. +`token_provider=` can supply a renewable token instead. A Deepnote app or project +marker overrides `local=True` and explicit tokens. Older launchers can also be +recognized by the request host or viewer cookie. Do not set `local=True` in an +unmarked hosting environment: that is an explicit choice to use local credentials. + +For Python code outside Streamlit, use `DeepnoteCloudRunner` from +`deepnote_toolkit.notebooks`. It accepts `token=`, `token_provider=`, or +`DEEPNOTE_TOKEN`. For a local `@deepnote/local-runner` sidecar, use +`DeepnoteLocalRunner(base_url="http://127.0.0.1:8787")`. + +## Inputs and outputs + +`render_inputs()` preserves saved defaults, including `0` and `False`. A select +without a valid saved choice starts empty. Unselected single selects and partially +selected date ranges are omitted from the returned dictionary; disable your Run +button until required fields are present. An omitted input uses the notebook's +value according to the API. Stale multi-select choices produce a warning. Invalid +slider bounds/defaults and duplicate variable names raise `ValueError`. + +`runner.info().matches_inputs(document.inputs)` compares static input definitions: +unique names, types, single/multiple selection, options, and slider bounds/steps. +It is a drift check, not a guarantee that every submitted value will be accepted. +Options populated from a variable cannot be checked against the saved file. + +Cloud results contain outputs from the executed notebook. `result.text()` returns +text and `result.first_dataframe()` returns the first table, if present. Tables +contain a preview page; check `row_count` and `is_truncated` before treating them as +complete data. Non-numeric cells, including booleans, can arrive as strings. +`DeepnoteDocument.load("report.snapshot.deepnote")` can display saved outputs +without network access. + +## Execution settings + +Streamlit runs are detached and use `storage_mode="readonly"`: they can read +persistent project files but cannot modify them. Use `storage_mode="read_write"` +only when the app intentionally needs to change those files. The general cloud +runner leaves storage mode to the API. + +`timeout` (600 seconds by default) is the elapsed-time budget for creation, +authentication, polling, and output retrieval. Each HTTP request and sleep is +limited to the remaining budget. Output retrieval also has its own +`snapshot_timeout` (10 seconds); only an explicitly pending snapshot is polled. +Requests uses socket timeouts, so OS DNS resolution or a server streaming bytes +can exceed a request budget; this is not hard cancellation of a running notebook. +Only GET polls are retried after transient failures, up to five consecutive +retries. Creating a run is never automatically retried. + +Pass `session=requests.Session()` to configure proxies or HTTP adapters. A custom +`credentials=` provider on `DeepnoteCloudRunner` receives a `timeout` keyword and +returns `ApiCredentials(token=..., api_origin=...)`. Providers should honor that +budget. API clients, HTTP helpers, and wire schemas are internal; supported names +are listed in each package's `__all__`. + +The existing `streamlit_data_apps` module handles database federation. Notebook +execution uses its viewer-cookie reader and does not replace its database APIs. diff --git a/docs/testing/streamlit-hosted.md b/docs/testing/streamlit-hosted.md new file mode 100644 index 00000000..4afcf057 --- /dev/null +++ b/docs/testing/streamlit-hosted.md @@ -0,0 +1,31 @@ +# Hosted Streamlit merge gate + +Run this on the **final PR commit** in a Deepnote test environment that supports +viewer API-token exchange. Local unit tests and Streamlit AppTest do not verify +platform permissions, storage mounts, or the deployed launcher. + +Record the commit, environment, app ID, notebook ID, test date, and results in the +PR before merging. Do not record tokens or cookies. + +1. Start the app through the deployed launcher. Confirm the app process sees its + UUID in `DEEPNOTE_STREAMLIT_APP_ID`. +2. As both owner and a workspace viewer, run a notebook with text and dataframe + outputs using `StreamlitCloudRunner`. Confirm success, available snapshot, and + outputs from only the executed notebook. +3. Have the notebook read a known project-storage file. Attempt a write to a + disposable path on the persistent storage mount and catch the expected + permission error. Confirm the default readonly run can read but cannot write. +4. Confirm each account cannot fetch the other account's run. An outsider must + not obtain a viewer token. +5. Repeat with a bogus `DEEPNOTE_TOKEN`, explicit `token=`, and a `token_provider` + that raises if called. Hosted execution must still use the viewer. +6. Disable project API access, retry, and confirm the server's reason is shown + without a fallback request. Restore the original project setting afterward. +7. Invoke the same runner from a worker thread. Confirm it raises before HTTP. +8. Stop the app, remove its app ID/cookie/host context in a local AppTest, and + confirm it fails closed unless local development is explicitly configured. + +The readonly literal is defined by the public API's +`DetachedRunStorageModeSchema` in `apps/webapp/server/public-api/v2/contracts/runs.ts` +(`read_write | readonly`) and is also sent by +`apps/webapp-client/src/features/static-files-app-client/connect.ts`. diff --git a/installer/module/server_process.py b/installer/module/server_process.py index 89a43407..102d2bd5 100644 --- a/installer/module/server_process.py +++ b/installer/module/server_process.py @@ -14,7 +14,13 @@ class ServerProcess: """A class to manage a server process.""" - def __init__(self, command: str, cwd: Optional[str] = None): + def __init__( + self, + command: str, + cwd: Optional[str] = None, + *, + env: Optional[dict[str, str]] = None, + ): """ Initialize the ServerProcess with the given command. @@ -22,6 +28,7 @@ def __init__(self, command: str, cwd: Optional[str] = None): """ self.command = command self.cwd = cwd + self.env = dict(env or {}) self.process = None self.stdout_thread = None self.stderr_thread = None @@ -53,7 +60,7 @@ def _start(self, retries: int = 3, delay: float = 0.2) -> subprocess.Popen: :return: The started process. :raises Exception: If the process fails to start after all retries. """ - env = os.environ.copy() + env = {**os.environ, **self.env} env["PYTHONUNBUFFERED"] = "1" attempt = 0 diff --git a/installer/module/streamlit.py b/installer/module/streamlit.py index 1fd91e81..d2136c73 100644 --- a/installer/module/streamlit.py +++ b/installer/module/streamlit.py @@ -3,17 +3,13 @@ import json import logging import os -import re +import shlex import urllib.request from typing import List from .helper import request_with_retries from .virtual_environment import VirtualEnvironment -_APP_ID_PATTERN = re.compile( - r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}", re.IGNORECASE -) - def get_webapp_url() -> str: """Returns the WebApp URL.""" @@ -125,15 +121,21 @@ def start_streamlit_servers( # The toolkit reads the app ID to run notebooks as the app's viewer. app_id = app.get("id") - is_app_id_valid = isinstance(app_id, str) and _APP_ID_PATTERN.fullmatch( - app_id - ) - env = f"DEEPNOTE_STREAMLIT_APP_ID={app_id} " if is_app_id_valid else "" + if not isinstance(app_id, str) or not app_id: + logger.warning( + "Streamlit app %r has no app ID; viewer authentication will fail", + entrypoint_path, + ) + # Always mark the process as hosted; the SDK validates the app ID. + env = { + "DEEPNOTE_STREAMLIT_APP_ID": app_id if isinstance(app_id, str) else "" + } processes.append( venv.start_server( - f"{env}streamlit run '{entrypoint_path}' {arg_str}", + f"streamlit run {shlex.quote(entrypoint_path)} {arg_str}", cwd=directory_path, + env=env, ) ) except Exception as e: diff --git a/installer/module/virtual_environment.py b/installer/module/virtual_environment.py index c2ccb6c3..edf74ae6 100644 --- a/installer/module/virtual_environment.py +++ b/installer/module/virtual_environment.py @@ -64,7 +64,13 @@ def execute(self, command: str) -> str: result = self._run_command(full_command, shell=True) return result.stdout - def start_server(self, command: str, cwd: Optional[str] = None) -> ServerProcess: + def start_server( + self, + command: str, + cwd: Optional[str] = None, + *, + env: Optional[dict[str, str]] = None, + ) -> ServerProcess: """ Start a server process using the virtual environment. @@ -73,7 +79,7 @@ def start_server(self, command: str, cwd: Optional[str] = None) -> ServerProcess :raises Exception: If the server fails to start. """ full_command = f". {self.activate_file_path} && {command}" - server_proc = ServerProcess(full_command, cwd=cwd) + server_proc = ServerProcess(full_command, cwd=cwd, env=env) # Start the server internally and handle any startup errors server_proc.start() diff --git a/tests/unit/helpers/notebook_api.py b/tests/unit/helpers/notebook_api.py new file mode 100644 index 00000000..468fca11 --- /dev/null +++ b/tests/unit/helpers/notebook_api.py @@ -0,0 +1,38 @@ +"""Reusable HTTP and clock fixtures for the notebook clients.""" + +import json + +import requests +import responses + + +class Clock: + def __init__(self): + self.now = 0.0 + self.sleeps = [] + + def __call__(self): + return self.now + + def sleep(self, seconds): + self.sleeps.append(seconds) + self.now += seconds + + +def run_response(status="success", **fields): + return {"run": {"runId": "run-1", "status": status, **fields}} + + +def session(): + http = requests.Session() + http.trust_env = False + return http + + +def body(call): + return json.loads(call.request.body) + + +def add_run(http, payload, *, create=False, origin="https://api.deepnote.com"): + path = "/v2/runs" if create else "/v2/runs/run-1?snapshotDelivery=blocks" + http.add(responses.POST if create else responses.GET, origin + path, json=payload) diff --git a/tests/unit/test_deepnote_streamlit_auth.py b/tests/unit/test_deepnote_streamlit_auth.py index 12c7d996..f961c1e8 100644 --- a/tests/unit/test_deepnote_streamlit_auth.py +++ b/tests/unit/test_deepnote_streamlit_auth.py @@ -1,356 +1,179 @@ -from __future__ import annotations - -import io -import json import sys import time -from http.client import RemoteDisconnected from types import SimpleNamespace -from typing import Any -from unittest.mock import patch -from urllib.error import HTTPError import pytest +import requests +import responses -from deepnote_toolkit.streamlit import ( - CurrentUserApiCredentials, - CurrentUserApiTokenError, - current_user_api_credentials, - current_user_api_token, -) -from deepnote_toolkit.streamlit.auth import _read_streamlit_app_id_from_context +from deepnote_toolkit.streamlit import auth +from tests.unit.helpers.notebook_api import session APP_ID = "3853c7f5-2048-4b57-946d-6c5592c3317e" +TOKEN_URL = f"http://localhost:19456/userpod-api/streamlit-apps/{APP_ID}/api-token" -class FakeResponse: - def __init__(self, payload: Any): - self.payload = payload +@pytest.fixture +def state(monkeypatch): + state = {} + monkeypatch.delenv("DEEPNOTE_STREAMLIT_APP_ID", raising=False) + monkeypatch.setattr(auth, "_read_streamlit_session_state", lambda: state) + return state - def __enter__(self) -> "FakeResponse": - return self - def __exit__(self, *_args: object) -> None: - return None +@pytest.fixture +def http(): + with responses.RequestsMock() as mock: + yield mock - def read(self) -> bytes: - return json.dumps(self.payload).encode() - -def test_reads_app_id_from_original_host_before_host() -> None: - streamlit = SimpleNamespace( - context=SimpleNamespace( - headers={ - "Host": "streamlit-00000000-0000-0000-0000-000000000000.example", - "X-Original-Host": f"streamlit-{APP_ID}.deepnote.com", - } - ) +def credentials(http_session, **kwargs): + return auth.current_user_api_credentials( + app_id=APP_ID, streamlit_token="cookie", session=http_session, **kwargs ) - with patch.dict(sys.modules, {"streamlit": streamlit}): - assert _read_streamlit_app_id_from_context() == APP_ID +def payload(**overrides): + return { + "token": "viewer", + "apiOrigin": "https://api.deepnote-staging.com/", + "expiresAtSeconds": time.time() + 900, + **overrides, + } -def test_reads_app_id_from_host_fallback() -> None: - streamlit = SimpleNamespace( - context=SimpleNamespace( - headers={"host": f"streamlit-{APP_ID}.deepnote.com:443"} - ) + +def test_exchange_uses_cookie_and_reuses_credentials_only_in_same_session(http, state): + http.post(TOKEN_URL, json=payload()) + transport = session() + first = credentials(transport) + assert credentials(transport) is first + assert ( + first.token == "viewer" + and first.api_origin == "https://api.deepnote-staging.com" + ) + assert len(http.calls) == 1 + assert http.calls[0].request.headers["StreamlitToken"] == "cookie" + assert "Authorization" not in http.calls[0].request.headers + state.clear() + assert credentials(transport) is not first + assert len(http.calls) == 2 + + +def test_changed_cookie_or_expiry_refreshes_credentials(http, state): + http.post(TOKEN_URL, json=payload(expiresAtSeconds=time.time() + 30)) + http.post(TOKEN_URL, json=payload(token="second")) + http.post(TOKEN_URL, json=payload(token="third")) + transport = session() + assert credentials(transport).token == "viewer" + assert credentials(transport).token == "second" + assert ( + auth.current_user_api_credentials( + app_id=APP_ID, streamlit_token="changed", session=transport + ).token + == "third" ) - with patch.dict(sys.modules, {"streamlit": streamlit}): - assert _read_streamlit_app_id_from_context() == APP_ID + +@pytest.mark.parametrize("value", ["bad/path", "../apps", "", "x?query", "x#fragment"]) +def test_explicit_app_id_is_validated_before_network(http, state, value): + with pytest.raises(auth.CurrentUserApiTokenError): + auth.current_user_api_credentials( + app_id=value, streamlit_token="cookie", session=session() + ) + assert not http.calls @pytest.mark.parametrize( - "streamlit", + "overrides", [ - SimpleNamespace(context=SimpleNamespace(headers={})), - SimpleNamespace(context=SimpleNamespace(headers={"host": "localhost:8501"})), + {"token": ""}, + {"token": 1}, + {"expiresAtSeconds": True}, + {"expiresAtSeconds": 0}, + {"expiresAtSeconds": "99999999999"}, + {"apiOrigin": "https://user:pass@example.com"}, + {"apiOrigin": "https://example.com/path"}, ], ) -def test_app_id_is_unavailable_outside_hosted_app(streamlit: object) -> None: - with patch.dict(sys.modules, {"streamlit": streamlit}): - assert _read_streamlit_app_id_from_context() is None - - -def test_exchanges_opaque_cookie_for_public_api_credentials() -> None: - captured = {} - - def open_request(request: Any, *, timeout: float) -> FakeResponse: - captured["url"] = request.full_url - captured["method"] = request.method - captured["headers"] = dict(request.header_items()) - captured["body"] = request.data - captured["timeout"] = timeout - return FakeResponse( - { - "token": "viewer-api-token", - "apiOrigin": "https://api.deepnote-staging.com/", - "expiresAtSeconds": 1_800_000_000, - } - ) +def test_malformed_credentials_are_not_cached(http, state, overrides): + http.post(TOKEN_URL, json=payload(**overrides)) + with pytest.raises(auth.CurrentUserApiTokenError): + credentials(session()) + assert state == {} - credentials = current_user_api_credentials( - app_id=APP_ID, - streamlit_token="opaque-cookie", - timeout=7, - opener=open_request, - ) - assert captured["url"] == ( - f"http://localhost:19456/userpod-api/streamlit-apps/{APP_ID}/api-token" +@pytest.mark.parametrize( + "status,transient", [(401, False), (403, False), (429, True), (503, True)] +) +def test_exchange_preserves_server_reason_and_retry_classification( + http, state, status, transient +): + http.post( + TOKEN_URL, + status=status, + json={"message": "API access is not available for this app"}, ) - assert captured["method"] == "POST" - assert captured["body"] == b"" - assert captured["timeout"] == 7 - headers = {key.lower(): value for key, value in captured["headers"].items()} - assert headers["streamlittoken"] == "opaque-cookie" - assert "authorization" not in headers - assert credentials.token == "viewer-api-token" - assert credentials.api_origin == "https://api.deepnote-staging.com" - assert credentials.expires_at_seconds == 1_800_000_000 - - -def _hosted_session_modules(session_state: dict[str, Any]) -> dict[str, Any]: - scriptrunner = SimpleNamespace(get_script_run_ctx=lambda: object()) - return { - "streamlit": SimpleNamespace(session_state=session_state), - "streamlit.runtime": SimpleNamespace(scriptrunner=scriptrunner), - "streamlit.runtime.scriptrunner": scriptrunner, - } - - -def _counting_opener(expires_at_seconds: float) -> tuple[list[Any], Any]: - requests: list[Any] = [] - - def open_request(request: Any, *, timeout: float) -> FakeResponse: - requests.append(request) - return FakeResponse( - { - "token": f"viewer-api-token-{len(requests)}", - "apiOrigin": "https://api.deepnote.com", - "expiresAtSeconds": expires_at_seconds, - } - ) - - return requests, open_request - - -def test_reuses_credentials_within_a_streamlit_session() -> None: - requests, open_request = _counting_opener(time.time() + 15 * 60) - - with patch.dict(sys.modules, _hosted_session_modules({})): - first = current_user_api_credentials( - app_id=APP_ID, streamlit_token="opaque-cookie", opener=open_request - ) - second = current_user_api_credentials( - app_id=APP_ID, streamlit_token="opaque-cookie", opener=open_request - ) - - assert len(requests) == 1 - assert second == first - - -def test_does_not_share_credentials_between_sessions() -> None: - requests, open_request = _counting_opener(time.time() + 15 * 60) - - for _session in range(2): - with patch.dict(sys.modules, _hosted_session_modules({})): - current_user_api_credentials( - app_id=APP_ID, streamlit_token="opaque-cookie", opener=open_request - ) - - assert len(requests) == 2 + with pytest.raises( + auth.CurrentUserApiTokenError, match="API access is not available" + ) as exc: + credentials(session()) + assert exc.value.transient is transient + assert len(http.calls) == 1 @pytest.mark.parametrize( - ("expires_in_seconds", "second_cookie"), - [(30, "opaque-cookie"), (15 * 60, "another-cookie")], + "failure", [requests.Timeout(), requests.ConnectionError("closed")] ) -def test_exchanges_again_near_expiry_or_for_another_cookie( - expires_in_seconds: int, second_cookie: str -) -> None: - requests, open_request = _counting_opener(time.time() + expires_in_seconds) +def test_exchange_network_failures_are_transient(http, state, failure): + http.post(TOKEN_URL, body=failure) + with pytest.raises(auth.CurrentUserApiTokenError) as exc: + credentials(session()) + assert exc.value.transient - with patch.dict(sys.modules, _hosted_session_modules({})): - current_user_api_credentials( - app_id=APP_ID, streamlit_token="opaque-cookie", opener=open_request - ) - current_user_api_credentials( - app_id=APP_ID, streamlit_token=second_cookie, opener=open_request - ) - - assert len(requests) == 2 - - -def test_public_token_provider_returns_the_current_credentials_token() -> None: - with patch( - "deepnote_toolkit.streamlit.auth.current_user_api_credentials" - ) as exchange: - exchange.side_effect = [ - SimpleNamespace(token="first"), - SimpleNamespace(token="second"), - ] - - assert current_user_api_token() == "first" - assert current_user_api_token() == "second" - assert exchange.call_count == 2 - - -def test_exchange_requires_hosted_streamlit_context() -> None: - with ( - patch( - "deepnote_toolkit.streamlit.auth._read_streamlit_app_id_from_context", - return_value=None, - ), - pytest.raises(CurrentUserApiTokenError, match="app ID"), - ): - current_user_api_token() - - -def test_exchange_requires_viewer_cookie() -> None: - with ( - patch( - "deepnote_toolkit.streamlit.auth._read_streamlit_app_id_from_context", - return_value=APP_ID, - ), - patch( - "deepnote_toolkit.streamlit.auth.read_streamlit_token_from_context", - return_value=None, - ), - pytest.raises(CurrentUserApiTokenError, match="streamlit-token"), - ): - current_user_api_token() +def test_exchange_never_follows_redirects_or_exposes_html(http, state): + http.post(TOKEN_URL, status=302, headers={"Location": "https://other.example"}) + with pytest.raises(auth.CurrentUserApiTokenError, match="Refused a redirect"): + credentials(session()) + http.replace(responses.POST, TOKEN_URL, status=502, body="private") + with pytest.raises(auth.CurrentUserApiTokenError) as exc: + credentials(session()) + assert "private" not in str(exc.value) @pytest.mark.parametrize( - "payload", + "headers,expected", [ - {}, - {"token": "token"}, - { - "token": "token", - "apiOrigin": "javascript:alert(1)", - "expiresAtSeconds": 123, - }, - { - "token": "token", - "apiOrigin": "https://api.deepnote.com/unexpected", - "expiresAtSeconds": 123, - }, - { - "token": "token", - "apiOrigin": "https://api.deepnote.com?secret=value", - "expiresAtSeconds": 123, - }, + ({"Host": f"streamlit-{APP_ID}.example"}, APP_ID), + ( + {"Host": "localhost", "X-Original-Host": f"streamlit-{APP_ID}.example"}, + APP_ID, + ), + ({"Host": "localhost:8501"}, None), + ({}, None), ], ) -def test_exchange_rejects_invalid_response(payload: dict[str, Any]) -> None: - with pytest.raises(CurrentUserApiTokenError): - current_user_api_credentials( - app_id=APP_ID, - streamlit_token="opaque-cookie", - opener=lambda *_args, **_kwargs: FakeResponse(payload), - ) - - -def test_exchange_error_includes_the_server_message() -> None: - def open_request(*_args: Any, **_kwargs: Any) -> FakeResponse: - raise HTTPError( - "http://localhost:19456/userpod-api/streamlit-apps/id/api-token", - 403, - "Forbidden", - {}, - io.BytesIO( - json.dumps( - {"error": "API access is not available for this app"} - ).encode() - ), - ) - - with pytest.raises(CurrentUserApiTokenError) as exc_info: - current_user_api_credentials( - app_id=APP_ID, - streamlit_token="opaque-cookie", - opener=open_request, - ) - - assert str(exc_info.value) == ( - "Current viewer API-token exchange returned HTTP 403: " - "API access is not available for this app" +def test_host_id_resolution(monkeypatch, headers, expected): + monkeypatch.setitem( + sys.modules, + "streamlit", + SimpleNamespace(context=SimpleNamespace(headers=headers)), ) + assert auth._read_streamlit_app_id_from_context() == expected -def test_exchange_error_does_not_expose_a_raw_response_body() -> None: - def open_request(*_args: Any, **_kwargs: Any) -> FakeResponse: - raise HTTPError( - "http://localhost:19456/userpod-api/streamlit-apps/id/api-token", - 502, - "Bad Gateway", - {}, - io.BytesIO(b"must-not-leak"), - ) - - with pytest.raises(CurrentUserApiTokenError) as exc_info: - current_user_api_credentials( - app_id=APP_ID, - streamlit_token="opaque-cookie", - opener=open_request, - ) - - assert str(exc_info.value) == "Current viewer API-token exchange returned HTTP 502." +def test_session_state_lookup_suppresses_missing_context_warning(monkeypatch): + calls = [] + def get_ctx(*, suppress_warning): + calls.append(suppress_warning) + return None -def test_credentials_repr_hides_the_token() -> None: - credentials = CurrentUserApiCredentials( - token="secret-token", - api_origin="https://api.deepnote.com", - expires_at_seconds=1_800_000_000, + monkeypatch.setitem(sys.modules, "streamlit", SimpleNamespace()) + monkeypatch.setitem( + sys.modules, + "streamlit.runtime.scriptrunner", + SimpleNamespace(get_script_run_ctx=get_ctx), ) - - assert "secret-token" not in repr(credentials) - - -def test_dropped_connection_during_exchange_is_transient() -> None: - def open_request(_request: Any, *, timeout: float) -> Any: - raise RemoteDisconnected("Remote end closed connection without response") - - with pytest.raises(CurrentUserApiTokenError) as exc_info: - current_user_api_credentials( - app_id="11111111-2222-3333-4444-555555555555", - streamlit_token="cookie", - opener=open_request, - ) - - assert exc_info.value.transient is True - - -def test_exchange_prefers_the_app_id_exported_by_the_launcher( - monkeypatch: pytest.MonkeyPatch, -) -> None: - monkeypatch.setenv("DEEPNOTE_STREAMLIT_APP_ID", APP_ID.upper()) - urls = [] - - def open_request(request: Any, *, timeout: float) -> FakeResponse: - urls.append(request.full_url) - return FakeResponse( - { - "token": "viewer-api-token", - "apiOrigin": "https://api.deepnote.com", - "expiresAtSeconds": 1_800_000_000, - } - ) - - with patch( - "deepnote_toolkit.streamlit.auth._read_streamlit_app_id_from_context", - return_value="00000000-0000-0000-0000-000000000000", - ): - current_user_api_credentials( - streamlit_token="opaque-cookie", opener=open_request - ) - - assert urls == [ - f"http://localhost:19456/userpod-api/streamlit-apps/{APP_ID}/api-token" - ] + assert auth._read_streamlit_session_state() is None + assert calls == [True] diff --git a/tests/unit/test_deepnote_streamlit_cloud_runner.py b/tests/unit/test_deepnote_streamlit_cloud_runner.py index e1224585..7412c75f 100644 --- a/tests/unit/test_deepnote_streamlit_cloud_runner.py +++ b/tests/unit/test_deepnote_streamlit_cloud_runner.py @@ -1,373 +1,197 @@ -import json -from typing import Any -from unittest.mock import MagicMock, patch +import time import pytest +import responses + +from deepnote_toolkit.notebooks import RunnerError +from deepnote_toolkit.streamlit import StreamlitCloudRunner, auth +from deepnote_toolkit.streamlit import viewer_credentials as viewer +from tests.unit.helpers.notebook_api import Clock, add_run, body, run_response, session + +APP_ID = "11111111-2222-3333-4444-555555555555" +TOKEN_URL = f"http://localhost:19456/userpod-api/streamlit-apps/{APP_ID}/api-token" + + +@pytest.fixture +def context(monkeypatch): + monkeypatch.delenv("DEEPNOTE_STREAMLIT_APP_ID", raising=False) + monkeypatch.delenv("DEEPNOTE_PROJECT_ID", raising=False) + monkeypatch.setenv("DEEPNOTE_TOKEN", "owner-token") + state = {"script": True, "hosted": False, "worker": False} + monkeypatch.setattr(viewer, "_has_script_run_context", lambda: state["script"]) + monkeypatch.setattr( + viewer, "_has_hosted_streamlit_context", lambda: state["hosted"] + ) + monkeypatch.setattr( + viewer, "_is_streamlit_thread_without_request", lambda: state["worker"] + ) + monkeypatch.setattr(auth, "_read_streamlit_app_id_from_context", lambda: None) + monkeypatch.setattr( + auth, "read_streamlit_token_from_context", lambda: "viewer-cookie" + ) + monkeypatch.setattr(auth, "_read_streamlit_session_state", lambda: {}) + return state -from deepnote_toolkit.notebooks import ApiCredentials, RunnerError, UrllibTransport -from deepnote_toolkit.streamlit import ( - CurrentUserApiCredentials, - CurrentUserApiTokenError, - StreamlitCloudRunner, - ViewerCredentials, -) - - -class FakeResponse: - def __init__(self, payload: Any): - self.payload = payload - - def __enter__(self) -> "FakeResponse": - return self - - def __exit__(self, *_: object) -> None: - return None - def read(self) -> bytes: - return json.dumps(self.payload).encode() +@pytest.fixture +def http(): + with responses.RequestsMock() as mock: + yield mock -def test_hosted_cloud_runner_exchanges_per_request_and_uses_api_origin( - monkeypatch: pytest.MonkeyPatch, -) -> None: - monkeypatch.setenv("DEEPNOTE_TOKEN", "must-not-be-used") - api_calls = [] - responses = iter( - [ - {"run": {"runId": "run-1", "status": "pending"}}, - {"run": {"runId": "run-1", "status": "success", "snapshotBlocks": []}}, - ] +@pytest.mark.parametrize( + "explicit", [{}, {"token": "owner"}, {"token_provider": lambda: "owner"}] +) +def test_missing_hosted_signals_fail_closed(context, http, explicit): + with pytest.raises(RunnerError, match="app ID"): + StreamlitCloudRunner("n", session=session(), **explicit).run({}) + assert len(http.calls) == 0 + + +@pytest.mark.parametrize("marker", ["DEEPNOTE_STREAMLIT_APP_ID", "DEEPNOTE_PROJECT_ID"]) +@pytest.mark.parametrize("local", [False, True]) +def test_hosted_run_uses_viewer_and_readonly_even_with_explicit_owner_token( + context, http, monkeypatch, marker, local +): + monkeypatch.setenv(marker, APP_ID) + monkeypatch.setattr(auth, "_read_streamlit_app_id_from_context", lambda: APP_ID) + http.post( + TOKEN_URL, + json={ + "token": "viewer", + "apiOrigin": "https://api.deepnote-staging.com", + "expiresAtSeconds": time.time() + 900, + }, ) - - def open_request(request: Any, *, timeout: float) -> FakeResponse: - api_calls.append( - ( - request.full_url, - request.headers["Authorization"], - timeout, - ) + add_run( + http, + run_response(snapshotBlocks=[]), + create=True, + origin="https://api.deepnote-staging.com", + ) + runner = StreamlitCloudRunner( + "n", + token="owner", + base_url="https://wrong.example", + local=local, + session=session(), + ) + assert runner.run({}).success + assert http.calls[0].request.headers["StreamlitToken"] == "viewer-cookie" + assert http.calls[1].request.headers["Authorization"] == "Bearer viewer" + assert body(http.calls[1])["detachedRunStorageMode"] == "readonly" + + +@pytest.mark.parametrize("marker", ["", "invalid", APP_ID]) +def test_malformed_app_marker_never_falls_back(context, http, monkeypatch, marker): + monkeypatch.setenv("DEEPNOTE_STREAMLIT_APP_ID", marker) + ( + http.post( + TOKEN_URL, + status=403, + json={"message": "API access is not available for this app"}, ) - return FakeResponse(next(responses)) - - credentials = [ - CurrentUserApiCredentials( - token="viewer-token-1", - api_origin="https://api.deepnote-staging.com", - expires_at_seconds=1_800_000_000, - ), - CurrentUserApiCredentials( - token="viewer-token-2", - api_origin="https://api.deepnote-staging.com", - expires_at_seconds=1_800_000_001, - ), - ] - with ( - patch( - "deepnote_toolkit.streamlit.viewer_credentials._has_script_run_context", - return_value=True, - ), - patch( - "deepnote_toolkit.streamlit.viewer_credentials._has_hosted_streamlit_context", - return_value=True, - ), - patch( - "deepnote_toolkit.streamlit.viewer_credentials.current_user_api_credentials", - side_effect=credentials, - ) as exchange, - ): - result = StreamlitCloudRunner( - "notebook-1", - transport=UrllibTransport(open_request), - sleep=lambda _delay: None, - ).run({}) - - assert result.success is True - assert exchange.call_count == 2 - assert api_calls == [ - ( - "https://api.deepnote-staging.com/v2/runs", - "Bearer viewer-token-1", - 30, - ), - ( - "https://api.deepnote-staging.com/v2/runs/run-1" "?snapshotDelivery=blocks", - "Bearer viewer-token-2", - 30, - ), - ] - - -def test_hosted_runner_never_falls_back_to_environment_token( - monkeypatch: pytest.MonkeyPatch, -) -> None: - monkeypatch.setenv("DEEPNOTE_TOKEN", "shared-token") - opener = MagicMock() - with ( - patch( - "deepnote_toolkit.streamlit.viewer_credentials._has_script_run_context", - return_value=True, - ), - patch( - "deepnote_toolkit.streamlit.viewer_credentials._has_hosted_streamlit_context", - return_value=True, - ), - patch( - "deepnote_toolkit.streamlit.viewer_credentials.current_user_api_credentials", - side_effect=CurrentUserApiTokenError("viewer token unavailable"), - ), - pytest.raises(RunnerError, match="viewer token unavailable"), - ): - StreamlitCloudRunner("notebook-1", transport=UrllibTransport(opener)).info() + if marker == APP_ID + else None + ) + with pytest.raises(RunnerError): + StreamlitCloudRunner("n", token="owner", local=True, session=session()).info() + assert all(c.request.url == TOKEN_URL for c in http.calls) - opener.assert_not_called() +@pytest.mark.parametrize( + "explicit", [{}, {"token": "owner"}, {"token_provider": lambda: "owner"}] +) +def test_worker_thread_fails_closed(context, http, explicit): + context.update(script=False, worker=True) + with pytest.raises(RunnerError, match="No viewer request"): + StreamlitCloudRunner("n", session=session(), **explicit).info() + assert not http.calls -def test_worker_thread_never_falls_back_to_environment_token( - monkeypatch: pytest.MonkeyPatch, -) -> None: - monkeypatch.setenv("DEEPNOTE_TOKEN", "shared-token") - opener = MagicMock() - with ( - patch( - "deepnote_toolkit.streamlit.viewer_credentials._has_hosted_streamlit_context", - return_value=False, - ), - patch( - "deepnote_toolkit.streamlit.viewer_credentials._is_streamlit_thread_without_request", - return_value=True, - ), - pytest.raises(RunnerError, match="No viewer request"), - ): - StreamlitCloudRunner("notebook-1", transport=UrllibTransport(opener)).info() - opener.assert_not_called() +def test_local_streamlit_requires_explicit_opt_in_and_token(context, http): + http.get( + "https://api.deepnote.com/v2/notebooks/n", json={"notebook": {"name": "N"}} + ) + runner = StreamlitCloudRunner("n", local=True, token="local", session=session()) + assert runner.info().notebook == "N" + assert http.calls[0].request.headers["Authorization"] == "Bearer local" + with pytest.raises(RunnerError, match="explicitly"): + StreamlitCloudRunner("n", local=True, session=session()).info() -def test_cloud_run_retries_a_transient_token_exchange_failure() -> None: - responses = iter( - [ - {"run": {"runId": "run-1", "status": "pending"}}, - {"run": {"runId": "run-1", "status": "success", "snapshotBlocks": []}}, - ] +def test_bare_python_uses_local_token_without_streamlit_lookups(context, http): + context["script"] = False + http.get( + "https://api.deepnote.com/v2/notebooks/n", json={"notebook": {"name": "N"}} ) - credentials = CurrentUserApiCredentials( - token="viewer-token", - api_origin="https://api.deepnote.com", - expires_at_seconds=1_800_000_000, + assert StreamlitCloudRunner("n", session=session()).info().notebook == "N" + + +def test_transient_exchange_failure_during_poll_is_retried(context, http, monkeypatch): + monkeypatch.setenv("DEEPNOTE_STREAMLIT_APP_ID", APP_ID) + payload = { + "token": "viewer", + "apiOrigin": "https://api.deepnote.com", + "expiresAtSeconds": time.time() + 900, + } + http.post(TOKEN_URL, json=payload) + http.post(TOKEN_URL, status=503) + http.post(TOKEN_URL, json=payload) + add_run(http, run_response("running"), create=True) + add_run(http, run_response(snapshotBlocks=[])) + clock = Clock() + assert ( + StreamlitCloudRunner("n", session=session(), clock=clock, sleep=clock.sleep) + .run({}) + .success ) - with ( - patch( - "deepnote_toolkit.streamlit.viewer_credentials._has_script_run_context", - return_value=True, - ), - patch( - "deepnote_toolkit.streamlit.viewer_credentials._has_hosted_streamlit_context", - return_value=True, - ), - patch( - "deepnote_toolkit.streamlit.viewer_credentials.current_user_api_credentials", - side_effect=[ - credentials, - CurrentUserApiTokenError("exchange timed out", transient=True), - credentials, - ], - ), - ): - result = StreamlitCloudRunner( - "notebook-1", - transport=UrllibTransport( - lambda _request, *, timeout: FakeResponse(next(responses)) - ), - sleep=lambda _delay: None, - ).run({}) - - assert result.success is True - - -def test_local_streamlit_runner_uses_environment_token( - monkeypatch: pytest.MonkeyPatch, -) -> None: - monkeypatch.setenv("DEEPNOTE_TOKEN", "local-token") - - def open_request(request: Any, *, timeout: float) -> FakeResponse: - assert request.headers["Authorization"] == "Bearer local-token" - return FakeResponse({"notebook": {"name": "Revenue", "inputs": []}}) - - with ( - patch( - "deepnote_toolkit.streamlit.viewer_credentials._has_hosted_streamlit_context", - return_value=False, - ), - patch( - "deepnote_toolkit.streamlit.viewer_credentials._is_streamlit_thread_without_request", - return_value=False, - ), - ): - info = StreamlitCloudRunner( - "notebook-1", transport=UrllibTransport(open_request) - ).info() - - assert info.notebook == "Revenue" - - -def test_hosted_runner_sends_the_viewer_token_only_to_the_returned_origin() -> None: - urls = [] - - def open_request(request: Any, *, timeout: float) -> FakeResponse: - urls.append(request.full_url) - return FakeResponse({"notebook": {"name": "Revenue", "inputs": []}}) - - with ( - patch( - "deepnote_toolkit.streamlit.viewer_credentials._has_script_run_context", - return_value=True, - ), - patch( - "deepnote_toolkit.streamlit.viewer_credentials._has_hosted_streamlit_context", - return_value=True, - ), - patch( - "deepnote_toolkit.streamlit.viewer_credentials.current_user_api_credentials", - return_value=CurrentUserApiCredentials( - token="viewer-token", - api_origin="https://api.deepnote.com", - expires_at_seconds=1_800_000_000, - ), - ), - ): - StreamlitCloudRunner( - "notebook-1", - base_url="https://elsewhere.example", - transport=UrllibTransport(open_request), - ).info() - - assert urls == ["https://api.deepnote.com/v2/notebooks/notebook-1"] - - -def test_runner_skips_streamlit_lookups_outside_a_script_run( - monkeypatch: pytest.MonkeyPatch, -) -> None: - monkeypatch.setenv("DEEPNOTE_TOKEN", "local-token") - - def open_request(request: Any, *, timeout: float) -> FakeResponse: - assert request.headers["Authorization"] == "Bearer local-token" - return FakeResponse({"notebook": {"name": "Revenue", "inputs": []}}) - - with ( - patch( - "deepnote_toolkit.streamlit.viewer_credentials._has_script_run_context", - return_value=False, - ), - patch( - "deepnote_toolkit.streamlit.viewer_credentials._has_hosted_streamlit_context" - ) as hosted_lookup, - patch( - "deepnote_toolkit.streamlit.viewer_credentials._is_streamlit_thread_without_request", - return_value=False, - ), - ): - StreamlitCloudRunner( - "notebook-1", transport=UrllibTransport(open_request) - ).info() - - hosted_lookup.assert_not_called() - - -def test_hosted_runner_ignores_an_explicit_token() -> None: - authorizations = [] - - def open_request(request: Any, *, timeout: float) -> FakeResponse: - authorizations.append(request.headers["Authorization"]) - return FakeResponse({"notebook": {"name": "Revenue", "inputs": []}}) - - with ( - patch( - "deepnote_toolkit.streamlit.viewer_credentials._has_script_run_context", - return_value=True, - ), - patch( - "deepnote_toolkit.streamlit.viewer_credentials._has_hosted_streamlit_context", - return_value=True, - ), - patch( - "deepnote_toolkit.streamlit.viewer_credentials.current_user_api_credentials", - return_value=CurrentUserApiCredentials( - token="viewer-token", - api_origin="https://api.deepnote.com", - expires_at_seconds=1_800_000_000, - ), - ), - ): - StreamlitCloudRunner( - "notebook-1", token="owner-token", transport=UrllibTransport(open_request) - ).info() - - assert authorizations == ["Bearer viewer-token"] - - -def test_streamlit_runs_are_readonly_by_default( - monkeypatch: pytest.MonkeyPatch, -) -> None: - monkeypatch.setenv("DEEPNOTE_TOKEN", "local-token") - bodies = [] + assert len(http.calls) == 5 + + +def test_real_streamlit_script_and_worker_keep_viewer_identity(monkeypatch, http): + pytest.importorskip("streamlit") + from streamlit.testing.v1 import AppTest + + monkeypatch.setenv("DEEPNOTE_STREAMLIT_APP_ID", APP_ID) + monkeypatch.setenv("DEEPNOTE_TOKEN", "owner-token") + monkeypatch.setattr(auth, "read_streamlit_token_from_context", lambda: "cookie") + http.post( + TOKEN_URL, + json={ + "token": "viewer", + "apiOrigin": "https://api.deepnote.com", + "expiresAtSeconds": time.time() + 900, + }, + ) + add_run(http, run_response(snapshotBlocks=[]), create=True) - def open_request(request: Any, *, timeout: float) -> FakeResponse: - bodies.append(json.loads(request.data)) - return FakeResponse( - {"run": {"runId": "run-1", "status": "success", "snapshotBlocks": []}} - ) + def app(): + import threading - with ( - patch( - "deepnote_toolkit.streamlit.viewer_credentials._has_script_run_context", - return_value=False, - ), - patch( - "deepnote_toolkit.streamlit.viewer_credentials._is_streamlit_thread_without_request", - return_value=False, - ), - ): - StreamlitCloudRunner("notebook-1", transport=UrllibTransport(open_request)).run( - {} - ) + import streamlit as st - assert bodies == [ - { - "notebookId": "notebook-1", - "detached": True, - "inputs": {}, - "detachedRunStorageMode": "readonly", - } - ] + from deepnote_toolkit.notebooks import RunnerError + from deepnote_toolkit.streamlit import StreamlitCloudRunner + runner = StreamlitCloudRunner("n", token="owner", local=True) + st.session_state["success"] = runner.run({}).success + errors = [] -def test_viewer_credentials_use_an_explicit_token_on_a_worker_thread() -> None: - with ( - patch( - "deepnote_toolkit.streamlit.viewer_credentials._has_script_run_context", - return_value=False, - ), - patch( - "deepnote_toolkit.streamlit.viewer_credentials._is_streamlit_thread_without_request", - return_value=True, - ), - ): - credentials = ViewerCredentials(token="local-token")() + def worker(): + try: + runner.run({}) + except RunnerError as error: + errors.append(str(error)) - assert credentials == ApiCredentials("local-token", "https://api.deepnote.com") + thread = threading.Thread(target=worker) + thread.start() + thread.join(timeout=5) + st.session_state["worker_errors"] = errors - -def test_hosted_process_raises_off_the_script_thread_even_with_a_token( - monkeypatch: pytest.MonkeyPatch, -) -> None: - monkeypatch.setenv( - "DEEPNOTE_STREAMLIT_APP_ID", "11111111-2222-3333-4444-555555555555" - ) - with ( - patch( - "deepnote_toolkit.streamlit.viewer_credentials._has_script_run_context", - return_value=False, - ), - pytest.raises(RunnerError, match="No viewer request"), - ): - ViewerCredentials(token="owner-token")() + at = AppTest.from_function(app).run() + assert not at.exception + assert at.session_state["success"] is True + assert "No viewer request" in at.session_state["worker_errors"][0] + assert len(http.calls) == 2 + assert http.calls[1].request.headers["Authorization"] == "Bearer viewer" diff --git a/tests/unit/test_deepnote_streamlit_widgets.py b/tests/unit/test_deepnote_streamlit_widgets.py index 0066dedc..425cc03f 100644 --- a/tests/unit/test_deepnote_streamlit_widgets.py +++ b/tests/unit/test_deepnote_streamlit_widgets.py @@ -8,6 +8,9 @@ class FakeContainer: + def warning(self, message): + pass + def checkbox(self, _label: str, **kwargs: Any) -> Any: return kwargs["value"] @@ -15,7 +18,7 @@ def multiselect(self, _label: str, _options: list[str], **kwargs: Any) -> Any: return kwargs["default"] def selectbox(self, _label: str, options: list[str], **kwargs: Any) -> Any: - return options[kwargs["index"]] + return options[kwargs["index"]] if kwargs["index"] is not None else None def slider(self, _label: str, **kwargs: Any) -> Any: return kwargs["value"] @@ -74,7 +77,7 @@ def date_input(self, _label: str, **_kwargs: Any) -> Any: IncompleteDateContainer(), ) - assert values == {"period": ["2026-08-17", "2026-08-17"]} + assert values == {} def test_slider_preserves_fractional_default_with_integer_bounds() -> None: @@ -226,16 +229,15 @@ def app() -> None: } -def test_inputs_sharing_a_variable_name_render_once() -> None: - values = render_inputs( - [ - InputBlock("region", "input-text", "EU"), - InputBlock("region", "input-text", "US"), - ], - FakeContainer(), - ) - - assert values == {"region": "EU"} +def test_duplicate_variable_names_are_rejected() -> None: + with pytest.raises(ValueError, match="unique"): + render_inputs( + [ + InputBlock("region", "input-text", "EU"), + InputBlock("region", "input-text", "US"), + ], + FakeContainer(), + ) def test_multiselect_treats_a_scalar_default_as_one_selection() -> None: @@ -252,3 +254,97 @@ def test_multiselect_treats_a_scalar_default_as_one_selection() -> None: ) assert values == {"regions": ["EU"], "empty": []} + + +@pytest.mark.parametrize("kind", ["input-text", "input-textarea", "input-file"]) +@pytest.mark.parametrize("value,expected", [(0, "0"), (False, "False"), (None, "")]) +def test_falsey_text_defaults_are_preserved(kind, value, expected): + assert render_inputs([InputBlock("x", kind, value)], FakeContainer()) == { + "x": expected + } + + +@pytest.mark.parametrize("value", [None, "stale"]) +def test_unselected_single_select_does_not_submit_first_option(value): + assert ( + render_inputs( + [InputBlock("x", "input-select", value, options=("first", "second"))], + FakeContainer(), + ) + == {} + ) + + +def test_stale_multiselect_default_warns(): + warnings = [] + container = FakeContainer() + container.warning = warnings.append + values = render_inputs( + [ + InputBlock( + "x", + "input-select", + ["old", "current"], + options=("current",), + multiple=True, + ) + ], + container, + ) + assert values == {"x": ["current"]} + assert len(warnings) == 1 + + +@pytest.mark.parametrize( + "value,min_value,max_value,step", + [ + (11, 0, 10, 1), + (-1, 0, 10, 1), + (3, 10, 0, 1), + (3, 0, 10, 0), + (3, 0, 10, -1), + ("bad", 0, 10, 1), + (float("nan"), 0, 10, 1), + (3, 0, float("inf"), 1), + ], +) +def test_invalid_slider_configuration_is_reported(value, min_value, max_value, step): + with pytest.raises(ValueError, match="[Ss]lider"): + render_inputs( + [ + InputBlock( + "x", "input-slider", value, min=min_value, max=max_value, step=step + ) + ], + FakeContainer(), + ) + + +def test_real_widgets_keep_falsey_defaults_and_require_selection(): + pytest.importorskip("streamlit") + from streamlit.testing.v1 import AppTest + + def app(): + import streamlit as st + + from deepnote_toolkit.notebooks import InputBlock + from deepnote_toolkit.streamlit import render_inputs + + st.session_state["values"] = render_inputs( + [ + InputBlock("zero", "input-text", 0), + InputBlock("false", "input-textarea", False), + InputBlock("choice", "input-select", "stale", options=("A", "B")), + InputBlock("period", "input-date-range", ["", ""]), + ] + ) + + at = AppTest.from_function(app).run() + assert not at.exception + assert at.text_input[0].value == "0" and at.text_area[0].value == "False" + assert "choice" not in at.session_state["values"] + at.selectbox[0].select("B").run() + assert at.session_state["values"]["choice"] == "B" + at.date_input[0].set_value((date(2026, 8, 17),)).run() + assert not at.exception + assert "period" not in at.session_state["values"] diff --git a/tests/unit/test_notebooks_document.py b/tests/unit/test_notebooks_document.py index 0883b4d2..ee3d78b2 100644 --- a/tests/unit/test_notebooks_document.py +++ b/tests/unit/test_notebooks_document.py @@ -3,28 +3,29 @@ from typing import Any import pytest +import responses from deepnote_toolkit.notebooks import ( - DATAFRAME_MIME, DeepnoteDataframe, DeepnoteDocument, - DeepnoteRunner, + DeepnoteLocalRunner, InputBlock, RunResult, - join_text, ) +from deepnote_toolkit.notebooks.models import DATAFRAME_MIME, join_text +from tests.unit.helpers.notebook_api import session -class FakeTransport: - def __init__(self, payload: Mapping[str, Any]): - self.payload = payload - - def request_json(self, *_args: Any, **_kwargs: Any) -> Mapping[str, Any]: - return self.payload +def run_locally(payload: Mapping[str, Any]) -> RunResult: + with responses.RequestsMock() as http: + http.post("http://127.0.0.1:8787/api/run", json=payload) + return DeepnoteLocalRunner(session=session()).run({}) -def run_locally(payload: Mapping[str, Any]) -> RunResult: - return DeepnoteRunner(transport=FakeTransport(payload)).run({}) +def local_info(payload): + with responses.RequestsMock() as http: + http.get("http://127.0.0.1:8787/api/info", json=payload) + return DeepnoteLocalRunner(session=session()).info() SNAPSHOT_YAML = """ @@ -126,22 +127,20 @@ def test_reads_input_metadata_from_file_and_api_shapes() -> None: } } ) - info = DeepnoteRunner( - transport=FakeTransport( - { - "inputs": [ - { - "variableName": "countries", - "type": "input-select", - "label": "Countries", - "value": ["Panama"], - "options": ["Panama", "Colombia"], - "multiple": True, - } - ] - } - ) - ).info() + info = local_info( + { + "inputs": [ + { + "variableName": "countries", + "type": "input-select", + "label": "Countries", + "value": ["Panama"], + "options": ["Panama", "Colombia"], + "multiple": True, + } + ] + } + ) (file_input,) = document.inputs (api_input,) = info.inputs diff --git a/tests/unit/test_notebooks_runners.py b/tests/unit/test_notebooks_runners.py index 566796a5..3fcc4daa 100644 --- a/tests/unit/test_notebooks_runners.py +++ b/tests/unit/test_notebooks_runners.py @@ -1,550 +1,429 @@ -import io import json import threading -from http.client import RemoteDisconnected from http.server import BaseHTTPRequestHandler, HTTPServer -from typing import Any -from urllib.error import HTTPError, URLError import pytest +import requests +import responses from deepnote_toolkit.notebooks import ( ApiCredentials, DeepnoteCloudRunner, - DeepnoteRunner, + DeepnoteLocalRunner, InputBlock, RunnerError, RunnerInfo, - UrllibTransport, ) +from tests.unit.helpers.notebook_api import Clock, add_run, body, run_response, session -class FakeResponse: - def __init__(self, payload: Any): - self.payload = payload +@pytest.fixture +def http(): + with responses.RequestsMock() as mock: + yield mock - def __enter__(self) -> "FakeResponse": - return self - def __exit__(self, *_: object) -> None: - return None +@pytest.fixture +def clock(): + return Clock() - def read(self) -> bytes: - return json.dumps(self.payload).encode() +@pytest.fixture +def runner(clock): + return DeepnoteCloudRunner( + "notebook-1", + token="token", + session=session(), + clock=clock, + sleep=clock.sleep, + poll_interval=0.5, + ) -def test_info_parses_runner_contract() -> None: - calls = [] - def open_request(request: Any, *, timeout: float) -> FakeResponse: - calls.append((request.full_url, request.method, timeout)) - return FakeResponse( - { - "notebook": "Revenue", - "runTarget": "cloud", +def test_cloud_info_preserves_input_contract_and_quotes_notebook_id(http): + http.get( + "https://api.deepnote.com/v2/notebooks/a%2Fb%3Fadmin%3Dtrue", + json={ + "notebook": { + "name": "Revenue", "inputs": [ - {"variableName": "region", "type": "input-select", "value": "All"} + { + "name": "region", + "type": "input-select", + "value": "EU", + "options": ["EU", "US"], + "multiple": True, + }, + { + "name": "limit", + "type": "input-slider", + "value": "5", + "min": 1, + "max": 9, + "step": 2, + }, + {"name": "future", "type": "input-unknown"}, ], } - ) - - info = DeepnoteRunner( - "http://runner/", timeout=12, transport=UrllibTransport(open_request) - ).info() - - assert calls == [("http://runner/api/info", "GET", 12)] - assert info.notebook == "Revenue" - assert info.run_target == "cloud" - assert info.inputs[0].variable_name == "region" - - -def test_runner_info_requires_matching_input_names_and_types() -> None: - info = RunnerInfo( - notebook="Revenue", - inputs=(InputBlock("region", "input-select", "All"),), - run_target="cloud", + }, ) - - assert info.accepts_inputs([InputBlock("region", "input-select", "Europe")]) - assert not info.accepts_inputs([InputBlock("market", "input-select", "Europe")]) - assert not info.accepts_inputs([InputBlock("region", "input-text", "Europe")]) - - -def test_run_posts_inputs_and_parses_one_result_shape() -> None: - def open_request(request: Any, *, timeout: float) -> FakeResponse: - assert timeout == 600 - assert request.method == "POST" - assert json.loads(request.data) == {"inputs": {"limit": 20}} - return FakeResponse({"target": "local", "success": True, "outputs": []}) - - result = DeepnoteRunner(transport=UrllibTransport(open_request)).run({"limit": 20}) - - assert result.target == "local" - assert result.success is True - - -def test_http_error_surfaces_runner_message() -> None: - def open_request(*_: Any, **__: Any) -> FakeResponse: - raise HTTPError( - "http://runner/api/run", - 500, - "Server error", - {}, - io.BytesIO(b'{"error":"DEEPNOTE_TOKEN is required"}'), - ) - - with pytest.raises(RunnerError, match="DEEPNOTE_TOKEN is required"): - DeepnoteRunner("http://runner", transport=UrllibTransport(open_request)).run({}) - - -def test_connection_error_names_runner_url() -> None: - def open_request(*_: Any, **__: Any) -> FakeResponse: - raise URLError("connection refused") - - with pytest.raises(RunnerError, match="http://runner"): - DeepnoteRunner("http://runner", transport=UrllibTransport(open_request)).info() - - -def test_timeout_names_runner_url_and_duration() -> None: - def open_request(*_: Any, **__: Any) -> FakeResponse: - raise TimeoutError - - with pytest.raises(RunnerError, match="http://runner.*12 seconds"): - DeepnoteRunner( - "http://runner", timeout=12, transport=UrllibTransport(open_request) - ).info() - - -def test_cloud_info_reads_public_notebook_contract() -> None: - def open_request(request: Any, *, timeout: float) -> FakeResponse: - assert request.full_url == "https://api.deepnote.com/v2/notebooks/notebook-1" - assert request.headers["Authorization"] == "Bearer token-1" - assert timeout == 30 - return FakeResponse( - { - "notebook": { - "name": "Revenue", - "inputs": [ - { - "name": "region", - "type": "input-select", - "value": "All", - "label": "Region", - } - ], - } - } - ) - info = DeepnoteCloudRunner( - "notebook-1", token="token-1", transport=UrllibTransport(open_request) + "a/b?admin=true", token="token", session=session() ).info() - assert info.notebook == "Revenue" - assert info.run_target == "cloud" - assert info.inputs[0].variable_name == "region" - - -def test_cloud_run_posts_inputs_polls_and_reads_the_executed_blocks() -> None: - calls = [] - responses = iter( - [ - {"run": {"runId": "run-1", "status": "pending"}}, - {"run": {"runId": "run-1", "status": "running"}}, - { - "run": { - "runId": "run-1", - "status": "success", - "snapshotBlocks": [ - { - "id": "code-1", - "type": "code", - "outputs": [{"output_type": "stream", "text": "done"}], - } - ], + assert info.inputs == ( + InputBlock("region", "input-select", "EU", options=("EU", "US"), multiple=True), + InputBlock("limit", "input-slider", "5", min=1, max=9, step=2), + ) + assert http.calls[0].request.headers["Authorization"] == "Bearer token" + + +def test_run_refreshes_credentials_normalizes_inputs_and_reads_only_run_blocks( + http, clock +): + add_run(http, run_response("pending"), create=True) + add_run(http, run_response("future-state")) + add_run(http, run_response(snapshotStatus="pending")) + add_run( + http, + run_response( + snapshotStatus="available", + snapshotBlocks=[ + { + "id": "b", + "type": "code", + "outputs": [{"output_type": "stream", "text": "done"}], } - }, - ] + ], + snapshotContent="must not be used", + ), ) - - def open_request(request: Any, *, timeout: float) -> FakeResponse: - calls.append( - ( - request.full_url, - request.method, - request.headers["Authorization"], - request.data, - timeout, - ) - ) - return FakeResponse(next(responses)) - - tokens = iter(["token-1", "token-2", "token-3"]) - sleeps = [] + tokens = iter(["one", "two", "three", "four"]) result = DeepnoteCloudRunner( "notebook-1", token_provider=lambda: next(tokens), - transport=UrllibTransport(open_request), - sleep=sleeps.append, - poll_interval=0.25, - ).run({"limit": 20, "enabled": True, "regions": ["EU"]}) - - assert json.loads(calls[0][3]) == { + session=session(), + clock=clock, + sleep=clock.sleep, + poll_interval=0.5, + storage_mode="readonly", + ).run({"limit": 20, "enabled": False, "regions": ("EU",)}) + assert result.success and result.text() == "done" and result.snapshot is None + assert body(http.calls[0]) == { "notebookId": "notebook-1", "detached": True, - "inputs": {"limit": "20", "enabled": True, "regions": ["EU"]}, + "inputs": {"limit": "20", "enabled": False, "regions": ["EU"]}, + "detachedRunStorageMode": "readonly", } - assert calls[1][0].endswith("/v2/runs/run-1?snapshotDelivery=blocks") - assert [call[2] for call in calls] == [ - "Bearer token-1", - "Bearer token-2", - "Bearer token-3", + assert [c.request.headers["Authorization"] for c in http.calls] == [ + f"Bearer {token}" for token in ("one", "two", "three", "four") ] - assert sleeps == [0.25, 0.25] - assert result.success is True - assert result.text() == "done" + assert clock.sleeps == [0.5, 0.5, 0.5] -def test_cloud_run_reads_sanitized_snapshot_blocks_without_raw_snapshot() -> None: - responses = iter( - [ - {"run": {"runId": "run-1", "status": "pending"}}, - { - "run": { - "runId": "run-1", - "status": "success", - "snapshotBlocks": [ - { - "id": "code-1", - "type": "code", - "outputs": [ - { - "output_type": "execute_result", - "data": { - "application/vnd.deepnote.dataframe.v3+json": { - "columns": [{"name": "revenue"}], - "rows": [{"revenue": 42}], - } - }, - } - ], - "metadata": {"deepnote_table_state": {}}, - }, - { - "id": "agent-1", - "type": "agent", - "outputs": [ - { - "output_type": "display_data", - "data": {"text/markdown": "**Done**"}, - } - ], - "metadata": {}, - }, - ], - } - }, - ] - ) - - def open_request(request: Any, *, timeout: float) -> FakeResponse: - assert timeout == 30 - if request.method == "POST": - assert json.loads(request.data) == { - "notebookId": "notebook-1", - "detached": True, - "inputs": {"region": "EU"}, - } - return FakeResponse(next(responses)) +@pytest.mark.parametrize( + "payload", + [ + {"run": {"runId": "r"}}, + {"run": {"runId": "r", "status": None}}, + {"run": {"runId": "r", "status": []}}, + {"run": {"runId": "r", "status": ""}}, + {"run": {"status": "success"}}, + ], +) +def test_malformed_run_fails_without_polling(http, runner, payload): + add_run(http, payload, create=True) + with pytest.raises(RunnerError, match="run (response|id)"): + runner.run({}) + assert len(http.calls) == 1 - result = DeepnoteCloudRunner( - "notebook-1", - token="token", - transport=UrllibTransport(open_request), - sleep=lambda _delay: None, - ).run({"region": "EU"}) - - assert result.snapshot is None - assert [output.block_id for output in result.outputs] == ["code-1", "agent-1"] - assert [output.block_type for output in result.outputs] == ["code", "agent"] - dataframe = result.first_dataframe() - assert dataframe is not None - assert dataframe.records() == [{"revenue": 42}] - assert result.agent_text() == "**Done**" - - -def test_cloud_run_surfaces_terminal_error() -> None: - def open_request(_request: Any, *, timeout: float) -> FakeResponse: - assert timeout == 30 - return FakeResponse( - { - "run": { - "id": "run-1", - "status": "error", - "error": {"message": "bad input"}, - } - } - ) - result = DeepnoteCloudRunner( - "notebook-1", - token="token", - transport=UrllibTransport(open_request), - sleep=lambda _delay: None, - ).run({}) +@pytest.mark.parametrize( + "snapshot_status", [None, "unavailable", "available", "future"] +) +def test_only_pending_snapshots_are_polled(http, runner, snapshot_status): + add_run( + http, + run_response("error", snapshotStatus=snapshot_status, error="bad input"), + create=True, + ) + result = runner.run({}) + assert not result.success and result.error == "bad input" + assert len(http.calls) == 1 - assert result.success is False - assert result.error == "bad input" +@pytest.mark.parametrize( + "failure", [429, 503, requests.ConnectionError("reset"), requests.Timeout()] +) +def test_transient_get_failure_is_retried(http, runner, failure): + add_run(http, run_response("running"), create=True) + kwargs = {"status": failure} if isinstance(failure, int) else {"body": failure} + http.get("https://api.deepnote.com/v2/runs/run-1?snapshotDelivery=blocks", **kwargs) + add_run(http, run_response(snapshotBlocks=[])) + assert runner.run({}).success + assert len(http.calls) == 3 + + +@pytest.mark.parametrize("status", [400, 401, 403, 404]) +def test_non_transient_poll_failure_is_not_retried(http, runner, status): + add_run(http, run_response("running"), create=True) + http.get( + "https://api.deepnote.com/v2/runs/run-1?snapshotDelivery=blocks", + status=status, + json={"message": "reason"}, + ) + with pytest.raises(RunnerError, match=f"HTTP {status}: reason"): + runner.run({}) + assert len(http.calls) == 2 -def test_cloud_runner_uses_environment_token( - monkeypatch: pytest.MonkeyPatch, -) -> None: - monkeypatch.setenv("DEEPNOTE_TOKEN", "local-token") - def open_request(request: Any, *, timeout: float) -> FakeResponse: - assert request.headers["Authorization"] == "Bearer local-token" - assert timeout == 30 - return FakeResponse({"notebook": {"name": "Revenue", "inputs": []}}) +def test_post_is_never_replayed(http, runner): + http.post("https://api.deepnote.com/v2/runs", body=requests.Timeout()) + with pytest.raises(RunnerError): + runner.run({}) + assert len(http.calls) == 1 - info = DeepnoteCloudRunner( - "notebook-1", transport=UrllibTransport(open_request) - ).info() - assert info.notebook == "Revenue" +def test_poll_retries_have_a_limit(http, runner): + add_run(http, run_response("running"), create=True) + http.get( + "https://api.deepnote.com/v2/runs/run-1?snapshotDelivery=blocks", status=503 + ) + with pytest.raises(RunnerError, match="503"): + runner.run({}) + assert len(http.calls) == 7 -def test_cloud_runner_requires_one_token_source( - monkeypatch: pytest.MonkeyPatch, -) -> None: - with pytest.raises(ValueError, match="not both"): - DeepnoteCloudRunner("notebook-1", token="token", token_provider=lambda: "other") +def test_request_and_sleep_time_count_against_run_deadline(http, clock): + observed = [] - monkeypatch.delenv("DEEPNOTE_TOKEN", raising=False) - with pytest.raises(RunnerError, match="token is required"): - DeepnoteCloudRunner( - "notebook-1", - token="", - transport=UrllibTransport(lambda *_args, **_kwargs: FakeResponse({})), - ).info() + def create(request): + observed.append(request.req_kwargs["timeout"].total) + clock.now += 3 + return 200, {}, json.dumps(run_response("running")) + def poll(request): + observed.append(request.req_kwargs["timeout"].total) + clock.now += 4 + return 200, {}, json.dumps(run_response("running")) -def test_cloud_run_retries_transient_poll_failures() -> None: - responses = iter( - [ - {"run": {"runId": "run-1", "status": "pending"}}, - HTTPError("http://api", 503, "Unavailable", {}, io.BytesIO(b"{}")), - URLError("connection reset"), - {"run": {"runId": "run-1", "status": "success", "snapshotBlocks": []}}, - ] + http.add_callback( + responses.POST, "https://api.deepnote.com/v2/runs", callback=create ) + http.add_callback( + responses.GET, + "https://api.deepnote.com/v2/runs/run-1?snapshotDelivery=blocks", + callback=poll, + ) + runner = DeepnoteCloudRunner( + "n", + token="t", + session=session(), + timeout=10, + poll_interval=2, + clock=clock, + sleep=clock.sleep, + ) + with pytest.raises(RunnerError, match="10 seconds"): + runner.run({}) + assert observed == [10, 5] + assert clock.now == 10 + assert clock.sleeps == [2, 1] + + +def test_snapshot_deadline_counts_slow_requests_and_caps_timeout(http, clock): + add_run(http, run_response(snapshotStatus="pending"), create=True) + timeouts = [] + + def poll(request): + timeouts.append(request.req_kwargs["timeout"].total) + clock.now += 4 + return 200, {}, json.dumps(run_response(snapshotStatus="pending")) + + http.add_callback( + responses.GET, + "https://api.deepnote.com/v2/runs/run-1?snapshotDelivery=blocks", + callback=poll, + ) + runner = DeepnoteCloudRunner( + "n", + token="t", + session=session(), + snapshot_timeout=5, + poll_interval=1, + clock=clock, + sleep=clock.sleep, + ) + assert runner.run({}).snapshot_status == "pending" + assert timeouts == [4] + assert clock.now == 5 - def open_request(_request: Any, *, timeout: float) -> FakeResponse: - response = next(responses) - if isinstance(response, Exception): - raise response - return FakeResponse(response) - - result = DeepnoteCloudRunner( - "notebook-1", - token="token", - transport=UrllibTransport(open_request), - sleep=lambda _delay: None, - ).run({}) - assert result.success is True +def test_no_request_starts_after_snapshot_deadline(http, clock): + add_run(http, run_response(snapshotStatus="pending"), create=True) + runner = DeepnoteCloudRunner( + "n", + token="t", + session=session(), + snapshot_timeout=0.1, + poll_interval=2, + clock=clock, + sleep=clock.sleep, + ) + assert runner.run({}).outputs == () + assert clock.sleeps == [0.1] + assert len(http.calls) == 1 -def test_cloud_run_raises_poll_failures_that_are_not_transient() -> None: - responses = iter( - [ - {"run": {"runId": "run-1", "status": "pending"}}, - HTTPError("http://api", 403, "Forbidden", {}, io.BytesIO(b"{}")), - ] +def test_unknown_status_is_bounded_by_deadline(http, clock): + add_run(http, run_response("future"), create=True) + add_run(http, run_response("future")) + runner = DeepnoteCloudRunner( + "n", + token="t", + session=session(), + timeout=2, + poll_interval=1, + clock=clock, + sleep=clock.sleep, ) + with pytest.raises(RunnerError, match="2 seconds"): + runner.run({}) + assert len(http.calls) == 2 - def open_request(_request: Any, *, timeout: float) -> FakeResponse: - response = next(responses) - if isinstance(response, Exception): - raise response - return FakeResponse(response) - - with pytest.raises(RunnerError, match="HTTP 403"): - DeepnoteCloudRunner( - "notebook-1", - token="token", - transport=UrllibTransport(open_request), - sleep=lambda _delay: None, - ).run({}) +@pytest.mark.parametrize( + "kwargs", + [ + {"timeout": 0}, + {"timeout": float("inf")}, + {"snapshot_timeout": -1}, + {"poll_interval": 0}, + ], +) +def test_invalid_timeouts_rejected(kwargs): + with pytest.raises(ValueError): + DeepnoteCloudRunner("n", **kwargs) -def test_cloud_run_stops_retrying_after_repeated_transient_failures() -> None: - calls = [] - def open_request(request: Any, *, timeout: float) -> FakeResponse: - calls.append(request.method) - if request.method == "POST": - return FakeResponse({"run": {"runId": "run-1", "status": "pending"}}) - raise URLError("connection reset") +def test_run_id_is_quoted(http, runner): + add_run(http, run_response("running", runId="a/b?x"), create=True) + http.get( + "https://api.deepnote.com/v2/runs/a%2Fb%3Fx?snapshotDelivery=blocks", + json=run_response(), + ) + assert runner.run({}).success - with pytest.raises(RunnerError, match="connection reset"): - DeepnoteCloudRunner( - "notebook-1", - token="token", - transport=UrllibTransport(open_request), - sleep=lambda _delay: None, - ).run({}) - assert calls == ["POST"] + ["GET"] * 6 +def test_custom_credentials_supply_origin_and_receive_budget(http): + budgets = [] + def credentials(*, timeout): + budgets.append(timeout) + return ApiCredentials("token", "https://api.example") -def test_cloud_run_waits_for_a_snapshot_that_lags_the_terminal_status() -> None: - responses = iter( - [ - {"run": {"runId": "run-1", "status": "success"}}, - {"run": {"runId": "run-1", "status": "success"}}, - { - "run": { - "runId": "run-1", - "status": "success", - "snapshotBlocks": [ - { - "id": "code-1", - "type": "code", - "outputs": [{"output_type": "stream", "text": "done"}], - } - ], - } - }, - ] + http.get("https://api.example/v2/notebooks/n", json={"notebook": {"name": "N"}}) + assert ( + DeepnoteCloudRunner("n", credentials=credentials, session=session()) + .info() + .notebook + == "N" ) - sleeps = [] + assert budgets == [30] - result = DeepnoteCloudRunner( - "notebook-1", - token="token", - transport=UrllibTransport( - lambda _request, *, timeout: FakeResponse(next(responses)) - ), - sleep=sleeps.append, - poll_interval=0.5, - ).run({}) - assert sleeps == [0.5, 0.5] - assert result.text() == "done" +def test_credentials_and_token_are_mutually_exclusive(): + with pytest.raises(ValueError): + DeepnoteCloudRunner("n", token="t", credentials=lambda **_: ApiCredentials("t")) -def test_info_skips_inputs_without_a_name_or_a_known_type() -> None: - def open_request(_request: Any, *, timeout: float) -> FakeResponse: - return FakeResponse( - { - "inputs": [ - {"type": "input-text"}, - {"variableName": "orphan"}, - {"variableName": "mystery", "type": "input-unknown"}, - {"variableName": "region", "type": "input-text"}, - ] - } - ) +def test_local_runner_contract(http): + http.get( + "http://127.0.0.1:8787/api/info", + json={"notebook": "N", "runTarget": "local", "inputs": []}, + ) + http.post( + "http://127.0.0.1:8787/api/run", + json={"target": "local", "success": True, "outputs": []}, + ) + runner = DeepnoteLocalRunner(session=session()) + assert runner.info().notebook == "N" + assert runner.run({"n": 3}).success + assert body(http.calls[1]) == {"inputs": {"n": 3}} - info = DeepnoteRunner(transport=UrllibTransport(open_request)).info() - assert info.inputs == (InputBlock("region", "input-text", None),) +@pytest.mark.parametrize("value", [None, {}, set(), b"x"]) +def test_invalid_input_is_rejected_before_sending(http, runner, value): + with pytest.raises(ValueError): + runner.run({"n": value}) + assert len(http.calls) == 0 -def test_cloud_run_retries_a_dropped_connection() -> None: - responses = iter( - [ - {"run": {"runId": "run-1", "status": "pending"}}, - RemoteDisconnected("Remote end closed connection without response"), - {"run": {"runId": "run-1", "status": "success", "snapshotBlocks": []}}, - ] +def test_error_page_does_not_leak_into_exception(http): + http.get( + "http://127.0.0.1:8787/api/info", + body="proxy internals", + status=502, ) + with pytest.raises(RunnerError) as exc: + DeepnoteLocalRunner(session=session()).info() + assert "proxy internals" not in str(exc.value) - def open_request(_request: Any, *, timeout: float) -> FakeResponse: - response = next(responses) - if isinstance(response, Exception): - raise response - return FakeResponse(response) - result = DeepnoteCloudRunner( - "notebook-1", - token="token", - transport=UrllibTransport(open_request), - sleep=lambda _delay: None, - ).run({}) +def test_real_http_redirect_never_receives_credentials(): + received = [] - assert result.success is True + class Handler(BaseHTTPRequestHandler): + def do_GET(self): + received.append( + (self.server.server_port, self.headers.get("Authorization")) + ) + self.send_response(302) + self.send_header("Location", f"http://127.0.0.1:{other.server_port}/") + self.end_headers() + def log_message(self, *_args): + pass -def test_cloud_run_does_not_wait_for_a_snapshot_that_will_not_come() -> None: - sleeps = [] - result = DeepnoteCloudRunner( - "notebook-1", - token="token", - transport=UrllibTransport( - lambda _request, *, timeout: FakeResponse( - { - "run": { - "runId": "run-1", - "status": "error", - "snapshotStatus": "unavailable", - } - } - ) - ), - sleep=sleeps.append, - ).run({}) - - assert sleeps == [] - assert result.snapshot_status == "unavailable" - - -def test_cloud_info_keeps_select_options_and_slider_bounds() -> None: - def open_request(_request: Any, *, timeout: float) -> FakeResponse: - return FakeResponse( - { - "notebook": { - "name": "Revenue", - "inputs": [ - { - "name": "region", - "type": "input-select", - "value": "EU", - "options": ["EU", "US"], - "multiple": True, - }, - { - "name": "limit", - "type": "input-slider", - "value": "5", - "min": 1, - "max": 9, - "step": 2, - }, - ], - } - } - ) + api = HTTPServer(("127.0.0.1", 0), Handler) + other = HTTPServer(("127.0.0.1", 0), Handler) + for server in (api, other): + threading.Thread(target=server.serve_forever, daemon=True).start() + try: + with pytest.raises(RunnerError, match="Refused a redirect"): + DeepnoteCloudRunner( + "n", + token="t", + session=session(), + base_url=f"http://127.0.0.1:{api.server_port}", + ).info() + finally: + for server in (api, other): + server.shutdown() + server.server_close() + assert received == [(api.server_port, "Bearer t")] - info = DeepnoteCloudRunner( - "notebook-1", token="token", transport=UrllibTransport(open_request) - ).info() - assert info.inputs == ( - InputBlock("region", "input-select", "EU", options=("EU", "US"), multiple=True), - InputBlock("limit", "input-slider", "5", min=1, max=9, step=2), +def test_runner_info_requires_matching_input_names_and_types() -> None: + info = RunnerInfo( + notebook="Revenue", + inputs=(InputBlock("region", "input-select", "All"),), + run_target="cloud", ) + assert info.matches_inputs([InputBlock("region", "input-select", "Europe")]) + assert not info.matches_inputs([InputBlock("market", "input-select", "Europe")]) + assert not info.matches_inputs([InputBlock("region", "input-text", "Europe")]) -def test_runner_info_ignores_repeated_input_names() -> None: + +def test_runner_info_rejects_repeated_input_names() -> None: info = RunnerInfo( notebook="Revenue", inputs=(InputBlock("region", "input-text", "EU"),), run_target="cloud", ) - assert info.accepts_inputs( + assert not info.matches_inputs( [ InputBlock("region", "input-text", "EU"), InputBlock("region", "input-text", "US"), @@ -552,32 +431,6 @@ def test_runner_info_ignores_repeated_input_names() -> None: ) -def test_cloud_run_sends_the_requested_storage_mode() -> None: - bodies = [] - - def open_request(request: Any, *, timeout: float) -> FakeResponse: - bodies.append(json.loads(request.data)) - return FakeResponse( - {"run": {"runId": "run-1", "status": "success", "snapshotBlocks": []}} - ) - - DeepnoteCloudRunner( - "notebook-1", - token="token", - storage_mode="readonly", - transport=UrllibTransport(open_request), - ).run({}) - - assert bodies == [ - { - "notebookId": "notebook-1", - "detached": True, - "inputs": {}, - "detachedRunStorageMode": "readonly", - } - ] - - @pytest.mark.parametrize( "changed", [ @@ -595,10 +448,10 @@ def test_runner_info_rejects_a_select_that_takes_other_values( run_target="cloud", ) - assert info.accepts_inputs( + assert info.matches_inputs( [InputBlock("region", "input-select", "US", options=("EU", "US"))] ) - assert not info.accepts_inputs([changed]) + assert not info.matches_inputs([changed]) def test_runner_info_ignores_select_options_filled_from_a_variable() -> None: @@ -608,7 +461,7 @@ def test_runner_info_ignores_select_options_filled_from_a_variable() -> None: run_target="cloud", ) - assert info.accepts_inputs( + assert info.matches_inputs( [ InputBlock( "region", @@ -628,182 +481,56 @@ def test_runner_info_compares_slider_bounds_with_the_defaults_filled_in() -> Non run_target="cloud", ) - assert info.accepts_inputs([InputBlock("limit", "input-slider", "20")]) - assert not info.accepts_inputs([InputBlock("limit", "input-slider", "20", max=50)]) - - -class FakeTransport: - def __init__(self, payload: Any): - self.payload = payload - self.calls: list[tuple[str, str, Any]] = [] - - def request_json( - self, method: str, url: str, *, headers: Any, body: Any, timeout: float - ) -> Any: - self.calls.append((method, url, headers)) - return self.payload - - -def test_cloud_runner_uses_injected_credentials_and_transport() -> None: - transport = FakeTransport({"notebook": {"name": "Revenue", "inputs": []}}) - - info = DeepnoteCloudRunner( - "notebook-1", - credentials=lambda: ApiCredentials("token", "https://api.example"), - transport=transport, - ).info() - - assert info.notebook == "Revenue" - assert transport.calls == [ - ( - "GET", - "https://api.example/v2/notebooks/notebook-1", - {"Authorization": "Bearer token"}, - ) - ] - - -def test_cloud_runner_rejects_credentials_together_with_a_token() -> None: - with pytest.raises(ValueError, match="not both"): - DeepnoteCloudRunner( - "notebook-1", - token="token", - credentials=lambda: ApiCredentials("other-token"), - ) - - -def test_default_transport_refuses_a_redirect_to_another_origin() -> None: - received = [] - - class Handler(BaseHTTPRequestHandler): - def do_GET(self) -> None: - received.append((self.server.server_port, self.headers["Authorization"])) - self.send_response(302) - self.send_header("Location", f"http://127.0.0.1:{other.server_port}/") - self.end_headers() - - def log_message(self, *_args: Any) -> None: - return None - - api = HTTPServer(("127.0.0.1", 0), Handler) - other = HTTPServer(("127.0.0.1", 0), Handler) - for server in (api, other): - threading.Thread(target=server.serve_forever, daemon=True).start() - try: - with pytest.raises(RunnerError, match="HTTP 302: Refused a redirect"): - DeepnoteCloudRunner( - "notebook-1", - token="token", - base_url=f"http://127.0.0.1:{api.server_port}", - ).info() - finally: - for server in (api, other): - server.shutdown() - server.server_close() - - assert received == [(api.server_port, "Bearer token")] - - -def test_http_error_keeps_a_body_that_is_not_json_out_of_the_message() -> None: - def open_request(*_: Any, **__: Any) -> FakeResponse: - raise HTTPError( - "http://runner/api/run", - 502, - "Bad Gateway", - {}, - io.BytesIO(b"proxy internals"), - ) - - with pytest.raises(RunnerError) as raised: - DeepnoteRunner("http://runner", transport=UrllibTransport(open_request)).run({}) + assert info.matches_inputs([InputBlock("limit", "input-slider", "20")]) + assert not info.matches_inputs([InputBlock("limit", "input-slider", "20", max=50)]) - assert str(raised.value) == "http://runner returned HTTP 502: Bad Gateway" - -def test_cloud_run_raises_when_the_run_outlasts_the_timeout() -> None: - transport = FakeTransport({"run": {"runId": "run-1", "status": "running"}}) - - with pytest.raises(RunnerError, match="run-1 did not finish in 0 seconds"): - DeepnoteCloudRunner( - "notebook-1", token="token", timeout=0, transport=transport - ).run({}) - - -def test_cloud_run_stops_waiting_for_a_snapshot_after_the_snapshot_timeout() -> None: - transport = FakeTransport( - {"run": {"runId": "run-1", "status": "success", "snapshotStatus": "pending"}} +@pytest.mark.parametrize( + "changed", + [ + InputBlock("n", "input-slider", "5", min=1, max=10, step=1), + InputBlock("n", "input-slider", "5", min=0, max=9, step=1), + InputBlock("n", "input-slider", "5", min=0, max=10, step=2), + ], +) +def test_input_match_detects_slider_constraints(changed): + info = RunnerInfo( + "N", (InputBlock("n", "input-slider", "5", min=0, max=10, step=1),), "cloud" ) - sleeps: list[float] = [] + assert not info.matches_inputs([changed]) - result = DeepnoteCloudRunner( - "notebook-1", - token="token", - snapshot_timeout=1, - poll_interval=0.5, - transport=transport, - sleep=sleeps.append, - ).run({}) - - assert sleeps == [0.5, 0.5] - assert result.success is True - assert result.snapshot_status == "pending" - assert result.outputs == () +def test_credential_exchange_time_reduces_http_budget(http, clock): + def credentials(*, timeout): + assert timeout == 4 + clock.now += 3 + return ApiCredentials("token") -def test_cloud_run_honors_a_snapshot_timeout_shorter_than_the_poll_interval() -> None: - transport = FakeTransport( - {"run": {"runId": "run-1", "status": "success", "snapshotStatus": "pending"}} + http.post("https://api.deepnote.com/v2/runs", json=run_response(snapshotBlocks=[])) + runner = DeepnoteCloudRunner( + "n", + credentials=credentials, + timeout=4, + clock=clock, + sleep=clock.sleep, + session=session(), ) - sleeps: list[float] = [] - - DeepnoteCloudRunner( - "notebook-1", - token="token", - snapshot_timeout=1, - poll_interval=2, - transport=transport, - sleep=sleeps.append, - ).run({}) + assert runner.run({}).success + assert http.calls[0].request.req_kwargs["timeout"].total == 1 - assert sleeps == [1] +def test_expired_credential_budget_does_not_send_request(http, clock): + def credentials(*, timeout): + clock.now += timeout + return ApiCredentials("token") -@pytest.mark.parametrize( - "run", - [{"runId": "run-1"}, {"runId": "run-1", "status": "cancelled"}], - ids=["missing", "unknown"], -) -def test_cloud_run_rejects_a_run_without_a_known_status(run: dict[str, str]) -> None: - sleeps: list[float] = [] - - with pytest.raises(RunnerError, match="run-1 has an unknown status"): + with pytest.raises(RunnerError, match="exhausted"): DeepnoteCloudRunner( - "notebook-1", - token="token", - transport=FakeTransport({"run": run}), - sleep=sleeps.append, + "n", + credentials=credentials, + timeout=4, + clock=clock, + sleep=clock.sleep, + session=session(), ).run({}) - - assert sleeps == [] - - -def test_cloud_run_sends_a_tuple_as_a_list_and_rejects_a_missing_value() -> None: - bodies = [] - - class RecordingTransport(FakeTransport): - def request_json(self, method: str, url: str, **kwargs: Any) -> Any: - bodies.append(kwargs["body"]) - return self.payload - - runner = DeepnoteCloudRunner( - "notebook-1", - token="token", - transport=RecordingTransport( - {"run": {"runId": "run-1", "status": "success", "snapshotBlocks": []}} - ), - ) - runner.run({"regions": ("EU", "US")}) - - assert bodies[0]["inputs"] == {"regions": ["EU", "US"]} - with pytest.raises(ValueError, match='Input "region" has a NoneType value'): - runner.run({"region": None}) + assert not http.calls diff --git a/tests/unit/test_notebooks_yaml_loader.py b/tests/unit/test_notebooks_yaml_loader.py index cd67fbee..0c68286d 100644 --- a/tests/unit/test_notebooks_yaml_loader.py +++ b/tests/unit/test_notebooks_yaml_loader.py @@ -85,3 +85,11 @@ def test_the_same_key_may_repeat_in_separate_mappings(load_yaml: Any) -> None: def test_mapping_tag_on_another_node_is_a_yaml_error(load_yaml: Any) -> None: with pytest.raises(yaml.YAMLError, match="expected a mapping node"): load_yaml("!!map [1, 2]") + + +@pytest.mark.parametrize( + "content", ["x: !!timestamp invalid", "x: !!binary [1]", "x: !!bool []"] +) +def test_malformed_explicit_tags_raise_yaml_error(load_yaml, content): + with pytest.raises(yaml.YAMLError): + load_yaml(content) diff --git a/tests/unit/test_streamlit.py b/tests/unit/test_streamlit.py index 819bc74f..06e81b53 100644 --- a/tests/unit/test_streamlit.py +++ b/tests/unit/test_streamlit.py @@ -98,8 +98,8 @@ def exists_side_effect(path: str) -> bool: assert mock_logger.warning.call_count == 2 assert mock_venv.start_server.call_count == 1 - def test_exports_a_valid_app_id_to_the_app_process(self): - """The app ID reaches the process only when it is a UUID, since it enters a shell command.""" + def test_passes_app_id_as_environment_data(self): + """App IDs are data, never shell syntax; validation belongs to the SDK.""" apps = [ { "id": "11111111-2222-3333-4444-555555555555", @@ -116,9 +116,24 @@ def test_exports_a_valid_app_id_to_the_app_process(self): ): start_streamlit_servers(mock_venv, MagicMock(spec=logging.Logger)) - commands = [call[0][0] for call in mock_venv.start_server.call_args_list] - assert commands[0].startswith( - "DEEPNOTE_STREAMLIT_APP_ID=11111111-2222-3333-4444-555555555555 " - "streamlit run '/work/a/app.py' " - ) - assert commands[1].startswith("streamlit run '/work/b/app.py' ") + calls = mock_venv.start_server.call_args_list + assert calls[0].args[0].startswith("streamlit run /work/a/app.py ") + assert calls[1].args[0].startswith("streamlit run /work/b/app.py ") + assert calls[0].kwargs["env"] == {"DEEPNOTE_STREAMLIT_APP_ID": apps[0]["id"]} + assert calls[1].kwargs["env"] == {"DEEPNOTE_STREAMLIT_APP_ID": apps[1]["id"]} + + def test_missing_app_id_warns_and_still_marks_process_as_hosted(self): + app = {"entrypoint": "app.py", "port": "8501"} + venv = MagicMock() + logger = MagicMock(spec=logging.Logger) + with ( + patch( + "installer.module.streamlit.fetch_streamlit_apps", return_value=[app] + ), + patch("installer.module.streamlit.os.path.exists", return_value=True), + ): + start_streamlit_servers(venv, logger) + logger.warning.assert_called_once() + assert venv.start_server.call_args.kwargs["env"] == { + "DEEPNOTE_STREAMLIT_APP_ID": "" + } diff --git a/tests/unit/test_virtual_environment.py b/tests/unit/test_virtual_environment.py index 841d0d91..54771ad9 100644 --- a/tests/unit/test_virtual_environment.py +++ b/tests/unit/test_virtual_environment.py @@ -98,3 +98,33 @@ def test_import_package_bundle_condition_env_and_priority_mutually_exclusive( condition_env="SOME_ENV_VAR", priority=True, ) + + +def test_server_environment_is_passed_as_data_and_inherits_parent( + tmp_path, monkeypatch +): + import json + import shlex + import sys + + from installer.module.virtual_environment import VirtualEnvironment + + monkeypatch.setenv("TOOLKIT_TEST_PARENT", "inherited") + venv_path = tmp_path / "venv" + (venv_path / "bin").mkdir(parents=True) + (venv_path / "bin" / "activate").write_text("") + result = tmp_path / "result.json" + script = tmp_path / "child.py" + script.write_text( + "import json, os\n" + f"with open({str(result)!r}, 'w') as f:\n" + " json.dump([os.environ['DEEPNOTE_STREAMLIT_APP_ID'], " + "os.environ['TOOLKIT_TEST_PARENT']], f)\n" + ) + app_id = "x; echo must-not-be-executed" + server = VirtualEnvironment(venv_path).start_server( + f"{shlex.quote(sys.executable)} {shlex.quote(str(script))}", + env={"DEEPNOTE_STREAMLIT_APP_ID": app_id}, + ) + assert server.wait(timeout=10) == 0 + assert json.loads(result.read_text()) == [app_id, "inherited"] From bc2d7d15d6556346a2aa3afee22f8c309784d665 Mon Sep 17 00:00:00 2001 From: Wojtek <6819634+voyti@users.noreply.github.com> Date: Mon, 21 Sep 2026 19:15:35 +0200 Subject: [PATCH 29/43] fix(streamlit): isolate AppTest state and redact credential validation errors --- deepnote_toolkit/streamlit/auth.py | 9 ++++++--- tests/unit/conftest.py | 11 +++++++++++ tests/unit/test_deepnote_streamlit_auth.py | 9 +++++++++ tests/unit/test_deepnote_streamlit_cloud_runner.py | 8 ++++---- tests/unit/test_deepnote_streamlit_widgets.py | 12 ++++-------- 5 files changed, 34 insertions(+), 15 deletions(-) diff --git a/deepnote_toolkit/streamlit/auth.py b/deepnote_toolkit/streamlit/auth.py index 6ffd5331..661df38e 100644 --- a/deepnote_toolkit/streamlit/auth.py +++ b/deepnote_toolkit/streamlit/auth.py @@ -83,7 +83,9 @@ def current_user_api_credentials( "Could not resolve a Deepnote Streamlit app ID from the request host." ) - if not re.fullmatch(_APP_ID, resolved_app_id, re.IGNORECASE): + if not isinstance(resolved_app_id, str) or not re.fullmatch( + _APP_ID, resolved_app_id, re.IGNORECASE + ): raise CurrentUserApiTokenError("app_id must be a UUID.") resolved_app_id = resolved_app_id.lower() @@ -132,10 +134,11 @@ def current_user_api_credentials( ) except RunnerError as error: raise CurrentUserApiTokenError(str(error), transient=error.transient) from error - except ValidationError as error: + except ValidationError: + # Pydantic validation errors can contain the bearer token as input data. raise CurrentUserApiTokenError( "Viewer API-token response is missing or has invalid required fields." - ) from error + ) from None finally: if owned_session: http.close() diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index 5f9179d8..4196a95c 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -1,6 +1,7 @@ """Pytest configuration and fixtures for unit tests.""" import os +import sys import tempfile from typing import Generator @@ -65,3 +66,13 @@ def test_log_directory() -> Generator[str, None, None]: os.environ.pop("DEEPNOTE_PATHS__LOG_DIR", None) else: os.environ["DEEPNOTE_PATHS__LOG_DIR"] = original_log_dir + + +@pytest.fixture +def streamlit_app_test(monkeypatch): + """Restore the main module that Streamlit replaces while executing an app.""" + pytest.importorskip("streamlit") + from streamlit.testing.v1 import AppTest + + monkeypatch.setitem(sys.modules, "__main__", sys.modules["__main__"]) + return AppTest diff --git a/tests/unit/test_deepnote_streamlit_auth.py b/tests/unit/test_deepnote_streamlit_auth.py index f961c1e8..807680df 100644 --- a/tests/unit/test_deepnote_streamlit_auth.py +++ b/tests/unit/test_deepnote_streamlit_auth.py @@ -177,3 +177,12 @@ def get_ctx(*, suppress_warning): ) assert auth._read_streamlit_session_state() is None assert calls == [True] + + +def test_credential_validation_traceback_does_not_expose_bearer(http, state): + import traceback + + http.post(TOKEN_URL, json=payload(token={"secret": "private-token"})) + with pytest.raises(auth.CurrentUserApiTokenError) as exc: + credentials(session()) + assert "private-token" not in "".join(traceback.format_exception(exc.value)) diff --git a/tests/unit/test_deepnote_streamlit_cloud_runner.py b/tests/unit/test_deepnote_streamlit_cloud_runner.py index 7412c75f..6d332079 100644 --- a/tests/unit/test_deepnote_streamlit_cloud_runner.py +++ b/tests/unit/test_deepnote_streamlit_cloud_runner.py @@ -149,9 +149,9 @@ def test_transient_exchange_failure_during_poll_is_retried(context, http, monkey assert len(http.calls) == 5 -def test_real_streamlit_script_and_worker_keep_viewer_identity(monkeypatch, http): - pytest.importorskip("streamlit") - from streamlit.testing.v1 import AppTest +def test_real_streamlit_script_and_worker_keep_viewer_identity( + monkeypatch, http, streamlit_app_test +): monkeypatch.setenv("DEEPNOTE_STREAMLIT_APP_ID", APP_ID) monkeypatch.setenv("DEEPNOTE_TOKEN", "owner-token") @@ -189,7 +189,7 @@ def worker(): thread.join(timeout=5) st.session_state["worker_errors"] = errors - at = AppTest.from_function(app).run() + at = streamlit_app_test.from_function(app).run() assert not at.exception assert at.session_state["success"] is True assert "No viewer request" in at.session_state["worker_errors"][0] diff --git a/tests/unit/test_deepnote_streamlit_widgets.py b/tests/unit/test_deepnote_streamlit_widgets.py index 425cc03f..53776c39 100644 --- a/tests/unit/test_deepnote_streamlit_widgets.py +++ b/tests/unit/test_deepnote_streamlit_widgets.py @@ -183,9 +183,7 @@ def test_relative_date_ranges_resolve_to_concrete_dates( } -def test_render_inputs_runs_on_real_streamlit_widgets() -> None: - pytest.importorskip("streamlit") - from streamlit.testing.v1 import AppTest +def test_render_inputs_runs_on_real_streamlit_widgets(streamlit_app_test) -> None: def app() -> None: import streamlit as st @@ -213,7 +211,7 @@ def app() -> None: ] ) - at = AppTest.from_function(app).run() + at = streamlit_app_test.from_function(app).run() assert not at.exception assert at.session_state["values"] == { @@ -320,9 +318,7 @@ def test_invalid_slider_configuration_is_reported(value, min_value, max_value, s ) -def test_real_widgets_keep_falsey_defaults_and_require_selection(): - pytest.importorskip("streamlit") - from streamlit.testing.v1 import AppTest +def test_real_widgets_keep_falsey_defaults_and_require_selection(streamlit_app_test): def app(): import streamlit as st @@ -339,7 +335,7 @@ def app(): ] ) - at = AppTest.from_function(app).run() + at = streamlit_app_test.from_function(app).run() assert not at.exception assert at.text_input[0].value == "0" and at.text_area[0].value == "False" assert "choice" not in at.session_state["values"] From 886d00882089d0f3d485f44ded05ebb0268a25e0 Mon Sep 17 00:00:00 2001 From: Wojtek <6819634+voyti@users.noreply.github.com> Date: Mon, 21 Sep 2026 19:20:03 +0200 Subject: [PATCH 30/43] test(notebooks): cover snapshot failures and malformed HTTP responses --- deepnote_toolkit/notebooks/yaml_loader.py | 5 ++-- docs/streamlit-apps.md | 5 ++-- tests/unit/test_notebooks_runners.py | 34 +++++++++++++++++++++++ 3 files changed, 40 insertions(+), 4 deletions(-) diff --git a/deepnote_toolkit/notebooks/yaml_loader.py b/deepnote_toolkit/notebooks/yaml_loader.py index e5aed9ec..91a3d532 100644 --- a/deepnote_toolkit/notebooks/yaml_loader.py +++ b/deepnote_toolkit/notebooks/yaml_loader.py @@ -11,11 +11,12 @@ class _CoreSchemaLoader(_BaseLoader): # type: ignore[misc,valid-type] - """A safe loader that resolves plain scalars by the YAML 1.2 core schema. + """A safe loader for the scalar conventions used by `.deepnote` files. `.deepnote` files are written as YAML 1.2, where `No`, `on`, `12:30` and `2026-08-17` are strings. PyYAML's YAML 1.1 rules read them as booleans, - numbers and dates. + numbers and dates. Leading-zero scalars intentionally remain strings, unlike + the core schema. Merge keys are treated as literal keys, not YAML 1.1 merges. """ yaml_implicit_resolvers: dict[str, Any] = {} diff --git a/docs/streamlit-apps.md b/docs/streamlit-apps.md index 2ab227b1..dcfa35c2 100644 --- a/docs/streamlit-apps.md +++ b/docs/streamlit-apps.md @@ -22,7 +22,7 @@ if st.button("Run"): elif result.snapshot_status == "pending": st.info("The run finished, but its outputs are not available yet.") elif (table := result.first_dataframe()) is not None: - st.dataframe(table.records()) + st.dataframe(table.records(include_index=False)) else: st.write(result.text()) except RunnerError as error: @@ -63,7 +63,8 @@ without a valid saved choice starts empty. Unselected single selects and partial selected date ranges are omitted from the returned dictionary; disable your Run button until required fields are present. An omitted input uses the notebook's value according to the API. Stale multi-select choices produce a warning. Invalid -slider bounds/defaults and duplicate variable names raise `ValueError`. +slider bounds/defaults and duplicate variable names raise `ValueError`. File inputs +render as text paths; this helper does not upload files. `runner.info().matches_inputs(document.inputs)` compares static input definitions: unique names, types, single/multiple selection, options, and slider bounds/steps. diff --git a/tests/unit/test_notebooks_runners.py b/tests/unit/test_notebooks_runners.py index 3fcc4daa..f8723e57 100644 --- a/tests/unit/test_notebooks_runners.py +++ b/tests/unit/test_notebooks_runners.py @@ -534,3 +534,37 @@ def credentials(*, timeout): session=session(), ).run({}) assert not http.calls + + +@pytest.mark.parametrize("status", [403, 503]) +def test_snapshot_poll_error_policy(http, runner, status): + add_run(http, run_response(snapshotStatus="pending"), create=True) + http.get( + "https://api.deepnote.com/v2/runs/run-1?snapshotDelivery=blocks", status=status + ) + if status == 403: + with pytest.raises(RunnerError, match="403"): + runner.run({}) + assert len(http.calls) == 2 + else: + add_run(http, run_response(snapshotStatus="available", snapshotBlocks=[])) + assert runner.run({}).snapshot_status == "available" + assert len(http.calls) == 3 + + +@pytest.mark.parametrize("body", ["not-json", "[]", "null"]) +def test_invalid_json_response_is_a_runner_error(http, body): + http.get("http://127.0.0.1:8787/api/info", body=body) + with pytest.raises(RunnerError, match="invalid JSON|non-object"): + DeepnoteLocalRunner(session=session()).info() + + +def test_same_origin_redirect_is_also_refused(http, runner): + http.post( + "https://api.deepnote.com/v2/runs", + status=307, + headers={"Location": "https://api.deepnote.com/other"}, + ) + with pytest.raises(RunnerError, match="Refused a redirect"): + runner.run({}) + assert len(http.calls) == 1 From 35cef2eba29898e59008edc8cd71566dbe5a54f8 Mon Sep 17 00:00:00 2001 From: Wojtek <6819634+voyti@users.noreply.github.com> Date: Mon, 21 Sep 2026 19:24:01 +0200 Subject: [PATCH 31/43] fix(notebooks): stop expired polls before credential lookup --- deepnote_toolkit/notebooks/api_client.py | 2 ++ tests/unit/test_notebooks_runners.py | 12 ++++++++++++ 2 files changed, 14 insertions(+) diff --git a/deepnote_toolkit/notebooks/api_client.py b/deepnote_toolkit/notebooks/api_client.py index 71998e34..fe00be83 100644 --- a/deepnote_toolkit/notebooks/api_client.py +++ b/deepnote_toolkit/notebooks/api_client.py @@ -129,6 +129,8 @@ def _request( if timeout is None else min(timeout, self._request_timeout) ) + if budget <= 0: + raise RunnerError("API request deadline expired", transient=True) deadline = self._clock() + budget credentials = self._credentials(timeout=budget) remaining = deadline - self._clock() diff --git a/tests/unit/test_notebooks_runners.py b/tests/unit/test_notebooks_runners.py index f8723e57..17813820 100644 --- a/tests/unit/test_notebooks_runners.py +++ b/tests/unit/test_notebooks_runners.py @@ -568,3 +568,15 @@ def test_same_origin_redirect_is_also_refused(http, runner): with pytest.raises(RunnerError, match="Refused a redirect"): runner.run({}) assert len(http.calls) == 1 + + +def test_exhausted_poll_budget_does_not_even_fetch_credentials(http): + from deepnote_toolkit.notebooks.api_client import DeepnoteApiClient + + def credentials(*, timeout): + pytest.fail("Expired requests must not fetch credentials") + + client = DeepnoteApiClient(credentials, session=session()) + with pytest.raises(RunnerError, match="deadline expired"): + client.get_run("r", timeout=0) + assert not http.calls From 170a36ae22ae6cff3d392927fe46a9aa330ccd3b Mon Sep 17 00:00:00 2001 From: Wojtek <6819634+voyti@users.noreply.github.com> Date: Mon, 21 Sep 2026 19:29:40 +0200 Subject: [PATCH 32/43] fix(streamlit): keep missing select defaults unselected --- deepnote_toolkit/streamlit/widgets.py | 2 +- tests/unit/test_deepnote_streamlit_widgets.py | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/deepnote_toolkit/streamlit/widgets.py b/deepnote_toolkit/streamlit/widgets.py index 028d8aaa..2ac508b0 100644 --- a/deepnote_toolkit/streamlit/widgets.py +++ b/deepnote_toolkit/streamlit/widgets.py @@ -68,7 +68,7 @@ def _render_one(container: Any, input_block: InputBlock, label: str, key: str) - return container.multiselect(label, options, default=defaults, key=key) index = ( options.index(str(input_block.value)) - if str(input_block.value) in options + if input_block.value is not None and str(input_block.value) in options else None ) return container.selectbox(label, options, index=index, key=key) diff --git a/tests/unit/test_deepnote_streamlit_widgets.py b/tests/unit/test_deepnote_streamlit_widgets.py index 53776c39..2df63e67 100644 --- a/tests/unit/test_deepnote_streamlit_widgets.py +++ b/tests/unit/test_deepnote_streamlit_widgets.py @@ -344,3 +344,13 @@ def app(): at.date_input[0].set_value((date(2026, 8, 17),)).run() assert not at.exception assert "period" not in at.session_state["values"] + + +def test_missing_select_value_does_not_select_literal_none_option(): + assert ( + render_inputs( + [InputBlock("x", "input-select", None, options=("None", "EU"))], + FakeContainer(), + ) + == {} + ) From 6fb1fa05072be5b0f225128301721361dd4bdb96 Mon Sep 17 00:00:00 2001 From: Wojtek <6819634+voyti@users.noreply.github.com> Date: Mon, 21 Sep 2026 19:54:23 +0200 Subject: [PATCH 33/43] fix(streamlit): close remaining authentication and snapshot edge cases --- deepnote_toolkit/notebooks/cloud_runner.py | 2 +- deepnote_toolkit/notebooks/local_runner.py | 5 +- deepnote_toolkit/streamlit/auth.py | 8 +- .../streamlit/viewer_credentials.py | 4 +- deepnote_toolkit/streamlit/widgets.py | 4 + docs/streamlit-apps.md | 2 + installer/module/server_process.py | 2 +- tests/unit/conftest.py | 7 +- tests/unit/test_deepnote_streamlit_auth.py | 1 + .../test_deepnote_streamlit_cloud_runner.py | 16 ++-- tests/unit/test_deepnote_streamlit_widgets.py | 75 ++++++++++++++++--- tests/unit/test_notebooks_document.py | 7 +- tests/unit/test_notebooks_runners.py | 22 +++++- tests/unit/test_streamlit.py | 5 +- tests/unit/test_virtual_environment.py | 5 +- 15 files changed, 133 insertions(+), 32 deletions(-) diff --git a/deepnote_toolkit/notebooks/cloud_runner.py b/deepnote_toolkit/notebooks/cloud_runner.py index 494beef5..e98963e3 100644 --- a/deepnote_toolkit/notebooks/cloud_runner.py +++ b/deepnote_toolkit/notebooks/cloud_runner.py @@ -158,9 +158,9 @@ def _settle_snapshot(self, run: CloudRun, deadline: float) -> CloudRun: updated = self._client.get_run( run.run_id, timeout=deadline - self._clock() ) + run = updated if self._clock() >= deadline: break - run = updated except RunnerError as error: if not error.transient: raise diff --git a/deepnote_toolkit/notebooks/local_runner.py b/deepnote_toolkit/notebooks/local_runner.py index f2f13ad7..0197a191 100644 --- a/deepnote_toolkit/notebooks/local_runner.py +++ b/deepnote_toolkit/notebooks/local_runner.py @@ -2,6 +2,7 @@ from __future__ import annotations +import logging from collections.abc import Mapping from typing import Any @@ -13,6 +14,8 @@ from .transport import request_json from .wire import decode_block_outputs, decode_inputs, optional_string +logger = logging.getLogger(__name__) + class DeepnoteLocalRunner: """Run a notebook through a local sidecar, configured for a local or cloud kernel.""" @@ -65,7 +68,7 @@ def _decode_run_result(payload: Mapping[str, Any]) -> RunResult: try: snapshot = DeepnoteDocument.parse(snapshot_yaml) except ValueError: - pass + logger.warning("Could not parse the run snapshot; using inline outputs") return RunResult( target=str(payload.get("target", "")), success=payload.get("success") is True, diff --git a/deepnote_toolkit/streamlit/auth.py b/deepnote_toolkit/streamlit/auth.py index 661df38e..b432ed45 100644 --- a/deepnote_toolkit/streamlit/auth.py +++ b/deepnote_toolkit/streamlit/auth.py @@ -235,7 +235,13 @@ def _is_streamlit_thread_without_request() -> bool: def _validated_origin(value: str, *, name: str) -> str: - parsed = urlparse(value) + """Validate an origin and normalize URL parser failures to authentication errors.""" + try: + parsed = urlparse(value) + except ValueError as error: + raise CurrentUserApiTokenError( + f"{name} must be a valid HTTP(S) origin." + ) from error if ( parsed.scheme not in {"http", "https"} or not parsed.netloc diff --git a/deepnote_toolkit/streamlit/viewer_credentials.py b/deepnote_toolkit/streamlit/viewer_credentials.py index f6d47b99..b0496567 100644 --- a/deepnote_toolkit/streamlit/viewer_credentials.py +++ b/deepnote_toolkit/streamlit/viewer_credentials.py @@ -76,9 +76,9 @@ def __call__(self, *, timeout: float = 30) -> ApiCredentials: if _is_streamlit_thread_without_request(): raise RunnerError(_NO_REQUEST + ".") - if has_request and self._local_mode and not self._local_token_explicit: + if not self._local_mode or not self._local_token_explicit: raise RunnerError( - "Viewer identity is unavailable. For local development, " + "Viewer identity is unavailable. For local execution, set local=True and " "pass token= or token_provider= explicitly." ) return self._local(timeout=timeout) diff --git a/deepnote_toolkit/streamlit/widgets.py b/deepnote_toolkit/streamlit/widgets.py index 2ac508b0..c02d4c4c 100644 --- a/deepnote_toolkit/streamlit/widgets.py +++ b/deepnote_toolkit/streamlit/widgets.py @@ -49,6 +49,7 @@ def render_inputs( def _render_one(container: Any, input_block: InputBlock, label: str, key: str) -> Any: + """Render one input block using its saved default and constraints.""" if input_block.type == "input-checkbox": return container.checkbox(label, value=_as_bool(input_block.value), key=key) @@ -129,12 +130,14 @@ def _render_one(container: Any, input_block: InputBlock, label: str, key: str) - def _as_bool(value: Any) -> bool: + """Decode checkbox defaults without treating the text false as truthy.""" if isinstance(value, bool): return value return str(value).lower() in {"true", "1"} def _as_number(value: Any, fallback: float | int) -> float | int: + """Decode numeric defaults while preserving fractional values.""" if value is None: return fallback try: @@ -177,4 +180,5 @@ def _as_date_range(value: Any) -> tuple[date, ...]: def _serialize_date(value: Any) -> str: + """Encode a chosen date, leaving an empty widget empty.""" return value.isoformat() if isinstance(value, date) else "" diff --git a/docs/streamlit-apps.md b/docs/streamlit-apps.md index dcfa35c2..ff656188 100644 --- a/docs/streamlit-apps.md +++ b/docs/streamlit-apps.md @@ -55,6 +55,8 @@ For Python code outside Streamlit, use `DeepnoteCloudRunner` from `deepnote_toolkit.notebooks`. It accepts `token=`, `token_provider=`, or `DEEPNOTE_TOKEN`. For a local `@deepnote/local-runner` sidecar, use `DeepnoteLocalRunner(base_url="http://127.0.0.1:8787")`. +`StreamlitCloudRunner` requires explicit local mode and credentials even when +called outside the Streamlit runtime. ## Inputs and outputs diff --git a/installer/module/server_process.py b/installer/module/server_process.py index 102d2bd5..e9e7f63d 100644 --- a/installer/module/server_process.py +++ b/installer/module/server_process.py @@ -20,7 +20,7 @@ def __init__( cwd: Optional[str] = None, *, env: Optional[dict[str, str]] = None, - ): + ) -> None: """ Initialize the ServerProcess with the given command. diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index 4196a95c..109ac8e0 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -3,10 +3,13 @@ import os import sys import tempfile -from typing import Generator +from typing import TYPE_CHECKING, Generator import pytest +if TYPE_CHECKING: + from streamlit.testing.v1 import AppTest + @pytest.fixture(autouse=True, scope="session") def apply_patches() -> None: @@ -69,7 +72,7 @@ def test_log_directory() -> Generator[str, None, None]: @pytest.fixture -def streamlit_app_test(monkeypatch): +def streamlit_app_test(monkeypatch: pytest.MonkeyPatch) -> "type[AppTest]": """Restore the main module that Streamlit replaces while executing an app.""" pytest.importorskip("streamlit") from streamlit.testing.v1 import AppTest diff --git a/tests/unit/test_deepnote_streamlit_auth.py b/tests/unit/test_deepnote_streamlit_auth.py index 807680df..dd1c2f67 100644 --- a/tests/unit/test_deepnote_streamlit_auth.py +++ b/tests/unit/test_deepnote_streamlit_auth.py @@ -93,6 +93,7 @@ def test_explicit_app_id_is_validated_before_network(http, state, value): {"expiresAtSeconds": "99999999999"}, {"apiOrigin": "https://user:pass@example.com"}, {"apiOrigin": "https://example.com/path"}, + {"apiOrigin": "https://[::1"}, ], ) def test_malformed_credentials_are_not_cached(http, state, overrides): diff --git a/tests/unit/test_deepnote_streamlit_cloud_runner.py b/tests/unit/test_deepnote_streamlit_cloud_runner.py index 6d332079..1370b14e 100644 --- a/tests/unit/test_deepnote_streamlit_cloud_runner.py +++ b/tests/unit/test_deepnote_streamlit_cloud_runner.py @@ -109,7 +109,10 @@ def test_worker_thread_fails_closed(context, http, explicit): assert not http.calls -def test_local_streamlit_requires_explicit_opt_in_and_token(context, http): +@pytest.mark.parametrize("script", [False, True]) +def test_local_streamlit_requires_explicit_opt_in_and_token(context, http, script): + """Explicit local credentials work with or without an active Streamlit script.""" + context["script"] = script http.get( "https://api.deepnote.com/v2/notebooks/n", json={"notebook": {"name": "N"}} ) @@ -120,12 +123,13 @@ def test_local_streamlit_requires_explicit_opt_in_and_token(context, http): StreamlitCloudRunner("n", local=True, session=session()).info() -def test_bare_python_uses_local_token_without_streamlit_lookups(context, http): +@pytest.mark.parametrize("explicit", [{}, {"token": "owner"}, {"local": True}]) +def test_bare_python_requires_explicit_local_credentials(context, http, explicit): + """The Streamlit adapter cannot use an ambient owner token outside the runtime.""" context["script"] = False - http.get( - "https://api.deepnote.com/v2/notebooks/n", json={"notebook": {"name": "N"}} - ) - assert StreamlitCloudRunner("n", session=session()).info().notebook == "N" + with pytest.raises(RunnerError, match="local=True.*explicitly"): + StreamlitCloudRunner("n", session=session(), **explicit).info() + assert not http.calls def test_transient_exchange_failure_during_poll_is_retried(context, http, monkeypatch): diff --git a/tests/unit/test_deepnote_streamlit_widgets.py b/tests/unit/test_deepnote_streamlit_widgets.py index 2df63e67..a148888c 100644 --- a/tests/unit/test_deepnote_streamlit_widgets.py +++ b/tests/unit/test_deepnote_streamlit_widgets.py @@ -1,39 +1,53 @@ from datetime import date -from typing import Any +from typing import TYPE_CHECKING, Any import pytest from deepnote_toolkit.notebooks import InputBlock from deepnote_toolkit.streamlit import render_inputs +if TYPE_CHECKING: + from streamlit.testing.v1 import AppTest + class FakeContainer: - def warning(self, message): + """Return widget defaults without starting Streamlit.""" + + def warning(self, message: str) -> None: + """Ignore warnings unless a test installs a recording callback.""" pass def checkbox(self, _label: str, **kwargs: Any) -> Any: + """Return the configured checkbox value.""" return kwargs["value"] def multiselect(self, _label: str, _options: list[str], **kwargs: Any) -> Any: + """Return the configured selection list.""" return kwargs["default"] def selectbox(self, _label: str, options: list[str], **kwargs: Any) -> Any: + """Return the selected option, preserving an empty selection.""" return options[kwargs["index"]] if kwargs["index"] is not None else None def slider(self, _label: str, **kwargs: Any) -> Any: + """Record or return the configured slider value.""" return kwargs["value"] def date_input(self, _label: str, **kwargs: Any) -> Any: + """Return the dates supplied by the test container.""" return kwargs["value"] def text_area(self, _label: str, **kwargs: Any) -> Any: + """Return the configured multiline text.""" return kwargs["value"] def text_input(self, _label: str, **kwargs: Any) -> Any: + """Return the configured text.""" return kwargs["value"] def test_render_inputs_maps_all_deepnote_input_types_to_api_values() -> None: + """Convert each supported widget value to its API representation.""" inputs = [ InputBlock("name", "input-text", "Ada"), InputBlock("notes", "input-textarea", "Hello"), @@ -64,8 +78,13 @@ def test_render_inputs_maps_all_deepnote_input_types_to_api_values() -> None: def test_incomplete_date_range_is_still_valid_for_runner_contract() -> None: + """Omit incomplete ranges until the user chooses both dates.""" + class IncompleteDateContainer(FakeContainer): + """Simulate a user selecting only the start of a range.""" + def date_input(self, _label: str, **_kwargs: Any) -> Any: + """Return the dates supplied by the test container.""" return (date(2026, 8, 17),) values = render_inputs( @@ -81,10 +100,15 @@ def date_input(self, _label: str, **_kwargs: Any) -> Any: def test_slider_preserves_fractional_default_with_integer_bounds() -> None: + """Keep fractional slider defaults and consistent numeric argument types.""" + class SliderContainer(FakeContainer): + """Record slider arguments for numeric consistency checks.""" + slider_kwargs: dict[str, Any] def slider(self, _label: str, **kwargs: Any) -> Any: + """Record or return the configured slider value.""" self.slider_kwargs = kwargs return kwargs["value"] @@ -105,6 +129,7 @@ def slider(self, _label: str, **kwargs: Any) -> Any: def test_multiselect_normalizes_and_filters_stale_defaults() -> None: + """Normalize saved selections and filter unavailable options.""" values = render_inputs( [ InputBlock( @@ -122,10 +147,14 @@ def test_multiselect_normalizes_and_filters_stale_defaults() -> None: def test_date_reads_timestamp_default_and_keeps_its_shape() -> None: + """Preserve timestamp compatibility for older date blocks.""" defaults = [] class RecordingContainer(FakeContainer): + """Record defaults and simulate a changed date.""" + def date_input(self, _label: str, **kwargs: Any) -> Any: + """Return the dates supplied by the test container.""" defaults.append(kwargs["value"]) return date(2026, 8, 20) @@ -142,6 +171,7 @@ def date_input(self, _label: str, **kwargs: Any) -> Any: def test_empty_dates_stay_empty_instead_of_becoming_today() -> None: + """Leave unspecified dates empty.""" values = render_inputs( [ InputBlock("as_of", "input-date", ""), @@ -154,14 +184,18 @@ def test_empty_dates_stay_empty_instead_of_becoming_today() -> None: class FrozenDate(date): + """Keep relative date calculations deterministic.""" + @classmethod def today(cls) -> "FrozenDate": + """Use a month end in a leap year.""" return cls(2024, 3, 31) def test_relative_date_ranges_resolve_to_concrete_dates( monkeypatch: pytest.MonkeyPatch, ) -> None: + """Resolve relative ranges and clamp dates at month boundaries.""" monkeypatch.setattr("deepnote_toolkit.streamlit.widgets.date", FrozenDate) values = render_inputs( @@ -183,9 +217,13 @@ def test_relative_date_ranges_resolve_to_concrete_dates( } -def test_render_inputs_runs_on_real_streamlit_widgets(streamlit_app_test) -> None: +def test_render_inputs_runs_on_real_streamlit_widgets( + streamlit_app_test: "type[AppTest]", +) -> None: + """Exercise every widget family using Streamlit AppTest.""" def app() -> None: + """Render the test inputs inside a real Streamlit script.""" import streamlit as st from deepnote_toolkit.notebooks import InputBlock @@ -228,6 +266,7 @@ def app() -> None: def test_duplicate_variable_names_are_rejected() -> None: + """Reject duplicate variables before widgets overwrite their values.""" with pytest.raises(ValueError, match="unique"): render_inputs( [ @@ -239,6 +278,7 @@ def test_duplicate_variable_names_are_rejected() -> None: def test_multiselect_treats_a_scalar_default_as_one_selection() -> None: + """Normalize scalar and absent multiselect defaults.""" values = render_inputs( [ InputBlock( @@ -256,14 +296,20 @@ def test_multiselect_treats_a_scalar_default_as_one_selection() -> None: @pytest.mark.parametrize("kind", ["input-text", "input-textarea", "input-file"]) @pytest.mark.parametrize("value,expected", [(0, "0"), (False, "False"), (None, "")]) -def test_falsey_text_defaults_are_preserved(kind, value, expected): +def test_falsey_text_defaults_are_preserved( + kind: str, value: Any, expected: str +) -> None: + """Keep zero and false defaults visible in text widgets.""" assert render_inputs([InputBlock("x", kind, value)], FakeContainer()) == { "x": expected } @pytest.mark.parametrize("value", [None, "stale"]) -def test_unselected_single_select_does_not_submit_first_option(value): +def test_unselected_single_select_does_not_submit_first_option( + value: str | None, +) -> None: + """Do not submit an option that the user has not selected.""" assert ( render_inputs( [InputBlock("x", "input-select", value, options=("first", "second"))], @@ -273,7 +319,8 @@ def test_unselected_single_select_does_not_submit_first_option(value): ) -def test_stale_multiselect_default_warns(): +def test_stale_multiselect_default_warns() -> None: + """Make unavailable saved selections visible to the user.""" warnings = [] container = FakeContainer() container.warning = warnings.append @@ -306,7 +353,10 @@ def test_stale_multiselect_default_warns(): (3, 0, float("inf"), 1), ], ) -def test_invalid_slider_configuration_is_reported(value, min_value, max_value, step): +def test_invalid_slider_configuration_is_reported( + value: Any, min_value: float, max_value: float, step: float +) -> None: + """Reject invalid bounds, steps, and defaults before rendering.""" with pytest.raises(ValueError, match="[Ss]lider"): render_inputs( [ @@ -318,9 +368,13 @@ def test_invalid_slider_configuration_is_reported(value, min_value, max_value, s ) -def test_real_widgets_keep_falsey_defaults_and_require_selection(streamlit_app_test): +def test_real_widgets_keep_falsey_defaults_and_require_selection( + streamlit_app_test: "type[AppTest]", +) -> None: + """Verify user interactions preserve defaults and omit partial ranges.""" - def app(): + def app() -> None: + """Render the test inputs inside a real Streamlit script.""" import streamlit as st from deepnote_toolkit.notebooks import InputBlock @@ -346,7 +400,8 @@ def app(): assert "period" not in at.session_state["values"] -def test_missing_select_value_does_not_select_literal_none_option(): +def test_missing_select_value_does_not_select_literal_none_option() -> None: + """Distinguish a missing default from an option containing the word None.""" assert ( render_inputs( [InputBlock("x", "input-select", None, options=("None", "EU"))], diff --git a/tests/unit/test_notebooks_document.py b/tests/unit/test_notebooks_document.py index ee3d78b2..74ac19f0 100644 --- a/tests/unit/test_notebooks_document.py +++ b/tests/unit/test_notebooks_document.py @@ -205,7 +205,10 @@ def test_run_result_falls_back_to_inline_outputs_without_snapshot() -> None: assert dataframe.records() == [{"value": 42}] -def test_run_result_falls_back_to_inline_outputs_for_malformed_snapshot() -> None: +def test_run_result_falls_back_to_inline_outputs_for_malformed_snapshot( + caplog: pytest.LogCaptureFixture, +) -> None: + """Keep inline outputs and report the snapshot fallback without its contents.""" result = run_locally( { "target": "cloud", @@ -227,6 +230,8 @@ def test_run_result_falls_back_to_inline_outputs_for_malformed_snapshot() -> Non assert result.snapshot is None assert result.text() == "fallback output" + assert "Could not parse the run snapshot; using inline outputs" in caplog.text + assert "not: a deepnote snapshot" not in caplog.text @pytest.mark.parametrize( diff --git a/tests/unit/test_notebooks_runners.py b/tests/unit/test_notebooks_runners.py index 17813820..270f9392 100644 --- a/tests/unit/test_notebooks_runners.py +++ b/tests/unit/test_notebooks_runners.py @@ -231,14 +231,27 @@ def poll(request): assert clock.sleeps == [2, 1] -def test_snapshot_deadline_counts_slow_requests_and_caps_timeout(http, clock): +@pytest.mark.parametrize("available", [False, True]) +def test_snapshot_deadline_counts_slow_requests_and_caps_timeout( + http, clock, available +): + """Keep received outputs at the deadline without starting another request.""" add_run(http, run_response(snapshotStatus="pending"), create=True) timeouts = [] def poll(request): timeouts.append(request.req_kwargs["timeout"].total) clock.now += 4 - return 200, {}, json.dumps(run_response(snapshotStatus="pending")) + payload = run_response(snapshotStatus="available" if available else "pending") + if available: + payload["run"]["snapshotBlocks"] = [ + { + "id": "b", + "type": "code", + "outputs": [{"output_type": "stream", "text": "done"}], + } + ] + return 200, {}, json.dumps(payload) http.add_callback( responses.GET, @@ -254,7 +267,10 @@ def poll(request): clock=clock, sleep=clock.sleep, ) - assert runner.run({}).snapshot_status == "pending" + result = runner.run({}) + assert result.snapshot_status == ("available" if available else "pending") + assert result.text() == ("done" if available else "") + assert len(http.calls) == 2 assert timeouts == [4] assert clock.now == 5 diff --git a/tests/unit/test_streamlit.py b/tests/unit/test_streamlit.py index 06e81b53..0c6bab77 100644 --- a/tests/unit/test_streamlit.py +++ b/tests/unit/test_streamlit.py @@ -98,7 +98,7 @@ def exists_side_effect(path: str) -> bool: assert mock_logger.warning.call_count == 2 assert mock_venv.start_server.call_count == 1 - def test_passes_app_id_as_environment_data(self): + def test_passes_app_id_as_environment_data(self) -> None: """App IDs are data, never shell syntax; validation belongs to the SDK.""" apps = [ { @@ -122,7 +122,8 @@ def test_passes_app_id_as_environment_data(self): assert calls[0].kwargs["env"] == {"DEEPNOTE_STREAMLIT_APP_ID": apps[0]["id"]} assert calls[1].kwargs["env"] == {"DEEPNOTE_STREAMLIT_APP_ID": apps[1]["id"]} - def test_missing_app_id_warns_and_still_marks_process_as_hosted(self): + def test_missing_app_id_warns_and_still_marks_process_as_hosted(self) -> None: + """Missing app IDs warn without permitting local credential fallback.""" app = {"entrypoint": "app.py", "port": "8501"} venv = MagicMock() logger = MagicMock(spec=logging.Logger) diff --git a/tests/unit/test_virtual_environment.py b/tests/unit/test_virtual_environment.py index 54771ad9..f646495e 100644 --- a/tests/unit/test_virtual_environment.py +++ b/tests/unit/test_virtual_environment.py @@ -101,8 +101,9 @@ def test_import_package_bundle_condition_env_and_priority_mutually_exclusive( def test_server_environment_is_passed_as_data_and_inherits_parent( - tmp_path, monkeypatch -): + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Pass app IDs as environment data while retaining the parent environment.""" import json import shlex import sys From 3d397a0f848910f7dfeb34fccfb7df304b2d8d84 Mon Sep 17 00:00:00 2001 From: Wojtek <6819634+voyti@users.noreply.github.com> Date: Mon, 21 Sep 2026 20:01:39 +0200 Subject: [PATCH 34/43] chore(notebooks): finish review typing and logging nits --- deepnote_toolkit/notebooks/local_runner.py | 6 +++++- tests/unit/test_deepnote_streamlit_cloud_runner.py | 11 ++++++++--- tests/unit/test_notebooks_document.py | 2 ++ tests/unit/test_notebooks_runners.py | 8 +++++--- 4 files changed, 20 insertions(+), 7 deletions(-) diff --git a/deepnote_toolkit/notebooks/local_runner.py b/deepnote_toolkit/notebooks/local_runner.py index 0197a191..7c4d32b2 100644 --- a/deepnote_toolkit/notebooks/local_runner.py +++ b/deepnote_toolkit/notebooks/local_runner.py @@ -68,7 +68,11 @@ def _decode_run_result(payload: Mapping[str, Any]) -> RunResult: try: snapshot = DeepnoteDocument.parse(snapshot_yaml) except ValueError: - logger.warning("Could not parse the run snapshot; using inline outputs") + logger.warning( + "Could not parse the run snapshot; using inline outputs (runId=%r, target=%r)", + optional_string(payload.get("runId")), + optional_string(payload.get("target")), + ) return RunResult( target=str(payload.get("target", "")), success=payload.get("success") is True, diff --git a/tests/unit/test_deepnote_streamlit_cloud_runner.py b/tests/unit/test_deepnote_streamlit_cloud_runner.py index 1370b14e..8907e396 100644 --- a/tests/unit/test_deepnote_streamlit_cloud_runner.py +++ b/tests/unit/test_deepnote_streamlit_cloud_runner.py @@ -1,4 +1,5 @@ import time +from typing import Any import pytest import responses @@ -110,7 +111,9 @@ def test_worker_thread_fails_closed(context, http, explicit): @pytest.mark.parametrize("script", [False, True]) -def test_local_streamlit_requires_explicit_opt_in_and_token(context, http, script): +def test_local_streamlit_requires_explicit_opt_in_and_token( + context: dict[str, bool], http: responses.RequestsMock, script: bool +) -> None: """Explicit local credentials work with or without an active Streamlit script.""" context["script"] = script http.get( @@ -124,10 +127,12 @@ def test_local_streamlit_requires_explicit_opt_in_and_token(context, http, scrip @pytest.mark.parametrize("explicit", [{}, {"token": "owner"}, {"local": True}]) -def test_bare_python_requires_explicit_local_credentials(context, http, explicit): +def test_bare_python_requires_explicit_local_credentials( + context: dict[str, bool], http: responses.RequestsMock, explicit: dict[str, Any] +) -> None: """The Streamlit adapter cannot use an ambient owner token outside the runtime.""" context["script"] = False - with pytest.raises(RunnerError, match="local=True.*explicitly"): + with pytest.raises(RunnerError, match=r"local=True.*explicitly"): StreamlitCloudRunner("n", session=session(), **explicit).info() assert not http.calls diff --git a/tests/unit/test_notebooks_document.py b/tests/unit/test_notebooks_document.py index 74ac19f0..281d395d 100644 --- a/tests/unit/test_notebooks_document.py +++ b/tests/unit/test_notebooks_document.py @@ -212,6 +212,7 @@ def test_run_result_falls_back_to_inline_outputs_for_malformed_snapshot( result = run_locally( { "target": "cloud", + "runId": "run-fallback", "success": True, "snapshotYaml": "not: a deepnote snapshot", "outputs": [ @@ -231,6 +232,7 @@ def test_run_result_falls_back_to_inline_outputs_for_malformed_snapshot( assert result.snapshot is None assert result.text() == "fallback output" assert "Could not parse the run snapshot; using inline outputs" in caplog.text + assert "run-fallback" in caplog.text and "cloud" in caplog.text assert "not: a deepnote snapshot" not in caplog.text diff --git a/tests/unit/test_notebooks_runners.py b/tests/unit/test_notebooks_runners.py index 270f9392..51c2cc14 100644 --- a/tests/unit/test_notebooks_runners.py +++ b/tests/unit/test_notebooks_runners.py @@ -1,6 +1,7 @@ import json import threading from http.server import BaseHTTPRequestHandler, HTTPServer +from typing import Any import pytest import requests @@ -233,13 +234,14 @@ def poll(request): @pytest.mark.parametrize("available", [False, True]) def test_snapshot_deadline_counts_slow_requests_and_caps_timeout( - http, clock, available -): + http: responses.RequestsMock, clock: Clock, available: bool +) -> None: """Keep received outputs at the deadline without starting another request.""" add_run(http, run_response(snapshotStatus="pending"), create=True) timeouts = [] - def poll(request): + def poll(request: Any) -> tuple[int, dict[str, str], str]: + """Return a snapshot as the monotonic request budget expires.""" timeouts.append(request.req_kwargs["timeout"].total) clock.now += 4 payload = run_response(snapshotStatus="available" if available else "pending") From 94655e1a8d83ada3326026f98b942ff6a90a0449 Mon Sep 17 00:00:00 2001 From: Wojtek <6819634+voyti@users.noreply.github.com> Date: Tue, 22 Sep 2026 02:01:03 +0200 Subject: [PATCH 35/43] fix(streamlit): preserve viewer credentials and open-ended date ranges --- deepnote_toolkit/notebooks/transport.py | 8 +++++ deepnote_toolkit/streamlit/widgets.py | 21 +++++++---- docs/streamlit-apps.md | 23 ++++++------ .../test_deepnote_streamlit_cloud_runner.py | 36 +++++++++++++++++++ tests/unit/test_deepnote_streamlit_widgets.py | 32 +++++++++++++++++ 5 files changed, 104 insertions(+), 16 deletions(-) diff --git a/deepnote_toolkit/notebooks/transport.py b/deepnote_toolkit/notebooks/transport.py index de6db4f1..85f57506 100644 --- a/deepnote_toolkit/notebooks/transport.py +++ b/deepnote_toolkit/notebooks/transport.py @@ -12,6 +12,13 @@ from .runner import RunnerError +def _preserve_authorization( + request: requests.PreparedRequest, +) -> requests.PreparedRequest: + """Keep resolved credentials instead of applying Session.auth or .netrc.""" + return request + + def request_json( session: requests.Session, method: str, @@ -29,6 +36,7 @@ def request_json( method, url, headers={"Accept": "application/json", **headers}, + auth=_preserve_authorization if "Authorization" in headers else None, json=body, timeout=Timeout(total=timeout), allow_redirects=False, diff --git a/deepnote_toolkit/streamlit/widgets.py b/deepnote_toolkit/streamlit/widgets.py index c02d4c4c..59adea15 100644 --- a/deepnote_toolkit/streamlit/widgets.py +++ b/deepnote_toolkit/streamlit/widgets.py @@ -105,9 +105,18 @@ def _render_one(container: Any, input_block: InputBlock, label: str, key: str) - return f"{selected}T00:00:00.000Z" if selected and is_timestamp else selected if input_block.type == "input-date-range": - selected = container.date_input( - label, value=_as_date_range(input_block.value), key=key - ) + defaults = _as_date_range(input_block.value) + # One range picker cannot represent an open start or end independently. + if None in defaults: + return [ + _serialize_date( + container.date_input( + f"{label} ({endpoint})", value=value, key=f"{key}:{endpoint}" + ) + ) + for endpoint, value in zip(("start", "end"), defaults) + ] + selected = container.date_input(label, value=defaults, key=key) if not isinstance(selected, (list, tuple)): return None serialized = [_serialize_date(value) for value in selected] @@ -162,12 +171,12 @@ def _as_date(value: Any) -> date | None: return None -def _as_date_range(value: Any) -> tuple[date, ...]: - """Resolve an absolute or relative Deepnote range. () leaves the widget empty.""" +def _as_date_range(value: Any) -> tuple[date | None, ...]: + """Resolve a range; None marks an open endpoint and () an entirely empty range.""" if isinstance(value, list): dates = tuple(_as_date(item) for item in value[:2]) - return dates if len(dates) == 2 and None not in dates else () + return dates if len(dates) == 2 and any(dates) else () today = date.today() if match := re.fullmatch(r"past(\d+)days|customDays(\d+)", str(value)): diff --git a/docs/streamlit-apps.md b/docs/streamlit-apps.md index ff656188..dc029ca9 100644 --- a/docs/streamlit-apps.md +++ b/docs/streamlit-apps.md @@ -61,12 +61,13 @@ called outside the Streamlit runtime. ## Inputs and outputs `render_inputs()` preserves saved defaults, including `0` and `False`. A select -without a valid saved choice starts empty. Unselected single selects and partially -selected date ranges are omitted from the returned dictionary; disable your Run -button until required fields are present. An omitted input uses the notebook's -value according to the API. Stale multi-select choices produce a warning. Invalid -slider bounds/defaults and duplicate variable names raise `ValueError`. File inputs -render as text paths; this helper does not upload files. +without a valid saved choice starts empty. Unselected single selects and incomplete +selections in the date-range picker are omitted from the returned dictionary; +disable your Run button until required fields are present. An omitted input uses +the notebook's value according to the API. Saved open-ended ranges use separate +start/end fields so their chosen endpoint is preserved. Stale multi-select choices +produce a warning. Invalid slider bounds/defaults and duplicate variable names +raise `ValueError`. File inputs render as text paths; this helper does not upload files. `runner.info().matches_inputs(document.inputs)` compares static input definitions: unique names, types, single/multiple selection, options, and slider bounds/steps. @@ -93,14 +94,16 @@ limited to the remaining budget. Output retrieval also has its own `snapshot_timeout` (10 seconds); only an explicitly pending snapshot is polled. Requests uses socket timeouts, so OS DNS resolution or a server streaming bytes can exceed a request budget; this is not hard cancellation of a running notebook. -Only GET polls are retried after transient failures, up to five consecutive -retries. Creating a run is never automatically retried. +Run-status GET polls retry transient failures up to five consecutive times; +snapshot GET polls retry within the snapshot budget. Creating a run is never +automatically retried. Pass `session=requests.Session()` to configure proxies or HTTP adapters. A custom `credentials=` provider on `DeepnoteCloudRunner` receives a `timeout` keyword and returns `ApiCredentials(token=..., api_origin=...)`. Providers should honor that -budget. API clients, HTTP helpers, and wire schemas are internal; supported names -are listed in each package's `__all__`. +budget. Resolved bearer credentials take precedence over `.netrc` and session +authentication. API clients, HTTP helpers, and wire schemas are internal; +supported names are listed in each package's `__all__`. The existing `streamlit_data_apps` module handles database federation. Notebook execution uses its viewer-cookie reader and does not replace its database APIs. diff --git a/tests/unit/test_deepnote_streamlit_cloud_runner.py b/tests/unit/test_deepnote_streamlit_cloud_runner.py index 8907e396..72ec2629 100644 --- a/tests/unit/test_deepnote_streamlit_cloud_runner.py +++ b/tests/unit/test_deepnote_streamlit_cloud_runner.py @@ -2,6 +2,7 @@ from typing import Any import pytest +import requests import responses from deepnote_toolkit.notebooks import RunnerError @@ -83,6 +84,41 @@ def test_hosted_run_uses_viewer_and_readonly_even_with_explicit_owner_token( assert body(http.calls[1])["detachedRunStorageMode"] == "readonly" +@pytest.mark.parametrize("ambient_auth", ["netrc", "session"]) +def test_resolved_viewer_token_overrides_requests_auth( + context, http, monkeypatch, tmp_path, ambient_auth +): + monkeypatch.setenv("DEEPNOTE_STREAMLIT_APP_ID", APP_ID) + http.post( + TOKEN_URL, + json={ + "token": "viewer", + "apiOrigin": "https://api.deepnote.com", + "expiresAtSeconds": time.time() + 900, + }, + ) + add_run(http, run_response(snapshotBlocks=[]), create=True) + transport = requests.Session() + if ambient_auth == "netrc": + netrc = tmp_path / "credentials.netrc" + netrc.write_text("machine api.deepnote.com login unrelated password dummy\n") + monkeypatch.setenv("NETRC", str(netrc)) + else: + + def owner_auth(request): + request.headers["Authorization"] = "Bearer owner" + return request + + transport.auth = owner_auth + + with transport: + result = StreamlitCloudRunner("n", session=transport).run({}) + + assert result.success + assert http.calls[1].request.headers["Authorization"] == "Bearer viewer" + assert transport.trust_env is True + + @pytest.mark.parametrize("marker", ["", "invalid", APP_ID]) def test_malformed_app_marker_never_falls_back(context, http, monkeypatch, marker): monkeypatch.setenv("DEEPNOTE_STREAMLIT_APP_ID", marker) diff --git a/tests/unit/test_deepnote_streamlit_widgets.py b/tests/unit/test_deepnote_streamlit_widgets.py index a148888c..8a1da46a 100644 --- a/tests/unit/test_deepnote_streamlit_widgets.py +++ b/tests/unit/test_deepnote_streamlit_widgets.py @@ -183,6 +183,38 @@ def test_empty_dates_stay_empty_instead_of_becoming_today() -> None: assert values == {"as_of": "", "period": ["", ""]} +@pytest.mark.parametrize("value", [["2026-08-01", ""], ["", "2026-08-17"]]) +def test_saved_open_ended_date_range_keeps_its_chosen_endpoint(value) -> None: + assert render_inputs( + [InputBlock("period", "input-date-range", value)], FakeContainer() + ) == {"period": value} + + +@pytest.mark.parametrize("value", [["2026-08-01", ""], ["", "2026-08-17"]]) +def test_real_widgets_preserve_and_edit_open_ended_date_ranges( + streamlit_app_test: "type[AppTest]", value +) -> None: + def app(value) -> None: + import streamlit as st + + from deepnote_toolkit.notebooks import InputBlock + from deepnote_toolkit.streamlit import render_inputs + + st.session_state["values"] = render_inputs( + [InputBlock("period", "input-date-range", value)] + ) + + at = streamlit_app_test.from_function(app, args=(value,)).run() + assert not at.exception + assert at.session_state["values"] == {"period": value} + assert len(at.date_input) == 2 + missing = value.index("") + chosen = date(2026, 8, 1 if missing == 0 else 17) + at.date_input[missing].set_value(chosen).run() + assert not at.exception + assert at.session_state["values"] == {"period": ["2026-08-01", "2026-08-17"]} + + class FrozenDate(date): """Keep relative date calculations deterministic.""" From 9eb073e10a689bb0d04ba856fd010b2945989101 Mon Sep 17 00:00:00 2001 From: Wojtek <6819634+voyti@users.noreply.github.com> Date: Tue, 22 Sep 2026 02:20:17 +0200 Subject: [PATCH 36/43] fix(streamlit): validate credential origins and header casing --- deepnote_toolkit/notebooks/transport.py | 9 ++++-- deepnote_toolkit/streamlit/auth.py | 1 + tests/unit/test_deepnote_streamlit_auth.py | 6 ++++ .../test_deepnote_streamlit_cloud_runner.py | 13 ++++++-- tests/unit/test_deepnote_streamlit_widgets.py | 12 ++++++-- tests/unit/test_notebooks_runners.py | 30 +++++++++++++++++++ 6 files changed, 63 insertions(+), 8 deletions(-) diff --git a/deepnote_toolkit/notebooks/transport.py b/deepnote_toolkit/notebooks/transport.py index 85f57506..bd5d1fdd 100644 --- a/deepnote_toolkit/notebooks/transport.py +++ b/deepnote_toolkit/notebooks/transport.py @@ -31,12 +31,17 @@ def request_json( """Send one request, without replaying POSTs or forwarding credentials on redirects.""" origin = urlsplit(url) origin_name = f"{origin.scheme}://{origin.netloc}" + request_headers = requests.structures.CaseInsensitiveDict( + {"Accept": "application/json", **headers} + ) try: with session.request( method, url, - headers={"Accept": "application/json", **headers}, - auth=_preserve_authorization if "Authorization" in headers else None, + headers=request_headers, + auth=( + _preserve_authorization if "Authorization" in request_headers else None + ), json=body, timeout=Timeout(total=timeout), allow_redirects=False, diff --git a/deepnote_toolkit/streamlit/auth.py b/deepnote_toolkit/streamlit/auth.py index b432ed45..52d85961 100644 --- a/deepnote_toolkit/streamlit/auth.py +++ b/deepnote_toolkit/streamlit/auth.py @@ -251,6 +251,7 @@ def _validated_origin(value: str, *, name: str) -> str: or parsed.params or parsed.query or parsed.fragment + or value.endswith(("?", "#", ";")) ): raise CurrentUserApiTokenError(f"{name} must be a valid HTTP(S) origin.") return value.rstrip("/") diff --git a/tests/unit/test_deepnote_streamlit_auth.py b/tests/unit/test_deepnote_streamlit_auth.py index dd1c2f67..5e377d25 100644 --- a/tests/unit/test_deepnote_streamlit_auth.py +++ b/tests/unit/test_deepnote_streamlit_auth.py @@ -94,6 +94,12 @@ def test_explicit_app_id_is_validated_before_network(http, state, value): {"apiOrigin": "https://user:pass@example.com"}, {"apiOrigin": "https://example.com/path"}, {"apiOrigin": "https://[::1"}, + {"apiOrigin": "https://example.com?"}, + {"apiOrigin": "https://example.com#"}, + {"apiOrigin": "https://example.com;"}, + {"apiOrigin": "https://example.com/?"}, + {"apiOrigin": "https://example.com/#"}, + {"apiOrigin": "https://example.com/;"}, ], ) def test_malformed_credentials_are_not_cached(http, state, overrides): diff --git a/tests/unit/test_deepnote_streamlit_cloud_runner.py b/tests/unit/test_deepnote_streamlit_cloud_runner.py index 72ec2629..95c05269 100644 --- a/tests/unit/test_deepnote_streamlit_cloud_runner.py +++ b/tests/unit/test_deepnote_streamlit_cloud_runner.py @@ -1,4 +1,5 @@ import time +from pathlib import Path from typing import Any import pytest @@ -86,8 +87,13 @@ def test_hosted_run_uses_viewer_and_readonly_even_with_explicit_owner_token( @pytest.mark.parametrize("ambient_auth", ["netrc", "session"]) def test_resolved_viewer_token_overrides_requests_auth( - context, http, monkeypatch, tmp_path, ambient_auth -): + context: dict[str, bool], + http: responses.RequestsMock, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + ambient_auth: str, +) -> None: + """Keep the viewer bearer authoritative over ambient Requests authentication.""" monkeypatch.setenv("DEEPNOTE_STREAMLIT_APP_ID", APP_ID) http.post( TOKEN_URL, @@ -105,7 +111,8 @@ def test_resolved_viewer_token_overrides_requests_auth( monkeypatch.setenv("NETRC", str(netrc)) else: - def owner_auth(request): + def owner_auth(request: requests.PreparedRequest) -> requests.PreparedRequest: + """Represent an injected session configured with an owner identity.""" request.headers["Authorization"] = "Bearer owner" return request diff --git a/tests/unit/test_deepnote_streamlit_widgets.py b/tests/unit/test_deepnote_streamlit_widgets.py index 8a1da46a..e7f5de4d 100644 --- a/tests/unit/test_deepnote_streamlit_widgets.py +++ b/tests/unit/test_deepnote_streamlit_widgets.py @@ -184,7 +184,10 @@ def test_empty_dates_stay_empty_instead_of_becoming_today() -> None: @pytest.mark.parametrize("value", [["2026-08-01", ""], ["", "2026-08-17"]]) -def test_saved_open_ended_date_range_keeps_its_chosen_endpoint(value) -> None: +def test_saved_open_ended_date_range_keeps_its_chosen_endpoint( + value: list[str], +) -> None: + """Preserve the saved endpoint instead of clearing an open-ended range.""" assert render_inputs( [InputBlock("period", "input-date-range", value)], FakeContainer() ) == {"period": value} @@ -192,9 +195,12 @@ def test_saved_open_ended_date_range_keeps_its_chosen_endpoint(value) -> None: @pytest.mark.parametrize("value", [["2026-08-01", ""], ["", "2026-08-17"]]) def test_real_widgets_preserve_and_edit_open_ended_date_ranges( - streamlit_app_test: "type[AppTest]", value + streamlit_app_test: "type[AppTest]", value: list[str] ) -> None: - def app(value) -> None: + """Keep open endpoints visible and allow completing them in real widgets.""" + + def app(value: list[str]) -> None: + """Render a saved open-ended range inside a Streamlit script.""" import streamlit as st from deepnote_toolkit.notebooks import InputBlock diff --git a/tests/unit/test_notebooks_runners.py b/tests/unit/test_notebooks_runners.py index 51c2cc14..d62ff1a5 100644 --- a/tests/unit/test_notebooks_runners.py +++ b/tests/unit/test_notebooks_runners.py @@ -588,6 +588,36 @@ def test_same_origin_redirect_is_also_refused(http, runner): assert len(http.calls) == 1 +@pytest.mark.parametrize( + "header", [None, "Authorization", "authorization", "aUtHoRiZaTiOn"] +) +def test_explicit_auth_headers_override_session_auth_regardless_of_case( + http: responses.RequestsMock, header: str | None +) -> None: + """Honor HTTP header casing while retaining session auth for unauthenticated calls.""" + from deepnote_toolkit.notebooks.transport import request_json + + def ambient_auth(request: requests.PreparedRequest) -> requests.PreparedRequest: + """Represent a session configured with a different API identity.""" + request.headers["Authorization"] = "Bearer ambient" + return request + + http.get("https://api.example/info", json={}) + with session() as transport: + transport.auth = ambient_auth + request_json( + transport, + "GET", + "https://api.example/info", + headers={header: "Bearer selected"} if header else {}, + timeout=1, + ) + + assert http.calls[0].request.headers["Authorization"] == ( + "Bearer selected" if header else "Bearer ambient" + ) + + def test_exhausted_poll_budget_does_not_even_fetch_credentials(http): from deepnote_toolkit.notebooks.api_client import DeepnoteApiClient From bc6bdfa0ef54b6002caf9d747fbff68ae7875872 Mon Sep 17 00:00:00 2001 From: Wojtek <6819634+voyti@users.noreply.github.com> Date: Tue, 22 Sep 2026 02:47:55 +0200 Subject: [PATCH 37/43] fix(streamlit): validate normalized credential origins --- deepnote_toolkit/streamlit/auth.py | 5 +++-- tests/unit/test_deepnote_streamlit_auth.py | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/deepnote_toolkit/streamlit/auth.py b/deepnote_toolkit/streamlit/auth.py index 52d85961..8f5b8387 100644 --- a/deepnote_toolkit/streamlit/auth.py +++ b/deepnote_toolkit/streamlit/auth.py @@ -242,6 +242,7 @@ def _validated_origin(value: str, *, name: str) -> str: raise CurrentUserApiTokenError( f"{name} must be a valid HTTP(S) origin." ) from error + normalized = value.rstrip("/") if ( parsed.scheme not in {"http", "https"} or not parsed.netloc @@ -251,7 +252,7 @@ def _validated_origin(value: str, *, name: str) -> str: or parsed.params or parsed.query or parsed.fragment - or value.endswith(("?", "#", ";")) + or normalized.endswith(("?", "#", ";")) ): raise CurrentUserApiTokenError(f"{name} must be a valid HTTP(S) origin.") - return value.rstrip("/") + return normalized diff --git a/tests/unit/test_deepnote_streamlit_auth.py b/tests/unit/test_deepnote_streamlit_auth.py index 5e377d25..28b88384 100644 --- a/tests/unit/test_deepnote_streamlit_auth.py +++ b/tests/unit/test_deepnote_streamlit_auth.py @@ -100,6 +100,7 @@ def test_explicit_app_id_is_validated_before_network(http, state, value): {"apiOrigin": "https://example.com/?"}, {"apiOrigin": "https://example.com/#"}, {"apiOrigin": "https://example.com/;"}, + {"apiOrigin": "https://example.com;/"}, ], ) def test_malformed_credentials_are_not_cached(http, state, overrides): From 06b9f613f5c37a972f48e14910be5e6506a8f1b0 Mon Sep 17 00:00:00 2001 From: Wojciech Apanowicz Date: Tue, 22 Sep 2026 14:13:33 +0200 Subject: [PATCH 38/43] refactor(notebooks): follow the API contracts and inject the Streamlit runtime Run and snapshot statuses are closed literals, the create-run response is decoded flat and the get-run response nested, and the "id" and "viewUrl" fields that no contract carries are gone. Input blocks are built from the validated notebook model. ViewerCredentials asks an injected StreamlitRuntime for the app ID, the viewer cookie and the thread state, so tests pass a fake instead of patching private functions. current_user_api_credentials is exported. A slider default outside its bounds is clamped with a warning, like a stale multi-select choice. The launcher passes the app ID through without validating it. --- deepnote_toolkit/notebooks/_schemas.py | 16 +- deepnote_toolkit/notebooks/api_client.py | 92 +++++----- deepnote_toolkit/notebooks/api_types.py | 2 - deepnote_toolkit/notebooks/cloud_runner.py | 1 - deepnote_toolkit/notebooks/local_runner.py | 2 +- deepnote_toolkit/notebooks/run_result.py | 4 +- deepnote_toolkit/notebooks/wire.py | 10 +- deepnote_toolkit/streamlit/__init__.py | 8 +- deepnote_toolkit/streamlit/auth.py | 173 +++++++++--------- deepnote_toolkit/streamlit/cloud_runner.py | 3 + .../streamlit/viewer_credentials.py | 26 +-- deepnote_toolkit/streamlit/widgets.py | 32 ++-- deepnote_toolkit/streamlit_data_apps.py | 8 +- docs/streamlit-apps.md | 74 ++++---- docs/testing/streamlit-hosted.md | 31 ---- installer/module/streamlit.py | 15 +- tests/unit/helpers/notebook_api.py | 10 +- tests/unit/helpers/streamlit_runtime.py | 28 +++ tests/unit/test_deepnote_streamlit_auth.py | 73 ++++---- .../test_deepnote_streamlit_cloud_runner.py | 142 +++++++------- tests/unit/test_deepnote_streamlit_widgets.py | 38 ++-- tests/unit/test_notebooks_runners.py | 54 +++--- tests/unit/test_streamlit.py | 7 +- tests/unit/test_streamlit_data_apps.py | 2 +- 24 files changed, 404 insertions(+), 447 deletions(-) delete mode 100644 docs/testing/streamlit-hosted.md create mode 100644 tests/unit/helpers/streamlit_runtime.py diff --git a/deepnote_toolkit/notebooks/_schemas.py b/deepnote_toolkit/notebooks/_schemas.py index 472d4a43..12a14695 100644 --- a/deepnote_toolkit/notebooks/_schemas.py +++ b/deepnote_toolkit/notebooks/_schemas.py @@ -1,7 +1,6 @@ """Consumed fields of the v2 API contracts (contracts/runs.ts and notebooks.ts). -Extra fields are intentionally ignored. Status strings are extensible: only the -known terminal run statuses stop polling. Missing or ill-typed statuses are errors. +Extra fields are ignored. """ from __future__ import annotations @@ -10,6 +9,8 @@ from pydantic import BaseModel, Field, StrictBool, StrictFloat, StrictInt, StrictStr +from .api_types import RunStatus, SnapshotStatus + class ApiInput(BaseModel): name: StrictStr @@ -34,13 +35,16 @@ class NotebookResponse(BaseModel): class ApiRun(BaseModel): run_id: StrictStr = Field(alias="runId", min_length=1) - status: StrictStr = Field(min_length=1) - snapshot_status: StrictStr | None = Field(default=None, alias="snapshotStatus") + status: RunStatus + snapshot_status: SnapshotStatus | None = Field(default=None, alias="snapshotStatus") snapshot_blocks: list[dict[str, Any]] | None = Field( default=None, alias="snapshotBlocks" ) - error: StrictStr | dict[str, Any] | None = None - view_url: StrictStr | None = Field(default=None, alias="viewUrl") + error: StrictStr | None = None + + +class GetRunResponse(BaseModel): + run: ApiRun class ViewerTokenResponse(BaseModel): diff --git a/deepnote_toolkit/notebooks/api_client.py b/deepnote_toolkit/notebooks/api_client.py index fe00be83..94944688 100644 --- a/deepnote_toolkit/notebooks/api_client.py +++ b/deepnote_toolkit/notebooks/api_client.py @@ -2,21 +2,22 @@ from __future__ import annotations -import json import time from collections.abc import Callable, Mapping from dataclasses import dataclass -from typing import Any +from typing import Any, TypeVar, cast from urllib.parse import quote import requests -from pydantic import ValidationError +from pydantic import BaseModel, ValidationError -from ._schemas import ApiRun, NotebookResponse +from ._schemas import ApiInput, ApiRun, GetRunResponse, NotebookResponse from .api_types import ( - SNAPSHOT_STATUSES, + INPUT_BLOCK_TYPES, TERMINAL_RUN_STATUSES, + InputBlockType, InputValue, + RunStatus, SnapshotStatus, StorageMode, ) @@ -24,7 +25,9 @@ from .models import InputBlock, NotebookOutput from .runner import RunnerError from .transport import request_json -from .wire import decode_block_outputs, decode_inputs +from .wire import decode_block_outputs + +Schema = TypeVar("Schema", bound=BaseModel) @dataclass(frozen=True) @@ -40,11 +43,10 @@ class CloudRun: """The state of one run. `outputs` is None until the run's snapshot is stored.""" run_id: str - status: str + status: RunStatus snapshot_status: SnapshotStatus | None outputs: tuple[NotebookOutput, ...] | None error: str | None - view_url: str | None @property def is_finished(self) -> bool: @@ -73,15 +75,14 @@ def get_notebook(self, notebook_id: str) -> CloudNotebook: """Read a notebook's name and input blocks.""" payload = self._request("GET", f"/v2/notebooks/{quote(notebook_id, safe='')}") - try: - notebook = NotebookResponse(**payload).notebook - except ValidationError as error: - raise RunnerError( - "Deepnote API returned an invalid notebook response" - ) from error + notebook = _validate(NotebookResponse, payload, "notebook").notebook return CloudNotebook( name=notebook.name, - inputs=decode_inputs(payload["notebook"].get("inputs"), name_key="name"), + inputs=tuple( + _input_block(value) + for value in notebook.inputs + if value.type in INPUT_BLOCK_TYPES + ), ) def create_run( @@ -103,7 +104,8 @@ def create_run( } if storage_mode is not None: body["detachedRunStorageMode"] = storage_mode - return _decode_run(self._request("POST", "/v2/runs", body, timeout=timeout)) + payload = self._request("POST", "/v2/runs", body, timeout=timeout) + return _cloud_run(_validate(ApiRun, payload, "run")) def get_run(self, run_id: str, *, timeout: float | None = None) -> CloudRun: """Read a run with the outputs of the notebook it executed.""" @@ -114,7 +116,7 @@ def get_run(self, run_id: str, *, timeout: float | None = None) -> CloudRun: f"/v2/runs/{quote(run_id, safe='')}?snapshotDelivery=blocks", timeout=timeout, ) - return _decode_run(payload, run_id=run_id) + return _cloud_run(_validate(GetRunResponse, payload, "run").run) def _request( self, @@ -148,6 +150,15 @@ def _request( ) +def _validate(schema: type[Schema], payload: Mapping[str, Any], what: str) -> Schema: + try: + return schema(**payload) + except ValidationError as error: + raise RunnerError( + f"Deepnote API returned an invalid {what} response" + ) from error + + def _encode_input(name: str, value: Any) -> InputValue: """Convert a value to the form the runs API accepts, or raise `ValueError`.""" @@ -163,34 +174,29 @@ def _encode_input(name: str, value: Any) -> InputValue: return str(value) -def _decode_run(payload: Mapping[str, Any], *, run_id: str | None = None) -> CloudRun: - nested = payload.get("run") - run = nested if isinstance(nested, Mapping) else payload - run_id = run.get("runId") or run.get("id") or run_id - if not isinstance(run_id, str) or not run_id: - raise RunnerError("Deepnote API response did not include a run id") - try: - parsed = ApiRun(**{**run, "runId": run_id}) - except ValidationError as error: - raise RunnerError( - f"Deepnote API returned an invalid run response for {run_id}" - ) from error - error = parsed.error - if isinstance(error, Mapping): - error = error.get("message") or json.dumps(error) +def _input_block(value: ApiInput) -> InputBlock: + return InputBlock( + variable_name=value.name, + type=cast(InputBlockType, value.type), + value=value.value, + label=value.label, + options=tuple(value.options), + multiple=value.multiple, + min=value.min, + max=value.max, + step=value.step, + ) + + +def _cloud_run(run: ApiRun) -> CloudRun: return CloudRun( - run_id=parsed.run_id, - status=parsed.status, - snapshot_status=( - parsed.snapshot_status - if parsed.snapshot_status in SNAPSHOT_STATUSES - else None - ), + run_id=run.run_id, + status=run.status, + snapshot_status=run.snapshot_status, outputs=( - decode_block_outputs(parsed.snapshot_blocks, id_key="id") - if parsed.snapshot_blocks is not None + decode_block_outputs(run.snapshot_blocks, id_key="id") + if run.snapshot_blocks is not None else None ), - error=str(error) if error is not None else None, - view_url=parsed.view_url, + error=run.error, ) diff --git a/deepnote_toolkit/notebooks/api_types.py b/deepnote_toolkit/notebooks/api_types.py index 15a9d648..1a9623e4 100644 --- a/deepnote_toolkit/notebooks/api_types.py +++ b/deepnote_toolkit/notebooks/api_types.py @@ -22,8 +22,6 @@ InputValue = Union[str, bool, list[str]] INPUT_BLOCK_TYPES: frozenset[InputBlockType] = frozenset(get_args(InputBlockType)) -RUN_STATUSES: frozenset[RunStatus] = frozenset(get_args(RunStatus)) TERMINAL_RUN_STATUSES: frozenset[RunStatus] = frozenset( {"success", "error", "internal_error", "stopped"} ) -SNAPSHOT_STATUSES: frozenset[SnapshotStatus] = frozenset(get_args(SnapshotStatus)) diff --git a/deepnote_toolkit/notebooks/cloud_runner.py b/deepnote_toolkit/notebooks/cloud_runner.py index e98963e3..f36f71c4 100644 --- a/deepnote_toolkit/notebooks/cloud_runner.py +++ b/deepnote_toolkit/notebooks/cloud_runner.py @@ -116,7 +116,6 @@ def run(self, inputs: Mapping[str, Any]) -> RunResult: status=run.status, error=run.error, snapshot_status=run.snapshot_status, - view_url=run.view_url, ) def _timed_out(self, run: CloudRun) -> None: diff --git a/deepnote_toolkit/notebooks/local_runner.py b/deepnote_toolkit/notebooks/local_runner.py index 7c4d32b2..db2d6a80 100644 --- a/deepnote_toolkit/notebooks/local_runner.py +++ b/deepnote_toolkit/notebooks/local_runner.py @@ -37,7 +37,7 @@ def info(self) -> RunnerInfo: payload = self._request("GET", "/api/info") return RunnerInfo( notebook=str(payload.get("notebook", "Untitled project")), - inputs=decode_inputs(payload.get("inputs"), name_key="variableName"), + inputs=decode_inputs(payload.get("inputs")), run_target=str(payload.get("runTarget", "")), ) diff --git a/deepnote_toolkit/notebooks/run_result.py b/deepnote_toolkit/notebooks/run_result.py index d818ff49..22af4fb4 100644 --- a/deepnote_toolkit/notebooks/run_result.py +++ b/deepnote_toolkit/notebooks/run_result.py @@ -14,8 +14,8 @@ class RunResult(OutputCollection): """What one run produced, whether it ran in Deepnote Cloud or locally. - `snapshot_status` is set for cloud runs. `snapshot` and `created` are set by - the local runner. + `snapshot_status` is set for cloud runs. `view_url`, `snapshot` and `created` + are set by the local runner. """ target: str diff --git a/deepnote_toolkit/notebooks/wire.py b/deepnote_toolkit/notebooks/wire.py index 28498939..be0c623d 100644 --- a/deepnote_toolkit/notebooks/wire.py +++ b/deepnote_toolkit/notebooks/wire.py @@ -1,4 +1,4 @@ -"""Decode the JSON shapes shared by the API, the sidecar and `.deepnote` files.""" +"""Decode the JSON shapes shared by the sidecar, the API and `.deepnote` files.""" from __future__ import annotations @@ -31,14 +31,14 @@ def string_tuple(value: Any) -> tuple[str, ...]: return tuple(str(item) for item in value) if isinstance(value, list) else () -def decode_inputs(values: Any, *, name_key: str) -> tuple[InputBlock, ...]: - """Read an API's camelCase inputs, skipping any without a name or a known type.""" +def decode_inputs(values: Any) -> tuple[InputBlock, ...]: + """Read the sidecar's inputs, skipping any without a name or a known type.""" if not isinstance(values, list): return () return tuple( InputBlock( - variable_name=value[name_key], + variable_name=value["variableName"], type=cast(InputBlockType, value["type"]), label=optional_string(value.get("label")), value=value.get("value"), @@ -50,7 +50,7 @@ def decode_inputs(values: Any, *, name_key: str) -> tuple[InputBlock, ...]: ) for value in values if isinstance(value, Mapping) - and isinstance(value.get(name_key), str) + and isinstance(value.get("variableName"), str) and isinstance(value.get("type"), str) and value["type"] in INPUT_BLOCK_TYPES ) diff --git a/deepnote_toolkit/streamlit/__init__.py b/deepnote_toolkit/streamlit/__init__.py index 9cd8fb83..83c8a556 100644 --- a/deepnote_toolkit/streamlit/__init__.py +++ b/deepnote_toolkit/streamlit/__init__.py @@ -1,6 +1,12 @@ """Helpers for Streamlit apps built on Deepnote notebooks.""" +from .auth import CurrentUserApiTokenError, current_user_api_credentials from .cloud_runner import StreamlitCloudRunner from .widgets import render_inputs -__all__ = ["StreamlitCloudRunner", "render_inputs"] +__all__ = [ + "CurrentUserApiTokenError", + "StreamlitCloudRunner", + "current_user_api_credentials", + "render_inputs", +] diff --git a/deepnote_toolkit/streamlit/auth.py b/deepnote_toolkit/streamlit/auth.py index 8f5b8387..4eeeb5c4 100644 --- a/deepnote_toolkit/streamlit/auth.py +++ b/deepnote_toolkit/streamlit/auth.py @@ -7,8 +7,9 @@ import os import re import time +from collections.abc import MutableMapping from dataclasses import dataclass, field -from typing import Any +from typing import Any, Protocol from urllib.parse import urlparse import requests @@ -27,7 +28,6 @@ STREAMLIT_APP_HOST_PATTERN = re.compile(rf"^streamlit-({_APP_ID})\.", re.IGNORECASE) STREAMLIT_APP_ID_ENV = "DEEPNOTE_STREAMLIT_APP_ID" - _SESSION_STATE_KEY = "_deepnote_current_user_api_credentials" _EXPIRY_MARGIN_SECONDS = 60 @@ -49,57 +49,94 @@ class CurrentUserApiCredentials: expires_at_seconds: float -def current_user_api_token() -> str: - """Return a short-lived public API bearer for the current Streamlit viewer. +class StreamlitRuntime(Protocol): + """What viewer authentication asks Streamlit about the process and the thread.""" - The opaque streamlit-token cookie is exchanged for a viewer-scoped token. - It is never itself used as a public API bearer. - """ + def has_request(self) -> bool: + """Whether this thread is running a script for a viewer.""" + + def is_worker_thread(self) -> bool: + """Whether Streamlit is running but this thread has no viewer request.""" + + def app_id(self) -> str | None: + """The hosted app's ID, from the launcher or from the request host.""" + + def viewer_cookie(self) -> str | None: + """The viewer's streamlit-token cookie.""" + + def session_state(self) -> MutableMapping[str, Any] | None: + """The viewer's session state, or None outside a script run.""" + + +class _DefaultStreamlitRuntime: + def has_request(self) -> bool: + try: + from streamlit.runtime.scriptrunner import ( # type: ignore[import-not-found] + get_script_run_ctx, + ) + except ImportError: + return False + + return get_script_run_ctx(suppress_warning=True) is not None + + def is_worker_thread(self) -> bool: + try: + from streamlit import runtime # type: ignore[import-not-found] + except ImportError: + return False + + return runtime.exists() and not self.has_request() + + def app_id(self) -> str | None: + hosted = os.environ.get(STREAMLIT_APP_ID_ENV) + if hosted is not None: + return hosted + return _read_streamlit_app_id_from_context() if self.has_request() else None + + def viewer_cookie(self) -> str | None: + return read_streamlit_token_from_context() if self.has_request() else None + + def session_state(self) -> MutableMapping[str, Any] | None: + if not self.has_request(): + return None + import streamlit as st # type: ignore[import-not-found] + + return st.session_state - return current_user_api_credentials().token + +streamlit_runtime: StreamlitRuntime = _DefaultStreamlitRuntime() def current_user_api_credentials( *, - app_id: str | None = None, - streamlit_token: str | None = None, timeout: float = 10, session: requests.Session | None = None, + runtime: StreamlitRuntime = streamlit_runtime, ) -> CurrentUserApiCredentials: - """Exchange the active viewer cookie for public API credentials. + """Exchange the viewer's cookie for short-lived public API credentials. - The bearer is only valid at the returned API origin. Credentials are reused + The token is only valid at the returned API origin. Credentials are reused within the current Streamlit session until shortly before they expire, and never shared between sessions. """ - resolved_app_id = ( - app_id - if app_id is not None - else (_read_hosted_app_id() or _read_streamlit_app_id_from_context()) - ) - if not resolved_app_id: + app_id = runtime.app_id() + if not app_id: raise CurrentUserApiTokenError( - "Could not resolve a Deepnote Streamlit app ID from the request host." + "Could not resolve the Deepnote Streamlit app ID." ) + if not re.fullmatch(_APP_ID, app_id, re.IGNORECASE): + raise CurrentUserApiTokenError("The Deepnote Streamlit app ID must be a UUID.") + app_id = app_id.lower() - if not isinstance(resolved_app_id, str) or not re.fullmatch( - _APP_ID, resolved_app_id, re.IGNORECASE - ): - raise CurrentUserApiTokenError("app_id must be a UUID.") - resolved_app_id = resolved_app_id.lower() - - viewer_token = streamlit_token or read_streamlit_token_from_context() + viewer_token = runtime.viewer_cookie() if not viewer_token: raise CurrentUserApiTokenError( "Could not read the current viewer's streamlit-token cookie." ) - session_state = _read_streamlit_session_state() - cache_key = ( - resolved_app_id, - hashlib.sha256(viewer_token.encode()).hexdigest(), - ) + session_state = runtime.session_state() + cache_key = (app_id, hashlib.sha256(viewer_token.encode()).hexdigest()) if session_state is not None: cached = session_state.get(_SESSION_STATE_KEY) if ( @@ -109,13 +146,26 @@ def current_user_api_credentials( ): return cached[1] + credentials = _exchange(app_id, viewer_token, timeout=timeout, session=session) + if session_state is not None: + session_state[_SESSION_STATE_KEY] = (cache_key, credentials) + return credentials + + +def _exchange( + app_id: str, + viewer_token: str, + *, + timeout: float, + session: requests.Session | None, +) -> CurrentUserApiCredentials: owned_session = session is None http = session if session is not None else requests.Session() try: payload = request_json( http, "POST", - get_absolute_userpod_api_url(f"streamlit-apps/{resolved_app_id}/api-token"), + get_absolute_userpod_api_url(f"streamlit-apps/{app_id}/api-token"), headers={"StreamlitToken": viewer_token, **get_project_auth_headers()}, timeout=timeout, ) @@ -142,33 +192,9 @@ def current_user_api_credentials( finally: if owned_session: http.close() - if session_state is not None: - session_state[_SESSION_STATE_KEY] = (cache_key, credentials) return credentials -def _read_streamlit_session_state() -> Any | None: - """Return the current session's state, or None outside a Streamlit script run.""" - - try: - import streamlit as st # type: ignore[import-not-found] - from streamlit.runtime.scriptrunner import ( # type: ignore[import-not-found] - get_script_run_ctx, - ) - except ImportError: - return None - - if get_script_run_ctx(suppress_warning=True) is None: - return None - return st.session_state - - -def _read_hosted_app_id() -> str | None: - """Return the app ID that Deepnote's launcher exports to a hosted app's process.""" - - return os.environ.get(STREAMLIT_APP_ID_ENV) - - def _read_streamlit_app_id_from_context() -> str | None: """Resolve the app UUID from the external Streamlit request hostname. @@ -199,41 +225,6 @@ def _read_streamlit_app_id_from_context() -> str | None: return None -def _has_hosted_streamlit_context() -> bool: - """Return whether this request carries either hosted-app identity signal.""" - - return bool( - _read_streamlit_app_id_from_context() or read_streamlit_token_from_context() - ) - - -def _has_script_run_context() -> bool: - """Return whether this thread is running a Streamlit script for a viewer.""" - - try: - from streamlit.runtime.scriptrunner import ( # type: ignore[import-not-found] - get_script_run_ctx, - ) - except ImportError: - return False - - return get_script_run_ctx(suppress_warning=True) is not None - - -def _is_streamlit_thread_without_request() -> bool: - """Return whether Streamlit is running but this thread has no viewer request. - - Worker threads see no headers or cookies, so they look identical to a local script. - """ - - try: - from streamlit import runtime # type: ignore[import-not-found] - except ImportError: - return False - - return runtime.exists() and not _has_script_run_context() - - def _validated_origin(value: str, *, name: str) -> str: """Validate an origin and normalize URL parser failures to authentication errors.""" try: diff --git a/deepnote_toolkit/streamlit/cloud_runner.py b/deepnote_toolkit/streamlit/cloud_runner.py index 1f70cd45..1d61c0ae 100644 --- a/deepnote_toolkit/streamlit/cloud_runner.py +++ b/deepnote_toolkit/streamlit/cloud_runner.py @@ -14,6 +14,7 @@ from deepnote_toolkit.notebooks.models import RunnerInfo from deepnote_toolkit.notebooks.run_result import RunResult +from .auth import StreamlitRuntime, streamlit_runtime from .viewer_credentials import ViewerCredentials @@ -41,6 +42,7 @@ def __init__( session: requests.Session | None = None, sleep: Sleep = time.sleep, clock: Callable[[], float] = time.monotonic, + runtime: StreamlitRuntime = streamlit_runtime, ): session = session if session is not None else requests.Session() self._runner = DeepnoteCloudRunner( @@ -52,6 +54,7 @@ def __init__( timeout=min(timeout, 10), session=session, local=local, + runtime=runtime, ), storage_mode=storage_mode, timeout=timeout, diff --git a/deepnote_toolkit/streamlit/viewer_credentials.py b/deepnote_toolkit/streamlit/viewer_credentials.py index b0496567..15f97584 100644 --- a/deepnote_toolkit/streamlit/viewer_credentials.py +++ b/deepnote_toolkit/streamlit/viewer_credentials.py @@ -16,16 +16,14 @@ from .auth import ( CurrentUserApiTokenError, - _has_hosted_streamlit_context, - _has_script_run_context, - _is_streamlit_thread_without_request, - _read_hosted_app_id, + StreamlitRuntime, current_user_api_credentials, + streamlit_runtime, ) _NO_REQUEST = ( "No viewer request is available on this thread. Call the runner from the " - "Streamlit script thread" + "Streamlit script thread." ) @@ -46,35 +44,39 @@ def __init__( timeout: float = 10, session: requests.Session | None = None, local: bool = False, + runtime: StreamlitRuntime = streamlit_runtime, ): self._local_mode = local self._local_token_explicit = token is not None or token_provider is not None self._local = token_credentials(token, token_provider, base_url=base_url) self._timeout = timeout self._session = session + self._runtime = runtime def __call__(self, *, timeout: float = 30) -> ApiCredentials: """Return the viewer's credentials, or the local ones outside hosting.""" - has_request = _has_script_run_context() + has_request = self._runtime.has_request() is_hosted = ( - _read_hosted_app_id() is not None + self._runtime.app_id() is not None or bool(os.environ.get("DEEPNOTE_PROJECT_ID")) - or (has_request and _has_hosted_streamlit_context()) + or self._runtime.viewer_cookie() is not None ) if is_hosted or (has_request and not self._local_mode): if not has_request: - raise RunnerError(_NO_REQUEST + ".") + raise RunnerError(_NO_REQUEST) try: viewer = current_user_api_credentials( - timeout=min(timeout, self._timeout), session=self._session + timeout=min(timeout, self._timeout), + session=self._session, + runtime=self._runtime, ) except CurrentUserApiTokenError as error: raise RunnerError(str(error), transient=error.transient) from error return ApiCredentials(token=viewer.token, api_origin=viewer.api_origin) - if _is_streamlit_thread_without_request(): - raise RunnerError(_NO_REQUEST + ".") + if self._runtime.is_worker_thread(): + raise RunnerError(_NO_REQUEST) if not self._local_mode or not self._local_token_explicit: raise RunnerError( diff --git a/deepnote_toolkit/streamlit/widgets.py b/deepnote_toolkit/streamlit/widgets.py index 59adea15..22d11da8 100644 --- a/deepnote_toolkit/streamlit/widgets.py +++ b/deepnote_toolkit/streamlit/widgets.py @@ -78,16 +78,22 @@ def _render_one(container: Any, input_block: InputBlock, label: str, key: str) - minimum = input_block.min if input_block.min is not None else 0 maximum = input_block.max if input_block.max is not None else 100 step = input_block.step if input_block.step is not None else 1 - value = _as_number(input_block.value, minimum) if ( - not all(math.isfinite(n) for n in (minimum, maximum, step, value)) + not all(math.isfinite(n) for n in (minimum, maximum, step)) or minimum >= maximum or step <= 0 - or not minimum <= value <= maximum ): raise ValueError( - f"{label}: slider needs finite ordered bounds, a positive step, and a default within its bounds" + f"{label}: slider needs finite ordered bounds and a positive step" ) + value = _as_number(input_block.value) + if input_block.value is not None and ( + value is None or not minimum <= value <= maximum + ): + container.warning( + f"{label}: the saved default is outside the slider's bounds. Review the value before running." + ) + value = min(max(value if value is not None else minimum, minimum), maximum) if any(isinstance(number, float) for number in (minimum, maximum, value, step)): minimum, maximum, value, step = ( float(number) for number in (minimum, maximum, value, step) @@ -145,19 +151,15 @@ def _as_bool(value: Any) -> bool: return str(value).lower() in {"true", "1"} -def _as_number(value: Any, fallback: float | int) -> float | int: - """Decode numeric defaults while preserving fractional values.""" - if value is None: - return fallback +def _as_number(value: Any) -> float | int | None: + """Read a saved slider value. None when it is missing or not a finite number.""" try: number = float(value) - return ( - number - if isinstance(fallback, float) or not number.is_integer() - else int(number) - ) - except (TypeError, ValueError) as error: - raise ValueError("Slider default must be a number") from error + except (TypeError, ValueError): + return None + if not math.isfinite(number): + return None + return int(number) if number.is_integer() else number def _as_date(value: Any) -> date | None: diff --git a/deepnote_toolkit/streamlit_data_apps.py b/deepnote_toolkit/streamlit_data_apps.py index fb87b6d4..f989e61f 100644 --- a/deepnote_toolkit/streamlit_data_apps.py +++ b/deepnote_toolkit/streamlit_data_apps.py @@ -87,12 +87,6 @@ def read_streamlit_token_from_context() -> Optional[str]: return token -def _read_streamlit_token_from_context() -> Optional[str]: - """Backward-compatible private alias for the public cookie helper.""" - - return read_streamlit_token_from_context() - - def get_federated_auth_token( integration_id: str, *, @@ -127,7 +121,7 @@ def get_federated_auth_token( if not integration_id: raise StreamlitFederatedAuthError("integration_id is required.") - token = streamlit_token or _read_streamlit_token_from_context() + token = streamlit_token or read_streamlit_token_from_context() if not token: raise StreamlitFederatedAuthError( "Could not read the `streamlit-token` cookie from the Streamlit context. " diff --git a/docs/streamlit-apps.md b/docs/streamlit-apps.md index dc029ca9..64ba57a2 100644 --- a/docs/streamlit-apps.md +++ b/docs/streamlit-apps.md @@ -31,10 +31,10 @@ if st.button("Run"): ## Authentication and local development -On Deepnote, `StreamlitCloudRunner` uses the current viewer's permissions. -The hosting environment must support viewer-token exchange. If the app ID, -viewer cookie, or exchange is unavailable, the call fails; it does not fall back -to an owner token. Call it on the Streamlit script thread, not a worker thread. +On Deepnote, `StreamlitCloudRunner` runs the notebook with the current viewer's +permissions. If the app ID, viewer cookie or token exchange is unavailable, the +call fails instead of falling back to an owner token. Call it on the Streamlit +script thread, not a worker thread. For a locally hosted Streamlit app, opt into local credentials explicitly: @@ -46,28 +46,29 @@ runner = StreamlitCloudRunner( ) ``` -`token_provider=` can supply a renewable token instead. A Deepnote app or project -marker overrides `local=True` and explicit tokens. Older launchers can also be -recognized by the request host or viewer cookie. Do not set `local=True` in an -unmarked hosting environment: that is an explicit choice to use local credentials. +`token_provider=` can supply a renewable token instead. On Deepnote, `local=True` +and explicit tokens are ignored and the viewer is used. + +To call other API endpoints as the viewer, `current_user_api_credentials()` +returns the viewer's short-lived token and the API origin it is valid at. It +raises `CurrentUserApiTokenError` outside a hosted request. For Python code outside Streamlit, use `DeepnoteCloudRunner` from `deepnote_toolkit.notebooks`. It accepts `token=`, `token_provider=`, or `DEEPNOTE_TOKEN`. For a local `@deepnote/local-runner` sidecar, use `DeepnoteLocalRunner(base_url="http://127.0.0.1:8787")`. -`StreamlitCloudRunner` requires explicit local mode and credentials even when -called outside the Streamlit runtime. ## Inputs and outputs -`render_inputs()` preserves saved defaults, including `0` and `False`. A select -without a valid saved choice starts empty. Unselected single selects and incomplete -selections in the date-range picker are omitted from the returned dictionary; -disable your Run button until required fields are present. An omitted input uses -the notebook's value according to the API. Saved open-ended ranges use separate -start/end fields so their chosen endpoint is preserved. Stale multi-select choices -produce a warning. Invalid slider bounds/defaults and duplicate variable names -raise `ValueError`. File inputs render as text paths; this helper does not upload files. +`render_inputs()` keeps saved defaults, including `0` and `False`. A select +without a valid saved choice starts empty. Unselected single selects and +incomplete date-range selections are left out of the returned dictionary, so +disable your Run button until the required values are present. An omitted input +runs with the notebook's saved value. A saved open-ended date range renders as +separate start and end fields. Stale multi-select choices and slider defaults +outside the bounds are adjusted with a warning. Invalid slider bounds and +duplicate variable names raise `ValueError`. File inputs render as text paths; +this helper does not upload files. `runner.info().matches_inputs(document.inputs)` compares static input definitions: unique names, types, single/multiple selection, options, and slider bounds/steps. @@ -83,27 +84,16 @@ without network access. ## Execution settings -Streamlit runs are detached and use `storage_mode="readonly"`: they can read -persistent project files but cannot modify them. Use `storage_mode="read_write"` -only when the app intentionally needs to change those files. The general cloud -runner leaves storage mode to the API. - -`timeout` (600 seconds by default) is the elapsed-time budget for creation, -authentication, polling, and output retrieval. Each HTTP request and sleep is -limited to the remaining budget. Output retrieval also has its own -`snapshot_timeout` (10 seconds); only an explicitly pending snapshot is polled. -Requests uses socket timeouts, so OS DNS resolution or a server streaming bytes -can exceed a request budget; this is not hard cancellation of a running notebook. -Run-status GET polls retry transient failures up to five consecutive times; -snapshot GET polls retry within the snapshot budget. Creating a run is never -automatically retried. - -Pass `session=requests.Session()` to configure proxies or HTTP adapters. A custom -`credentials=` provider on `DeepnoteCloudRunner` receives a `timeout` keyword and -returns `ApiCredentials(token=..., api_origin=...)`. Providers should honor that -budget. Resolved bearer credentials take precedence over `.netrc` and session -authentication. API clients, HTTP helpers, and wire schemas are internal; -supported names are listed in each package's `__all__`. - -The existing `streamlit_data_apps` module handles database federation. Notebook -execution uses its viewer-cookie reader and does not replace its database APIs. +Streamlit runs use `storage_mode="readonly"`: the notebook can read the project's +files but not change them. Pass `storage_mode="read_write"` when the app needs to +write them. `DeepnoteCloudRunner` leaves the choice to the API. + +`timeout` (600 seconds by default) bounds the whole run, from creating it to +reading its outputs. Outputs can arrive after the run finishes; `snapshot_timeout` +(10 seconds) is how long to wait for them, and a result whose `snapshot_status` is +still `pending` has none. + +Pass `session=requests.Session()` to configure proxies or HTTP adapters. On +`DeepnoteCloudRunner`, `credentials=` accepts any callable that takes a `timeout` +keyword and returns `ApiCredentials(token=..., api_origin=...)`. Supported names +are listed in each package's `__all__`. diff --git a/docs/testing/streamlit-hosted.md b/docs/testing/streamlit-hosted.md deleted file mode 100644 index 4afcf057..00000000 --- a/docs/testing/streamlit-hosted.md +++ /dev/null @@ -1,31 +0,0 @@ -# Hosted Streamlit merge gate - -Run this on the **final PR commit** in a Deepnote test environment that supports -viewer API-token exchange. Local unit tests and Streamlit AppTest do not verify -platform permissions, storage mounts, or the deployed launcher. - -Record the commit, environment, app ID, notebook ID, test date, and results in the -PR before merging. Do not record tokens or cookies. - -1. Start the app through the deployed launcher. Confirm the app process sees its - UUID in `DEEPNOTE_STREAMLIT_APP_ID`. -2. As both owner and a workspace viewer, run a notebook with text and dataframe - outputs using `StreamlitCloudRunner`. Confirm success, available snapshot, and - outputs from only the executed notebook. -3. Have the notebook read a known project-storage file. Attempt a write to a - disposable path on the persistent storage mount and catch the expected - permission error. Confirm the default readonly run can read but cannot write. -4. Confirm each account cannot fetch the other account's run. An outsider must - not obtain a viewer token. -5. Repeat with a bogus `DEEPNOTE_TOKEN`, explicit `token=`, and a `token_provider` - that raises if called. Hosted execution must still use the viewer. -6. Disable project API access, retry, and confirm the server's reason is shown - without a fallback request. Restore the original project setting afterward. -7. Invoke the same runner from a worker thread. Confirm it raises before HTTP. -8. Stop the app, remove its app ID/cookie/host context in a local AppTest, and - confirm it fails closed unless local development is explicitly configured. - -The readonly literal is defined by the public API's -`DetachedRunStorageModeSchema` in `apps/webapp/server/public-api/v2/contracts/runs.ts` -(`read_write | readonly`) and is also sent by -`apps/webapp-client/src/features/static-files-app-client/connect.ts`. diff --git a/installer/module/streamlit.py b/installer/module/streamlit.py index d2136c73..8debe7e7 100644 --- a/installer/module/streamlit.py +++ b/installer/module/streamlit.py @@ -119,23 +119,12 @@ def start_streamlit_servers( arg_str = " ".join(args) - # The toolkit reads the app ID to run notebooks as the app's viewer. - app_id = app.get("id") - if not isinstance(app_id, str) or not app_id: - logger.warning( - "Streamlit app %r has no app ID; viewer authentication will fail", - entrypoint_path, - ) - # Always mark the process as hosted; the SDK validates the app ID. - env = { - "DEEPNOTE_STREAMLIT_APP_ID": app_id if isinstance(app_id, str) else "" - } - processes.append( venv.start_server( f"streamlit run {shlex.quote(entrypoint_path)} {arg_str}", cwd=directory_path, - env=env, + # The toolkit reads the app ID to run notebooks as the app's viewer. + env={"DEEPNOTE_STREAMLIT_APP_ID": str(app.get("id") or "")}, ) ) except Exception as e: diff --git a/tests/unit/helpers/notebook_api.py b/tests/unit/helpers/notebook_api.py index 468fca11..c0d7161e 100644 --- a/tests/unit/helpers/notebook_api.py +++ b/tests/unit/helpers/notebook_api.py @@ -20,7 +20,7 @@ def sleep(self, seconds): def run_response(status="success", **fields): - return {"run": {"runId": "run-1", "status": status, **fields}} + return {"runId": "run-1", "status": status, **fields} def session(): @@ -34,5 +34,9 @@ def body(call): def add_run(http, payload, *, create=False, origin="https://api.deepnote.com"): - path = "/v2/runs" if create else "/v2/runs/run-1?snapshotDelivery=blocks" - http.add(responses.POST if create else responses.GET, origin + path, json=payload) + """Register the flat POST /v2/runs response or the nested GET /v2/runs/{id} one.""" + if create: + http.add(responses.POST, origin + "/v2/runs", json=payload) + else: + path = "/v2/runs/run-1?snapshotDelivery=blocks" + http.add(responses.GET, origin + path, json={"run": payload}) diff --git a/tests/unit/helpers/streamlit_runtime.py b/tests/unit/helpers/streamlit_runtime.py new file mode 100644 index 00000000..747ef1b8 --- /dev/null +++ b/tests/unit/helpers/streamlit_runtime.py @@ -0,0 +1,28 @@ +"""A Streamlit runtime whose answers a test sets directly.""" + +from dataclasses import dataclass, field +from typing import Any, Optional + + +@dataclass +class FakeStreamlitRuntime: + request: bool = True + worker: bool = False + app: Optional[str] = None + cookie: Optional[str] = None + state: Optional[dict[str, Any]] = field(default_factory=dict) + + def has_request(self) -> bool: + return self.request + + def is_worker_thread(self) -> bool: + return self.worker + + def app_id(self) -> Optional[str]: + return self.app + + def viewer_cookie(self) -> Optional[str]: + return self.cookie + + def session_state(self) -> Optional[dict[str, Any]]: + return self.state diff --git a/tests/unit/test_deepnote_streamlit_auth.py b/tests/unit/test_deepnote_streamlit_auth.py index 28b88384..c1ac4349 100644 --- a/tests/unit/test_deepnote_streamlit_auth.py +++ b/tests/unit/test_deepnote_streamlit_auth.py @@ -8,17 +8,15 @@ from deepnote_toolkit.streamlit import auth from tests.unit.helpers.notebook_api import session +from tests.unit.helpers.streamlit_runtime import FakeStreamlitRuntime APP_ID = "3853c7f5-2048-4b57-946d-6c5592c3317e" TOKEN_URL = f"http://localhost:19456/userpod-api/streamlit-apps/{APP_ID}/api-token" @pytest.fixture -def state(monkeypatch): - state = {} - monkeypatch.delenv("DEEPNOTE_STREAMLIT_APP_ID", raising=False) - monkeypatch.setattr(auth, "_read_streamlit_session_state", lambda: state) - return state +def runtime(): + return FakeStreamlitRuntime(app=APP_ID, cookie="cookie") @pytest.fixture @@ -27,10 +25,8 @@ def http(): yield mock -def credentials(http_session, **kwargs): - return auth.current_user_api_credentials( - app_id=APP_ID, streamlit_token="cookie", session=http_session, **kwargs - ) +def credentials(http_session, runtime): + return auth.current_user_api_credentials(session=http_session, runtime=runtime) def payload(**overrides): @@ -42,11 +38,13 @@ def payload(**overrides): } -def test_exchange_uses_cookie_and_reuses_credentials_only_in_same_session(http, state): +def test_exchange_uses_cookie_and_reuses_credentials_only_in_same_session( + http, runtime +): http.post(TOKEN_URL, json=payload()) transport = session() - first = credentials(transport) - assert credentials(transport) is first + first = credentials(transport, runtime) + assert credentials(transport, runtime) is first assert ( first.token == "viewer" and first.api_origin == "https://api.deepnote-staging.com" @@ -54,32 +52,27 @@ def test_exchange_uses_cookie_and_reuses_credentials_only_in_same_session(http, assert len(http.calls) == 1 assert http.calls[0].request.headers["StreamlitToken"] == "cookie" assert "Authorization" not in http.calls[0].request.headers - state.clear() - assert credentials(transport) is not first + runtime.state.clear() + assert credentials(transport, runtime) is not first assert len(http.calls) == 2 -def test_changed_cookie_or_expiry_refreshes_credentials(http, state): +def test_changed_cookie_or_expiry_refreshes_credentials(http, runtime): http.post(TOKEN_URL, json=payload(expiresAtSeconds=time.time() + 30)) http.post(TOKEN_URL, json=payload(token="second")) http.post(TOKEN_URL, json=payload(token="third")) transport = session() - assert credentials(transport).token == "viewer" - assert credentials(transport).token == "second" - assert ( - auth.current_user_api_credentials( - app_id=APP_ID, streamlit_token="changed", session=transport - ).token - == "third" - ) + assert credentials(transport, runtime).token == "viewer" + assert credentials(transport, runtime).token == "second" + runtime.cookie = "changed" + assert credentials(transport, runtime).token == "third" @pytest.mark.parametrize("value", ["bad/path", "../apps", "", "x?query", "x#fragment"]) -def test_explicit_app_id_is_validated_before_network(http, state, value): +def test_app_id_is_validated_before_network(http, runtime, value): + runtime.app = value with pytest.raises(auth.CurrentUserApiTokenError): - auth.current_user_api_credentials( - app_id=value, streamlit_token="cookie", session=session() - ) + credentials(session(), runtime) assert not http.calls @@ -103,18 +96,18 @@ def test_explicit_app_id_is_validated_before_network(http, state, value): {"apiOrigin": "https://example.com;/"}, ], ) -def test_malformed_credentials_are_not_cached(http, state, overrides): +def test_malformed_credentials_are_not_cached(http, runtime, overrides): http.post(TOKEN_URL, json=payload(**overrides)) with pytest.raises(auth.CurrentUserApiTokenError): - credentials(session()) - assert state == {} + credentials(session(), runtime) + assert runtime.state == {} @pytest.mark.parametrize( "status,transient", [(401, False), (403, False), (429, True), (503, True)] ) def test_exchange_preserves_server_reason_and_retry_classification( - http, state, status, transient + http, runtime, status, transient ): http.post( TOKEN_URL, @@ -124,7 +117,7 @@ def test_exchange_preserves_server_reason_and_retry_classification( with pytest.raises( auth.CurrentUserApiTokenError, match="API access is not available" ) as exc: - credentials(session()) + credentials(session(), runtime) assert exc.value.transient is transient assert len(http.calls) == 1 @@ -132,20 +125,20 @@ def test_exchange_preserves_server_reason_and_retry_classification( @pytest.mark.parametrize( "failure", [requests.Timeout(), requests.ConnectionError("closed")] ) -def test_exchange_network_failures_are_transient(http, state, failure): +def test_exchange_network_failures_are_transient(http, runtime, failure): http.post(TOKEN_URL, body=failure) with pytest.raises(auth.CurrentUserApiTokenError) as exc: - credentials(session()) + credentials(session(), runtime) assert exc.value.transient -def test_exchange_never_follows_redirects_or_exposes_html(http, state): +def test_exchange_never_follows_redirects_or_exposes_html(http, runtime): http.post(TOKEN_URL, status=302, headers={"Location": "https://other.example"}) with pytest.raises(auth.CurrentUserApiTokenError, match="Refused a redirect"): - credentials(session()) + credentials(session(), runtime) http.replace(responses.POST, TOKEN_URL, status=502, body="private") with pytest.raises(auth.CurrentUserApiTokenError) as exc: - credentials(session()) + credentials(session(), runtime) assert "private" not in str(exc.value) @@ -183,14 +176,14 @@ def get_ctx(*, suppress_warning): "streamlit.runtime.scriptrunner", SimpleNamespace(get_script_run_ctx=get_ctx), ) - assert auth._read_streamlit_session_state() is None + assert auth.streamlit_runtime.session_state() is None assert calls == [True] -def test_credential_validation_traceback_does_not_expose_bearer(http, state): +def test_credential_validation_traceback_does_not_expose_bearer(http, runtime): import traceback http.post(TOKEN_URL, json=payload(token={"secret": "private-token"})) with pytest.raises(auth.CurrentUserApiTokenError) as exc: - credentials(session()) + credentials(session(), runtime) assert "private-token" not in "".join(traceback.format_exception(exc.value)) diff --git a/tests/unit/test_deepnote_streamlit_cloud_runner.py b/tests/unit/test_deepnote_streamlit_cloud_runner.py index 95c05269..2cfb4642 100644 --- a/tests/unit/test_deepnote_streamlit_cloud_runner.py +++ b/tests/unit/test_deepnote_streamlit_cloud_runner.py @@ -8,32 +8,23 @@ from deepnote_toolkit.notebooks import RunnerError from deepnote_toolkit.streamlit import StreamlitCloudRunner, auth -from deepnote_toolkit.streamlit import viewer_credentials as viewer from tests.unit.helpers.notebook_api import Clock, add_run, body, run_response, session +from tests.unit.helpers.streamlit_runtime import FakeStreamlitRuntime APP_ID = "11111111-2222-3333-4444-555555555555" TOKEN_URL = f"http://localhost:19456/userpod-api/streamlit-apps/{APP_ID}/api-token" +VIEWER_TOKEN = { + "token": "viewer", + "apiOrigin": "https://api.deepnote.com", + "expiresAtSeconds": time.time() + 900, +} -@pytest.fixture -def context(monkeypatch): +@pytest.fixture(autouse=True) +def env(monkeypatch): monkeypatch.delenv("DEEPNOTE_STREAMLIT_APP_ID", raising=False) monkeypatch.delenv("DEEPNOTE_PROJECT_ID", raising=False) monkeypatch.setenv("DEEPNOTE_TOKEN", "owner-token") - state = {"script": True, "hosted": False, "worker": False} - monkeypatch.setattr(viewer, "_has_script_run_context", lambda: state["script"]) - monkeypatch.setattr( - viewer, "_has_hosted_streamlit_context", lambda: state["hosted"] - ) - monkeypatch.setattr( - viewer, "_is_streamlit_thread_without_request", lambda: state["worker"] - ) - monkeypatch.setattr(auth, "_read_streamlit_app_id_from_context", lambda: None) - monkeypatch.setattr( - auth, "read_streamlit_token_from_context", lambda: "viewer-cookie" - ) - monkeypatch.setattr(auth, "_read_streamlit_session_state", lambda: {}) - return state @pytest.fixture @@ -42,29 +33,31 @@ def http(): yield mock +def hosted(**overrides): + return FakeStreamlitRuntime( + **{"app": APP_ID, "cookie": "viewer-cookie", **overrides} + ) + + @pytest.mark.parametrize( "explicit", [{}, {"token": "owner"}, {"token_provider": lambda: "owner"}] ) -def test_missing_hosted_signals_fail_closed(context, http, explicit): +def test_script_thread_without_local_mode_fails_closed(http, explicit): + runner = StreamlitCloudRunner( + "n", session=session(), runtime=FakeStreamlitRuntime(), **explicit + ) with pytest.raises(RunnerError, match="app ID"): - StreamlitCloudRunner("n", session=session(), **explicit).run({}) + runner.run({}) assert len(http.calls) == 0 -@pytest.mark.parametrize("marker", ["DEEPNOTE_STREAMLIT_APP_ID", "DEEPNOTE_PROJECT_ID"]) @pytest.mark.parametrize("local", [False, True]) def test_hosted_run_uses_viewer_and_readonly_even_with_explicit_owner_token( - context, http, monkeypatch, marker, local + http, local ): - monkeypatch.setenv(marker, APP_ID) - monkeypatch.setattr(auth, "_read_streamlit_app_id_from_context", lambda: APP_ID) http.post( TOKEN_URL, - json={ - "token": "viewer", - "apiOrigin": "https://api.deepnote-staging.com", - "expiresAtSeconds": time.time() + 900, - }, + json={**VIEWER_TOKEN, "apiOrigin": "https://api.deepnote-staging.com"}, ) add_run( http, @@ -78,6 +71,7 @@ def test_hosted_run_uses_viewer_and_readonly_even_with_explicit_owner_token( base_url="https://wrong.example", local=local, session=session(), + runtime=hosted(), ) assert runner.run({}).success assert http.calls[0].request.headers["StreamlitToken"] == "viewer-cookie" @@ -85,24 +79,26 @@ def test_hosted_run_uses_viewer_and_readonly_even_with_explicit_owner_token( assert body(http.calls[1])["detachedRunStorageMode"] == "readonly" +@pytest.mark.parametrize("explicit", [{}, {"local": True, "token": "owner"}]) +def test_project_marker_fails_closed_without_an_app_id(http, monkeypatch, explicit): + monkeypatch.setenv("DEEPNOTE_PROJECT_ID", "project") + runner = StreamlitCloudRunner( + "n", session=session(), runtime=FakeStreamlitRuntime(), **explicit + ) + with pytest.raises(RunnerError, match="app ID"): + runner.run({}) + assert len(http.calls) == 0 + + @pytest.mark.parametrize("ambient_auth", ["netrc", "session"]) def test_resolved_viewer_token_overrides_requests_auth( - context: dict[str, bool], http: responses.RequestsMock, monkeypatch: pytest.MonkeyPatch, tmp_path: Path, ambient_auth: str, ) -> None: """Keep the viewer bearer authoritative over ambient Requests authentication.""" - monkeypatch.setenv("DEEPNOTE_STREAMLIT_APP_ID", APP_ID) - http.post( - TOKEN_URL, - json={ - "token": "viewer", - "apiOrigin": "https://api.deepnote.com", - "expiresAtSeconds": time.time() + 900, - }, - ) + http.post(TOKEN_URL, json=VIEWER_TOKEN) add_run(http, run_response(snapshotBlocks=[]), create=True) transport = requests.Session() if ambient_auth == "netrc": @@ -112,14 +108,13 @@ def test_resolved_viewer_token_overrides_requests_auth( else: def owner_auth(request: requests.PreparedRequest) -> requests.PreparedRequest: - """Represent an injected session configured with an owner identity.""" request.headers["Authorization"] = "Bearer owner" return request transport.auth = owner_auth with transport: - result = StreamlitCloudRunner("n", session=transport).run({}) + result = StreamlitCloudRunner("n", session=transport, runtime=hosted()).run({}) assert result.success assert http.calls[1].request.headers["Authorization"] == "Bearer viewer" @@ -127,95 +122,84 @@ def owner_auth(request: requests.PreparedRequest) -> requests.PreparedRequest: @pytest.mark.parametrize("marker", ["", "invalid", APP_ID]) -def test_malformed_app_marker_never_falls_back(context, http, monkeypatch, marker): - monkeypatch.setenv("DEEPNOTE_STREAMLIT_APP_ID", marker) - ( +def test_malformed_app_marker_never_falls_back(http, marker): + if marker == APP_ID: http.post( TOKEN_URL, status=403, json={"message": "API access is not available for this app"}, ) - if marker == APP_ID - else None + runner = StreamlitCloudRunner( + "n", token="owner", local=True, session=session(), runtime=hosted(app=marker) ) with pytest.raises(RunnerError): - StreamlitCloudRunner("n", token="owner", local=True, session=session()).info() + runner.info() assert all(c.request.url == TOKEN_URL for c in http.calls) @pytest.mark.parametrize( "explicit", [{}, {"token": "owner"}, {"token_provider": lambda: "owner"}] ) -def test_worker_thread_fails_closed(context, http, explicit): - context.update(script=False, worker=True) +def test_worker_thread_fails_closed(http, explicit): + runtime = FakeStreamlitRuntime(request=False, worker=True) with pytest.raises(RunnerError, match="No viewer request"): - StreamlitCloudRunner("n", session=session(), **explicit).info() + StreamlitCloudRunner("n", session=session(), runtime=runtime, **explicit).info() assert not http.calls @pytest.mark.parametrize("script", [False, True]) def test_local_streamlit_requires_explicit_opt_in_and_token( - context: dict[str, bool], http: responses.RequestsMock, script: bool + http: responses.RequestsMock, script: bool ) -> None: """Explicit local credentials work with or without an active Streamlit script.""" - context["script"] = script + runtime = FakeStreamlitRuntime(request=script) http.get( "https://api.deepnote.com/v2/notebooks/n", json={"notebook": {"name": "N"}} ) - runner = StreamlitCloudRunner("n", local=True, token="local", session=session()) + runner = StreamlitCloudRunner( + "n", local=True, token="local", session=session(), runtime=runtime + ) assert runner.info().notebook == "N" assert http.calls[0].request.headers["Authorization"] == "Bearer local" with pytest.raises(RunnerError, match="explicitly"): - StreamlitCloudRunner("n", local=True, session=session()).info() + StreamlitCloudRunner("n", local=True, session=session(), runtime=runtime).info() @pytest.mark.parametrize("explicit", [{}, {"token": "owner"}, {"local": True}]) def test_bare_python_requires_explicit_local_credentials( - context: dict[str, bool], http: responses.RequestsMock, explicit: dict[str, Any] + http: responses.RequestsMock, explicit: dict[str, Any] ) -> None: """The Streamlit adapter cannot use an ambient owner token outside the runtime.""" - context["script"] = False + runtime = FakeStreamlitRuntime(request=False) with pytest.raises(RunnerError, match=r"local=True.*explicitly"): - StreamlitCloudRunner("n", session=session(), **explicit).info() + StreamlitCloudRunner("n", session=session(), runtime=runtime, **explicit).info() assert not http.calls -def test_transient_exchange_failure_during_poll_is_retried(context, http, monkeypatch): - monkeypatch.setenv("DEEPNOTE_STREAMLIT_APP_ID", APP_ID) - payload = { - "token": "viewer", - "apiOrigin": "https://api.deepnote.com", - "expiresAtSeconds": time.time() + 900, - } - http.post(TOKEN_URL, json=payload) +def test_transient_exchange_failure_during_poll_is_retried(http): + http.post(TOKEN_URL, json=VIEWER_TOKEN) http.post(TOKEN_URL, status=503) - http.post(TOKEN_URL, json=payload) + http.post(TOKEN_URL, json=VIEWER_TOKEN) add_run(http, run_response("running"), create=True) add_run(http, run_response(snapshotBlocks=[])) clock = Clock() - assert ( - StreamlitCloudRunner("n", session=session(), clock=clock, sleep=clock.sleep) - .run({}) - .success + runner = StreamlitCloudRunner( + "n", + session=session(), + clock=clock, + sleep=clock.sleep, + runtime=hosted(state=None), ) + assert runner.run({}).success assert len(http.calls) == 5 def test_real_streamlit_script_and_worker_keep_viewer_identity( monkeypatch, http, streamlit_app_test ): - monkeypatch.setenv("DEEPNOTE_STREAMLIT_APP_ID", APP_ID) - monkeypatch.setenv("DEEPNOTE_TOKEN", "owner-token") monkeypatch.setattr(auth, "read_streamlit_token_from_context", lambda: "cookie") - http.post( - TOKEN_URL, - json={ - "token": "viewer", - "apiOrigin": "https://api.deepnote.com", - "expiresAtSeconds": time.time() + 900, - }, - ) + http.post(TOKEN_URL, json=VIEWER_TOKEN) add_run(http, run_response(snapshotBlocks=[]), create=True) def app(): diff --git a/tests/unit/test_deepnote_streamlit_widgets.py b/tests/unit/test_deepnote_streamlit_widgets.py index e7f5de4d..c50b47a0 100644 --- a/tests/unit/test_deepnote_streamlit_widgets.py +++ b/tests/unit/test_deepnote_streamlit_widgets.py @@ -379,33 +379,39 @@ def test_stale_multiselect_default_warns() -> None: @pytest.mark.parametrize( - "value,min_value,max_value,step", - [ - (11, 0, 10, 1), - (-1, 0, 10, 1), - (3, 10, 0, 1), - (3, 0, 10, 0), - (3, 0, 10, -1), - ("bad", 0, 10, 1), - (float("nan"), 0, 10, 1), - (3, 0, float("inf"), 1), - ], + "min_value,max_value,step", + [(10, 0, 1), (0, 10, 0), (0, 10, -1), (0, float("inf"), 1)], ) -def test_invalid_slider_configuration_is_reported( - value: Any, min_value: float, max_value: float, step: float +def test_invalid_slider_constraints_are_rejected( + min_value: float, max_value: float, step: float ) -> None: - """Reject invalid bounds, steps, and defaults before rendering.""" - with pytest.raises(ValueError, match="[Ss]lider"): + with pytest.raises(ValueError, match="slider"): render_inputs( [ InputBlock( - "x", "input-slider", value, min=min_value, max=max_value, step=step + "x", "input-slider", 3, min=min_value, max=max_value, step=step ) ], FakeContainer(), ) +@pytest.mark.parametrize( + "value,expected", [(11, 10), (-1, 0), ("bad", 0), (float("nan"), 0), (None, 0)] +) +def test_slider_default_outside_bounds_is_clamped_with_a_warning( + value: Any, expected: int +) -> None: + warnings = [] + container = FakeContainer() + container.warning = warnings.append + values = render_inputs( + [InputBlock("x", "input-slider", value, min=0, max=10, step=1)], container + ) + assert values == {"x": expected} + assert len(warnings) == (0 if value is None else 1) + + def test_real_widgets_keep_falsey_defaults_and_require_selection( streamlit_app_test: "type[AppTest]", ) -> None: diff --git a/tests/unit/test_notebooks_runners.py b/tests/unit/test_notebooks_runners.py index d62ff1a5..675d7b1a 100644 --- a/tests/unit/test_notebooks_runners.py +++ b/tests/unit/test_notebooks_runners.py @@ -83,7 +83,7 @@ def test_run_refreshes_credentials_normalizes_inputs_and_reads_only_run_blocks( http, clock ): add_run(http, run_response("pending"), create=True) - add_run(http, run_response("future-state")) + add_run(http, run_response("running")) add_run(http, run_response(snapshotStatus="pending")) add_run( http, @@ -125,23 +125,32 @@ def test_run_refreshes_credentials_normalizes_inputs_and_reads_only_run_blocks( @pytest.mark.parametrize( "payload", [ - {"run": {"runId": "r"}}, - {"run": {"runId": "r", "status": None}}, - {"run": {"runId": "r", "status": []}}, - {"run": {"runId": "r", "status": ""}}, - {"run": {"status": "success"}}, + {"runId": "r"}, + {"runId": "r", "status": None}, + {"runId": "r", "status": ""}, + {"runId": "r", "status": "future"}, + {"runId": "r", "status": "success", "snapshotStatus": "future"}, + {"status": "success"}, + {"run": {"runId": "r", "status": "success"}}, ], ) def test_malformed_run_fails_without_polling(http, runner, payload): add_run(http, payload, create=True) - with pytest.raises(RunnerError, match="run (response|id)"): + with pytest.raises(RunnerError, match="invalid run response"): runner.run({}) assert len(http.calls) == 1 -@pytest.mark.parametrize( - "snapshot_status", [None, "unavailable", "available", "future"] -) +@pytest.mark.parametrize("payload", [{"runId": "run-1", "status": "future"}, {}]) +def test_malformed_poll_stops_the_run(http, runner, payload): + add_run(http, run_response("running"), create=True) + add_run(http, payload) + with pytest.raises(RunnerError, match="invalid run response"): + runner.run({}) + assert len(http.calls) == 2 + + +@pytest.mark.parametrize("snapshot_status", [None, "unavailable", "available"]) def test_only_pending_snapshots_are_polled(http, runner, snapshot_status): add_run( http, @@ -206,7 +215,7 @@ def create(request): def poll(request): observed.append(request.req_kwargs["timeout"].total) clock.now += 4 - return 200, {}, json.dumps(run_response("running")) + return 200, {}, json.dumps({"run": run_response("running")}) http.add_callback( responses.POST, "https://api.deepnote.com/v2/runs", callback=create @@ -246,14 +255,14 @@ def poll(request: Any) -> tuple[int, dict[str, str], str]: clock.now += 4 payload = run_response(snapshotStatus="available" if available else "pending") if available: - payload["run"]["snapshotBlocks"] = [ + payload["snapshotBlocks"] = [ { "id": "b", "type": "code", "outputs": [{"output_type": "stream", "text": "done"}], } ] - return 200, {}, json.dumps(payload) + return 200, {}, json.dumps({"run": payload}) http.add_callback( responses.GET, @@ -293,23 +302,6 @@ def test_no_request_starts_after_snapshot_deadline(http, clock): assert len(http.calls) == 1 -def test_unknown_status_is_bounded_by_deadline(http, clock): - add_run(http, run_response("future"), create=True) - add_run(http, run_response("future")) - runner = DeepnoteCloudRunner( - "n", - token="t", - session=session(), - timeout=2, - poll_interval=1, - clock=clock, - sleep=clock.sleep, - ) - with pytest.raises(RunnerError, match="2 seconds"): - runner.run({}) - assert len(http.calls) == 2 - - @pytest.mark.parametrize( "kwargs", [ @@ -328,7 +320,7 @@ def test_run_id_is_quoted(http, runner): add_run(http, run_response("running", runId="a/b?x"), create=True) http.get( "https://api.deepnote.com/v2/runs/a%2Fb%3Fx?snapshotDelivery=blocks", - json=run_response(), + json={"run": run_response()}, ) assert runner.run({}).success diff --git a/tests/unit/test_streamlit.py b/tests/unit/test_streamlit.py index 0c6bab77..94531a1a 100644 --- a/tests/unit/test_streamlit.py +++ b/tests/unit/test_streamlit.py @@ -122,19 +122,16 @@ def test_passes_app_id_as_environment_data(self) -> None: assert calls[0].kwargs["env"] == {"DEEPNOTE_STREAMLIT_APP_ID": apps[0]["id"]} assert calls[1].kwargs["env"] == {"DEEPNOTE_STREAMLIT_APP_ID": apps[1]["id"]} - def test_missing_app_id_warns_and_still_marks_process_as_hosted(self) -> None: - """Missing app IDs warn without permitting local credential fallback.""" + def test_missing_app_id_still_marks_process_as_hosted(self) -> None: app = {"entrypoint": "app.py", "port": "8501"} venv = MagicMock() - logger = MagicMock(spec=logging.Logger) with ( patch( "installer.module.streamlit.fetch_streamlit_apps", return_value=[app] ), patch("installer.module.streamlit.os.path.exists", return_value=True), ): - start_streamlit_servers(venv, logger) - logger.warning.assert_called_once() + start_streamlit_servers(venv, MagicMock(spec=logging.Logger)) assert venv.start_server.call_args.kwargs["env"] == { "DEEPNOTE_STREAMLIT_APP_ID": "" } diff --git a/tests/unit/test_streamlit_data_apps.py b/tests/unit/test_streamlit_data_apps.py index f1fc2c69..631bdc22 100644 --- a/tests/unit/test_streamlit_data_apps.py +++ b/tests/unit/test_streamlit_data_apps.py @@ -81,7 +81,7 @@ def test_get_federated_auth_token_raises_when_token_missing(tmp_path, monkeypatc _setup_attached_config(tmp_path, monkeypatch) with patch( - "deepnote_toolkit.streamlit_data_apps._read_streamlit_token_from_context", + "deepnote_toolkit.streamlit_data_apps.read_streamlit_token_from_context", return_value=None, ): with pytest.raises(StreamlitFederatedAuthError) as excinfo: From c6f41d1e3e1ca2e28647e48e69502108b43bac97 Mon Sep 17 00:00:00 2001 From: Wojciech Apanowicz Date: Tue, 22 Sep 2026 14:47:38 +0200 Subject: [PATCH 39/43] refactor(notebooks): trim unused surface and duplicated error handling - Drop OutputCollection.outputs_for_mime and DeepnoteDataframe.data_columns, which no consumer used; agent_text is a comprehension. - DeepnoteDocument.load delegates to parse and prefixes the path. - CurrentUserApiTokenError subclasses RunnerError, so ViewerCredentials no longer re-wraps it. - Reduce apiOrigin to scheme://host[:port] instead of enumerating what a trusted origin must not contain. - _settle_snapshot relies on _pause for the deadline check. - Build the viewer-token fixture per test so its expiry is never stale. Co-Authored-By: Claude Fable 5.1 --- deepnote_toolkit/notebooks/cloud_runner.py | 7 +- deepnote_toolkit/notebooks/document.py | 11 ++- deepnote_toolkit/notebooks/models.py | 10 --- deepnote_toolkit/notebooks/outputs.py | 19 ++--- deepnote_toolkit/streamlit/auth.py | 69 ++++++++----------- .../streamlit/viewer_credentials.py | 20 ++---- tests/unit/test_deepnote_streamlit_auth.py | 17 +++-- .../test_deepnote_streamlit_cloud_runner.py | 24 ++++--- tests/unit/test_notebooks_document.py | 20 ------ 9 files changed, 68 insertions(+), 129 deletions(-) diff --git a/deepnote_toolkit/notebooks/cloud_runner.py b/deepnote_toolkit/notebooks/cloud_runner.py index f36f71c4..898f18f7 100644 --- a/deepnote_toolkit/notebooks/cloud_runner.py +++ b/deepnote_toolkit/notebooks/cloud_runner.py @@ -154,12 +154,7 @@ def _settle_snapshot(self, run: CloudRun, deadline: float) -> CloudRun: if not self._pause(deadline): break try: - updated = self._client.get_run( - run.run_id, timeout=deadline - self._clock() - ) - run = updated - if self._clock() >= deadline: - break + run = self._client.get_run(run.run_id, timeout=deadline - self._clock()) except RunnerError as error: if not error.transient: raise diff --git a/deepnote_toolkit/notebooks/document.py b/deepnote_toolkit/notebooks/document.py index 58fe6e51..e2417ccf 100644 --- a/deepnote_toolkit/notebooks/document.py +++ b/deepnote_toolkit/notebooks/document.py @@ -48,12 +48,11 @@ def load( source = Path(path) try: - raw = load_yaml(source.read_text(encoding="utf-8")) - except yaml.YAMLError as error: - raise ValueError(f"Could not parse {source}: {error}") from error - if not isinstance(raw, Mapping): - raise ValueError(f"Expected {source} to contain a YAML object") - return cls(raw, notebook_id=notebook_id) + return cls.parse( + source.read_text(encoding="utf-8"), notebook_id=notebook_id + ) + except ValueError as error: + raise ValueError(f"{source}: {error}") from None @classmethod def parse(cls, content: str, *, notebook_id: str | None = None) -> DeepnoteDocument: diff --git a/deepnote_toolkit/notebooks/models.py b/deepnote_toolkit/notebooks/models.py index cc9825b2..d41c344f 100644 --- a/deepnote_toolkit/notebooks/models.py +++ b/deepnote_toolkit/notebooks/models.py @@ -75,16 +75,6 @@ def is_truncated(self) -> bool: return self.row_count > len(self.rows) - @property - def data_columns(self) -> tuple[str, ...]: - """Column names without Deepnote's index column.""" - - return tuple( - str(column.get("name")) - for column in self.columns - if column.get("name") not in (None, INDEX_COLUMN) - ) - def records(self, *, include_index: bool = True) -> list[dict[str, Any]]: """Return rows as plain dicts, optionally without the index column.""" diff --git a/deepnote_toolkit/notebooks/outputs.py b/deepnote_toolkit/notebooks/outputs.py index 23c27e58..252d4d0f 100644 --- a/deepnote_toolkit/notebooks/outputs.py +++ b/deepnote_toolkit/notebooks/outputs.py @@ -10,11 +10,6 @@ class OutputCollection: outputs: tuple[NotebookOutput, ...] - def outputs_for_mime(self, mime: str) -> list[NotebookOutput]: - """Outputs that carry the given MIME type.""" - - return [output for output in self.outputs if mime in output.data] - def first_dataframe(self) -> DeepnoteDataframe | None: """The first dataframe output, or None when there is none.""" @@ -40,12 +35,8 @@ def text(self, mime: str = "text/plain") -> str: def agent_text(self) -> str: """The text written by agent blocks, preferring Markdown over plain text.""" - chunks: list[str] = [] - for output in self.outputs: - if output.block_type != "agent": - continue - if output.output_type == "stream": - chunks.append(output.text()) - else: - chunks.append(output.text("text/markdown") or output.text()) - return "".join(chunks).strip() + return "".join( + output.text("text/markdown") or output.text() + for output in self.outputs + if output.block_type == "agent" + ).strip() diff --git a/deepnote_toolkit/streamlit/auth.py b/deepnote_toolkit/streamlit/auth.py index 4eeeb5c4..fe9f97d0 100644 --- a/deepnote_toolkit/streamlit/auth.py +++ b/deepnote_toolkit/streamlit/auth.py @@ -10,7 +10,7 @@ from collections.abc import MutableMapping from dataclasses import dataclass, field from typing import Any, Protocol -from urllib.parse import urlparse +from urllib.parse import urlsplit import requests from pydantic import ValidationError @@ -32,13 +32,9 @@ _EXPIRY_MARGIN_SECONDS = 60 -class CurrentUserApiTokenError(RuntimeError): +class CurrentUserApiTokenError(RunnerError): """Raised when a hosted app cannot obtain the current viewer's API token.""" - def __init__(self, message: str, *, transient: bool = False): - super().__init__(message) - self.transient = transient - @dataclass(frozen=True) class CurrentUserApiCredentials: @@ -159,7 +155,6 @@ def _exchange( timeout: float, session: requests.Session | None, ) -> CurrentUserApiCredentials: - owned_session = session is None http = session if session is not None else requests.Session() try: payload = request_json( @@ -170,18 +165,6 @@ def _exchange( timeout=timeout, ) parsed = ViewerTokenResponse(**payload) - credentials = CurrentUserApiCredentials( - token=parsed.token, - api_origin=_validated_origin(parsed.api_origin, name="apiOrigin"), - expires_at_seconds=float(parsed.expires_at_seconds), - ) - if ( - not math.isfinite(credentials.expires_at_seconds) - or credentials.expires_at_seconds <= time.time() - ): - raise CurrentUserApiTokenError( - "Viewer API credentials have already expired." - ) except RunnerError as error: raise CurrentUserApiTokenError(str(error), transient=error.transient) from error except ValidationError: @@ -190,9 +173,16 @@ def _exchange( "Viewer API-token response is missing or has invalid required fields." ) from None finally: - if owned_session: + if http is not session: http.close() - return credentials + expires_at = float(parsed.expires_at_seconds) + if not math.isfinite(expires_at) or expires_at <= time.time(): + raise CurrentUserApiTokenError("Viewer API credentials have already expired.") + return CurrentUserApiCredentials( + token=parsed.token, + api_origin=_origin(parsed.api_origin), + expires_at_seconds=expires_at, + ) def _read_streamlit_app_id_from_context() -> str | None: @@ -225,25 +215,20 @@ def _read_streamlit_app_id_from_context() -> str | None: return None -def _validated_origin(value: str, *, name: str) -> str: - """Validate an origin and normalize URL parser failures to authentication errors.""" +def _origin(value: str) -> str: + """Reduce `apiOrigin` to `scheme://host[:port]`, rejecting anything else in it.""" + try: - parsed = urlparse(value) - except ValueError as error: - raise CurrentUserApiTokenError( - f"{name} must be a valid HTTP(S) origin." - ) from error - normalized = value.rstrip("/") - if ( - parsed.scheme not in {"http", "https"} - or not parsed.netloc - or parsed.username - or parsed.password - or parsed.path not in {"", "/"} - or parsed.params - or parsed.query - or parsed.fragment - or normalized.endswith(("?", "#", ";")) - ): - raise CurrentUserApiTokenError(f"{name} must be a valid HTTP(S) origin.") - return normalized + parts = urlsplit(value) + bare = ( + parts.scheme in {"http", "https"} + and bool(parts.hostname) + and parts.username is None + and not parts.path.strip("/") + and not (parts.query or parts.fragment) + ) + except ValueError: + bare = False + if not bare: + raise CurrentUserApiTokenError("apiOrigin must be a valid HTTP(S) origin.") + return f"{parts.scheme}://{parts.netloc}" diff --git a/deepnote_toolkit/streamlit/viewer_credentials.py b/deepnote_toolkit/streamlit/viewer_credentials.py index 15f97584..8b0d61a8 100644 --- a/deepnote_toolkit/streamlit/viewer_credentials.py +++ b/deepnote_toolkit/streamlit/viewer_credentials.py @@ -14,12 +14,7 @@ ) from deepnote_toolkit.notebooks.runner import RunnerError -from .auth import ( - CurrentUserApiTokenError, - StreamlitRuntime, - current_user_api_credentials, - streamlit_runtime, -) +from .auth import StreamlitRuntime, current_user_api_credentials, streamlit_runtime _NO_REQUEST = ( "No viewer request is available on this thread. Call the runner from the " @@ -65,14 +60,11 @@ def __call__(self, *, timeout: float = 30) -> ApiCredentials: if is_hosted or (has_request and not self._local_mode): if not has_request: raise RunnerError(_NO_REQUEST) - try: - viewer = current_user_api_credentials( - timeout=min(timeout, self._timeout), - session=self._session, - runtime=self._runtime, - ) - except CurrentUserApiTokenError as error: - raise RunnerError(str(error), transient=error.transient) from error + viewer = current_user_api_credentials( + timeout=min(timeout, self._timeout), + session=self._session, + runtime=self._runtime, + ) return ApiCredentials(token=viewer.token, api_origin=viewer.api_origin) if self._runtime.is_worker_thread(): diff --git a/tests/unit/test_deepnote_streamlit_auth.py b/tests/unit/test_deepnote_streamlit_auth.py index c1ac4349..f3601144 100644 --- a/tests/unit/test_deepnote_streamlit_auth.py +++ b/tests/unit/test_deepnote_streamlit_auth.py @@ -86,14 +86,9 @@ def test_app_id_is_validated_before_network(http, runtime, value): {"expiresAtSeconds": "99999999999"}, {"apiOrigin": "https://user:pass@example.com"}, {"apiOrigin": "https://example.com/path"}, + {"apiOrigin": "https://example.com/?next=1"}, + {"apiOrigin": "ftp://example.com"}, {"apiOrigin": "https://[::1"}, - {"apiOrigin": "https://example.com?"}, - {"apiOrigin": "https://example.com#"}, - {"apiOrigin": "https://example.com;"}, - {"apiOrigin": "https://example.com/?"}, - {"apiOrigin": "https://example.com/#"}, - {"apiOrigin": "https://example.com/;"}, - {"apiOrigin": "https://example.com;/"}, ], ) def test_malformed_credentials_are_not_cached(http, runtime, overrides): @@ -103,6 +98,14 @@ def test_malformed_credentials_are_not_cached(http, runtime, overrides): assert runtime.state == {} +@pytest.mark.parametrize("suffix", ["", "/", "/?", "/#", "?#"]) +def test_api_origin_is_reduced_to_scheme_and_host(http, runtime, suffix): + http.post( + TOKEN_URL, json=payload(apiOrigin="HTTPS://api.example.com:8443" + suffix) + ) + assert credentials(session(), runtime).api_origin == "https://api.example.com:8443" + + @pytest.mark.parametrize( "status,transient", [(401, False), (403, False), (429, True), (503, True)] ) diff --git a/tests/unit/test_deepnote_streamlit_cloud_runner.py b/tests/unit/test_deepnote_streamlit_cloud_runner.py index 2cfb4642..c3e02777 100644 --- a/tests/unit/test_deepnote_streamlit_cloud_runner.py +++ b/tests/unit/test_deepnote_streamlit_cloud_runner.py @@ -13,11 +13,15 @@ APP_ID = "11111111-2222-3333-4444-555555555555" TOKEN_URL = f"http://localhost:19456/userpod-api/streamlit-apps/{APP_ID}/api-token" -VIEWER_TOKEN = { - "token": "viewer", - "apiOrigin": "https://api.deepnote.com", - "expiresAtSeconds": time.time() + 900, -} + + +def viewer_token(**overrides): + return { + "token": "viewer", + "apiOrigin": "https://api.deepnote.com", + "expiresAtSeconds": time.time() + 900, + **overrides, + } @pytest.fixture(autouse=True) @@ -57,7 +61,7 @@ def test_hosted_run_uses_viewer_and_readonly_even_with_explicit_owner_token( ): http.post( TOKEN_URL, - json={**VIEWER_TOKEN, "apiOrigin": "https://api.deepnote-staging.com"}, + json=viewer_token(apiOrigin="https://api.deepnote-staging.com"), ) add_run( http, @@ -98,7 +102,7 @@ def test_resolved_viewer_token_overrides_requests_auth( ambient_auth: str, ) -> None: """Keep the viewer bearer authoritative over ambient Requests authentication.""" - http.post(TOKEN_URL, json=VIEWER_TOKEN) + http.post(TOKEN_URL, json=viewer_token()) add_run(http, run_response(snapshotBlocks=[]), create=True) transport = requests.Session() if ambient_auth == "netrc": @@ -177,9 +181,9 @@ def test_bare_python_requires_explicit_local_credentials( def test_transient_exchange_failure_during_poll_is_retried(http): - http.post(TOKEN_URL, json=VIEWER_TOKEN) + http.post(TOKEN_URL, json=viewer_token()) http.post(TOKEN_URL, status=503) - http.post(TOKEN_URL, json=VIEWER_TOKEN) + http.post(TOKEN_URL, json=viewer_token()) add_run(http, run_response("running"), create=True) add_run(http, run_response(snapshotBlocks=[])) clock = Clock() @@ -199,7 +203,7 @@ def test_real_streamlit_script_and_worker_keep_viewer_identity( ): monkeypatch.setenv("DEEPNOTE_STREAMLIT_APP_ID", APP_ID) monkeypatch.setattr(auth, "read_streamlit_token_from_context", lambda: "cookie") - http.post(TOKEN_URL, json=VIEWER_TOKEN) + http.post(TOKEN_URL, json=viewer_token()) add_run(http, run_response(snapshotBlocks=[]), create=True) def app(): diff --git a/tests/unit/test_notebooks_document.py b/tests/unit/test_notebooks_document.py index 281d395d..b4834c95 100644 --- a/tests/unit/test_notebooks_document.py +++ b/tests/unit/test_notebooks_document.py @@ -79,30 +79,10 @@ def test_loads_inputs_and_structured_outputs(tmp_path: Path) -> None: ) dataframe = snapshot.first_dataframe() assert dataframe is not None - assert dataframe.data_columns == ("Revenue",) assert dataframe.records(include_index=False) == [{"Revenue": 42}] assert snapshot.agent_text() == "**Done**" -def test_dataframe_ignores_columns_without_names() -> None: - dataframe = DeepnoteDocument.parse(""" -project: - notebooks: - - blocks: - - id: table - type: code - outputs: - - output_type: execute_result - data: - application/vnd.deepnote.dataframe.v3+json: - columns: [{}, {name: value}] - rows: [{value: 42}] -""").first_dataframe() - - assert dataframe is not None - assert dataframe.data_columns == ("value",) - - def test_reads_input_metadata_from_file_and_api_shapes() -> None: document = DeepnoteDocument( { From 1e1dac905a42ad3f466b6767efc4aeff13e90742 Mon Sep 17 00:00:00 2001 From: Wojtek <6819634+voyti@users.noreply.github.com> Date: Wed, 23 Sep 2026 15:23:51 +0200 Subject: [PATCH 40/43] fix(streamlit): reject malformed credential authorities --- deepnote_toolkit/notebooks/_schemas.py | 2 ++ deepnote_toolkit/notebooks/api_client.py | 6 +++++ deepnote_toolkit/streamlit/auth.py | 1 + tests/unit/test_deepnote_streamlit_auth.py | 27 ++++++++++++++++++---- tests/unit/test_notebooks_runners.py | 7 +++++- 5 files changed, 37 insertions(+), 6 deletions(-) diff --git a/deepnote_toolkit/notebooks/_schemas.py b/deepnote_toolkit/notebooks/_schemas.py index 12a14695..d63c1253 100644 --- a/deepnote_toolkit/notebooks/_schemas.py +++ b/deepnote_toolkit/notebooks/_schemas.py @@ -44,6 +44,8 @@ class ApiRun(BaseModel): class GetRunResponse(BaseModel): + """The response envelope returned when fetching an existing run.""" + run: ApiRun diff --git a/deepnote_toolkit/notebooks/api_client.py b/deepnote_toolkit/notebooks/api_client.py index 94944688..6ba2dfdf 100644 --- a/deepnote_toolkit/notebooks/api_client.py +++ b/deepnote_toolkit/notebooks/api_client.py @@ -151,6 +151,8 @@ def _request( def _validate(schema: type[Schema], payload: Mapping[str, Any], what: str) -> Schema: + """Validate an API response, reporting schema failures as `RunnerError`.""" + try: return schema(**payload) except ValidationError as error: @@ -175,6 +177,8 @@ def _encode_input(name: str, value: Any) -> InputValue: def _input_block(value: ApiInput) -> InputBlock: + """Convert validated API input metadata to a notebook input block.""" + return InputBlock( variable_name=value.name, type=cast(InputBlockType, value.type), @@ -189,6 +193,8 @@ def _input_block(value: ApiInput) -> InputBlock: def _cloud_run(run: ApiRun) -> CloudRun: + """Convert a validated run and any available snapshot blocks to runner data.""" + return CloudRun( run_id=run.run_id, status=run.status, diff --git a/deepnote_toolkit/streamlit/auth.py b/deepnote_toolkit/streamlit/auth.py index fe9f97d0..84c6fb13 100644 --- a/deepnote_toolkit/streamlit/auth.py +++ b/deepnote_toolkit/streamlit/auth.py @@ -223,6 +223,7 @@ def _origin(value: str) -> str: bare = ( parts.scheme in {"http", "https"} and bool(parts.hostname) + and ";" not in parts.netloc and parts.username is None and not parts.path.strip("/") and not (parts.query or parts.fragment) diff --git a/tests/unit/test_deepnote_streamlit_auth.py b/tests/unit/test_deepnote_streamlit_auth.py index f3601144..dbe18ec4 100644 --- a/tests/unit/test_deepnote_streamlit_auth.py +++ b/tests/unit/test_deepnote_streamlit_auth.py @@ -89,6 +89,10 @@ def test_app_id_is_validated_before_network(http, runtime, value): {"apiOrigin": "https://example.com/?next=1"}, {"apiOrigin": "ftp://example.com"}, {"apiOrigin": "https://[::1"}, + {"apiOrigin": "https://example.com;/"}, + {"apiOrigin": "https://example.com;"}, + {"apiOrigin": "https://exam;ple.com/"}, + {"apiOrigin": "https://example.com:8443;/"}, ], ) def test_malformed_credentials_are_not_cached(http, runtime, overrides): @@ -99,11 +103,24 @@ def test_malformed_credentials_are_not_cached(http, runtime, overrides): @pytest.mark.parametrize("suffix", ["", "/", "/?", "/#", "?#"]) -def test_api_origin_is_reduced_to_scheme_and_host(http, runtime, suffix): - http.post( - TOKEN_URL, json=payload(apiOrigin="HTTPS://api.example.com:8443" + suffix) - ) - assert credentials(session(), runtime).api_origin == "https://api.example.com:8443" +@pytest.mark.parametrize( + "origin,expected", + [ + ("HTTPS://api.example.com:8443", "https://api.example.com:8443"), + ("http://localhost:8080", "http://localhost:8080"), + ("https://[::1]:8443", "https://[::1]:8443"), + ], +) +def test_api_origin_is_reduced_to_scheme_and_host( + http: responses.RequestsMock, + runtime: FakeStreamlitRuntime, + suffix: str, + origin: str, + expected: str, +) -> None: + """Preserve hosts and ports while removing empty URL components.""" + http.post(TOKEN_URL, json=payload(apiOrigin=origin + suffix)) + assert credentials(session(), runtime).api_origin == expected @pytest.mark.parametrize( diff --git a/tests/unit/test_notebooks_runners.py b/tests/unit/test_notebooks_runners.py index 675d7b1a..1803ece7 100644 --- a/tests/unit/test_notebooks_runners.py +++ b/tests/unit/test_notebooks_runners.py @@ -142,7 +142,12 @@ def test_malformed_run_fails_without_polling(http, runner, payload): @pytest.mark.parametrize("payload", [{"runId": "run-1", "status": "future"}, {}]) -def test_malformed_poll_stops_the_run(http, runner, payload): +def test_malformed_poll_stops_the_run( + http: responses.RequestsMock, + runner: DeepnoteCloudRunner, + payload: dict[str, Any], +) -> None: + """Stop polling when the API returns a response outside the run contract.""" add_run(http, run_response("running"), create=True) add_run(http, payload) with pytest.raises(RunnerError, match="invalid run response"): From dd240379c4777ee6700b7ab86d6f5e7395fa0d74 Mon Sep 17 00:00:00 2001 From: Wojciech Apanowicz Date: Thu, 24 Sep 2026 11:08:31 +0200 Subject: [PATCH 41/43] fix(notebooks): enforce run contracts and refresh malformed credentials --- deepnote_toolkit/notebooks/_schemas.py | 11 ++- deepnote_toolkit/notebooks/api_client.py | 17 +++- deepnote_toolkit/notebooks/cloud_runner.py | 3 +- deepnote_toolkit/notebooks/yaml_loader.py | 6 +- deepnote_toolkit/streamlit/auth.py | 2 + tests/unit/helpers/notebook_api.py | 16 +++- tests/unit/test_deepnote_streamlit_auth.py | 35 ++++++++ .../test_deepnote_streamlit_cloud_runner.py | 26 ++++-- tests/unit/test_notebooks_runners.py | 88 +++++++++++++++---- tests/unit/test_notebooks_yaml_loader.py | 2 +- 10 files changed, 173 insertions(+), 33 deletions(-) diff --git a/deepnote_toolkit/notebooks/_schemas.py b/deepnote_toolkit/notebooks/_schemas.py index d63c1253..f9833bb5 100644 --- a/deepnote_toolkit/notebooks/_schemas.py +++ b/deepnote_toolkit/notebooks/_schemas.py @@ -33,10 +33,17 @@ class NotebookResponse(BaseModel): notebook: ApiNotebook -class ApiRun(BaseModel): +class CreateRunResponse(BaseModel): + """The run identity and execution status returned by POST /v2/runs.""" + run_id: StrictStr = Field(alias="runId", min_length=1) status: RunStatus - snapshot_status: SnapshotStatus | None = Field(default=None, alias="snapshotStatus") + + +class ApiRun(CreateRunResponse): + """GET run details, including the required snapshot lifecycle status.""" + + snapshot_status: SnapshotStatus = Field(alias="snapshotStatus") snapshot_blocks: list[dict[str, Any]] | None = Field( default=None, alias="snapshotBlocks" ) diff --git a/deepnote_toolkit/notebooks/api_client.py b/deepnote_toolkit/notebooks/api_client.py index 6ba2dfdf..78968c89 100644 --- a/deepnote_toolkit/notebooks/api_client.py +++ b/deepnote_toolkit/notebooks/api_client.py @@ -11,7 +11,13 @@ import requests from pydantic import BaseModel, ValidationError -from ._schemas import ApiInput, ApiRun, GetRunResponse, NotebookResponse +from ._schemas import ( + ApiInput, + ApiRun, + CreateRunResponse, + GetRunResponse, + NotebookResponse, +) from .api_types import ( INPUT_BLOCK_TYPES, TERMINAL_RUN_STATUSES, @@ -105,7 +111,14 @@ def create_run( if storage_mode is not None: body["detachedRunStorageMode"] = storage_mode payload = self._request("POST", "/v2/runs", body, timeout=timeout) - return _cloud_run(_validate(ApiRun, payload, "run")) + run = _validate(CreateRunResponse, payload, "run") + return CloudRun( + run_id=run.run_id, + status=run.status, + snapshot_status=None, + outputs=None, + error=None, + ) def get_run(self, run_id: str, *, timeout: float | None = None) -> CloudRun: """Read a run with the outputs of the notebook it executed.""" diff --git a/deepnote_toolkit/notebooks/cloud_runner.py b/deepnote_toolkit/notebooks/cloud_runner.py index 898f18f7..f3b92ef9 100644 --- a/deepnote_toolkit/notebooks/cloud_runner.py +++ b/deepnote_toolkit/notebooks/cloud_runner.py @@ -150,7 +150,8 @@ def _wait_until_finished(self, run: CloudRun, deadline: float) -> CloudRun: return run def _settle_snapshot(self, run: CloudRun, deadline: float) -> CloudRun: - while run.outputs is None and run.snapshot_status == "pending": + # POST has no snapshot metadata, even when it reports a completed run. + while run.outputs is None and run.snapshot_status in {None, "pending"}: if not self._pause(deadline): break try: diff --git a/deepnote_toolkit/notebooks/yaml_loader.py b/deepnote_toolkit/notebooks/yaml_loader.py index 91a3d532..911fd9c8 100644 --- a/deepnote_toolkit/notebooks/yaml_loader.py +++ b/deepnote_toolkit/notebooks/yaml_loader.py @@ -10,7 +10,7 @@ _BaseLoader: type = getattr(yaml, "CSafeLoader", yaml.SafeLoader) -class _CoreSchemaLoader(_BaseLoader): # type: ignore[misc,valid-type] +class _DeepnoteSchemaLoader(_BaseLoader): # type: ignore[misc,valid-type] """A safe loader for the scalar conventions used by `.deepnote` files. `.deepnote` files are written as YAML 1.2, where `No`, `on`, `12:30` and @@ -60,7 +60,7 @@ def construct_mapping( list("-+0123456789."), ), ): - _CoreSchemaLoader.add_implicit_resolver( + _DeepnoteSchemaLoader.add_implicit_resolver( f"tag:yaml.org,2002:{_tag}", re.compile(_pattern), _first ) @@ -68,7 +68,7 @@ def construct_mapping( def load_yaml(content: str) -> Any: """Parse one YAML document. Raises `yaml.YAMLError` when it is malformed.""" - loader = _CoreSchemaLoader(content) + loader = _DeepnoteSchemaLoader(content) try: return loader.get_single_data() except (ValueError, AttributeError, TypeError) as error: diff --git a/deepnote_toolkit/streamlit/auth.py b/deepnote_toolkit/streamlit/auth.py index 84c6fb13..8c889f4f 100644 --- a/deepnote_toolkit/streamlit/auth.py +++ b/deepnote_toolkit/streamlit/auth.py @@ -137,7 +137,9 @@ def current_user_api_credentials( cached = session_state.get(_SESSION_STATE_KEY) if ( isinstance(cached, tuple) + and len(cached) == 2 and cached[0] == cache_key + and isinstance(cached[1], CurrentUserApiCredentials) and cached[1].expires_at_seconds - _EXPIRY_MARGIN_SECONDS > time.time() ): return cached[1] diff --git a/tests/unit/helpers/notebook_api.py b/tests/unit/helpers/notebook_api.py index c0d7161e..025ef514 100644 --- a/tests/unit/helpers/notebook_api.py +++ b/tests/unit/helpers/notebook_api.py @@ -1,6 +1,7 @@ """Reusable HTTP and clock fixtures for the notebook clients.""" import json +from typing import Any import requests import responses @@ -19,10 +20,23 @@ def sleep(self, seconds): self.now += seconds -def run_response(status="success", **fields): +def create_run_response(status: str = "success", **fields: Any) -> dict[str, Any]: + """Build the run identity returned when creating a run.""" return {"runId": "run-1", "status": status, **fields} +def run_response(status: str = "success", **fields: Any) -> dict[str, Any]: + """Build GET run details with the required snapshot lifecycle status.""" + return { + "runId": "run-1", + "status": status, + "snapshotStatus": ( + "pending" if status in {"pending", "running"} else "available" + ), + **fields, + } + + def session(): http = requests.Session() http.trust_env = False diff --git a/tests/unit/test_deepnote_streamlit_auth.py b/tests/unit/test_deepnote_streamlit_auth.py index dbe18ec4..4f4ef928 100644 --- a/tests/unit/test_deepnote_streamlit_auth.py +++ b/tests/unit/test_deepnote_streamlit_auth.py @@ -68,6 +68,41 @@ def test_changed_cookie_or_expiry_refreshes_credentials(http, runtime): assert credentials(transport, runtime).token == "third" +@pytest.mark.parametrize( + "cache_shape", + ["empty", "short", "long", "none", "text", "mapping", "old_type", "list"], +) +def test_malformed_cached_credentials_are_refreshed( + http: responses.RequestsMock, + runtime: FakeStreamlitRuntime, + cache_shape: str, +) -> None: + """Replace malformed or stale session state with a fresh credential exchange.""" + http.post(TOKEN_URL, json=payload(token="old")) + http.post(TOKEN_URL, json=payload(token="fresh")) + with session() as transport: + initial = credentials(transport, runtime) + assert runtime.state is not None + key, _ = runtime.state[auth._SESSION_STATE_KEY] + malformed = { + "empty": (), + "short": (key,), + "long": (key, initial, "extra"), + "none": (key, None), + "text": (key, "old-token"), + "mapping": (key, {"expires_at_seconds": time.time() + 900}), + "old_type": (key, SimpleNamespace(**vars(initial))), + "list": [key, initial], + } + runtime.state[auth._SESSION_STATE_KEY] = malformed[cache_shape] + + refreshed = credentials(transport, runtime) + assert refreshed.token == "fresh" + assert runtime.state[auth._SESSION_STATE_KEY] == (key, refreshed) + assert credentials(transport, runtime) is refreshed + assert len(http.calls) == 2 + + @pytest.mark.parametrize("value", ["bad/path", "../apps", "", "x?query", "x#fragment"]) def test_app_id_is_validated_before_network(http, runtime, value): runtime.app = value diff --git a/tests/unit/test_deepnote_streamlit_cloud_runner.py b/tests/unit/test_deepnote_streamlit_cloud_runner.py index c3e02777..67b4cd15 100644 --- a/tests/unit/test_deepnote_streamlit_cloud_runner.py +++ b/tests/unit/test_deepnote_streamlit_cloud_runner.py @@ -8,7 +8,14 @@ from deepnote_toolkit.notebooks import RunnerError from deepnote_toolkit.streamlit import StreamlitCloudRunner, auth -from tests.unit.helpers.notebook_api import Clock, add_run, body, run_response, session +from tests.unit.helpers.notebook_api import ( + Clock, + add_run, + body, + create_run_response, + run_response, + session, +) from tests.unit.helpers.streamlit_runtime import FakeStreamlitRuntime APP_ID = "11111111-2222-3333-4444-555555555555" @@ -65,10 +72,15 @@ def test_hosted_run_uses_viewer_and_readonly_even_with_explicit_owner_token( ) add_run( http, - run_response(snapshotBlocks=[]), + create_run_response("running"), create=True, origin="https://api.deepnote-staging.com", ) + add_run( + http, + run_response(snapshotBlocks=[]), + origin="https://api.deepnote-staging.com", + ) runner = StreamlitCloudRunner( "n", token="owner", @@ -103,7 +115,8 @@ def test_resolved_viewer_token_overrides_requests_auth( ) -> None: """Keep the viewer bearer authoritative over ambient Requests authentication.""" http.post(TOKEN_URL, json=viewer_token()) - add_run(http, run_response(snapshotBlocks=[]), create=True) + add_run(http, create_run_response("running"), create=True) + add_run(http, run_response(snapshotBlocks=[])) transport = requests.Session() if ambient_auth == "netrc": netrc = tmp_path / "credentials.netrc" @@ -184,7 +197,7 @@ def test_transient_exchange_failure_during_poll_is_retried(http): http.post(TOKEN_URL, json=viewer_token()) http.post(TOKEN_URL, status=503) http.post(TOKEN_URL, json=viewer_token()) - add_run(http, run_response("running"), create=True) + add_run(http, create_run_response("running"), create=True) add_run(http, run_response(snapshotBlocks=[])) clock = Clock() runner = StreamlitCloudRunner( @@ -204,7 +217,8 @@ def test_real_streamlit_script_and_worker_keep_viewer_identity( monkeypatch.setenv("DEEPNOTE_STREAMLIT_APP_ID", APP_ID) monkeypatch.setattr(auth, "read_streamlit_token_from_context", lambda: "cookie") http.post(TOKEN_URL, json=viewer_token()) - add_run(http, run_response(snapshotBlocks=[]), create=True) + add_run(http, create_run_response("running"), create=True) + add_run(http, run_response(snapshotBlocks=[])) def app(): import threading @@ -233,5 +247,5 @@ def worker(): assert not at.exception assert at.session_state["success"] is True assert "No viewer request" in at.session_state["worker_errors"][0] - assert len(http.calls) == 2 + assert len(http.calls) == 3 assert http.calls[1].request.headers["Authorization"] == "Bearer viewer" diff --git a/tests/unit/test_notebooks_runners.py b/tests/unit/test_notebooks_runners.py index 1803ece7..c0db39dc 100644 --- a/tests/unit/test_notebooks_runners.py +++ b/tests/unit/test_notebooks_runners.py @@ -15,7 +15,14 @@ RunnerError, RunnerInfo, ) -from tests.unit.helpers.notebook_api import Clock, add_run, body, run_response, session +from tests.unit.helpers.notebook_api import ( + Clock, + add_run, + body, + create_run_response, + run_response, + session, +) @pytest.fixture @@ -82,7 +89,7 @@ def test_cloud_info_preserves_input_contract_and_quotes_notebook_id(http): def test_run_refreshes_credentials_normalizes_inputs_and_reads_only_run_blocks( http, clock ): - add_run(http, run_response("pending"), create=True) + add_run(http, create_run_response("pending"), create=True) add_run(http, run_response("running")) add_run(http, run_response(snapshotStatus="pending")) add_run( @@ -129,7 +136,6 @@ def test_run_refreshes_credentials_normalizes_inputs_and_reads_only_run_blocks( {"runId": "r", "status": None}, {"runId": "r", "status": ""}, {"runId": "r", "status": "future"}, - {"runId": "r", "status": "success", "snapshotStatus": "future"}, {"status": "success"}, {"run": {"runId": "r", "status": "success"}}, ], @@ -141,37 +147,85 @@ def test_malformed_run_fails_without_polling(http, runner, payload): assert len(http.calls) == 1 -@pytest.mark.parametrize("payload", [{"runId": "run-1", "status": "future"}, {}]) +@pytest.mark.parametrize( + "payload", + [ + {"runId": "run-1", "status": "future", "snapshotStatus": "pending"}, + {"runId": "run-1", "status": "success", "snapshotStatus": "future"}, + {}, + ], +) def test_malformed_poll_stops_the_run( http: responses.RequestsMock, runner: DeepnoteCloudRunner, payload: dict[str, Any], ) -> None: """Stop polling when the API returns a response outside the run contract.""" - add_run(http, run_response("running"), create=True) + add_run(http, create_run_response("running"), create=True) add_run(http, payload) with pytest.raises(RunnerError, match="invalid run response"): runner.run({}) assert len(http.calls) == 2 -@pytest.mark.parametrize("snapshot_status", [None, "unavailable", "available"]) +@pytest.mark.parametrize("status", ["running", "success"]) +@pytest.mark.parametrize("snapshot", [{}, {"snapshotStatus": None}]) +def test_get_run_requires_snapshot_status( + http: responses.RequestsMock, + runner: DeepnoteCloudRunner, + status: str, + snapshot: dict[str, Any], +) -> None: + """Reject missing or null GET snapshot metadata instead of returning no outputs.""" + add_run(http, create_run_response("running"), create=True) + add_run(http, {"runId": "run-1", "status": status, **snapshot}) + with pytest.raises(RunnerError, match="invalid run response"): + runner.run({}) + assert len(http.calls) == 2 + + +def test_finished_create_fetches_snapshot_metadata( + http: responses.RequestsMock, runner: DeepnoteCloudRunner +) -> None: + """A completed POST still needs GET metadata to discover its outputs.""" + add_run(http, create_run_response(), create=True) + add_run( + http, + run_response( + snapshotStatus="available", + snapshotBlocks=[ + { + "id": "b", + "type": "code", + "outputs": [{"output_type": "stream", "text": "done"}], + } + ], + ), + ) + result = runner.run({}) + assert result.success + assert result.snapshot_status == "available" + assert result.text() == "done" + assert len(http.calls) == 2 + + +@pytest.mark.parametrize("snapshot_status", ["unavailable", "available"]) def test_only_pending_snapshots_are_polled(http, runner, snapshot_status): + add_run(http, create_run_response("running"), create=True) add_run( http, run_response("error", snapshotStatus=snapshot_status, error="bad input"), - create=True, ) result = runner.run({}) assert not result.success and result.error == "bad input" - assert len(http.calls) == 1 + assert len(http.calls) == 2 @pytest.mark.parametrize( "failure", [429, 503, requests.ConnectionError("reset"), requests.Timeout()] ) def test_transient_get_failure_is_retried(http, runner, failure): - add_run(http, run_response("running"), create=True) + add_run(http, create_run_response("running"), create=True) kwargs = {"status": failure} if isinstance(failure, int) else {"body": failure} http.get("https://api.deepnote.com/v2/runs/run-1?snapshotDelivery=blocks", **kwargs) add_run(http, run_response(snapshotBlocks=[])) @@ -181,7 +235,7 @@ def test_transient_get_failure_is_retried(http, runner, failure): @pytest.mark.parametrize("status", [400, 401, 403, 404]) def test_non_transient_poll_failure_is_not_retried(http, runner, status): - add_run(http, run_response("running"), create=True) + add_run(http, create_run_response("running"), create=True) http.get( "https://api.deepnote.com/v2/runs/run-1?snapshotDelivery=blocks", status=status, @@ -200,7 +254,7 @@ def test_post_is_never_replayed(http, runner): def test_poll_retries_have_a_limit(http, runner): - add_run(http, run_response("running"), create=True) + add_run(http, create_run_response("running"), create=True) http.get( "https://api.deepnote.com/v2/runs/run-1?snapshotDelivery=blocks", status=503 ) @@ -215,7 +269,7 @@ def test_request_and_sleep_time_count_against_run_deadline(http, clock): def create(request): observed.append(request.req_kwargs["timeout"].total) clock.now += 3 - return 200, {}, json.dumps(run_response("running")) + return 200, {}, json.dumps(create_run_response("running")) def poll(request): observed.append(request.req_kwargs["timeout"].total) @@ -251,7 +305,7 @@ def test_snapshot_deadline_counts_slow_requests_and_caps_timeout( http: responses.RequestsMock, clock: Clock, available: bool ) -> None: """Keep received outputs at the deadline without starting another request.""" - add_run(http, run_response(snapshotStatus="pending"), create=True) + add_run(http, create_run_response(), create=True) timeouts = [] def poll(request: Any) -> tuple[int, dict[str, str], str]: @@ -292,7 +346,7 @@ def poll(request: Any) -> tuple[int, dict[str, str], str]: def test_no_request_starts_after_snapshot_deadline(http, clock): - add_run(http, run_response(snapshotStatus="pending"), create=True) + add_run(http, create_run_response(), create=True) runner = DeepnoteCloudRunner( "n", token="t", @@ -322,7 +376,7 @@ def test_invalid_timeouts_rejected(kwargs): def test_run_id_is_quoted(http, runner): - add_run(http, run_response("running", runId="a/b?x"), create=True) + add_run(http, create_run_response("running", runId="a/b?x"), create=True) http.get( "https://api.deepnote.com/v2/runs/a%2Fb%3Fx?snapshotDelivery=blocks", json={"run": run_response()}, @@ -521,7 +575,7 @@ def credentials(*, timeout): clock.now += 3 return ApiCredentials("token") - http.post("https://api.deepnote.com/v2/runs", json=run_response(snapshotBlocks=[])) + http.post("https://api.deepnote.com/v2/runs", json=create_run_response()) runner = DeepnoteCloudRunner( "n", credentials=credentials, @@ -553,7 +607,7 @@ def credentials(*, timeout): @pytest.mark.parametrize("status", [403, 503]) def test_snapshot_poll_error_policy(http, runner, status): - add_run(http, run_response(snapshotStatus="pending"), create=True) + add_run(http, create_run_response(), create=True) http.get( "https://api.deepnote.com/v2/runs/run-1?snapshotDelivery=blocks", status=status ) diff --git a/tests/unit/test_notebooks_yaml_loader.py b/tests/unit/test_notebooks_yaml_loader.py index 0c68286d..55bf013e 100644 --- a/tests/unit/test_notebooks_yaml_loader.py +++ b/tests/unit/test_notebooks_yaml_loader.py @@ -53,7 +53,7 @@ def load_yaml( ('"true"', "true"), ], ) -def test_plain_scalars_resolve_by_the_yaml_1_2_core_schema( +def test_plain_scalars_follow_deepnote_schema_conventions( load_yaml: Any, scalar: str, expected: object ) -> None: assert load_yaml(f"value: {scalar}\n") == {"value": expected} From c1e5aab86c1d868d3b4cde1413b885ec03c8de2f Mon Sep 17 00:00:00 2001 From: Wojciech Apanowicz Date: Thu, 24 Sep 2026 12:12:05 +0200 Subject: [PATCH 42/43] fix(streamlit): validate API origins with the HTTP URL parser --- deepnote_toolkit/notebooks/cloud_runner.py | 4 ++++ deepnote_toolkit/streamlit/auth.py | 16 +++++++++------- docs/streamlit-apps.md | 10 ++++++---- tests/unit/test_deepnote_streamlit_auth.py | 5 +++++ 4 files changed, 24 insertions(+), 11 deletions(-) diff --git a/deepnote_toolkit/notebooks/cloud_runner.py b/deepnote_toolkit/notebooks/cloud_runner.py index f3b92ef9..3e35faea 100644 --- a/deepnote_toolkit/notebooks/cloud_runner.py +++ b/deepnote_toolkit/notebooks/cloud_runner.py @@ -37,6 +37,10 @@ class DeepnoteCloudRunner: `storage_mode="readonly"` keeps the run from changing the project's stored files. None leaves the choice to the API, which allows writes. + `timeout` is a polling budget, including credentials and API requests. Network + operations or custom credential providers can overrun it. Expiry stops polling + but does not cancel the notebook. + The outputs can arrive after the run finishes. `snapshot_timeout` is how many seconds to wait for them. A result whose `snapshot_status` is still `pending` has none because that wait ran out. diff --git a/deepnote_toolkit/streamlit/auth.py b/deepnote_toolkit/streamlit/auth.py index 8c889f4f..3f4ca458 100644 --- a/deepnote_toolkit/streamlit/auth.py +++ b/deepnote_toolkit/streamlit/auth.py @@ -10,10 +10,11 @@ from collections.abc import MutableMapping from dataclasses import dataclass, field from typing import Any, Protocol -from urllib.parse import urlsplit import requests from pydantic import ValidationError +from urllib3.exceptions import LocationParseError +from urllib3.util import parse_url from deepnote_toolkit.get_webapp_url import ( get_absolute_userpod_api_url, @@ -221,16 +222,17 @@ def _origin(value: str) -> str: """Reduce `apiOrigin` to `scheme://host[:port]`, rejecting anything else in it.""" try: - parts = urlsplit(value) + # Use the same URL parser as Requests, including its port validation. + parts = parse_url(value) bare = ( parts.scheme in {"http", "https"} - and bool(parts.hostname) - and ";" not in parts.netloc - and parts.username is None - and not parts.path.strip("/") + and bool(parts.host) + and not re.search(r"[\s;]", parts.host) + and parts.auth is None + and not (parts.path or "").strip("/") and not (parts.query or parts.fragment) ) - except ValueError: + except LocationParseError: bare = False if not bare: raise CurrentUserApiTokenError("apiOrigin must be a valid HTTP(S) origin.") diff --git a/docs/streamlit-apps.md b/docs/streamlit-apps.md index 64ba57a2..30796ae7 100644 --- a/docs/streamlit-apps.md +++ b/docs/streamlit-apps.md @@ -88,10 +88,12 @@ Streamlit runs use `storage_mode="readonly"`: the notebook can read the project' files but not change them. Pass `storage_mode="read_write"` when the app needs to write them. `DeepnoteCloudRunner` leaves the choice to the API. -`timeout` (600 seconds by default) bounds the whole run, from creating it to -reading its outputs. Outputs can arrive after the run finishes; `snapshot_timeout` -(10 seconds) is how long to wait for them, and a result whose `snapshot_status` is -still `pending` has none. +`timeout` (600 seconds by default) is the polling budget, including time spent +creating the run, obtaining credentials and reading outputs. Network operations +or custom credential providers can overrun it. Expiry stops polling but does not +cancel the notebook. Outputs can arrive after the run finishes; `snapshot_timeout` +(10 seconds) limits the remaining wait for them within that budget. A result whose +`snapshot_status` is still `pending` has no outputs yet. Pass `session=requests.Session()` to configure proxies or HTTP adapters. On `DeepnoteCloudRunner`, `credentials=` accepts any callable that takes a `timeout` diff --git a/tests/unit/test_deepnote_streamlit_auth.py b/tests/unit/test_deepnote_streamlit_auth.py index 4f4ef928..8e201747 100644 --- a/tests/unit/test_deepnote_streamlit_auth.py +++ b/tests/unit/test_deepnote_streamlit_auth.py @@ -128,6 +128,11 @@ def test_app_id_is_validated_before_network(http, runtime, value): {"apiOrigin": "https://example.com;"}, {"apiOrigin": "https://exam;ple.com/"}, {"apiOrigin": "https://example.com:8443;/"}, + {"apiOrigin": "https://example.com:invalid/"}, + {"apiOrigin": "https://example.com:65536/"}, + {"apiOrigin": "https://exa mple.com/"}, + {"apiOrigin": "https://example.com\t/"}, + {"apiOrigin": "https://example.com\\other/"}, ], ) def test_malformed_credentials_are_not_cached(http, runtime, overrides): From 4b0464e64c9d3c190d1364bd91fe373d0441f25d Mon Sep 17 00:00:00 2001 From: Wojciech Apanowicz Date: Thu, 24 Sep 2026 13:00:09 +0200 Subject: [PATCH 43/43] fix(notebooks): follow snapshot lifecycle and reject unnamed inputs --- deepnote_toolkit/notebooks/api_client.py | 2 +- deepnote_toolkit/notebooks/cloud_runner.py | 2 +- deepnote_toolkit/notebooks/models.py | 2 +- deepnote_toolkit/notebooks/wire.py | 1 + deepnote_toolkit/streamlit/widgets.py | 2 + docs/streamlit-apps.md | 6 +- tests/unit/test_deepnote_streamlit_widgets.py | 6 ++ tests/unit/test_notebooks_runners.py | 87 +++++++++++++++++++ 8 files changed, 102 insertions(+), 6 deletions(-) diff --git a/deepnote_toolkit/notebooks/api_client.py b/deepnote_toolkit/notebooks/api_client.py index 78968c89..43494b67 100644 --- a/deepnote_toolkit/notebooks/api_client.py +++ b/deepnote_toolkit/notebooks/api_client.py @@ -87,7 +87,7 @@ def get_notebook(self, notebook_id: str) -> CloudNotebook: inputs=tuple( _input_block(value) for value in notebook.inputs - if value.type in INPUT_BLOCK_TYPES + if value.type in INPUT_BLOCK_TYPES and value.name ), ) diff --git a/deepnote_toolkit/notebooks/cloud_runner.py b/deepnote_toolkit/notebooks/cloud_runner.py index 3e35faea..95d9deb1 100644 --- a/deepnote_toolkit/notebooks/cloud_runner.py +++ b/deepnote_toolkit/notebooks/cloud_runner.py @@ -155,7 +155,7 @@ def _wait_until_finished(self, run: CloudRun, deadline: float) -> CloudRun: def _settle_snapshot(self, run: CloudRun, deadline: float) -> CloudRun: # POST has no snapshot metadata, even when it reports a completed run. - while run.outputs is None and run.snapshot_status in {None, "pending"}: + while run.snapshot_status in {None, "pending"}: if not self._pause(deadline): break try: diff --git a/deepnote_toolkit/notebooks/models.py b/deepnote_toolkit/notebooks/models.py index d41c344f..b53ea26a 100644 --- a/deepnote_toolkit/notebooks/models.py +++ b/deepnote_toolkit/notebooks/models.py @@ -152,7 +152,7 @@ def matches_inputs(self, inputs: Iterable[InputBlock]) -> bool: expected = tuple(inputs) for blocks in (expected, self.inputs): names = [block.variable_name for block in blocks] - if len(names) != len(set(names)): + if any(not name for name in names) or len(names) != len(set(names)): return False dynamic = frozenset( input_block.variable_name diff --git a/deepnote_toolkit/notebooks/wire.py b/deepnote_toolkit/notebooks/wire.py index be0c623d..a9ef3c03 100644 --- a/deepnote_toolkit/notebooks/wire.py +++ b/deepnote_toolkit/notebooks/wire.py @@ -51,6 +51,7 @@ def decode_inputs(values: Any) -> tuple[InputBlock, ...]: for value in values if isinstance(value, Mapping) and isinstance(value.get("variableName"), str) + and value["variableName"] and isinstance(value.get("type"), str) and value["type"] in INPUT_BLOCK_TYPES ) diff --git a/deepnote_toolkit/streamlit/widgets.py b/deepnote_toolkit/streamlit/widgets.py index 22d11da8..40780e48 100644 --- a/deepnote_toolkit/streamlit/widgets.py +++ b/deepnote_toolkit/streamlit/widgets.py @@ -36,6 +36,8 @@ def render_inputs( inputs = tuple(inputs) names = [block.variable_name for block in inputs] + if any(not name for name in names): + raise ValueError("Input variable names must not be empty") if len(names) != len(set(names)): raise ValueError("Input variable names must be unique") values: dict[str, Any] = {} diff --git a/docs/streamlit-apps.md b/docs/streamlit-apps.md index 30796ae7..279f0356 100644 --- a/docs/streamlit-apps.md +++ b/docs/streamlit-apps.md @@ -66,9 +66,9 @@ incomplete date-range selections are left out of the returned dictionary, so disable your Run button until the required values are present. An omitted input runs with the notebook's saved value. A saved open-ended date range renders as separate start and end fields. Stale multi-select choices and slider defaults -outside the bounds are adjusted with a warning. Invalid slider bounds and -duplicate variable names raise `ValueError`. File inputs render as text paths; -this helper does not upload files. +outside the bounds are adjusted with a warning. Invalid slider bounds, empty +variable names and duplicate variable names raise `ValueError`. File inputs render +as text paths; this helper does not upload files. `runner.info().matches_inputs(document.inputs)` compares static input definitions: unique names, types, single/multiple selection, options, and slider bounds/steps. diff --git a/tests/unit/test_deepnote_streamlit_widgets.py b/tests/unit/test_deepnote_streamlit_widgets.py index c50b47a0..b65de96c 100644 --- a/tests/unit/test_deepnote_streamlit_widgets.py +++ b/tests/unit/test_deepnote_streamlit_widgets.py @@ -315,6 +315,12 @@ def test_duplicate_variable_names_are_rejected() -> None: ) +def test_empty_variable_name_is_rejected() -> None: + """Do not render a hand-built input that cannot be submitted to the API.""" + with pytest.raises(ValueError, match="must not be empty"): + render_inputs([InputBlock("", "input-text", "value")], FakeContainer()) + + def test_multiselect_treats_a_scalar_default_as_one_selection() -> None: """Normalize scalar and absent multiselect defaults.""" values = render_inputs( diff --git a/tests/unit/test_notebooks_runners.py b/tests/unit/test_notebooks_runners.py index c0db39dc..76bcdf75 100644 --- a/tests/unit/test_notebooks_runners.py +++ b/tests/unit/test_notebooks_runners.py @@ -209,6 +209,86 @@ def test_finished_create_fetches_snapshot_metadata( assert len(http.calls) == 2 +@pytest.mark.parametrize( + "pending_blocks", + [ + [], + [{"id": "b", "outputs": []}], + [{"id": "b", "outputs": [{"output_type": "stream", "text": "partial"}]}], + ], +) +def test_pending_snapshot_blocks_do_not_stop_polling( + http: responses.RequestsMock, + runner: DeepnoteCloudRunner, + pending_blocks: list[dict[str, Any]], +) -> None: + """Use snapshot lifecycle status even when a pending response has blocks.""" + add_run(http, create_run_response("running"), create=True) + add_run( + http, + run_response(snapshotStatus="pending", snapshotBlocks=pending_blocks), + ) + add_run( + http, + run_response( + snapshotStatus="available", + snapshotBlocks=[ + { + "id": "b", + "outputs": [{"output_type": "stream", "text": "complete"}], + } + ], + ), + ) + result = runner.run({}) + assert result.snapshot_status == "available" + assert result.text() == "complete" + assert len(http.calls) == 3 + + +@pytest.mark.parametrize("target", ["cloud", "local"]) +def test_runner_info_skips_unnamed_inputs( + http: responses.RequestsMock, target: str +) -> None: + """Keep decoded inputs consistent with .deepnote files and valid API keys.""" + name_key = "name" if target == "cloud" else "variableName" + metadata = { + "inputs": [ + {name_key: "", "type": "input-text", "value": "unnamed"}, + {name_key: "region", "type": "input-text", "value": "EU"}, + ] + } + if target == "cloud": + http.get("https://api.deepnote.com/v2/notebooks/n", json={"notebook": metadata}) + info = DeepnoteCloudRunner("n", token="t", session=session()).info() + else: + http.get("http://127.0.0.1:8787/api/info", json=metadata) + info = DeepnoteLocalRunner(session=session()).info() + assert info.inputs == (InputBlock("region", "input-text", "EU"),) + + +def test_pending_empty_blocks_still_respect_snapshot_deadline( + http: responses.RequestsMock, clock: Clock +) -> None: + """Waiting for a pending snapshot remains bounded even when it has blocks.""" + add_run(http, create_run_response("running"), create=True) + add_run(http, run_response(snapshotStatus="pending", snapshotBlocks=[])) + runner = DeepnoteCloudRunner( + "n", + token="t", + session=session(), + snapshot_timeout=1, + poll_interval=0.25, + clock=clock, + sleep=clock.sleep, + ) + result = runner.run({}) + assert result.snapshot_status == "pending" + assert result.outputs == () + assert clock.now == 1.25 + assert len(http.calls) == 5 + + @pytest.mark.parametrize("snapshot_status", ["unavailable", "available"]) def test_only_pending_snapshots_are_polled(http, runner, snapshot_status): add_run(http, create_run_response("running"), create=True) @@ -500,6 +580,13 @@ def test_runner_info_rejects_repeated_input_names() -> None: ) +def test_runner_info_rejects_matching_empty_input_names() -> None: + """Matching unnamed inputs still cannot form a valid execution contract.""" + inputs = (InputBlock("", "input-text", "EU"),) + info = RunnerInfo(notebook="N", inputs=inputs, run_target="cloud") + assert not info.matches_inputs(inputs) + + @pytest.mark.parametrize( "changed", [