From 5d2512b16d18c75f08ee11c9125362616cd51c04 Mon Sep 17 00:00:00 2001 From: tomas Date: Wed, 18 Nov 2026 11:38:43 +0000 Subject: [PATCH 1/4] feat(artifacts): store DataFrames under DEEPNOTE_ARTIFACTS_ROOT when it is set store_dataframe and delete_dataframe now keep frames under the DEEPNOTE_ARTIFACTS_ROOT environment variable when it is non-empty, instead of under the notebook root. A leading ~ is expanded. Nothing else moves, and with the variable unset or empty every path is unchanged. The variable is read from the environment only, not from config files, so it never reaches the effective config that other servers' kernels read. _dataframe_storage.resolve_project_root() is renamed to resolve_artifacts_root() to match. Fixes #152 --- .../artifacts/_dataframe_storage.py | 12 +++- deepnote_toolkit/artifacts/dataframes.py | 4 +- docs/user/configuration.md | 7 ++ tests/unit/artifacts/conftest.py | 3 +- tests/unit/artifacts/test_project_root.py | 71 ++++++++++++++++--- 5 files changed, 82 insertions(+), 15 deletions(-) diff --git a/deepnote_toolkit/artifacts/_dataframe_storage.py b/deepnote_toolkit/artifacts/_dataframe_storage.py index c8c6b987..f443abc4 100644 --- a/deepnote_toolkit/artifacts/_dataframe_storage.py +++ b/deepnote_toolkit/artifacts/_dataframe_storage.py @@ -12,6 +12,7 @@ import pandas as pd from typing_extensions import TypeAlias, assert_never +from deepnote_toolkit import env from deepnote_toolkit.config import resolve_notebook_root from deepnote_toolkit.ocelots.types import PandasDF, PolarsEagerDF @@ -31,6 +32,7 @@ if TYPE_CHECKING: import pyarrow as pa +ARTIFACTS_ROOT_ENV = "DEEPNOTE_ARTIFACTS_ROOT" FRAMES_DIR = Path("deepnote_dataframes") STALE_FRAME_AGE_SECONDS = 24 * 60 * 60 MAX_BATCH_ROWS = 65_536 @@ -49,8 +51,14 @@ def is_safe_id(value: object) -> bool: return isinstance(value, str) and _SAFE_ID_REGEX.fullmatch(value) is not None -def resolve_project_root() -> Path: - """Return the project's root directory, the one set_notebook_path() uses.""" +def resolve_artifacts_root() -> Path: + """Return the root that stored DataFrames live under. + + That is `DEEPNOTE_ARTIFACTS_ROOT` when it is non-empty, otherwise the notebook root. + """ + override = env.get_env(ARTIFACTS_ROOT_ENV) + if override: + return Path(override).expanduser() return resolve_notebook_root() diff --git a/deepnote_toolkit/artifacts/dataframes.py b/deepnote_toolkit/artifacts/dataframes.py index 17492e47..9ae1334f 100644 --- a/deepnote_toolkit/artifacts/dataframes.py +++ b/deepnote_toolkit/artifacts/dataframes.py @@ -68,7 +68,7 @@ def store_dataframe(df: Any, name: str) -> Optional[str]: f"got {type(df).__qualname__}" ) - root = _dataframe_storage.resolve_project_root() + root = _dataframe_storage.resolve_artifacts_root() return _dataframe_storage.store_frame(df, name, root) @@ -85,7 +85,7 @@ def delete_dataframe(name: str) -> bool: files. A symbolic link in place of the name's directory raises too. """ _check_name(name) - root = _dataframe_storage.resolve_project_root() + root = _dataframe_storage.resolve_artifacts_root() directory = root / _dataframe_storage.FRAMES_DIR / name if not directory.is_dir(): return False diff --git a/docs/user/configuration.md b/docs/user/configuration.md index 933abd02..d7993878 100644 --- a/docs/user/configuration.md +++ b/docs/user/configuration.md @@ -97,6 +97,13 @@ Legacy variables are still accepted as input aliases (not precedence), e.g.: - `DEEPNOTE_ROOT_DIR`, `ROOT_DIR` - `TOOLKIT_VERSION`, `TOOLKIT_INDEX_URL`, `TOOLKIT_BUNDLE_PATH`, `TOOLKIT_CACHE_PATH` +`DEEPNOTE_ARTIFACTS_ROOT` sets where stored DataFrames go: + +- When it is set, `store_dataframe` writes to `/deepnote_dataframes//` instead of under the notebook root, and `delete_dataframe` deletes from there. +- The value must be an absolute path. A leading `~` is expanded. +- The toolkit reads it from the environment only. Config files don't set it. +- When it is unset or empty, frames stay under the notebook root. + Secrets/tokens (e.g., Jupyter token, integration credentials) are intentionally not stored in config files; provide them via environment. ## Effective Config (kernel/server) diff --git a/tests/unit/artifacts/conftest.py b/tests/unit/artifacts/conftest.py index e6081eec..5c0154bc 100644 --- a/tests/unit/artifacts/conftest.py +++ b/tests/unit/artifacts/conftest.py @@ -16,7 +16,7 @@ def tmp_project_root(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: project_root = tmp_path / "project" project_root.mkdir() monkeypatch.setattr( - _dataframe_storage, "resolve_project_root", lambda: project_root + _dataframe_storage, "resolve_artifacts_root", lambda: project_root ) return project_root @@ -114,6 +114,7 @@ def open_failing_on_close( @pytest.fixture def no_root_settings(monkeypatch: pytest.MonkeyPatch) -> None: for name in ( + "DEEPNOTE_ARTIFACTS_ROOT", "DEEPNOTE_PATHS__NOTEBOOK_ROOT", "DEEPNOTE_PATHS__HOME_DIR", "DEEPNOTE_HOME_DIR", diff --git a/tests/unit/artifacts/test_project_root.py b/tests/unit/artifacts/test_project_root.py index be5c53e5..be6c79bc 100644 --- a/tests/unit/artifacts/test_project_root.py +++ b/tests/unit/artifacts/test_project_root.py @@ -9,14 +9,19 @@ from deepnote_toolkit import artifacts from deepnote_toolkit import config as toolkit_config from deepnote_toolkit.artifacts import _dataframe_storage -from tests.unit.helpers.dataframe_storage import NAME, arrow_files, stored_manifest +from tests.unit.helpers.dataframe_storage import ( + NAME, + arrow_files, + frame_dir, + stored_manifest, +) def test_store_creates_a_missing_project_root( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: project = tmp_path / "work" - monkeypatch.setattr(_dataframe_storage, "resolve_project_root", lambda: project) + monkeypatch.setattr(_dataframe_storage, "resolve_artifacts_root", lambda: project) reference_id = artifacts.store_dataframe(pd.DataFrame({"a": [1]}), NAME) @@ -29,7 +34,24 @@ def _broken_config() -> Any: @pytest.mark.usefixtures("no_root_settings") class TestProjectRoot: - """resolve_project_root() follows the precedence of set_notebook_path().""" + """resolve_artifacts_root() prefers DEEPNOTE_ARTIFACTS_ROOT, then follows the + precedence of set_notebook_path().""" + + def test_empty_artifacts_root_falls_back_to_notebook_root( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setenv("DEEPNOTE_ARTIFACTS_ROOT", "") + monkeypatch.setenv("DEEPNOTE_PATHS__NOTEBOOK_ROOT", str(tmp_path / "notebooks")) + + assert _dataframe_storage.resolve_artifacts_root() == tmp_path / "notebooks" + + def test_artifacts_root_expands_a_leading_tilde( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setenv("HOME", str(tmp_path)) + monkeypatch.setenv("DEEPNOTE_ARTIFACTS_ROOT", "~/proj") + + assert _dataframe_storage.resolve_artifacts_root() == tmp_path / "proj" def test_notebook_root_wins( self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch @@ -37,26 +59,26 @@ def test_notebook_root_wins( monkeypatch.setenv("DEEPNOTE_PATHS__NOTEBOOK_ROOT", str(tmp_path / "notebooks")) monkeypatch.setenv("DEEPNOTE_HOME_DIR", str(tmp_path / "home")) - assert _dataframe_storage.resolve_project_root() == tmp_path / "notebooks" + assert _dataframe_storage.resolve_artifacts_root() == tmp_path / "notebooks" def test_home_dir_is_used_as_is( self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: monkeypatch.setenv("DEEPNOTE_HOME_DIR", str(tmp_path / "home")) - assert _dataframe_storage.resolve_project_root() == tmp_path / "home" + assert _dataframe_storage.resolve_artifacts_root() == tmp_path / "home" def test_dev_mode_uses_slash_work(self, monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv("DEEPNOTE_RUNNING_IN_DEV_MODE", "true") - assert _dataframe_storage.resolve_project_root() == Path("/work") + assert _dataframe_storage.resolve_artifacts_root() == Path("/work") def test_default_is_work_under_home( self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: monkeypatch.setenv("HOME", str(tmp_path)) - assert _dataframe_storage.resolve_project_root() == tmp_path / "work" + assert _dataframe_storage.resolve_artifacts_root() == tmp_path / "work" def test_unreadable_config_falls_back_to_dev_mode_env( self, monkeypatch: pytest.MonkeyPatch @@ -64,7 +86,7 @@ def test_unreadable_config_falls_back_to_dev_mode_env( monkeypatch.setattr(toolkit_config, "get_config", _broken_config) monkeypatch.setenv("DEEPNOTE_RUNNING_IN_DEV_MODE", "true") - assert _dataframe_storage.resolve_project_root() == Path("/work") + assert _dataframe_storage.resolve_artifacts_root() == Path("/work") def test_unreadable_config_falls_back_to_work_under_home( self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch @@ -72,7 +94,7 @@ def test_unreadable_config_falls_back_to_work_under_home( monkeypatch.setattr(toolkit_config, "get_config", _broken_config) monkeypatch.setenv("HOME", str(tmp_path)) - assert _dataframe_storage.resolve_project_root() == tmp_path / "work" + assert _dataframe_storage.resolve_artifacts_root() == tmp_path / "work" @pytest.mark.usefixtures("no_root_settings") @@ -92,11 +114,40 @@ def test_frame_lands_under_the_project_root_not_the_working_directory( assert list(notebook_directory.iterdir()) == [] +@pytest.mark.usefixtures("no_root_settings") +def test_store_writes_under_the_artifacts_root_not_the_notebook_root( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + artifacts_root = tmp_path / "artifacts" + notebook_root = tmp_path / "notebooks" + notebook_root.mkdir() + monkeypatch.setenv("DEEPNOTE_ARTIFACTS_ROOT", str(artifacts_root)) + monkeypatch.setenv("DEEPNOTE_PATHS__NOTEBOOK_ROOT", str(notebook_root)) + + reference_id = artifacts.store_dataframe(pd.DataFrame({"a": [1]}), NAME) + + assert list(notebook_root.iterdir()) == [] + assert stored_manifest(artifacts_root).file == f"{reference_id}.arrow" + + +@pytest.mark.usefixtures("no_root_settings") +def test_delete_removes_the_frames_under_the_artifacts_root( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + artifacts_root = tmp_path / "artifacts" + frame_dir(artifacts_root).mkdir(parents=True) + monkeypatch.setenv("DEEPNOTE_ARTIFACTS_ROOT", str(artifacts_root)) + monkeypatch.setenv("DEEPNOTE_PATHS__NOTEBOOK_ROOT", str(tmp_path / "notebooks")) + + assert artifacts.delete_dataframe(NAME) is True + assert not frame_dir(artifacts_root).exists() + + def test_delete_returns_false_when_the_project_root_is_missing( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: project = tmp_path / "work" - monkeypatch.setattr(_dataframe_storage, "resolve_project_root", lambda: project) + monkeypatch.setattr(_dataframe_storage, "resolve_artifacts_root", lambda: project) assert artifacts.delete_dataframe(NAME) is False assert not project.exists() From efe6a0ab12f4ac4902aa2a873c170e9a3a06c52c Mon Sep 17 00:00:00 2001 From: tomas Date: Wed, 18 Nov 2026 19:09:13 +0000 Subject: [PATCH 2/4] fix(artifacts): reject a relative DEEPNOTE_ARTIFACTS_ROOT A relative value resolved against the kernel's working directory, which set_notebook_path and %cd change at runtime. Frames stored before a directory change could then no longer be found or deleted. store_dataframe and delete_dataframe now raise ValueError when the value is still relative after a leading ~ is expanded. --- .../artifacts/_dataframe_storage.py | 11 +++++-- deepnote_toolkit/artifacts/dataframes.py | 9 +++--- docs/user/configuration.md | 2 +- tests/unit/artifacts/test_project_root.py | 29 +++++++++++++++++++ 4 files changed, 43 insertions(+), 8 deletions(-) diff --git a/deepnote_toolkit/artifacts/_dataframe_storage.py b/deepnote_toolkit/artifacts/_dataframe_storage.py index f443abc4..aa46ee1a 100644 --- a/deepnote_toolkit/artifacts/_dataframe_storage.py +++ b/deepnote_toolkit/artifacts/_dataframe_storage.py @@ -57,9 +57,14 @@ def resolve_artifacts_root() -> Path: That is `DEEPNOTE_ARTIFACTS_ROOT` when it is non-empty, otherwise the notebook root. """ override = env.get_env(ARTIFACTS_ROOT_ENV) - if override: - return Path(override).expanduser() - return resolve_notebook_root() + if not override: + return resolve_notebook_root() + root = Path(override).expanduser() + if not root.is_absolute(): + raise ValueError( + f"{ARTIFACTS_ROOT_ENV} must be an absolute path, got {override!r}" + ) + return root def _select_compression() -> Optional[str]: diff --git a/deepnote_toolkit/artifacts/dataframes.py b/deepnote_toolkit/artifacts/dataframes.py index 9ae1334f..d5a39481 100644 --- a/deepnote_toolkit/artifacts/dataframes.py +++ b/deepnote_toolkit/artifacts/dataframes.py @@ -33,9 +33,9 @@ def store_dataframe(df: Any, name: str) -> Optional[str]: read-only, in which case nothing is stored. Raises: - ValueError: `name` is not valid, or `df` needs more record batches than - its manifest can list, which takes some 600M rows or 70 GiB of Arrow - data. + ValueError: `name` is not valid, `DEEPNOTE_ARTIFACTS_ROOT` is set to a + relative path, or `df` needs more record batches than its manifest can + list, which takes some 600M rows or 70 GiB of Arrow data. TypeError: `df` is a SQL query preview or a frame derived from one, or is not a pandas or polars DataFrame. That includes PySpark and pandas-on-Spark DataFrames and polars LazyFrames. Also raised for any @@ -80,7 +80,8 @@ def delete_dataframe(name: str) -> bool: nothing is deleted. Raises: - ValueError: `name` is not valid. + ValueError: `name` is not valid, or `DEEPNOTE_ARTIFACTS_ROOT` is set to a + relative path. OSError: The delete failed, possibly part-way, leaving some of the name's files. A symbolic link in place of the name's directory raises too. """ diff --git a/docs/user/configuration.md b/docs/user/configuration.md index d7993878..50536f55 100644 --- a/docs/user/configuration.md +++ b/docs/user/configuration.md @@ -100,7 +100,7 @@ Legacy variables are still accepted as input aliases (not precedence), e.g.: `DEEPNOTE_ARTIFACTS_ROOT` sets where stored DataFrames go: - When it is set, `store_dataframe` writes to `/deepnote_dataframes//` instead of under the notebook root, and `delete_dataframe` deletes from there. -- The value must be an absolute path. A leading `~` is expanded. +- The value must be an absolute path after a leading `~` is expanded. Otherwise `store_dataframe` and `delete_dataframe` raise `ValueError`. - The toolkit reads it from the environment only. Config files don't set it. - When it is unset or empty, frames stay under the notebook root. diff --git a/tests/unit/artifacts/test_project_root.py b/tests/unit/artifacts/test_project_root.py index be6c79bc..729ff7ed 100644 --- a/tests/unit/artifacts/test_project_root.py +++ b/tests/unit/artifacts/test_project_root.py @@ -143,6 +143,35 @@ def test_delete_removes_the_frames_under_the_artifacts_root( assert not frame_dir(artifacts_root).exists() +@pytest.mark.usefixtures("no_root_settings") +def test_store_refuses_a_relative_artifacts_root( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("DEEPNOTE_ARTIFACTS_ROOT", "artifacts") + monkeypatch.setenv("DEEPNOTE_PATHS__NOTEBOOK_ROOT", str(tmp_path / "notebooks")) + + with pytest.raises(ValueError, match="DEEPNOTE_ARTIFACTS_ROOT"): + artifacts.store_dataframe(pd.DataFrame({"a": [1]}), NAME) + + assert list(tmp_path.iterdir()) == [] + + +@pytest.mark.usefixtures("no_root_settings") +def test_delete_refuses_a_relative_artifacts_root( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + frame_dir(tmp_path / "artifacts").mkdir(parents=True) + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("DEEPNOTE_ARTIFACTS_ROOT", "artifacts") + monkeypatch.setenv("DEEPNOTE_PATHS__NOTEBOOK_ROOT", str(tmp_path / "notebooks")) + + with pytest.raises(ValueError, match="DEEPNOTE_ARTIFACTS_ROOT"): + artifacts.delete_dataframe(NAME) + + assert frame_dir(tmp_path / "artifacts").is_dir() + + def test_delete_returns_false_when_the_project_root_is_missing( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: From cf5630cb107081c577182214322c0ef5fe175025 Mon Sep 17 00:00:00 2001 From: tomas Date: Wed, 18 Nov 2026 20:15:10 +0000 Subject: [PATCH 3/4] refactor(artifacts): rename names left over from resolve_project_root The resolver's local variable becomes artifacts_root and the test class becomes TestResolveArtifactsRoot, matching resolve_artifacts_root(). --- deepnote_toolkit/artifacts/_dataframe_storage.py | 8 ++++---- tests/unit/artifacts/test_project_root.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/deepnote_toolkit/artifacts/_dataframe_storage.py b/deepnote_toolkit/artifacts/_dataframe_storage.py index aa46ee1a..915b42f5 100644 --- a/deepnote_toolkit/artifacts/_dataframe_storage.py +++ b/deepnote_toolkit/artifacts/_dataframe_storage.py @@ -56,13 +56,13 @@ def resolve_artifacts_root() -> Path: That is `DEEPNOTE_ARTIFACTS_ROOT` when it is non-empty, otherwise the notebook root. """ - override = env.get_env(ARTIFACTS_ROOT_ENV) - if not override: + artifacts_root = env.get_env(ARTIFACTS_ROOT_ENV) + if not artifacts_root: return resolve_notebook_root() - root = Path(override).expanduser() + root = Path(artifacts_root).expanduser() if not root.is_absolute(): raise ValueError( - f"{ARTIFACTS_ROOT_ENV} must be an absolute path, got {override!r}" + f"{ARTIFACTS_ROOT_ENV} must be an absolute path, got {artifacts_root!r}" ) return root diff --git a/tests/unit/artifacts/test_project_root.py b/tests/unit/artifacts/test_project_root.py index 729ff7ed..30ac05e6 100644 --- a/tests/unit/artifacts/test_project_root.py +++ b/tests/unit/artifacts/test_project_root.py @@ -33,7 +33,7 @@ def _broken_config() -> Any: @pytest.mark.usefixtures("no_root_settings") -class TestProjectRoot: +class TestResolveArtifactsRoot: """resolve_artifacts_root() prefers DEEPNOTE_ARTIFACTS_ROOT, then follows the precedence of set_notebook_path().""" From bacf0a0733bcaecf2f9897dfd87cc63e72ac7dba Mon Sep 17 00:00:00 2001 From: tomas Date: Wed, 18 Nov 2026 22:17:54 +0000 Subject: [PATCH 4/4] test(artifacts): check the stored Arrow file exists under DEEPNOTE_ARTIFACTS_ROOT --- tests/unit/artifacts/test_project_root.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/unit/artifacts/test_project_root.py b/tests/unit/artifacts/test_project_root.py index 30ac05e6..617ee1f1 100644 --- a/tests/unit/artifacts/test_project_root.py +++ b/tests/unit/artifacts/test_project_root.py @@ -127,6 +127,9 @@ def test_store_writes_under_the_artifacts_root_not_the_notebook_root( reference_id = artifacts.store_dataframe(pd.DataFrame({"a": [1]}), NAME) assert list(notebook_root.iterdir()) == [] + assert arrow_files(artifacts_root) == [ + frame_dir(artifacts_root) / f"{reference_id}.arrow" + ] assert stored_manifest(artifacts_root).file == f"{reference_id}.arrow"