diff --git a/deepnote_toolkit/artifacts/_dataframe_storage.py b/deepnote_toolkit/artifacts/_dataframe_storage.py index c8c6b987..915b42f5 100644 --- a/deepnote_toolkit/artifacts/_dataframe_storage.py +++ b/deepnote_toolkit/artifacts/_dataframe_storage.py @@ -12,6 +12,7 @@ import pandas as pd from typing_extensions import TypeAlias, assert_never +from deepnote_toolkit import env from deepnote_toolkit.config import resolve_notebook_root from deepnote_toolkit.ocelots.types import PandasDF, PolarsEagerDF @@ -31,6 +32,7 @@ if TYPE_CHECKING: import pyarrow as pa +ARTIFACTS_ROOT_ENV = "DEEPNOTE_ARTIFACTS_ROOT" FRAMES_DIR = Path("deepnote_dataframes") STALE_FRAME_AGE_SECONDS = 24 * 60 * 60 MAX_BATCH_ROWS = 65_536 @@ -49,9 +51,20 @@ def is_safe_id(value: object) -> bool: return isinstance(value, str) and _SAFE_ID_REGEX.fullmatch(value) is not None -def resolve_project_root() -> Path: - """Return the project's root directory, the one set_notebook_path() uses.""" - return resolve_notebook_root() +def resolve_artifacts_root() -> Path: + """Return the root that stored DataFrames live under. + + That is `DEEPNOTE_ARTIFACTS_ROOT` when it is non-empty, otherwise the notebook root. + """ + artifacts_root = env.get_env(ARTIFACTS_ROOT_ENV) + if not artifacts_root: + return resolve_notebook_root() + root = Path(artifacts_root).expanduser() + if not root.is_absolute(): + raise ValueError( + f"{ARTIFACTS_ROOT_ENV} must be an absolute path, got {artifacts_root!r}" + ) + return root def _select_compression() -> Optional[str]: diff --git a/deepnote_toolkit/artifacts/dataframes.py b/deepnote_toolkit/artifacts/dataframes.py index 17492e47..d5a39481 100644 --- a/deepnote_toolkit/artifacts/dataframes.py +++ b/deepnote_toolkit/artifacts/dataframes.py @@ -33,9 +33,9 @@ def store_dataframe(df: Any, name: str) -> Optional[str]: read-only, in which case nothing is stored. Raises: - ValueError: `name` is not valid, or `df` needs more record batches than - its manifest can list, which takes some 600M rows or 70 GiB of Arrow - data. + ValueError: `name` is not valid, `DEEPNOTE_ARTIFACTS_ROOT` is set to a + relative path, or `df` needs more record batches than its manifest can + list, which takes some 600M rows or 70 GiB of Arrow data. TypeError: `df` is a SQL query preview or a frame derived from one, or is not a pandas or polars DataFrame. That includes PySpark and pandas-on-Spark DataFrames and polars LazyFrames. Also raised for any @@ -68,7 +68,7 @@ def store_dataframe(df: Any, name: str) -> Optional[str]: f"got {type(df).__qualname__}" ) - root = _dataframe_storage.resolve_project_root() + root = _dataframe_storage.resolve_artifacts_root() return _dataframe_storage.store_frame(df, name, root) @@ -80,12 +80,13 @@ def delete_dataframe(name: str) -> bool: nothing is deleted. Raises: - ValueError: `name` is not valid. + ValueError: `name` is not valid, or `DEEPNOTE_ARTIFACTS_ROOT` is set to a + relative path. OSError: The delete failed, possibly part-way, leaving some of the name's files. A symbolic link in place of the name's directory raises too. """ _check_name(name) - root = _dataframe_storage.resolve_project_root() + root = _dataframe_storage.resolve_artifacts_root() directory = root / _dataframe_storage.FRAMES_DIR / name if not directory.is_dir(): return False diff --git a/docs/user/configuration.md b/docs/user/configuration.md index 933abd02..50536f55 100644 --- a/docs/user/configuration.md +++ b/docs/user/configuration.md @@ -97,6 +97,13 @@ Legacy variables are still accepted as input aliases (not precedence), e.g.: - `DEEPNOTE_ROOT_DIR`, `ROOT_DIR` - `TOOLKIT_VERSION`, `TOOLKIT_INDEX_URL`, `TOOLKIT_BUNDLE_PATH`, `TOOLKIT_CACHE_PATH` +`DEEPNOTE_ARTIFACTS_ROOT` sets where stored DataFrames go: + +- When it is set, `store_dataframe` writes to `/deepnote_dataframes//` instead of under the notebook root, and `delete_dataframe` deletes from there. +- The value must be an absolute path after a leading `~` is expanded. Otherwise `store_dataframe` and `delete_dataframe` raise `ValueError`. +- The toolkit reads it from the environment only. Config files don't set it. +- When it is unset or empty, frames stay under the notebook root. + Secrets/tokens (e.g., Jupyter token, integration credentials) are intentionally not stored in config files; provide them via environment. ## Effective Config (kernel/server) diff --git a/tests/unit/artifacts/conftest.py b/tests/unit/artifacts/conftest.py index e6081eec..5c0154bc 100644 --- a/tests/unit/artifacts/conftest.py +++ b/tests/unit/artifacts/conftest.py @@ -16,7 +16,7 @@ def tmp_project_root(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: project_root = tmp_path / "project" project_root.mkdir() monkeypatch.setattr( - _dataframe_storage, "resolve_project_root", lambda: project_root + _dataframe_storage, "resolve_artifacts_root", lambda: project_root ) return project_root @@ -114,6 +114,7 @@ def open_failing_on_close( @pytest.fixture def no_root_settings(monkeypatch: pytest.MonkeyPatch) -> None: for name in ( + "DEEPNOTE_ARTIFACTS_ROOT", "DEEPNOTE_PATHS__NOTEBOOK_ROOT", "DEEPNOTE_PATHS__HOME_DIR", "DEEPNOTE_HOME_DIR", diff --git a/tests/unit/artifacts/test_project_root.py b/tests/unit/artifacts/test_project_root.py index be5c53e5..617ee1f1 100644 --- a/tests/unit/artifacts/test_project_root.py +++ b/tests/unit/artifacts/test_project_root.py @@ -9,14 +9,19 @@ from deepnote_toolkit import artifacts from deepnote_toolkit import config as toolkit_config from deepnote_toolkit.artifacts import _dataframe_storage -from tests.unit.helpers.dataframe_storage import NAME, arrow_files, stored_manifest +from tests.unit.helpers.dataframe_storage import ( + NAME, + arrow_files, + frame_dir, + stored_manifest, +) def test_store_creates_a_missing_project_root( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: project = tmp_path / "work" - monkeypatch.setattr(_dataframe_storage, "resolve_project_root", lambda: project) + monkeypatch.setattr(_dataframe_storage, "resolve_artifacts_root", lambda: project) reference_id = artifacts.store_dataframe(pd.DataFrame({"a": [1]}), NAME) @@ -28,8 +33,25 @@ def _broken_config() -> Any: @pytest.mark.usefixtures("no_root_settings") -class TestProjectRoot: - """resolve_project_root() follows the precedence of set_notebook_path().""" +class TestResolveArtifactsRoot: + """resolve_artifacts_root() prefers DEEPNOTE_ARTIFACTS_ROOT, then follows the + precedence of set_notebook_path().""" + + def test_empty_artifacts_root_falls_back_to_notebook_root( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setenv("DEEPNOTE_ARTIFACTS_ROOT", "") + monkeypatch.setenv("DEEPNOTE_PATHS__NOTEBOOK_ROOT", str(tmp_path / "notebooks")) + + assert _dataframe_storage.resolve_artifacts_root() == tmp_path / "notebooks" + + def test_artifacts_root_expands_a_leading_tilde( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setenv("HOME", str(tmp_path)) + monkeypatch.setenv("DEEPNOTE_ARTIFACTS_ROOT", "~/proj") + + assert _dataframe_storage.resolve_artifacts_root() == tmp_path / "proj" def test_notebook_root_wins( self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch @@ -37,26 +59,26 @@ def test_notebook_root_wins( monkeypatch.setenv("DEEPNOTE_PATHS__NOTEBOOK_ROOT", str(tmp_path / "notebooks")) monkeypatch.setenv("DEEPNOTE_HOME_DIR", str(tmp_path / "home")) - assert _dataframe_storage.resolve_project_root() == tmp_path / "notebooks" + assert _dataframe_storage.resolve_artifacts_root() == tmp_path / "notebooks" def test_home_dir_is_used_as_is( self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: monkeypatch.setenv("DEEPNOTE_HOME_DIR", str(tmp_path / "home")) - assert _dataframe_storage.resolve_project_root() == tmp_path / "home" + assert _dataframe_storage.resolve_artifacts_root() == tmp_path / "home" def test_dev_mode_uses_slash_work(self, monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv("DEEPNOTE_RUNNING_IN_DEV_MODE", "true") - assert _dataframe_storage.resolve_project_root() == Path("/work") + assert _dataframe_storage.resolve_artifacts_root() == Path("/work") def test_default_is_work_under_home( self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: monkeypatch.setenv("HOME", str(tmp_path)) - assert _dataframe_storage.resolve_project_root() == tmp_path / "work" + assert _dataframe_storage.resolve_artifacts_root() == tmp_path / "work" def test_unreadable_config_falls_back_to_dev_mode_env( self, monkeypatch: pytest.MonkeyPatch @@ -64,7 +86,7 @@ def test_unreadable_config_falls_back_to_dev_mode_env( monkeypatch.setattr(toolkit_config, "get_config", _broken_config) monkeypatch.setenv("DEEPNOTE_RUNNING_IN_DEV_MODE", "true") - assert _dataframe_storage.resolve_project_root() == Path("/work") + assert _dataframe_storage.resolve_artifacts_root() == Path("/work") def test_unreadable_config_falls_back_to_work_under_home( self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch @@ -72,7 +94,7 @@ def test_unreadable_config_falls_back_to_work_under_home( monkeypatch.setattr(toolkit_config, "get_config", _broken_config) monkeypatch.setenv("HOME", str(tmp_path)) - assert _dataframe_storage.resolve_project_root() == tmp_path / "work" + assert _dataframe_storage.resolve_artifacts_root() == tmp_path / "work" @pytest.mark.usefixtures("no_root_settings") @@ -92,11 +114,72 @@ def test_frame_lands_under_the_project_root_not_the_working_directory( assert list(notebook_directory.iterdir()) == [] +@pytest.mark.usefixtures("no_root_settings") +def test_store_writes_under_the_artifacts_root_not_the_notebook_root( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + artifacts_root = tmp_path / "artifacts" + notebook_root = tmp_path / "notebooks" + notebook_root.mkdir() + monkeypatch.setenv("DEEPNOTE_ARTIFACTS_ROOT", str(artifacts_root)) + monkeypatch.setenv("DEEPNOTE_PATHS__NOTEBOOK_ROOT", str(notebook_root)) + + reference_id = artifacts.store_dataframe(pd.DataFrame({"a": [1]}), NAME) + + assert list(notebook_root.iterdir()) == [] + assert arrow_files(artifacts_root) == [ + frame_dir(artifacts_root) / f"{reference_id}.arrow" + ] + assert stored_manifest(artifacts_root).file == f"{reference_id}.arrow" + + +@pytest.mark.usefixtures("no_root_settings") +def test_delete_removes_the_frames_under_the_artifacts_root( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + artifacts_root = tmp_path / "artifacts" + frame_dir(artifacts_root).mkdir(parents=True) + monkeypatch.setenv("DEEPNOTE_ARTIFACTS_ROOT", str(artifacts_root)) + monkeypatch.setenv("DEEPNOTE_PATHS__NOTEBOOK_ROOT", str(tmp_path / "notebooks")) + + assert artifacts.delete_dataframe(NAME) is True + assert not frame_dir(artifacts_root).exists() + + +@pytest.mark.usefixtures("no_root_settings") +def test_store_refuses_a_relative_artifacts_root( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("DEEPNOTE_ARTIFACTS_ROOT", "artifacts") + monkeypatch.setenv("DEEPNOTE_PATHS__NOTEBOOK_ROOT", str(tmp_path / "notebooks")) + + with pytest.raises(ValueError, match="DEEPNOTE_ARTIFACTS_ROOT"): + artifacts.store_dataframe(pd.DataFrame({"a": [1]}), NAME) + + assert list(tmp_path.iterdir()) == [] + + +@pytest.mark.usefixtures("no_root_settings") +def test_delete_refuses_a_relative_artifacts_root( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + frame_dir(tmp_path / "artifacts").mkdir(parents=True) + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("DEEPNOTE_ARTIFACTS_ROOT", "artifacts") + monkeypatch.setenv("DEEPNOTE_PATHS__NOTEBOOK_ROOT", str(tmp_path / "notebooks")) + + with pytest.raises(ValueError, match="DEEPNOTE_ARTIFACTS_ROOT"): + artifacts.delete_dataframe(NAME) + + assert frame_dir(tmp_path / "artifacts").is_dir() + + def test_delete_returns_false_when_the_project_root_is_missing( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: project = tmp_path / "work" - monkeypatch.setattr(_dataframe_storage, "resolve_project_root", lambda: project) + monkeypatch.setattr(_dataframe_storage, "resolve_artifacts_root", lambda: project) assert artifacts.delete_dataframe(NAME) is False assert not project.exists()