Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 16 additions & 3 deletions deepnote_toolkit/artifacts/_dataframe_storage.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
import pandas as pd
from typing_extensions import TypeAlias, assert_never

from deepnote_toolkit import env
from deepnote_toolkit.config import resolve_notebook_root
from deepnote_toolkit.ocelots.types import PandasDF, PolarsEagerDF

Expand All @@ -31,6 +32,7 @@
if TYPE_CHECKING:
import pyarrow as pa

ARTIFACTS_ROOT_ENV = "DEEPNOTE_ARTIFACTS_ROOT"
FRAMES_DIR = Path("deepnote_dataframes")
STALE_FRAME_AGE_SECONDS = 24 * 60 * 60
MAX_BATCH_ROWS = 65_536
Expand All @@ -49,9 +51,20 @@ def is_safe_id(value: object) -> bool:
return isinstance(value, str) and _SAFE_ID_REGEX.fullmatch(value) is not None


def resolve_project_root() -> Path:
"""Return the project's root directory, the one set_notebook_path() uses."""
return resolve_notebook_root()
def resolve_artifacts_root() -> Path:
"""Return the root that stored DataFrames live under.

That is `DEEPNOTE_ARTIFACTS_ROOT` when it is non-empty, otherwise the notebook root.
"""
artifacts_root = env.get_env(ARTIFACTS_ROOT_ENV)
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
if not artifacts_root:
return resolve_notebook_root()
root = Path(artifacts_root).expanduser()
if not root.is_absolute():
raise ValueError(
f"{ARTIFACTS_ROOT_ENV} must be an absolute path, got {artifacts_root!r}"
)
return root


def _select_compression() -> Optional[str]:
Expand Down
13 changes: 7 additions & 6 deletions deepnote_toolkit/artifacts/dataframes.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,9 +33,9 @@ def store_dataframe(df: Any, name: str) -> Optional[str]:
read-only, in which case nothing is stored.

Raises:
ValueError: `name` is not valid, or `df` needs more record batches than
its manifest can list, which takes some 600M rows or 70 GiB of Arrow
data.
ValueError: `name` is not valid, `DEEPNOTE_ARTIFACTS_ROOT` is set to a
relative path, or `df` needs more record batches than its manifest can
list, which takes some 600M rows or 70 GiB of Arrow data.
TypeError: `df` is a SQL query preview or a frame derived from one, or is
not a pandas or polars DataFrame. That includes PySpark and
pandas-on-Spark DataFrames and polars LazyFrames. Also raised for any
Expand Down Expand Up @@ -68,7 +68,7 @@ def store_dataframe(df: Any, name: str) -> Optional[str]:
f"got {type(df).__qualname__}"
)

root = _dataframe_storage.resolve_project_root()
root = _dataframe_storage.resolve_artifacts_root()
return _dataframe_storage.store_frame(df, name, root)


Expand All @@ -80,12 +80,13 @@ def delete_dataframe(name: str) -> bool:
nothing is deleted.

Raises:
ValueError: `name` is not valid.
ValueError: `name` is not valid, or `DEEPNOTE_ARTIFACTS_ROOT` is set to a
relative path.
OSError: The delete failed, possibly part-way, leaving some of the name's
files. A symbolic link in place of the name's directory raises too.
"""
_check_name(name)
root = _dataframe_storage.resolve_project_root()
root = _dataframe_storage.resolve_artifacts_root()
directory = root / _dataframe_storage.FRAMES_DIR / name
if not directory.is_dir():
return False
Expand Down
7 changes: 7 additions & 0 deletions docs/user/configuration.md
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,13 @@ Legacy variables are still accepted as input aliases (not precedence), e.g.:
- `DEEPNOTE_ROOT_DIR`, `ROOT_DIR`
- `TOOLKIT_VERSION`, `TOOLKIT_INDEX_URL`, `TOOLKIT_BUNDLE_PATH`, `TOOLKIT_CACHE_PATH`

`DEEPNOTE_ARTIFACTS_ROOT` sets where stored DataFrames go:

- When it is set, `store_dataframe` writes to `<value>/deepnote_dataframes/<name>/` instead of under the notebook root, and `delete_dataframe` deletes from there.
- The value must be an absolute path after a leading `~` is expanded. Otherwise `store_dataframe` and `delete_dataframe` raise `ValueError`.
- The toolkit reads it from the environment only. Config files don't set it.
- When it is unset or empty, frames stay under the notebook root.

Secrets/tokens (e.g., Jupyter token, integration credentials) are intentionally not stored in config files; provide them via environment.

## Effective Config (kernel/server)
Expand Down
3 changes: 2 additions & 1 deletion tests/unit/artifacts/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ def tmp_project_root(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path:
project_root = tmp_path / "project"
project_root.mkdir()
monkeypatch.setattr(
_dataframe_storage, "resolve_project_root", lambda: project_root
_dataframe_storage, "resolve_artifacts_root", lambda: project_root
)
return project_root

Expand Down Expand Up @@ -114,6 +114,7 @@ def open_failing_on_close(
@pytest.fixture
def no_root_settings(monkeypatch: pytest.MonkeyPatch) -> None:
for name in (
"DEEPNOTE_ARTIFACTS_ROOT",
"DEEPNOTE_PATHS__NOTEBOOK_ROOT",
"DEEPNOTE_PATHS__HOME_DIR",
"DEEPNOTE_HOME_DIR",
Expand Down
105 changes: 94 additions & 11 deletions tests/unit/artifacts/test_project_root.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,14 +9,19 @@
from deepnote_toolkit import artifacts
from deepnote_toolkit import config as toolkit_config
from deepnote_toolkit.artifacts import _dataframe_storage
from tests.unit.helpers.dataframe_storage import NAME, arrow_files, stored_manifest
from tests.unit.helpers.dataframe_storage import (
NAME,
arrow_files,
frame_dir,
stored_manifest,
)


def test_store_creates_a_missing_project_root(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
project = tmp_path / "work"
monkeypatch.setattr(_dataframe_storage, "resolve_project_root", lambda: project)
monkeypatch.setattr(_dataframe_storage, "resolve_artifacts_root", lambda: project)

reference_id = artifacts.store_dataframe(pd.DataFrame({"a": [1]}), NAME)

Expand All @@ -28,51 +33,68 @@ def _broken_config() -> Any:


@pytest.mark.usefixtures("no_root_settings")
class TestProjectRoot:
"""resolve_project_root() follows the precedence of set_notebook_path()."""
class TestResolveArtifactsRoot:
"""resolve_artifacts_root() prefers DEEPNOTE_ARTIFACTS_ROOT, then follows the
precedence of set_notebook_path()."""

def test_empty_artifacts_root_falls_back_to_notebook_root(
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setenv("DEEPNOTE_ARTIFACTS_ROOT", "")
monkeypatch.setenv("DEEPNOTE_PATHS__NOTEBOOK_ROOT", str(tmp_path / "notebooks"))

assert _dataframe_storage.resolve_artifacts_root() == tmp_path / "notebooks"

def test_artifacts_root_expands_a_leading_tilde(
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setenv("HOME", str(tmp_path))
monkeypatch.setenv("DEEPNOTE_ARTIFACTS_ROOT", "~/proj")

assert _dataframe_storage.resolve_artifacts_root() == tmp_path / "proj"

def test_notebook_root_wins(
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setenv("DEEPNOTE_PATHS__NOTEBOOK_ROOT", str(tmp_path / "notebooks"))
monkeypatch.setenv("DEEPNOTE_HOME_DIR", str(tmp_path / "home"))

assert _dataframe_storage.resolve_project_root() == tmp_path / "notebooks"
assert _dataframe_storage.resolve_artifacts_root() == tmp_path / "notebooks"

def test_home_dir_is_used_as_is(
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setenv("DEEPNOTE_HOME_DIR", str(tmp_path / "home"))

assert _dataframe_storage.resolve_project_root() == tmp_path / "home"
assert _dataframe_storage.resolve_artifacts_root() == tmp_path / "home"

def test_dev_mode_uses_slash_work(self, monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("DEEPNOTE_RUNNING_IN_DEV_MODE", "true")

assert _dataframe_storage.resolve_project_root() == Path("/work")
assert _dataframe_storage.resolve_artifacts_root() == Path("/work")

def test_default_is_work_under_home(
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setenv("HOME", str(tmp_path))

assert _dataframe_storage.resolve_project_root() == tmp_path / "work"
assert _dataframe_storage.resolve_artifacts_root() == tmp_path / "work"

def test_unreadable_config_falls_back_to_dev_mode_env(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setattr(toolkit_config, "get_config", _broken_config)
monkeypatch.setenv("DEEPNOTE_RUNNING_IN_DEV_MODE", "true")

assert _dataframe_storage.resolve_project_root() == Path("/work")
assert _dataframe_storage.resolve_artifacts_root() == Path("/work")

def test_unreadable_config_falls_back_to_work_under_home(
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setattr(toolkit_config, "get_config", _broken_config)
monkeypatch.setenv("HOME", str(tmp_path))

assert _dataframe_storage.resolve_project_root() == tmp_path / "work"
assert _dataframe_storage.resolve_artifacts_root() == tmp_path / "work"


@pytest.mark.usefixtures("no_root_settings")
Expand All @@ -92,11 +114,72 @@ def test_frame_lands_under_the_project_root_not_the_working_directory(
assert list(notebook_directory.iterdir()) == []


@pytest.mark.usefixtures("no_root_settings")
def test_store_writes_under_the_artifacts_root_not_the_notebook_root(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
artifacts_root = tmp_path / "artifacts"
notebook_root = tmp_path / "notebooks"
notebook_root.mkdir()
monkeypatch.setenv("DEEPNOTE_ARTIFACTS_ROOT", str(artifacts_root))
monkeypatch.setenv("DEEPNOTE_PATHS__NOTEBOOK_ROOT", str(notebook_root))

reference_id = artifacts.store_dataframe(pd.DataFrame({"a": [1]}), NAME)

assert list(notebook_root.iterdir()) == []
assert arrow_files(artifacts_root) == [
frame_dir(artifacts_root) / f"{reference_id}.arrow"
]
assert stored_manifest(artifacts_root).file == f"{reference_id}.arrow"


@pytest.mark.usefixtures("no_root_settings")
def test_delete_removes_the_frames_under_the_artifacts_root(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
artifacts_root = tmp_path / "artifacts"
frame_dir(artifacts_root).mkdir(parents=True)
monkeypatch.setenv("DEEPNOTE_ARTIFACTS_ROOT", str(artifacts_root))
monkeypatch.setenv("DEEPNOTE_PATHS__NOTEBOOK_ROOT", str(tmp_path / "notebooks"))

assert artifacts.delete_dataframe(NAME) is True
assert not frame_dir(artifacts_root).exists()


@pytest.mark.usefixtures("no_root_settings")
def test_store_refuses_a_relative_artifacts_root(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.chdir(tmp_path)
monkeypatch.setenv("DEEPNOTE_ARTIFACTS_ROOT", "artifacts")
monkeypatch.setenv("DEEPNOTE_PATHS__NOTEBOOK_ROOT", str(tmp_path / "notebooks"))

with pytest.raises(ValueError, match="DEEPNOTE_ARTIFACTS_ROOT"):
artifacts.store_dataframe(pd.DataFrame({"a": [1]}), NAME)

assert list(tmp_path.iterdir()) == []


@pytest.mark.usefixtures("no_root_settings")
def test_delete_refuses_a_relative_artifacts_root(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
frame_dir(tmp_path / "artifacts").mkdir(parents=True)
monkeypatch.chdir(tmp_path)
monkeypatch.setenv("DEEPNOTE_ARTIFACTS_ROOT", "artifacts")
monkeypatch.setenv("DEEPNOTE_PATHS__NOTEBOOK_ROOT", str(tmp_path / "notebooks"))

with pytest.raises(ValueError, match="DEEPNOTE_ARTIFACTS_ROOT"):
artifacts.delete_dataframe(NAME)

assert frame_dir(tmp_path / "artifacts").is_dir()


def test_delete_returns_false_when_the_project_root_is_missing(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
project = tmp_path / "work"
monkeypatch.setattr(_dataframe_storage, "resolve_project_root", lambda: project)
monkeypatch.setattr(_dataframe_storage, "resolve_artifacts_root", lambda: project)

assert artifacts.delete_dataframe(NAME) is False
assert not project.exists()
Loading