From 64f54dd7978073e77c40c75fc2ffcd97d190da0e Mon Sep 17 00:00:00 2001 From: Yue Chao Qin Date: Tue, 17 Nov 2026 17:24:43 -0800 Subject: [PATCH 1/5] feat: GSM Secrets - Save secret values through an optional store hook --- cloud_pipelines_backend/api_router.py | 17 +- cloud_pipelines_backend/api_server_sql.py | 38 ++- cloud_pipelines_backend/errors.py | 4 + cloud_pipelines_backend/secret_value_store.py | 126 +++++++++ tests/test_secret_value_store.py | 256 ++++++++++++++++++ 5 files changed, 438 insertions(+), 3 deletions(-) create mode 100644 cloud_pipelines_backend/secret_value_store.py create mode 100644 tests/test_secret_value_store.py diff --git a/cloud_pipelines_backend/api_router.py b/cloud_pipelines_backend/api_router.py index 0e8fa41a..5614afb0 100644 --- a/cloud_pipelines_backend/api_router.py +++ b/cloud_pipelines_backend/api_router.py @@ -18,6 +18,7 @@ from .instrumentation import contextual_logging if typing.TYPE_CHECKING: + from . import secret_value_store from .launchers import interfaces as launcher_interfaces @@ -40,6 +41,7 @@ def setup_routes( container_launcher_for_log_streaming: "launcher_interfaces.ContainerTaskLauncher[launcher_interfaces.LaunchedContainer] | None" = None, default_component_library_owner_username: str = "admin", do_skip_backfill: bool = False, + secret_value_store: "secret_value_store.SecretValueStore | None" = None, ): def get_session(): with orm.Session(autocommit=False, autoflush=False, bind=db_engine) as session: @@ -76,6 +78,7 @@ async def lifespan(app: fastapi.FastAPI): user_details_getter=user_details_getter, get_launcher=get_launcher, lifespan=lifespan, + secret_value_store=secret_value_store, ) @@ -93,6 +96,7 @@ def _setup_routes_internal( | typing.Callable[..., abc.Iterator[typing.Any]] | None ) = None, + secret_value_store: "secret_value_store.SecretValueStore | None" = None, ): # We request `app: fastapi.FastAPI` instead of just returning the router # because we want to add exception handler which is only supported for `FastAPI`. @@ -150,6 +154,15 @@ def handle_api_validation_error( content={"detail": str(exc)}, ) + @app.exception_handler(errors.SecretStorageTypeMismatchError) + def handle_secret_storage_type_mismatch_error( + request: fastapi.Request, exc: errors.SecretStorageTypeMismatchError + ): + return fastapi.responses.JSONResponse( + status_code=fastapi.status.HTTP_503_SERVICE_UNAVAILABLE, + content={"message": str(exc)}, + ) + @app.exception_handler(NotImplementedError) def handle_not_implemented_error( request: fastapi.Request, exc: NotImplementedError @@ -462,7 +475,9 @@ def get_current_user( ) ### Secrets routes - secrets_service = api_server_sql.SecretsApiService() + secrets_service = api_server_sql.SecretsApiService( + secret_value_store=secret_value_store + ) router.get("/api/secrets/", tags=["secrets"], **default_config)( inject_session_dependency( diff --git a/cloud_pipelines_backend/api_server_sql.py b/cloud_pipelines_backend/api_server_sql.py index 8277606e..c48aba51 100644 --- a/cloud_pipelines_backend/api_server_sql.py +++ b/cloud_pipelines_backend/api_server_sql.py @@ -13,6 +13,7 @@ from .access_control import checks as access_control_checks from .access_control import runs as access_control_runs from . import filter_query_sql +from . import secret_value_store if typing.TYPE_CHECKING: from cloud_pipelines.orchestration.storage_providers import ( @@ -1256,6 +1257,13 @@ class ListSecretsResponse: class SecretsApiService: + def __init__( + self, + *, + secret_value_store: secret_value_store.SecretValueStore | None = None, + ): + self._secret_value_store = secret_value_store + def create_secret( self, *, @@ -1318,7 +1326,9 @@ def _create_or_update_secret( raise errors.ItemAlreadyExistsError( f"Secret with name '{secret_name}' already exists." ) - secret.secret_value = secret_value + secret_value_store.raise_if_storage_type_mismatch( + secret=secret, store=self._secret_value_store + ) secret.updated_at = current_time else: if raise_if_not_exists: @@ -1328,17 +1338,30 @@ def _create_or_update_secret( secret = bts.Secret( user_id=user_id, secret_name=secret_name, - secret_value=secret_value, + # Set below, through the store when there is one. + secret_value="", created_at=current_time, updated_at=current_time, ) session.add(secret) + # The new value is saved before the row commits, and the one it replaces is retired after, + # so the row never names a value that is already gone. + previous_storage_details = secret_value_store.store_secret_value( + secret=secret, value=secret_value, store=self._secret_value_store + ) if description: secret.description = description if expires_at: secret.expires_at = expires_at response = SecretInfoResponse.from_db(secret) session.commit() + if previous_storage_details is not None: + secret_value_store.retire_secret_value( + store=self._secret_value_store, + storage_details=previous_storage_details, + user_id=user_id, + secret_name=secret_name, + ) return response def delete_secret( @@ -1353,8 +1376,19 @@ def delete_secret( raise errors.ItemNotFoundError( f"Secret with name '{secret_name}' does not exist." ) + secret_value_store.raise_if_storage_type_mismatch( + secret=secret, store=self._secret_value_store + ) + storage_details = secret_value_store.get_storage_details(secret=secret) session.delete(secret) session.commit() + if storage_details is not None: + secret_value_store.retire_secret_value( + store=self._secret_value_store, + storage_details=storage_details, + user_id=user_id, + secret_name=secret_name, + ) def list_secrets( self, diff --git a/cloud_pipelines_backend/errors.py b/cloud_pipelines_backend/errors.py index dee86527..9c684fea 100644 --- a/cloud_pipelines_backend/errors.py +++ b/cloud_pipelines_backend/errors.py @@ -22,6 +22,10 @@ class PermissionError(Exception): pass +class SecretStorageTypeMismatchError(Exception): + """A secret's value is in a store of another type than this server's, or the server has none, so it cannot be changed.""" + + class ApiValidationError(Exception): """Base for all filter/annotation validation errors -> 422.""" diff --git a/cloud_pipelines_backend/secret_value_store.py b/cloud_pipelines_backend/secret_value_store.py new file mode 100644 index 00000000..254ef451 --- /dev/null +++ b/cloud_pipelines_backend/secret_value_store.py @@ -0,0 +1,126 @@ +import logging +import typing +from typing import Any, Final + +from . import backend_types_sql as bts +from . import errors + +_logger = logging.getLogger(__name__) + +EXTRA_DATA_STORAGE_KEY: Final[str] = "storage" +STORAGE_TYPE_KEY: Final[str] = "type" +STORAGE_DETAILS_KEY: Final[str] = "details" + + +class SecretValueStore(typing.Protocol): + """Keeps secret values outside the database row, for example in a cloud secret manager.""" + + # Recorded in the row's `extra_data`, so later reads and writes know which store holds the value. + type: str + + def save( + self, *, user_id: str, secret_name: str, value: str + ) -> tuple[str, dict[str, Any]]: + """Saves the value; returns what the row's `secret_value` keeps, and the storage details for `get` and `delete`. + + Example, a store that keeps numbered versions: `save(user_id="alice", secret_name="HF_TOKEN", + value="hf_FAKE123")` returns `("", {"version": "4"})` — the row keeps `""`, and `{"version": "4"}` + is what `get` and `delete` are later given as `storage_details`. + """ + ... + + def get( + self, *, storage_details: dict[str, Any], user_id: str, secret_name: str + ) -> str: + """Returns the value that `save` saved with these storage details.""" + ... + + def delete( + self, *, storage_details: dict[str, Any], user_id: str, secret_name: str + ) -> None: + """Retires the value that `save` saved with these storage details.""" + ... + + +def get_storage_type(*, secret: bts.Secret) -> str | None: + """Returns the type of the store holding the row's value, or None when the value is plain in the row. + + Example: with `extra_data = {"storage": {"type": "gsm", "details": {"version": "4"}}}` it returns `"gsm"`. + """ + storage = (secret.extra_data or {}).get(EXTRA_DATA_STORAGE_KEY) + return storage[STORAGE_TYPE_KEY] if storage else None + + +def get_storage_details(*, secret: bts.Secret) -> dict[str, Any] | None: + """Returns what the store needs to find the row's value, or None when the value is plain in the row. + + Example: with `extra_data = {"storage": {"type": "gsm", "details": {"version": "4"}}}` it returns + `{"version": "4"}`. + """ + storage = (secret.extra_data or {}).get(EXTRA_DATA_STORAGE_KEY) + return storage[STORAGE_DETAILS_KEY] if storage else None + + +def store_secret_value( + *, secret: bts.Secret, value: str, store: SecretValueStore | None +) -> dict[str, Any] | None: + """Sets the row's value, through the store if one is given. + + Returns the storage details of the value it replaces in the store, or None when there was none. + """ + if store is None: + secret.secret_value = value + return None + previous_storage_details = get_storage_details(secret=secret) + secret.secret_value, storage_details = store.save( + user_id=secret.user_id, secret_name=secret.secret_name, value=value + ) + # A merge: the copy keeps every other key, so only the storage entry changes. + secret.extra_data = { + **(secret.extra_data or {}), + EXTRA_DATA_STORAGE_KEY: { + STORAGE_TYPE_KEY: store.type, + STORAGE_DETAILS_KEY: storage_details, + }, + } + return previous_storage_details + + +def raise_if_storage_type_mismatch( + *, secret: bts.Secret, store: SecretValueStore | None +) -> None: + """Raises when the row's value is in a store other than this server's, leaving the row unchanged. + + Writing plain text over it, or deleting the row, would leave the stored value behind and still readable. + """ + storage_type = get_storage_type(secret=secret) + if storage_type is None: + return + configured = store.type if store else None + if storage_type != configured: + raise errors.SecretStorageTypeMismatchError( + f"Secret '{secret.secret_name}' is kept in the {storage_type!r} store, " + f"but this server's secret store is {configured!r}; it cannot be changed or deleted here." + ) + + +def retire_secret_value( + *, + store: SecretValueStore, + storage_details: dict[str, Any], + user_id: str, + secret_name: str, +) -> None: + """Retires a stored value the row no longer names, logging a failure instead of raising it. + + A helper for after the row commits: the change is already saved, so raising would report a successful save + as a failure, and the only harm of a failed retire is an old value left enabled in the store. + """ + try: + store.delete( + storage_details=storage_details, user_id=user_id, secret_name=secret_name + ) + except Exception: + _logger.exception( + f"Could not retire the previous stored value of secret '{secret_name}'." + ) diff --git a/tests/test_secret_value_store.py b/tests/test_secret_value_store.py new file mode 100644 index 00000000..9ff1c6f8 --- /dev/null +++ b/tests/test_secret_value_store.py @@ -0,0 +1,256 @@ +import logging +import pathlib +from typing import Any + +import fastapi +import pytest +import sqlalchemy +from sqlalchemy import orm +from starlette.testclient import TestClient + +from cloud_pipelines_backend import api_router +from cloud_pipelines_backend import api_server_sql +from cloud_pipelines_backend import backend_types_sql as bts +from cloud_pipelines_backend import database_ops +from cloud_pipelines_backend import errors + + +class _FakeStore: + """Keeps values in memory, one numbered version per `save`, like a cloud secret manager.""" + + def __init__( + self, + *, + type: str = "fake", + engine: sqlalchemy.Engine | None = None, + fail_delete: bool = False, + ) -> None: + self.type = type + self.versions: dict[str, str] = {} + self.retired: list[str] = [] + # With an engine, `delete` records which version the committed row names at that moment. + self._engine = engine + self.row_version_at_delete: list[str | None] = [] + self._fail_delete = fail_delete + + def save( + self, *, user_id: str, secret_name: str, value: str + ) -> tuple[str, dict[str, Any]]: + """Saves the value as the next version.""" + version = str(len(self.versions) + 1) + self.versions[version] = value + return "", {"version": version} + + def get( + self, *, storage_details: dict[str, Any], user_id: str, secret_name: str + ) -> str: + """Returns the saved version.""" + return self.versions[storage_details["version"]] + + def delete( + self, *, storage_details: dict[str, Any], user_id: str, secret_name: str + ) -> None: + """Retires the version, or fails when told to.""" + if self._engine is not None: + with orm.Session(bind=self._engine) as session: + row = session.get(bts.Secret, (user_id, secret_name)) + self.row_version_at_delete.append( + row.extra_data["storage"]["details"]["version"] if row else None + ) + if self._fail_delete: + raise RuntimeError("store unavailable") + self.retired.append(storage_details["version"]) + + +def _engine(*, tmp_path: pathlib.Path) -> sqlalchemy.Engine: + """Returns a file database, so a second session sees only committed rows.""" + return database_ops.create_db_engine_and_migrate_db( + database_uri=f"sqlite:///{tmp_path / 'db.sqlite'}" + ) + + +def _save( + *, + engine: sqlalchemy.Engine, + store: _FakeStore | None, + value: str, + update: bool = False, +) -> None: + """Creates or updates alice's TOKEN through the service.""" + service = api_server_sql.SecretsApiService(secret_value_store=store) + method = service.update_secret if update else service.create_secret + with orm.Session(bind=engine) as session: + method( + session=session, user_id="alice", secret_name="TOKEN", secret_value=value + ) + + +def _row(*, engine: sqlalchemy.Engine) -> bts.Secret | None: + """Reads alice's TOKEN as committed.""" + with orm.Session(bind=engine, expire_on_commit=False) as session: + return session.get(bts.Secret, ("alice", "TOKEN")) + + +def test_without_store_keeps_the_value_in_the_row(tmp_path: pathlib.Path): + engine = _engine(tmp_path=tmp_path) + _save(engine=engine, store=None, value="s3cret") + row = _row(engine=engine) + assert row.secret_value == "s3cret" + assert row.extra_data is None + + +def test_without_store_update_and_delete_work_as_before(tmp_path: pathlib.Path): + engine = _engine(tmp_path=tmp_path) + _save(engine=engine, store=None, value="s3cret") + _save(engine=engine, store=None, value="n3w", update=True) + row = _row(engine=engine) + assert (row.secret_value, row.extra_data) == ("n3w", None) + with orm.Session(bind=engine) as session: + api_server_sql.SecretsApiService().delete_secret( + session=session, user_id="alice", secret_name="TOKEN" + ) + assert _row(engine=engine) is None + + +def test_with_store_the_row_keeps_only_what_save_returned(tmp_path: pathlib.Path): + engine = _engine(tmp_path=tmp_path) + store = _FakeStore() + _save(engine=engine, store=store, value="s3cret") + row = _row(engine=engine) + assert row.secret_value == "" + assert row.extra_data == {"storage": {"type": "fake", "details": {"version": "1"}}} + assert store.versions == {"1": "s3cret"} + + +def test_update_retires_the_previous_version_after_the_row_commits( + tmp_path: pathlib.Path, +): + engine = _engine(tmp_path=tmp_path) + store = _FakeStore(engine=engine) + _save(engine=engine, store=store, value="v1") + _save(engine=engine, store=store, value="v2", update=True) + assert _row(engine=engine).extra_data["storage"]["details"] == {"version": "2"} + assert store.retired == ["1"] + # When version 1 was retired, the committed row already named version 2. + assert store.row_version_at_delete == ["2"] + + +def test_delete_retires_the_version_after_the_row_is_gone(tmp_path: pathlib.Path): + engine = _engine(tmp_path=tmp_path) + store = _FakeStore(engine=engine) + _save(engine=engine, store=store, value="v1") + with orm.Session(bind=engine) as session: + api_server_sql.SecretsApiService(secret_value_store=store).delete_secret( + session=session, user_id="alice", secret_name="TOKEN" + ) + assert _row(engine=engine) is None + assert store.retired == ["1"] + assert store.row_version_at_delete == [None] + + +def test_a_plain_row_moves_into_the_store_on_update(tmp_path: pathlib.Path): + engine = _engine(tmp_path=tmp_path) + store = _FakeStore() + _save(engine=engine, store=None, value="plain") + _save(engine=engine, store=store, value="v2", update=True) + row = _row(engine=engine) + assert row.secret_value == "" + assert row.extra_data["storage"]["details"] == {"version": "1"} + assert store.retired == [] + + +def test_other_extra_data_keys_survive(tmp_path: pathlib.Path): + engine = _engine(tmp_path=tmp_path) + _save(engine=engine, store=None, value="plain") + with orm.Session(bind=engine) as session: + session.get(bts.Secret, ("alice", "TOKEN")).extra_data = {"owner_team": "data"} + session.commit() + _save(engine=engine, store=_FakeStore(), value="v2", update=True) + extra_data = _row(engine=engine).extra_data + assert extra_data["owner_team"] == "data" + assert "storage" in extra_data + + +@pytest.mark.parametrize("server_store", [None, _FakeStore(type="other")]) +def test_save_and_delete_refused_when_the_row_is_in_another_store( + tmp_path: pathlib.Path, server_store: _FakeStore | None +): + engine = _engine(tmp_path=tmp_path) + _save(engine=engine, store=_FakeStore(), value="v1") + before = _row(engine=engine) + with pytest.raises(errors.SecretStorageTypeMismatchError, match="'fake' store"): + _save(engine=engine, store=server_store, value="plain", update=True) + with orm.Session(bind=engine) as session: + with pytest.raises(errors.SecretStorageTypeMismatchError): + api_server_sql.SecretsApiService( + secret_value_store=server_store + ).delete_secret(session=session, user_id="alice", secret_name="TOKEN") + after = _row(engine=engine) + assert (after.secret_value, after.extra_data) == ( + before.secret_value, + before.extra_data, + ) + if server_store is not None: + assert server_store.versions == {} + assert server_store.retired == [] + + +def test_create_of_an_existing_name_saves_nothing_in_the_store( + tmp_path: pathlib.Path, +): + engine = _engine(tmp_path=tmp_path) + store = _FakeStore() + _save(engine=engine, store=store, value="v1") + with pytest.raises(errors.ItemAlreadyExistsError): + _save(engine=engine, store=store, value="v2") + assert store.versions == {"1": "v1"} + + +def test_a_failed_retire_is_logged_and_the_update_stands( + tmp_path: pathlib.Path, caplog: pytest.LogCaptureFixture +): + engine = _engine(tmp_path=tmp_path) + store = _FakeStore(fail_delete=True) + _save(engine=engine, store=store, value="v1") + with caplog.at_level(logging.ERROR): + _save(engine=engine, store=store, value="v2", update=True) + assert _row(engine=engine).extra_data["storage"]["details"] == {"version": "2"} + assert "Could not retire the previous stored value of secret 'TOKEN'" in caplog.text + assert "v1" not in caplog.text and "v2" not in caplog.text + + +def _client(*, engine: sqlalchemy.Engine, store: _FakeStore | None) -> TestClient: + """Returns an API client for alice, with the given store passed to setup_routes.""" + app = fastapi.FastAPI() + api_router.setup_routes( + app=app, + db_engine=engine, + user_details_getter=lambda: api_router.UserDetails( + name="alice", permissions={"read": True, "write": True, "admin": False} + ), + secret_value_store=store, + ) + return TestClient(app) + + +def test_setup_routes_passes_the_store_to_secret_routes(tmp_path: pathlib.Path): + engine = _engine(tmp_path=tmp_path) + store = _FakeStore() + response = _client(engine=engine, store=store).post( + "/api/secrets/", + params={"secret_name": "TOKEN"}, + json={"secret_value": "s3cret"}, + ) + assert response.status_code == 200, response.text + assert _row(engine=engine).secret_value == "" + assert store.versions == {"1": "s3cret"} + + +def test_a_refusal_is_a_503_naming_both_stores(tmp_path: pathlib.Path): + engine = _engine(tmp_path=tmp_path) + _save(engine=engine, store=_FakeStore(), value="v1") + response = _client(engine=engine, store=None).delete("/api/secrets/TOKEN") + assert response.status_code == 503, response.text + assert "'fake' store" in response.json()["message"] + assert "None" in response.json()["message"] + assert _row(engine=engine) is not None From 18a021acbcfd0c50f9bf4e52acbd83f7a71414ef Mon Sep 17 00:00:00 2001 From: Yue Chao Qin Date: Wed, 18 Nov 2026 20:05:21 -0800 Subject: [PATCH 2/5] feat: GSM Secrets - Keep secrets in an optional store; move table rows on first use SecretStore is an abstract base class whose subclass methods are wrapped to move the caller's rows out of the secret table first. Replaces the storage-type hook: no storage flag, no 503 handler. --- cloud_pipelines_backend/api_router.py | 21 +- cloud_pipelines_backend/api_server_sql.py | 78 ++-- cloud_pipelines_backend/errors.py | 4 - cloud_pipelines_backend/secret_store.py | 139 +++++++ cloud_pipelines_backend/secret_value_store.py | 126 ------- tests/test_secret_store.py | 348 ++++++++++++++++++ tests/test_secret_value_store.py | 256 ------------- 7 files changed, 535 insertions(+), 437 deletions(-) create mode 100644 cloud_pipelines_backend/secret_store.py delete mode 100644 cloud_pipelines_backend/secret_value_store.py create mode 100644 tests/test_secret_store.py delete mode 100644 tests/test_secret_value_store.py diff --git a/cloud_pipelines_backend/api_router.py b/cloud_pipelines_backend/api_router.py index 5614afb0..ab77087c 100644 --- a/cloud_pipelines_backend/api_router.py +++ b/cloud_pipelines_backend/api_router.py @@ -18,7 +18,7 @@ from .instrumentation import contextual_logging if typing.TYPE_CHECKING: - from . import secret_value_store + from . import secret_store from .launchers import interfaces as launcher_interfaces @@ -41,7 +41,7 @@ def setup_routes( container_launcher_for_log_streaming: "launcher_interfaces.ContainerTaskLauncher[launcher_interfaces.LaunchedContainer] | None" = None, default_component_library_owner_username: str = "admin", do_skip_backfill: bool = False, - secret_value_store: "secret_value_store.SecretValueStore | None" = None, + secret_store: "secret_store.SecretStore | None" = None, ): def get_session(): with orm.Session(autocommit=False, autoflush=False, bind=db_engine) as session: @@ -78,7 +78,7 @@ async def lifespan(app: fastapi.FastAPI): user_details_getter=user_details_getter, get_launcher=get_launcher, lifespan=lifespan, - secret_value_store=secret_value_store, + secret_store=secret_store, ) @@ -96,7 +96,7 @@ def _setup_routes_internal( | typing.Callable[..., abc.Iterator[typing.Any]] | None ) = None, - secret_value_store: "secret_value_store.SecretValueStore | None" = None, + secret_store: "secret_store.SecretStore | None" = None, ): # We request `app: fastapi.FastAPI` instead of just returning the router # because we want to add exception handler which is only supported for `FastAPI`. @@ -154,15 +154,6 @@ def handle_api_validation_error( content={"detail": str(exc)}, ) - @app.exception_handler(errors.SecretStorageTypeMismatchError) - def handle_secret_storage_type_mismatch_error( - request: fastapi.Request, exc: errors.SecretStorageTypeMismatchError - ): - return fastapi.responses.JSONResponse( - status_code=fastapi.status.HTTP_503_SERVICE_UNAVAILABLE, - content={"message": str(exc)}, - ) - @app.exception_handler(NotImplementedError) def handle_not_implemented_error( request: fastapi.Request, exc: NotImplementedError @@ -475,9 +466,7 @@ def get_current_user( ) ### Secrets routes - secrets_service = api_server_sql.SecretsApiService( - secret_value_store=secret_value_store - ) + secrets_service = api_server_sql.SecretsApiService(secret_store=secret_store) router.get("/api/secrets/", tags=["secrets"], **default_config)( inject_session_dependency( diff --git a/cloud_pipelines_backend/api_server_sql.py b/cloud_pipelines_backend/api_server_sql.py index f528a7b2..d43de839 100644 --- a/cloud_pipelines_backend/api_server_sql.py +++ b/cloud_pipelines_backend/api_server_sql.py @@ -14,7 +14,7 @@ from .access_control import checks as access_control_checks from .access_control import runs as access_control_runs from . import filter_query_sql -from . import secret_value_store +from . import secret_store if typing.TYPE_CHECKING: from cloud_pipelines.orchestration.storage_providers import ( @@ -1252,6 +1252,15 @@ def from_db(cls, secret_row: bts.Secret) -> "SecretInfoResponse": description=secret_row.description, ) + @classmethod + def from_store(cls, secret: secret_store.StoredSecret) -> "SecretInfoResponse": + return SecretInfoResponse( + secret_name=secret.secret_name, + created_at=secret.created_at, + updated_at=secret.updated_at, + description=secret.description, + ) + @dataclasses.dataclass(kw_only=True) class ListSecretsResponse: @@ -1259,12 +1268,8 @@ class ListSecretsResponse: class SecretsApiService: - def __init__( - self, - *, - secret_value_store: secret_value_store.SecretValueStore | None = None, - ): - self._secret_value_store = secret_value_store + def __init__(self, *, secret_store: secret_store.SecretStore | None = None): + self._secret_store = secret_store def create_secret( self, @@ -1323,15 +1328,32 @@ def _create_or_update_secret( raise_if_exists: bool = False, ) -> SecretInfoResponse: current_time = _get_current_time() + if self._secret_store: + store_method = ( + self._secret_store.create + if raise_if_exists + else self._secret_store.update + ) + saved = store_method( + session=session, + secret=secret_store.StoredSecret( + user_id=user_id, + secret_name=secret_name, + value=secret_value, + # None keeps the stored description, as an empty one leaves the row's alone below. + description=description or None, + created_at=current_time, + updated_at=current_time, + ), + ) + return SecretInfoResponse.from_store(saved) secret = session.get(bts.Secret, (user_id, secret_name)) if secret: if raise_if_exists: raise errors.ItemAlreadyExistsError( f"Secret with name '{secret_name}' already exists." ) - secret_value_store.raise_if_storage_type_mismatch( - secret=secret, store=self._secret_value_store - ) + secret.secret_value = secret_value secret.updated_at = current_time else: if raise_if_not_exists: @@ -1341,30 +1363,17 @@ def _create_or_update_secret( secret = bts.Secret( user_id=user_id, secret_name=secret_name, - # Set below, through the store when there is one. - secret_value="", + secret_value=secret_value, created_at=current_time, updated_at=current_time, ) session.add(secret) - # The new value is saved before the row commits, and the one it replaces is retired after, - # so the row never names a value that is already gone. - previous_storage_details = secret_value_store.store_secret_value( - secret=secret, value=secret_value, store=self._secret_value_store - ) if description: secret.description = description # if expires_at: # secret.expires_at = expires_at response = SecretInfoResponse.from_db(secret) session.commit() - if previous_storage_details is not None: - secret_value_store.retire_secret_value( - store=self._secret_value_store, - storage_details=previous_storage_details, - user_id=user_id, - secret_name=secret_name, - ) return response def delete_secret( @@ -1374,24 +1383,18 @@ def delete_secret( user_id: str, secret_name: str, ) -> None: + if self._secret_store: + self._secret_store.delete( + session=session, user_id=user_id, secret_name=secret_name + ) + return secret = session.get(bts.Secret, (user_id, secret_name)) if not secret: raise errors.ItemNotFoundError( f"Secret with name '{secret_name}' does not exist." ) - secret_value_store.raise_if_storage_type_mismatch( - secret=secret, store=self._secret_value_store - ) - storage_details = secret_value_store.get_storage_details(secret=secret) session.delete(secret) session.commit() - if storage_details is not None: - secret_value_store.retire_secret_value( - store=self._secret_value_store, - storage_details=storage_details, - user_id=user_id, - secret_name=secret_name, - ) def list_secrets( self, @@ -1399,6 +1402,11 @@ def list_secrets( session: orm.Session, user_id: str, ) -> ListSecretsResponse: + if self._secret_store: + stored = self._secret_store.list_secrets(session=session, user_id=user_id) + return ListSecretsResponse( + secrets=[SecretInfoResponse.from_store(secret) for secret in stored] + ) secrets = session.scalars( sql.select(bts.Secret).where(bts.Secret.user_id == user_id) ).all() diff --git a/cloud_pipelines_backend/errors.py b/cloud_pipelines_backend/errors.py index 9c684fea..dee86527 100644 --- a/cloud_pipelines_backend/errors.py +++ b/cloud_pipelines_backend/errors.py @@ -22,10 +22,6 @@ class PermissionError(Exception): pass -class SecretStorageTypeMismatchError(Exception): - """A secret's value is in a store of another type than this server's, or the server has none, so it cannot be changed.""" - - class ApiValidationError(Exception): """Base for all filter/annotation validation errors -> 422.""" diff --git a/cloud_pipelines_backend/secret_store.py b/cloud_pipelines_backend/secret_store.py new file mode 100644 index 00000000..99206326 --- /dev/null +++ b/cloud_pipelines_backend/secret_store.py @@ -0,0 +1,139 @@ +import abc +import dataclasses +import datetime +import functools +from typing import Any, Callable, Final + +import sqlalchemy as sql +from sqlalchemy import orm + +from . import backend_types_sql as bts +from . import errors + +# The `SecretStore` methods that move the caller's rows out of the `secret` table first. +_STORE_METHODS: Final[tuple[str, ...]] = ( + "create", + "update", + "get", + "list_secrets", + "delete", +) + + +@dataclasses.dataclass(frozen=True, kw_only=True) +class StoredSecret: + """A secret as a `SecretStore` keeps it; `value` is None when not loaded, and never shown by repr, str or logs.""" + + user_id: str + secret_name: str + value: str | None = dataclasses.field(default=None, repr=False) + description: str | None = None + created_at: datetime.datetime + updated_at: datetime.datetime + extra_data: dict[str, Any] | None = None + + +def _moves_rows_first(method: Callable[..., Any]) -> Callable[..., Any]: + """Wraps a store method so that, called with a `session`, it first moves the caller's rows into the store.""" + + @functools.wraps(method) + def wrapper( + self: "SecretStore", *, session: orm.Session | None = None, **kwargs: Any + ) -> Any: + # `move_rows` itself calls `create` without a session, so it never moves twice. + if session is not None: + secret = kwargs.get("secret") + move_rows( + session=session, + secret_store=self, + user_id=secret.user_id if secret else kwargs["user_id"], + secret_name=secret.secret_name if secret else kwargs.get("secret_name"), + ) + return method(self, **kwargs) + + return wrapper + + +class SecretStore(abc.ABC): + """Keeps secrets outside the database, for example in a cloud secret manager. + + With a store, the `secret` table only holds rows not yet moved. Each method below, as a subclass defines + it, is wrapped so that a call with `session=` first moves the caller's rows from the table into the store; + the subclass's own code never sees the session or the table. + """ + + def __init_subclass__(cls, **kwargs: Any) -> None: + super().__init_subclass__(**kwargs) + for name in _STORE_METHODS: + if name in cls.__dict__: + setattr(cls, name, _moves_rows_first(cls.__dict__[name])) + + @abc.abstractmethod + def create(self, *, secret: StoredSecret) -> StoredSecret: + """Saves a new secret and returns it without its value; raises `errors.ItemAlreadyExistsError` if it exists.""" + + @abc.abstractmethod + def update(self, *, secret: StoredSecret) -> StoredSecret: + """Saves a new value for an existing secret and returns it without its value. + + Keeps the stored `created_at` and name, and the stored description when `description` is None. + Raises `errors.ItemNotFoundError` when there is no such secret. + """ + + @abc.abstractmethod + def get(self, *, user_id: str, secret_name: str) -> str | None: + """Returns the secret's latest value, or None when there is no such secret.""" + + @abc.abstractmethod + def list_secrets(self, *, user_id: str) -> list[StoredSecret]: + """Returns the user's secrets, without values.""" + + @abc.abstractmethod + def delete(self, *, user_id: str, secret_name: str) -> None: + """Deletes the secret and all its values; raises `errors.ItemNotFoundError` when there is none.""" + + +def move_rows( + *, + session: orm.Session, + secret_store: SecretStore, + user_id: str, + secret_name: str | None = None, +) -> None: + """Moves the user's secret (all of the user's secrets when `secret_name` is None) from the `secret` table + into the store, committing after each. + + Each row is deleted in the same commit, after the store has saved it, so a failure leaves the row to move + next time. The store gets the row's own name, which the database may have matched ignoring case. + """ + if secret_name is None: + names = session.scalars( + sql.select(bts.Secret.secret_name).filter_by(user_id=user_id) + ).all() + else: + names = [secret_name] + for name in names: + secret = session.scalars( + sql.select(bts.Secret) + .filter_by(user_id=user_id, secret_name=name) + .with_for_update() + ).one_or_none() + if secret is None: + continue + try: + secret_store.create( + secret=StoredSecret( + user_id=secret.user_id, + secret_name=secret.secret_name, + value=secret.secret_value, + description=secret.description, + created_at=secret.created_at, + updated_at=secret.updated_at, + extra_data=secret.extra_data, + ) + ) + except errors.ItemAlreadyExistsError: + # An earlier move saved it but failed before deleting the row; the store's copy is the same or newer. + pass + session.delete(secret) + session.commit() diff --git a/cloud_pipelines_backend/secret_value_store.py b/cloud_pipelines_backend/secret_value_store.py deleted file mode 100644 index 254ef451..00000000 --- a/cloud_pipelines_backend/secret_value_store.py +++ /dev/null @@ -1,126 +0,0 @@ -import logging -import typing -from typing import Any, Final - -from . import backend_types_sql as bts -from . import errors - -_logger = logging.getLogger(__name__) - -EXTRA_DATA_STORAGE_KEY: Final[str] = "storage" -STORAGE_TYPE_KEY: Final[str] = "type" -STORAGE_DETAILS_KEY: Final[str] = "details" - - -class SecretValueStore(typing.Protocol): - """Keeps secret values outside the database row, for example in a cloud secret manager.""" - - # Recorded in the row's `extra_data`, so later reads and writes know which store holds the value. - type: str - - def save( - self, *, user_id: str, secret_name: str, value: str - ) -> tuple[str, dict[str, Any]]: - """Saves the value; returns what the row's `secret_value` keeps, and the storage details for `get` and `delete`. - - Example, a store that keeps numbered versions: `save(user_id="alice", secret_name="HF_TOKEN", - value="hf_FAKE123")` returns `("", {"version": "4"})` — the row keeps `""`, and `{"version": "4"}` - is what `get` and `delete` are later given as `storage_details`. - """ - ... - - def get( - self, *, storage_details: dict[str, Any], user_id: str, secret_name: str - ) -> str: - """Returns the value that `save` saved with these storage details.""" - ... - - def delete( - self, *, storage_details: dict[str, Any], user_id: str, secret_name: str - ) -> None: - """Retires the value that `save` saved with these storage details.""" - ... - - -def get_storage_type(*, secret: bts.Secret) -> str | None: - """Returns the type of the store holding the row's value, or None when the value is plain in the row. - - Example: with `extra_data = {"storage": {"type": "gsm", "details": {"version": "4"}}}` it returns `"gsm"`. - """ - storage = (secret.extra_data or {}).get(EXTRA_DATA_STORAGE_KEY) - return storage[STORAGE_TYPE_KEY] if storage else None - - -def get_storage_details(*, secret: bts.Secret) -> dict[str, Any] | None: - """Returns what the store needs to find the row's value, or None when the value is plain in the row. - - Example: with `extra_data = {"storage": {"type": "gsm", "details": {"version": "4"}}}` it returns - `{"version": "4"}`. - """ - storage = (secret.extra_data or {}).get(EXTRA_DATA_STORAGE_KEY) - return storage[STORAGE_DETAILS_KEY] if storage else None - - -def store_secret_value( - *, secret: bts.Secret, value: str, store: SecretValueStore | None -) -> dict[str, Any] | None: - """Sets the row's value, through the store if one is given. - - Returns the storage details of the value it replaces in the store, or None when there was none. - """ - if store is None: - secret.secret_value = value - return None - previous_storage_details = get_storage_details(secret=secret) - secret.secret_value, storage_details = store.save( - user_id=secret.user_id, secret_name=secret.secret_name, value=value - ) - # A merge: the copy keeps every other key, so only the storage entry changes. - secret.extra_data = { - **(secret.extra_data or {}), - EXTRA_DATA_STORAGE_KEY: { - STORAGE_TYPE_KEY: store.type, - STORAGE_DETAILS_KEY: storage_details, - }, - } - return previous_storage_details - - -def raise_if_storage_type_mismatch( - *, secret: bts.Secret, store: SecretValueStore | None -) -> None: - """Raises when the row's value is in a store other than this server's, leaving the row unchanged. - - Writing plain text over it, or deleting the row, would leave the stored value behind and still readable. - """ - storage_type = get_storage_type(secret=secret) - if storage_type is None: - return - configured = store.type if store else None - if storage_type != configured: - raise errors.SecretStorageTypeMismatchError( - f"Secret '{secret.secret_name}' is kept in the {storage_type!r} store, " - f"but this server's secret store is {configured!r}; it cannot be changed or deleted here." - ) - - -def retire_secret_value( - *, - store: SecretValueStore, - storage_details: dict[str, Any], - user_id: str, - secret_name: str, -) -> None: - """Retires a stored value the row no longer names, logging a failure instead of raising it. - - A helper for after the row commits: the change is already saved, so raising would report a successful save - as a failure, and the only harm of a failed retire is an old value left enabled in the store. - """ - try: - store.delete( - storage_details=storage_details, user_id=user_id, secret_name=secret_name - ) - except Exception: - _logger.exception( - f"Could not retire the previous stored value of secret '{secret_name}'." - ) diff --git a/tests/test_secret_store.py b/tests/test_secret_store.py new file mode 100644 index 00000000..468f572c --- /dev/null +++ b/tests/test_secret_store.py @@ -0,0 +1,348 @@ +import dataclasses +import pathlib + +import fastapi +import pytest +import sqlalchemy +from sqlalchemy import orm +from starlette.testclient import TestClient + +from cloud_pipelines_backend import api_router +from cloud_pipelines_backend import api_server_sql +from cloud_pipelines_backend import backend_types_sql as bts +from cloud_pipelines_backend import database_ops +from cloud_pipelines_backend import errors +from cloud_pipelines_backend import secret_store + + +class _FakeStore(secret_store.SecretStore): + """An in-memory `SecretStore` that follows the base class's contract, keyed by user and name.""" + + def __init__(self, *, fail_create: bool = False) -> None: + self.secrets: dict[tuple[str, str], secret_store.StoredSecret] = {} + self._fail_create = fail_create + + def create(self, *, secret: secret_store.StoredSecret) -> secret_store.StoredSecret: + if self._fail_create: + raise RuntimeError("store unavailable") + key = (secret.user_id, secret.secret_name) + if key in self.secrets: + raise errors.ItemAlreadyExistsError("exists") + self.secrets[key] = secret + return dataclasses.replace(secret, value=None) + + def update(self, *, secret: secret_store.StoredSecret) -> secret_store.StoredSecret: + key = (secret.user_id, secret.secret_name) + old = self.secrets.get(key) + if old is None: + raise errors.ItemNotFoundError("missing") + self.secrets[key] = dataclasses.replace( + secret, + secret_name=old.secret_name, + created_at=old.created_at, + description=( + old.description if secret.description is None else secret.description + ), + ) + return dataclasses.replace(self.secrets[key], value=None) + + def get(self, *, user_id: str, secret_name: str) -> str | None: + secret = self.secrets.get((user_id, secret_name)) + return secret.value if secret else None + + def list_secrets(self, *, user_id: str) -> list[secret_store.StoredSecret]: + return [ + dataclasses.replace(secret, value=None) + for (owner, _), secret in self.secrets.items() + if owner == user_id + ] + + def delete(self, *, user_id: str, secret_name: str) -> None: + if self.secrets.pop((user_id, secret_name), None) is None: + raise errors.ItemNotFoundError("missing") + + +def _engine(*, tmp_path: pathlib.Path) -> sqlalchemy.Engine: + """Returns a file database, so a second session sees only committed rows.""" + return database_ops.create_db_engine_and_migrate_db( + database_uri=f"sqlite:///{tmp_path / 'db.sqlite'}" + ) + + +def _service(*, store: _FakeStore | None) -> api_server_sql.SecretsApiService: + return api_server_sql.SecretsApiService(secret_store=store) + + +def _create( + *, + engine: sqlalchemy.Engine, + store: _FakeStore | None, + user_id: str = "alice", + secret_name: str = "TOKEN", + value: str = "v1", + description: str | None = None, +) -> api_server_sql.SecretInfoResponse: + with orm.Session(bind=engine) as session: + return _service(store=store).create_secret( + session=session, + user_id=user_id, + secret_name=secret_name, + secret_value=value, + description=description, + ) + + +def _update( + *, + engine: sqlalchemy.Engine, + store: _FakeStore | None, + value: str, + description: str | None = None, +) -> api_server_sql.SecretInfoResponse: + with orm.Session(bind=engine) as session: + return _service(store=store).update_secret( + session=session, + user_id="alice", + secret_name="TOKEN", + secret_value=value, + description=description, + ) + + +def _rows(*, engine: sqlalchemy.Engine) -> list[bts.Secret]: + """Reads every committed row.""" + with orm.Session(bind=engine, expire_on_commit=False) as session: + return list(session.scalars(sqlalchemy.select(bts.Secret)).all()) + + +def test_without_store_the_table_works_as_before(tmp_path: pathlib.Path) -> None: + engine = _engine(tmp_path=tmp_path) + _create(engine=engine, store=None, value="v1") + _update(engine=engine, store=None, value="v2") + [row] = _rows(engine=engine) + assert row.secret_value == "v2" + with orm.Session(bind=engine) as session: + _service(store=None).delete_secret( + session=session, user_id="alice", secret_name="TOKEN" + ) + assert _rows(engine=engine) == [] + + +def test_create_saves_in_the_store_and_writes_no_row(tmp_path: pathlib.Path) -> None: + engine = _engine(tmp_path=tmp_path) + store = _FakeStore() + response = _create(engine=engine, store=store, value="v1", description="d") + assert store.get(user_id="alice", secret_name="TOKEN") == "v1" + assert response.secret_name == "TOKEN" + assert response.description == "d" + assert _rows(engine=engine) == [] + + +def test_create_of_a_name_still_in_the_table_moves_it_and_conflicts( + tmp_path: pathlib.Path, +) -> None: + engine = _engine(tmp_path=tmp_path) + _create(engine=engine, store=None, value="old") + store = _FakeStore() + with pytest.raises(errors.ItemAlreadyExistsError): + _create(engine=engine, store=store, value="new") + assert store.get(user_id="alice", secret_name="TOKEN") == "old" + assert _rows(engine=engine) == [] + + +def test_update_moves_the_row_then_saves_the_new_value(tmp_path: pathlib.Path) -> None: + engine = _engine(tmp_path=tmp_path) + _create(engine=engine, store=None, value="old", description="from the row") + [row] = _rows(engine=engine) + store = _FakeStore() + response = _update(engine=engine, store=store, value="new") + assert store.get(user_id="alice", secret_name="TOKEN") == "new" + assert response.created_at == row.created_at + assert response.updated_at > row.updated_at + assert response.description == "from the row" + assert _rows(engine=engine) == [] + + +def test_update_with_an_empty_description_keeps_the_stored_one( + tmp_path: pathlib.Path, +) -> None: + engine = _engine(tmp_path=tmp_path) + store = _FakeStore() + _create(engine=engine, store=store, description="keep me") + response = _update(engine=engine, store=store, value="v2", description="") + assert response.description == "keep me" + + +def test_update_of_a_missing_secret_is_not_found(tmp_path: pathlib.Path) -> None: + with pytest.raises(errors.ItemNotFoundError): + _update(engine=_engine(tmp_path=tmp_path), store=_FakeStore(), value="v") + + +def test_delete_removes_a_secret_still_in_the_table(tmp_path: pathlib.Path) -> None: + engine = _engine(tmp_path=tmp_path) + _create(engine=engine, store=None) + store = _FakeStore() + with orm.Session(bind=engine) as session: + _service(store=store).delete_secret( + session=session, user_id="alice", secret_name="TOKEN" + ) + assert store.secrets == {} + assert _rows(engine=engine) == [] + with orm.Session(bind=engine) as session, pytest.raises(errors.ItemNotFoundError): + _service(store=store).delete_secret( + session=session, user_id="alice", secret_name="TOKEN" + ) + + +def test_list_moves_only_the_callers_rows(tmp_path: pathlib.Path) -> None: + engine = _engine(tmp_path=tmp_path) + _create(engine=engine, store=None, secret_name="A") + _create(engine=engine, store=None, secret_name="B") + _create(engine=engine, store=None, user_id="bob", secret_name="C") + store = _FakeStore() + _create(engine=engine, store=store, secret_name="D") + with orm.Session(bind=engine) as session: + listed = _service(store=store).list_secrets(session=session, user_id="alice") + assert sorted(secret.secret_name for secret in listed.secrets) == ["A", "B", "D"] + assert [(row.user_id, row.secret_name) for row in _rows(engine=engine)] == [ + ("bob", "C") + ] + + +def test_move_rows_carries_every_column(tmp_path: pathlib.Path) -> None: + engine = _engine(tmp_path=tmp_path) + _create(engine=engine, store=None, value="v1", description="d") + with orm.Session(bind=engine) as session: + session.get(bts.Secret, ("alice", "TOKEN")).extra_data = {"k": "v"} + session.commit() + [row] = _rows(engine=engine) + store = _FakeStore() + with orm.Session(bind=engine) as session: + secret_store.move_rows( + session=session, secret_store=store, user_id="alice", secret_name="TOKEN" + ) + assert store.secrets[("alice", "TOKEN")] == secret_store.StoredSecret( + user_id="alice", + secret_name="TOKEN", + value="v1", + description="d", + created_at=row.created_at, + updated_at=row.updated_at, + extra_data={"k": "v"}, + ) + assert _rows(engine=engine) == [] + + +def test_move_rows_keeps_the_row_when_the_store_fails(tmp_path: pathlib.Path) -> None: + engine = _engine(tmp_path=tmp_path) + _create(engine=engine, store=None, value="v1") + with orm.Session(bind=engine) as session, pytest.raises(RuntimeError): + secret_store.move_rows( + session=session, + secret_store=_FakeStore(fail_create=True), + user_id="alice", + secret_name="TOKEN", + ) + [row] = _rows(engine=engine) + assert row.secret_value == "v1" + + +def test_stored_secret_never_shows_its_value() -> None: + secret = secret_store.StoredSecret( + user_id="alice", + secret_name="TOKEN", + value="hunter2", + created_at=api_server_sql._get_current_time(), + updated_at=api_server_sql._get_current_time(), + ) + assert "hunter2" not in repr(secret) + assert "hunter2" not in f"{secret}" + + +def test_every_store_method_a_subclass_defines_moves_rows_first() -> None: + for name in ("create", "update", "get", "list_secrets", "delete"): + assert ( + getattr(_FakeStore, name).__wrapped__.__qualname__ == f"_FakeStore.{name}" + ) + + class _Incomplete(secret_store.SecretStore): + def create( + self, *, secret: secret_store.StoredSecret + ) -> secret_store.StoredSecret: + return secret + + with pytest.raises(TypeError, match="abstract"): + _Incomplete() + + +def test_a_call_without_a_session_leaves_the_table_alone( + tmp_path: pathlib.Path, +) -> None: + engine = _engine(tmp_path=tmp_path) + _create(engine=engine, store=None, value="v1") + store = _FakeStore() + assert store.get(user_id="alice", secret_name="TOKEN") is None + assert len(_rows(engine=engine)) == 1 + + +def test_get_with_a_session_moves_the_row_first(tmp_path: pathlib.Path) -> None: + engine = _engine(tmp_path=tmp_path) + _create(engine=engine, store=None, value="v1") + store = _FakeStore() + with orm.Session(bind=engine) as session: + value = store.get(session=session, user_id="alice", secret_name="TOKEN") + assert value == "v1" + assert _rows(engine=engine) == [] + + +def test_a_move_that_failed_before_deleting_the_row_finishes_next_time( + tmp_path: pathlib.Path, +) -> None: + engine = _engine(tmp_path=tmp_path) + _create(engine=engine, store=None, value="old") + [row] = _rows(engine=engine) + store = _FakeStore() + store.create( + secret=secret_store.StoredSecret( + user_id="alice", + secret_name="TOKEN", + value="old", + created_at=row.created_at, + updated_at=row.updated_at, + ) + ) + _update(engine=engine, store=store, value="new") + assert store.get(user_id="alice", secret_name="TOKEN") == "new" + assert _rows(engine=engine) == [] + + +def _client(*, engine: sqlalchemy.Engine, store: _FakeStore | None) -> TestClient: + """Returns an API client for alice, with the given store passed to setup_routes.""" + app = fastapi.FastAPI() + api_router.setup_routes( + app=app, + db_engine=engine, + user_details_getter=lambda: api_router.UserDetails( + name="alice", permissions={"read": True, "write": True, "admin": False} + ), + secret_store=store, + ) + return TestClient(app) + + +def test_setup_routes_passes_the_store_to_secret_routes(tmp_path: pathlib.Path) -> None: + engine = _engine(tmp_path=tmp_path) + store = _FakeStore() + client = _client(engine=engine, store=store) + created = client.post( + "/api/secrets/", params={"secret_name": "TOKEN"}, json={"secret_value": "v1"} + ) + assert created.status_code == 200, created.text + again = client.post( + "/api/secrets/", params={"secret_name": "TOKEN"}, json={"secret_value": "v2"} + ) + assert again.status_code == 409, again.text + listed = client.get("/api/secrets/") + assert [s["secret_name"] for s in listed.json()["secrets"]] == ["TOKEN"] + assert store.get(user_id="alice", secret_name="TOKEN") == "v1" + assert _rows(engine=engine) == [] diff --git a/tests/test_secret_value_store.py b/tests/test_secret_value_store.py deleted file mode 100644 index 9ff1c6f8..00000000 --- a/tests/test_secret_value_store.py +++ /dev/null @@ -1,256 +0,0 @@ -import logging -import pathlib -from typing import Any - -import fastapi -import pytest -import sqlalchemy -from sqlalchemy import orm -from starlette.testclient import TestClient - -from cloud_pipelines_backend import api_router -from cloud_pipelines_backend import api_server_sql -from cloud_pipelines_backend import backend_types_sql as bts -from cloud_pipelines_backend import database_ops -from cloud_pipelines_backend import errors - - -class _FakeStore: - """Keeps values in memory, one numbered version per `save`, like a cloud secret manager.""" - - def __init__( - self, - *, - type: str = "fake", - engine: sqlalchemy.Engine | None = None, - fail_delete: bool = False, - ) -> None: - self.type = type - self.versions: dict[str, str] = {} - self.retired: list[str] = [] - # With an engine, `delete` records which version the committed row names at that moment. - self._engine = engine - self.row_version_at_delete: list[str | None] = [] - self._fail_delete = fail_delete - - def save( - self, *, user_id: str, secret_name: str, value: str - ) -> tuple[str, dict[str, Any]]: - """Saves the value as the next version.""" - version = str(len(self.versions) + 1) - self.versions[version] = value - return "", {"version": version} - - def get( - self, *, storage_details: dict[str, Any], user_id: str, secret_name: str - ) -> str: - """Returns the saved version.""" - return self.versions[storage_details["version"]] - - def delete( - self, *, storage_details: dict[str, Any], user_id: str, secret_name: str - ) -> None: - """Retires the version, or fails when told to.""" - if self._engine is not None: - with orm.Session(bind=self._engine) as session: - row = session.get(bts.Secret, (user_id, secret_name)) - self.row_version_at_delete.append( - row.extra_data["storage"]["details"]["version"] if row else None - ) - if self._fail_delete: - raise RuntimeError("store unavailable") - self.retired.append(storage_details["version"]) - - -def _engine(*, tmp_path: pathlib.Path) -> sqlalchemy.Engine: - """Returns a file database, so a second session sees only committed rows.""" - return database_ops.create_db_engine_and_migrate_db( - database_uri=f"sqlite:///{tmp_path / 'db.sqlite'}" - ) - - -def _save( - *, - engine: sqlalchemy.Engine, - store: _FakeStore | None, - value: str, - update: bool = False, -) -> None: - """Creates or updates alice's TOKEN through the service.""" - service = api_server_sql.SecretsApiService(secret_value_store=store) - method = service.update_secret if update else service.create_secret - with orm.Session(bind=engine) as session: - method( - session=session, user_id="alice", secret_name="TOKEN", secret_value=value - ) - - -def _row(*, engine: sqlalchemy.Engine) -> bts.Secret | None: - """Reads alice's TOKEN as committed.""" - with orm.Session(bind=engine, expire_on_commit=False) as session: - return session.get(bts.Secret, ("alice", "TOKEN")) - - -def test_without_store_keeps_the_value_in_the_row(tmp_path: pathlib.Path): - engine = _engine(tmp_path=tmp_path) - _save(engine=engine, store=None, value="s3cret") - row = _row(engine=engine) - assert row.secret_value == "s3cret" - assert row.extra_data is None - - -def test_without_store_update_and_delete_work_as_before(tmp_path: pathlib.Path): - engine = _engine(tmp_path=tmp_path) - _save(engine=engine, store=None, value="s3cret") - _save(engine=engine, store=None, value="n3w", update=True) - row = _row(engine=engine) - assert (row.secret_value, row.extra_data) == ("n3w", None) - with orm.Session(bind=engine) as session: - api_server_sql.SecretsApiService().delete_secret( - session=session, user_id="alice", secret_name="TOKEN" - ) - assert _row(engine=engine) is None - - -def test_with_store_the_row_keeps_only_what_save_returned(tmp_path: pathlib.Path): - engine = _engine(tmp_path=tmp_path) - store = _FakeStore() - _save(engine=engine, store=store, value="s3cret") - row = _row(engine=engine) - assert row.secret_value == "" - assert row.extra_data == {"storage": {"type": "fake", "details": {"version": "1"}}} - assert store.versions == {"1": "s3cret"} - - -def test_update_retires_the_previous_version_after_the_row_commits( - tmp_path: pathlib.Path, -): - engine = _engine(tmp_path=tmp_path) - store = _FakeStore(engine=engine) - _save(engine=engine, store=store, value="v1") - _save(engine=engine, store=store, value="v2", update=True) - assert _row(engine=engine).extra_data["storage"]["details"] == {"version": "2"} - assert store.retired == ["1"] - # When version 1 was retired, the committed row already named version 2. - assert store.row_version_at_delete == ["2"] - - -def test_delete_retires_the_version_after_the_row_is_gone(tmp_path: pathlib.Path): - engine = _engine(tmp_path=tmp_path) - store = _FakeStore(engine=engine) - _save(engine=engine, store=store, value="v1") - with orm.Session(bind=engine) as session: - api_server_sql.SecretsApiService(secret_value_store=store).delete_secret( - session=session, user_id="alice", secret_name="TOKEN" - ) - assert _row(engine=engine) is None - assert store.retired == ["1"] - assert store.row_version_at_delete == [None] - - -def test_a_plain_row_moves_into_the_store_on_update(tmp_path: pathlib.Path): - engine = _engine(tmp_path=tmp_path) - store = _FakeStore() - _save(engine=engine, store=None, value="plain") - _save(engine=engine, store=store, value="v2", update=True) - row = _row(engine=engine) - assert row.secret_value == "" - assert row.extra_data["storage"]["details"] == {"version": "1"} - assert store.retired == [] - - -def test_other_extra_data_keys_survive(tmp_path: pathlib.Path): - engine = _engine(tmp_path=tmp_path) - _save(engine=engine, store=None, value="plain") - with orm.Session(bind=engine) as session: - session.get(bts.Secret, ("alice", "TOKEN")).extra_data = {"owner_team": "data"} - session.commit() - _save(engine=engine, store=_FakeStore(), value="v2", update=True) - extra_data = _row(engine=engine).extra_data - assert extra_data["owner_team"] == "data" - assert "storage" in extra_data - - -@pytest.mark.parametrize("server_store", [None, _FakeStore(type="other")]) -def test_save_and_delete_refused_when_the_row_is_in_another_store( - tmp_path: pathlib.Path, server_store: _FakeStore | None -): - engine = _engine(tmp_path=tmp_path) - _save(engine=engine, store=_FakeStore(), value="v1") - before = _row(engine=engine) - with pytest.raises(errors.SecretStorageTypeMismatchError, match="'fake' store"): - _save(engine=engine, store=server_store, value="plain", update=True) - with orm.Session(bind=engine) as session: - with pytest.raises(errors.SecretStorageTypeMismatchError): - api_server_sql.SecretsApiService( - secret_value_store=server_store - ).delete_secret(session=session, user_id="alice", secret_name="TOKEN") - after = _row(engine=engine) - assert (after.secret_value, after.extra_data) == ( - before.secret_value, - before.extra_data, - ) - if server_store is not None: - assert server_store.versions == {} - assert server_store.retired == [] - - -def test_create_of_an_existing_name_saves_nothing_in_the_store( - tmp_path: pathlib.Path, -): - engine = _engine(tmp_path=tmp_path) - store = _FakeStore() - _save(engine=engine, store=store, value="v1") - with pytest.raises(errors.ItemAlreadyExistsError): - _save(engine=engine, store=store, value="v2") - assert store.versions == {"1": "v1"} - - -def test_a_failed_retire_is_logged_and_the_update_stands( - tmp_path: pathlib.Path, caplog: pytest.LogCaptureFixture -): - engine = _engine(tmp_path=tmp_path) - store = _FakeStore(fail_delete=True) - _save(engine=engine, store=store, value="v1") - with caplog.at_level(logging.ERROR): - _save(engine=engine, store=store, value="v2", update=True) - assert _row(engine=engine).extra_data["storage"]["details"] == {"version": "2"} - assert "Could not retire the previous stored value of secret 'TOKEN'" in caplog.text - assert "v1" not in caplog.text and "v2" not in caplog.text - - -def _client(*, engine: sqlalchemy.Engine, store: _FakeStore | None) -> TestClient: - """Returns an API client for alice, with the given store passed to setup_routes.""" - app = fastapi.FastAPI() - api_router.setup_routes( - app=app, - db_engine=engine, - user_details_getter=lambda: api_router.UserDetails( - name="alice", permissions={"read": True, "write": True, "admin": False} - ), - secret_value_store=store, - ) - return TestClient(app) - - -def test_setup_routes_passes_the_store_to_secret_routes(tmp_path: pathlib.Path): - engine = _engine(tmp_path=tmp_path) - store = _FakeStore() - response = _client(engine=engine, store=store).post( - "/api/secrets/", - params={"secret_name": "TOKEN"}, - json={"secret_value": "s3cret"}, - ) - assert response.status_code == 200, response.text - assert _row(engine=engine).secret_value == "" - assert store.versions == {"1": "s3cret"} - - -def test_a_refusal_is_a_503_naming_both_stores(tmp_path: pathlib.Path): - engine = _engine(tmp_path=tmp_path) - _save(engine=engine, store=_FakeStore(), value="v1") - response = _client(engine=engine, store=None).delete("/api/secrets/TOKEN") - assert response.status_code == 503, response.text - assert "'fake' store" in response.json()["message"] - assert "None" in response.json()["message"] - assert _row(engine=engine) is not None From dbeb0eb232b08a1cdd1c8abc8018b687af1e7a97 Mon Sep 17 00:00:00 2001 From: Yue Chao Qin Date: Wed, 18 Nov 2026 20:15:17 -0800 Subject: [PATCH 3/5] feat: GSM Secrets - get only reads; it never moves table rows Launches read the table first, then the store, without writing; get no longer takes a session. --- cloud_pipelines_backend/secret_store.py | 10 +++++----- tests/test_secret_store.py | 13 ++++++------- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/cloud_pipelines_backend/secret_store.py b/cloud_pipelines_backend/secret_store.py index 99206326..6752a2bf 100644 --- a/cloud_pipelines_backend/secret_store.py +++ b/cloud_pipelines_backend/secret_store.py @@ -10,11 +10,10 @@ from . import backend_types_sql as bts from . import errors -# The `SecretStore` methods that move the caller's rows out of the `secret` table first. +# The `SecretStore` methods that move the caller's rows out of the `secret` table first; `get` only reads. _STORE_METHODS: Final[tuple[str, ...]] = ( "create", "update", - "get", "list_secrets", "delete", ) @@ -57,9 +56,10 @@ def wrapper( class SecretStore(abc.ABC): """Keeps secrets outside the database, for example in a cloud secret manager. - With a store, the `secret` table only holds rows not yet moved. Each method below, as a subclass defines - it, is wrapped so that a call with `session=` first moves the caller's rows from the table into the store; - the subclass's own code never sees the session or the table. + With a store, the `secret` table only holds rows not yet moved. Each method below except `get`, as a + subclass defines it, is wrapped so that a call with `session=` first moves the caller's rows from the + table into the store; the subclass's own code never sees the session or the table. `get` never moves, + so reading a secret writes nothing. """ def __init_subclass__(cls, **kwargs: Any) -> None: diff --git a/tests/test_secret_store.py b/tests/test_secret_store.py index 468f572c..98bb44d6 100644 --- a/tests/test_secret_store.py +++ b/tests/test_secret_store.py @@ -260,10 +260,11 @@ def test_stored_secret_never_shows_its_value() -> None: def test_every_store_method_a_subclass_defines_moves_rows_first() -> None: - for name in ("create", "update", "get", "list_secrets", "delete"): + for name in ("create", "update", "list_secrets", "delete"): assert ( getattr(_FakeStore, name).__wrapped__.__qualname__ == f"_FakeStore.{name}" ) + assert not hasattr(_FakeStore.get, "__wrapped__") class _Incomplete(secret_store.SecretStore): def create( @@ -285,14 +286,12 @@ def test_a_call_without_a_session_leaves_the_table_alone( assert len(_rows(engine=engine)) == 1 -def test_get_with_a_session_moves_the_row_first(tmp_path: pathlib.Path) -> None: +def test_get_never_moves_and_refuses_a_session(tmp_path: pathlib.Path) -> None: engine = _engine(tmp_path=tmp_path) _create(engine=engine, store=None, value="v1") - store = _FakeStore() - with orm.Session(bind=engine) as session: - value = store.get(session=session, user_id="alice", secret_name="TOKEN") - assert value == "v1" - assert _rows(engine=engine) == [] + with orm.Session(bind=engine) as session, pytest.raises(TypeError): + _FakeStore().get(session=session, user_id="alice", secret_name="TOKEN") + assert len(_rows(engine=engine)) == 1 def test_a_move_that_failed_before_deleting_the_row_finishes_next_time( From ca09d2b3c26a208fca04059e5d5edff657a8b73c Mon Sep 17 00:00:00 2001 From: Yue Chao Qin Date: Wed, 18 Nov 2026 22:17:38 -0800 Subject: [PATCH 4/5] feat: GSM Secrets - Routes keep rows where they are; new secrets go to the store SecretStore is a Protocol again; no move on use. PUT/DELETE lock the row only with a store; DELETE also removes a store copy; list reads the store. --- cloud_pipelines_backend/api_server_sql.py | 31 +++- cloud_pipelines_backend/secret_store.py | 108 +---------- tests/test_secret_store.py | 212 +++++++++------------- 3 files changed, 114 insertions(+), 237 deletions(-) diff --git a/cloud_pipelines_backend/api_server_sql.py b/cloud_pipelines_backend/api_server_sql.py index d43de839..b87609de 100644 --- a/cloud_pipelines_backend/api_server_sql.py +++ b/cloud_pipelines_backend/api_server_sql.py @@ -1328,14 +1328,20 @@ def _create_or_update_secret( raise_if_exists: bool = False, ) -> SecretInfoResponse: current_time = _get_current_time() - if self._secret_store: + # With a store, lock the row so that a backfill moving it waits, or has already finished. + secret = session.get( + bts.Secret, + (user_id, secret_name), + with_for_update=self._secret_store is not None, + ) + # Not in the table (new, or already moved): the store keeps it. A row is updated where it is. + if not secret and self._secret_store: store_method = ( self._secret_store.create if raise_if_exists else self._secret_store.update ) saved = store_method( - session=session, secret=secret_store.StoredSecret( user_id=user_id, secret_name=secret_name, @@ -1347,7 +1353,6 @@ def _create_or_update_secret( ), ) return SecretInfoResponse.from_store(saved) - secret = session.get(bts.Secret, (user_id, secret_name)) if secret: if raise_if_exists: raise errors.ItemAlreadyExistsError( @@ -1383,12 +1388,20 @@ def delete_secret( user_id: str, secret_name: str, ) -> None: + secret = session.get( + bts.Secret, + (user_id, secret_name), + with_for_update=self._secret_store is not None, + ) if self._secret_store: - self._secret_store.delete( - session=session, user_id=user_id, secret_name=secret_name - ) - return - secret = session.get(bts.Secret, (user_id, secret_name)) + # Also deletes a copy left by a backfill that stopped before deleting the row. + try: + self._secret_store.delete(user_id=user_id, secret_name=secret_name) + except errors.ItemNotFoundError: + if not secret: + raise + if not secret: + return if not secret: raise errors.ItemNotFoundError( f"Secret with name '{secret_name}' does not exist." @@ -1403,7 +1416,7 @@ def list_secrets( user_id: str, ) -> ListSecretsResponse: if self._secret_store: - stored = self._secret_store.list_secrets(session=session, user_id=user_id) + stored = self._secret_store.list_secrets(user_id=user_id) return ListSecretsResponse( secrets=[SecretInfoResponse.from_store(secret) for secret in stored] ) diff --git a/cloud_pipelines_backend/secret_store.py b/cloud_pipelines_backend/secret_store.py index 6752a2bf..8f3edc79 100644 --- a/cloud_pipelines_backend/secret_store.py +++ b/cloud_pipelines_backend/secret_store.py @@ -1,22 +1,6 @@ -import abc import dataclasses import datetime -import functools -from typing import Any, Callable, Final - -import sqlalchemy as sql -from sqlalchemy import orm - -from . import backend_types_sql as bts -from . import errors - -# The `SecretStore` methods that move the caller's rows out of the `secret` table first; `get` only reads. -_STORE_METHODS: Final[tuple[str, ...]] = ( - "create", - "update", - "list_secrets", - "delete", -) +from typing import Any, Protocol @dataclasses.dataclass(frozen=True, kw_only=True) @@ -32,108 +16,32 @@ class StoredSecret: extra_data: dict[str, Any] | None = None -def _moves_rows_first(method: Callable[..., Any]) -> Callable[..., Any]: - """Wraps a store method so that, called with a `session`, it first moves the caller's rows into the store.""" - - @functools.wraps(method) - def wrapper( - self: "SecretStore", *, session: orm.Session | None = None, **kwargs: Any - ) -> Any: - # `move_rows` itself calls `create` without a session, so it never moves twice. - if session is not None: - secret = kwargs.get("secret") - move_rows( - session=session, - secret_store=self, - user_id=secret.user_id if secret else kwargs["user_id"], - secret_name=secret.secret_name if secret else kwargs.get("secret_name"), - ) - return method(self, **kwargs) - - return wrapper - - -class SecretStore(abc.ABC): +class SecretStore(Protocol): """Keeps secrets outside the database, for example in a cloud secret manager. - With a store, the `secret` table only holds rows not yet moved. Each method below except `get`, as a - subclass defines it, is wrapped so that a call with `session=` first moves the caller's rows from the - table into the store; the subclass's own code never sees the session or the table. `get` never moves, - so reading a secret writes nothing. + With a store, new secrets go to the store; a secret still in the `secret` table stays there until moved. """ - def __init_subclass__(cls, **kwargs: Any) -> None: - super().__init_subclass__(**kwargs) - for name in _STORE_METHODS: - if name in cls.__dict__: - setattr(cls, name, _moves_rows_first(cls.__dict__[name])) - - @abc.abstractmethod def create(self, *, secret: StoredSecret) -> StoredSecret: """Saves a new secret and returns it without its value; raises `errors.ItemAlreadyExistsError` if it exists.""" + ... - @abc.abstractmethod def update(self, *, secret: StoredSecret) -> StoredSecret: """Saves a new value for an existing secret and returns it without its value. Keeps the stored `created_at` and name, and the stored description when `description` is None. Raises `errors.ItemNotFoundError` when there is no such secret. """ + ... - @abc.abstractmethod def get(self, *, user_id: str, secret_name: str) -> str | None: """Returns the secret's latest value, or None when there is no such secret.""" + ... - @abc.abstractmethod def list_secrets(self, *, user_id: str) -> list[StoredSecret]: """Returns the user's secrets, without values.""" + ... - @abc.abstractmethod def delete(self, *, user_id: str, secret_name: str) -> None: """Deletes the secret and all its values; raises `errors.ItemNotFoundError` when there is none.""" - - -def move_rows( - *, - session: orm.Session, - secret_store: SecretStore, - user_id: str, - secret_name: str | None = None, -) -> None: - """Moves the user's secret (all of the user's secrets when `secret_name` is None) from the `secret` table - into the store, committing after each. - - Each row is deleted in the same commit, after the store has saved it, so a failure leaves the row to move - next time. The store gets the row's own name, which the database may have matched ignoring case. - """ - if secret_name is None: - names = session.scalars( - sql.select(bts.Secret.secret_name).filter_by(user_id=user_id) - ).all() - else: - names = [secret_name] - for name in names: - secret = session.scalars( - sql.select(bts.Secret) - .filter_by(user_id=user_id, secret_name=name) - .with_for_update() - ).one_or_none() - if secret is None: - continue - try: - secret_store.create( - secret=StoredSecret( - user_id=secret.user_id, - secret_name=secret.secret_name, - value=secret.secret_value, - description=secret.description, - created_at=secret.created_at, - updated_at=secret.updated_at, - extra_data=secret.extra_data, - ) - ) - except errors.ItemAlreadyExistsError: - # An earlier move saved it but failed before deleting the row; the store's copy is the same or newer. - pass - session.delete(secret) - session.commit() + ... diff --git a/tests/test_secret_store.py b/tests/test_secret_store.py index 98bb44d6..71e57de5 100644 --- a/tests/test_secret_store.py +++ b/tests/test_secret_store.py @@ -16,15 +16,12 @@ class _FakeStore(secret_store.SecretStore): - """An in-memory `SecretStore` that follows the base class's contract, keyed by user and name.""" + """An in-memory `SecretStore` that follows its contract, keyed by user and name.""" - def __init__(self, *, fail_create: bool = False) -> None: + def __init__(self) -> None: self.secrets: dict[tuple[str, str], secret_store.StoredSecret] = {} - self._fail_create = fail_create def create(self, *, secret: secret_store.StoredSecret) -> secret_store.StoredSecret: - if self._fail_create: - raise RuntimeError("store unavailable") key = (secret.user_id, secret.secret_name) if key in self.secrets: raise errors.ItemAlreadyExistsError("exists") @@ -138,28 +135,53 @@ def test_create_saves_in_the_store_and_writes_no_row(tmp_path: pathlib.Path) -> assert _rows(engine=engine) == [] -def test_create_of_a_name_still_in_the_table_moves_it_and_conflicts( - tmp_path: pathlib.Path, -) -> None: +def _stored(*, value: str) -> secret_store.StoredSecret: + now = api_server_sql._get_current_time() + return secret_store.StoredSecret( + user_id="alice", + secret_name="TOKEN", + value=value, + created_at=now, + updated_at=now, + ) + + +def _delete(*, engine: sqlalchemy.Engine, store: _FakeStore | None) -> None: + with orm.Session(bind=engine) as session: + _service(store=store).delete_secret( + session=session, user_id="alice", secret_name="TOKEN" + ) + + +def test_create_of_a_name_still_in_the_table_conflicts(tmp_path: pathlib.Path) -> None: engine = _engine(tmp_path=tmp_path) _create(engine=engine, store=None, value="old") store = _FakeStore() with pytest.raises(errors.ItemAlreadyExistsError): _create(engine=engine, store=store, value="new") - assert store.get(user_id="alice", secret_name="TOKEN") == "old" - assert _rows(engine=engine) == [] + assert store.secrets == {} + assert [row.secret_value for row in _rows(engine=engine)] == ["old"] -def test_update_moves_the_row_then_saves_the_new_value(tmp_path: pathlib.Path) -> None: +def test_update_of_a_row_updates_it_where_it_is(tmp_path: pathlib.Path) -> None: + engine = _engine(tmp_path=tmp_path) + _create(engine=engine, store=None, value="old") + store = _FakeStore() + _update(engine=engine, store=store, value="new") + assert store.secrets == {} + assert [row.secret_value for row in _rows(engine=engine)] == ["new"] + + +def test_update_of_a_secret_in_the_store_updates_the_store( + tmp_path: pathlib.Path, +) -> None: engine = _engine(tmp_path=tmp_path) - _create(engine=engine, store=None, value="old", description="from the row") - [row] = _rows(engine=engine) store = _FakeStore() + created = _create(engine=engine, store=store, value="old", description="d") response = _update(engine=engine, store=store, value="new") assert store.get(user_id="alice", secret_name="TOKEN") == "new" - assert response.created_at == row.created_at - assert response.updated_at > row.updated_at - assert response.description == "from the row" + assert response.created_at == created.created_at + assert response.description == "d" assert _rows(engine=engine) == [] @@ -178,143 +200,77 @@ def test_update_of_a_missing_secret_is_not_found(tmp_path: pathlib.Path) -> None _update(engine=_engine(tmp_path=tmp_path), store=_FakeStore(), value="v") -def test_delete_removes_a_secret_still_in_the_table(tmp_path: pathlib.Path) -> None: +def test_delete_of_a_row_deletes_the_row(tmp_path: pathlib.Path) -> None: + engine = _engine(tmp_path=tmp_path) + _create(engine=engine, store=None) + _delete(engine=engine, store=_FakeStore()) + assert _rows(engine=engine) == [] + + +def test_delete_of_a_row_also_deletes_a_copy_left_in_the_store( + tmp_path: pathlib.Path, +) -> None: engine = _engine(tmp_path=tmp_path) _create(engine=engine, store=None) store = _FakeStore() - with orm.Session(bind=engine) as session: - _service(store=store).delete_secret( - session=session, user_id="alice", secret_name="TOKEN" - ) + store.create(secret=_stored(value="copy")) + _delete(engine=engine, store=store) assert store.secrets == {} assert _rows(engine=engine) == [] - with orm.Session(bind=engine) as session, pytest.raises(errors.ItemNotFoundError): - _service(store=store).delete_secret( - session=session, user_id="alice", secret_name="TOKEN" - ) -def test_list_moves_only_the_callers_rows(tmp_path: pathlib.Path) -> None: +def test_delete_of_a_secret_in_the_store(tmp_path: pathlib.Path) -> None: engine = _engine(tmp_path=tmp_path) - _create(engine=engine, store=None, secret_name="A") - _create(engine=engine, store=None, secret_name="B") - _create(engine=engine, store=None, user_id="bob", secret_name="C") store = _FakeStore() - _create(engine=engine, store=store, secret_name="D") - with orm.Session(bind=engine) as session: - listed = _service(store=store).list_secrets(session=session, user_id="alice") - assert sorted(secret.secret_name for secret in listed.secrets) == ["A", "B", "D"] - assert [(row.user_id, row.secret_name) for row in _rows(engine=engine)] == [ - ("bob", "C") - ] + _create(engine=engine, store=store) + _delete(engine=engine, store=store) + assert store.secrets == {} + with pytest.raises(errors.ItemNotFoundError): + _delete(engine=engine, store=store) -def test_move_rows_carries_every_column(tmp_path: pathlib.Path) -> None: +def test_list_reads_only_the_store(tmp_path: pathlib.Path) -> None: engine = _engine(tmp_path=tmp_path) - _create(engine=engine, store=None, value="v1", description="d") - with orm.Session(bind=engine) as session: - session.get(bts.Secret, ("alice", "TOKEN")).extra_data = {"k": "v"} - session.commit() - [row] = _rows(engine=engine) + _create(engine=engine, store=None, secret_name="IN_TABLE") store = _FakeStore() + _create(engine=engine, store=store, secret_name="IN_STORE") + _create(engine=engine, store=store, user_id="bob", secret_name="BOBS") with orm.Session(bind=engine) as session: - secret_store.move_rows( - session=session, secret_store=store, user_id="alice", secret_name="TOKEN" - ) - assert store.secrets[("alice", "TOKEN")] == secret_store.StoredSecret( - user_id="alice", - secret_name="TOKEN", - value="v1", - description="d", - created_at=row.created_at, - updated_at=row.updated_at, - extra_data={"k": "v"}, - ) - assert _rows(engine=engine) == [] + listed = _service(store=store).list_secrets(session=session, user_id="alice") + assert [secret.secret_name for secret in listed.secrets] == ["IN_STORE"] + assert [row.secret_name for row in _rows(engine=engine)] == ["IN_TABLE"] -def test_move_rows_keeps_the_row_when_the_store_fails(tmp_path: pathlib.Path) -> None: +@pytest.mark.parametrize("with_store", [True, False]) +def test_the_row_is_locked_only_with_a_store( + tmp_path: pathlib.Path, with_store: bool +) -> None: + """A backfill moving the row then waits; without a store the read stays as before.""" engine = _engine(tmp_path=tmp_path) - _create(engine=engine, store=None, value="v1") - with orm.Session(bind=engine) as session, pytest.raises(RuntimeError): - secret_store.move_rows( - session=session, - secret_store=_FakeStore(fail_create=True), - user_id="alice", - secret_name="TOKEN", + _create(engine=engine, store=None) + store = _FakeStore() if with_store else None + locks = [] + with orm.Session(bind=engine) as session: + sqlalchemy.event.listen( + session, + "do_orm_execute", + lambda state: locks.append(state.statement._for_update_arg is not None), ) - [row] = _rows(engine=engine) - assert row.secret_value == "v1" + service = _service(store=store) + service.update_secret( + session=session, user_id="alice", secret_name="TOKEN", secret_value="v2" + ) + session.expunge_all() + service.delete_secret(session=session, user_id="alice", secret_name="TOKEN") + assert locks[:1] + locks[-1:] == [with_store, with_store] def test_stored_secret_never_shows_its_value() -> None: - secret = secret_store.StoredSecret( - user_id="alice", - secret_name="TOKEN", - value="hunter2", - created_at=api_server_sql._get_current_time(), - updated_at=api_server_sql._get_current_time(), - ) + secret = _stored(value="hunter2") assert "hunter2" not in repr(secret) assert "hunter2" not in f"{secret}" -def test_every_store_method_a_subclass_defines_moves_rows_first() -> None: - for name in ("create", "update", "list_secrets", "delete"): - assert ( - getattr(_FakeStore, name).__wrapped__.__qualname__ == f"_FakeStore.{name}" - ) - assert not hasattr(_FakeStore.get, "__wrapped__") - - class _Incomplete(secret_store.SecretStore): - def create( - self, *, secret: secret_store.StoredSecret - ) -> secret_store.StoredSecret: - return secret - - with pytest.raises(TypeError, match="abstract"): - _Incomplete() - - -def test_a_call_without_a_session_leaves_the_table_alone( - tmp_path: pathlib.Path, -) -> None: - engine = _engine(tmp_path=tmp_path) - _create(engine=engine, store=None, value="v1") - store = _FakeStore() - assert store.get(user_id="alice", secret_name="TOKEN") is None - assert len(_rows(engine=engine)) == 1 - - -def test_get_never_moves_and_refuses_a_session(tmp_path: pathlib.Path) -> None: - engine = _engine(tmp_path=tmp_path) - _create(engine=engine, store=None, value="v1") - with orm.Session(bind=engine) as session, pytest.raises(TypeError): - _FakeStore().get(session=session, user_id="alice", secret_name="TOKEN") - assert len(_rows(engine=engine)) == 1 - - -def test_a_move_that_failed_before_deleting_the_row_finishes_next_time( - tmp_path: pathlib.Path, -) -> None: - engine = _engine(tmp_path=tmp_path) - _create(engine=engine, store=None, value="old") - [row] = _rows(engine=engine) - store = _FakeStore() - store.create( - secret=secret_store.StoredSecret( - user_id="alice", - secret_name="TOKEN", - value="old", - created_at=row.created_at, - updated_at=row.updated_at, - ) - ) - _update(engine=engine, store=store, value="new") - assert store.get(user_id="alice", secret_name="TOKEN") == "new" - assert _rows(engine=engine) == [] - - def _client(*, engine: sqlalchemy.Engine, store: _FakeStore | None) -> TestClient: """Returns an API client for alice, with the given store passed to setup_routes.""" app = fastapi.FastAPI() From 6cfaccd1ac5972a80f563e7200be7e92086f1fe7 Mon Sep 17 00:00:00 2001 From: Yue Chao Qin Date: Thu, 19 Nov 2026 13:55:56 -0800 Subject: [PATCH 5/5] refactor: GSM Secrets - Rename secret_store to secret_interfaces, list_secrets to list, store parameters to _secret_store The module name clashed with the parameter (secret_store: secret_store.SecretStore). The store's list returns builtins.list, because a plain list there means the method itself on Python 3.14. The Protocol methods drop their redundant ellipses; a docstring is a full body. --- cloud_pipelines_backend/api_router.py | 10 ++++---- cloud_pipelines_backend/api_server_sql.py | 12 ++++----- .../{secret_store.py => secret_interfaces.py} | 8 ++---- tests/test_secret_store.py | 25 +++++++++++-------- 4 files changed, 28 insertions(+), 27 deletions(-) rename cloud_pipelines_backend/{secret_store.py => secret_interfaces.py} (92%) diff --git a/cloud_pipelines_backend/api_router.py b/cloud_pipelines_backend/api_router.py index ab77087c..e2288a4e 100644 --- a/cloud_pipelines_backend/api_router.py +++ b/cloud_pipelines_backend/api_router.py @@ -18,7 +18,7 @@ from .instrumentation import contextual_logging if typing.TYPE_CHECKING: - from . import secret_store + from . import secret_interfaces from .launchers import interfaces as launcher_interfaces @@ -41,7 +41,7 @@ def setup_routes( container_launcher_for_log_streaming: "launcher_interfaces.ContainerTaskLauncher[launcher_interfaces.LaunchedContainer] | None" = None, default_component_library_owner_username: str = "admin", do_skip_backfill: bool = False, - secret_store: "secret_store.SecretStore | None" = None, + _secret_store: "secret_interfaces.SecretStore | None" = None, ): def get_session(): with orm.Session(autocommit=False, autoflush=False, bind=db_engine) as session: @@ -78,7 +78,7 @@ async def lifespan(app: fastapi.FastAPI): user_details_getter=user_details_getter, get_launcher=get_launcher, lifespan=lifespan, - secret_store=secret_store, + _secret_store=_secret_store, ) @@ -96,7 +96,7 @@ def _setup_routes_internal( | typing.Callable[..., abc.Iterator[typing.Any]] | None ) = None, - secret_store: "secret_store.SecretStore | None" = None, + _secret_store: "secret_interfaces.SecretStore | None" = None, ): # We request `app: fastapi.FastAPI` instead of just returning the router # because we want to add exception handler which is only supported for `FastAPI`. @@ -466,7 +466,7 @@ def get_current_user( ) ### Secrets routes - secrets_service = api_server_sql.SecretsApiService(secret_store=secret_store) + secrets_service = api_server_sql.SecretsApiService(_secret_store=_secret_store) router.get("/api/secrets/", tags=["secrets"], **default_config)( inject_session_dependency( diff --git a/cloud_pipelines_backend/api_server_sql.py b/cloud_pipelines_backend/api_server_sql.py index b87609de..77c70e12 100644 --- a/cloud_pipelines_backend/api_server_sql.py +++ b/cloud_pipelines_backend/api_server_sql.py @@ -14,7 +14,7 @@ from .access_control import checks as access_control_checks from .access_control import runs as access_control_runs from . import filter_query_sql -from . import secret_store +from . import secret_interfaces if typing.TYPE_CHECKING: from cloud_pipelines.orchestration.storage_providers import ( @@ -1253,7 +1253,7 @@ def from_db(cls, secret_row: bts.Secret) -> "SecretInfoResponse": ) @classmethod - def from_store(cls, secret: secret_store.StoredSecret) -> "SecretInfoResponse": + def from_store(cls, secret: secret_interfaces.StoredSecret) -> "SecretInfoResponse": return SecretInfoResponse( secret_name=secret.secret_name, created_at=secret.created_at, @@ -1268,8 +1268,8 @@ class ListSecretsResponse: class SecretsApiService: - def __init__(self, *, secret_store: secret_store.SecretStore | None = None): - self._secret_store = secret_store + def __init__(self, *, _secret_store: secret_interfaces.SecretStore | None = None): + self._secret_store = _secret_store def create_secret( self, @@ -1342,7 +1342,7 @@ def _create_or_update_secret( else self._secret_store.update ) saved = store_method( - secret=secret_store.StoredSecret( + secret=secret_interfaces.StoredSecret( user_id=user_id, secret_name=secret_name, value=secret_value, @@ -1416,7 +1416,7 @@ def list_secrets( user_id: str, ) -> ListSecretsResponse: if self._secret_store: - stored = self._secret_store.list_secrets(user_id=user_id) + stored = self._secret_store.list(user_id=user_id) return ListSecretsResponse( secrets=[SecretInfoResponse.from_store(secret) for secret in stored] ) diff --git a/cloud_pipelines_backend/secret_store.py b/cloud_pipelines_backend/secret_interfaces.py similarity index 92% rename from cloud_pipelines_backend/secret_store.py rename to cloud_pipelines_backend/secret_interfaces.py index 8f3edc79..e8339a9d 100644 --- a/cloud_pipelines_backend/secret_store.py +++ b/cloud_pipelines_backend/secret_interfaces.py @@ -1,3 +1,4 @@ +import builtins import dataclasses import datetime from typing import Any, Protocol @@ -24,7 +25,6 @@ class SecretStore(Protocol): def create(self, *, secret: StoredSecret) -> StoredSecret: """Saves a new secret and returns it without its value; raises `errors.ItemAlreadyExistsError` if it exists.""" - ... def update(self, *, secret: StoredSecret) -> StoredSecret: """Saves a new value for an existing secret and returns it without its value. @@ -32,16 +32,12 @@ def update(self, *, secret: StoredSecret) -> StoredSecret: Keeps the stored `created_at` and name, and the stored description when `description` is None. Raises `errors.ItemNotFoundError` when there is no such secret. """ - ... def get(self, *, user_id: str, secret_name: str) -> str | None: """Returns the secret's latest value, or None when there is no such secret.""" - ... - def list_secrets(self, *, user_id: str) -> list[StoredSecret]: + def list(self, *, user_id: str) -> builtins.list[StoredSecret]: """Returns the user's secrets, without values.""" - ... def delete(self, *, user_id: str, secret_name: str) -> None: """Deletes the secret and all its values; raises `errors.ItemNotFoundError` when there is none.""" - ... diff --git a/tests/test_secret_store.py b/tests/test_secret_store.py index 71e57de5..5fa19416 100644 --- a/tests/test_secret_store.py +++ b/tests/test_secret_store.py @@ -1,3 +1,4 @@ +import builtins import dataclasses import pathlib @@ -12,23 +13,27 @@ from cloud_pipelines_backend import backend_types_sql as bts from cloud_pipelines_backend import database_ops from cloud_pipelines_backend import errors -from cloud_pipelines_backend import secret_store +from cloud_pipelines_backend import secret_interfaces -class _FakeStore(secret_store.SecretStore): +class _FakeStore(secret_interfaces.SecretStore): """An in-memory `SecretStore` that follows its contract, keyed by user and name.""" def __init__(self) -> None: - self.secrets: dict[tuple[str, str], secret_store.StoredSecret] = {} + self.secrets: dict[tuple[str, str], secret_interfaces.StoredSecret] = {} - def create(self, *, secret: secret_store.StoredSecret) -> secret_store.StoredSecret: + def create( + self, *, secret: secret_interfaces.StoredSecret + ) -> secret_interfaces.StoredSecret: key = (secret.user_id, secret.secret_name) if key in self.secrets: raise errors.ItemAlreadyExistsError("exists") self.secrets[key] = secret return dataclasses.replace(secret, value=None) - def update(self, *, secret: secret_store.StoredSecret) -> secret_store.StoredSecret: + def update( + self, *, secret: secret_interfaces.StoredSecret + ) -> secret_interfaces.StoredSecret: key = (secret.user_id, secret.secret_name) old = self.secrets.get(key) if old is None: @@ -47,7 +52,7 @@ def get(self, *, user_id: str, secret_name: str) -> str | None: secret = self.secrets.get((user_id, secret_name)) return secret.value if secret else None - def list_secrets(self, *, user_id: str) -> list[secret_store.StoredSecret]: + def list(self, *, user_id: str) -> builtins.list[secret_interfaces.StoredSecret]: return [ dataclasses.replace(secret, value=None) for (owner, _), secret in self.secrets.items() @@ -67,7 +72,7 @@ def _engine(*, tmp_path: pathlib.Path) -> sqlalchemy.Engine: def _service(*, store: _FakeStore | None) -> api_server_sql.SecretsApiService: - return api_server_sql.SecretsApiService(secret_store=store) + return api_server_sql.SecretsApiService(_secret_store=store) def _create( @@ -135,9 +140,9 @@ def test_create_saves_in_the_store_and_writes_no_row(tmp_path: pathlib.Path) -> assert _rows(engine=engine) == [] -def _stored(*, value: str) -> secret_store.StoredSecret: +def _stored(*, value: str) -> secret_interfaces.StoredSecret: now = api_server_sql._get_current_time() - return secret_store.StoredSecret( + return secret_interfaces.StoredSecret( user_id="alice", secret_name="TOKEN", value=value, @@ -280,7 +285,7 @@ def _client(*, engine: sqlalchemy.Engine, store: _FakeStore | None) -> TestClien user_details_getter=lambda: api_router.UserDetails( name="alice", permissions={"read": True, "write": True, "admin": False} ), - secret_store=store, + _secret_store=store, ) return TestClient(app)