From eecea995f2aab9201fab957deeb323588d04ef86 Mon Sep 17 00:00:00 2001 From: Yue Chao Qin Date: Tue, 17 Nov 2026 21:37:19 -0800 Subject: [PATCH 1/3] feat: GSM Secrets - Backfill API to move plain-text secrets into the store --- cloud_pipelines_backend/api_router.py | 16 ++ cloud_pipelines_backend/api_server_sql.py | 55 +++++++ tests/test_secret_value_store.py | 185 +++++++++++++++++++++- 3 files changed, 253 insertions(+), 3 deletions(-) diff --git a/cloud_pipelines_backend/api_router.py b/cloud_pipelines_backend/api_router.py index 5614afb..22c24ef 100644 --- a/cloud_pipelines_backend/api_router.py +++ b/cloud_pipelines_backend/api_router.py @@ -634,6 +634,22 @@ def admin_set_execution_node_status( execution_node = session.get(backend_types_sql.ExecutionNode, id) execution_node.container_execution_status = status + @router.post( + "/api/admin/secrets/backfill_storage", + tags=["admin"], + dependencies=[ensure_admin_user_dependency], + **default_config, + ) + def admin_backfill_secret_storage( + session: typing.Annotated[orm.Session, fastapi.Depends(get_session)], + limit: typing.Annotated[int, fastapi.Query(ge=1, le=1000)] = 50, + user_id: str | None = None, + secret_name: str | None = None, + ) -> dict[str, int]: + return secrets_service.backfill_secret_storage( + session=session, limit=limit, user_id=user_id, secret_name=secret_name + ) + @router.get("/api/admin/sql_engine_connection_pool_status") async def get_sql_engine_connection_pool_status( session: typing.Annotated[orm.Session, fastapi.Depends(get_session)], diff --git a/cloud_pipelines_backend/api_server_sql.py b/cloud_pipelines_backend/api_server_sql.py index c48aba5..48ac65e 100644 --- a/cloud_pipelines_backend/api_server_sql.py +++ b/cloud_pipelines_backend/api_server_sql.py @@ -1364,6 +1364,61 @@ def _create_or_update_secret( ) return response + def backfill_secret_storage( + self, + *, + session: orm.Session, + limit: int, + user_id: str | None = None, + secret_name: str | None = None, + ) -> dict[str, int]: + """Moves up to `limit` plain-text secrets into the store, committing each row, and reports how many are left. + + `user_id` and `secret_name`, when given, move only matching rows; both together pick one secret. + """ + if not self._secret_value_store: + raise errors.ApiValidationError("No secret value store is configured.") + is_plaintext = sql.func.json_extract( + bts.Secret.extra_data, f"$.{secret_value_store.EXTRA_DATA_STORAGE_KEY}" + ).is_(None) + secrets_for_backfill = session.execute( + sql.select(bts.Secret.user_id, bts.Secret.secret_name) + .where(is_plaintext) + .filter_by( + **({"user_id": user_id} if user_id is not None else {}), + **({"secret_name": secret_name} if secret_name is not None else {}), + ) + .order_by(bts.Secret.user_id, bts.Secret.secret_name) + .limit(limit) + ).all() + moved_to_store = 0 + for user_id, secret_name in secrets_for_backfill: + # Re-read under a row lock: an update or another backfill may have moved or deleted it since. + secret = session.scalars( + sql.select(bts.Secret) + .where( + bts.Secret.user_id == user_id, + bts.Secret.secret_name == secret_name, + is_plaintext, + ) + .with_for_update() + ).one_or_none() + if secret is None: + continue + secret_value_store.store_secret_value( + secret=secret, value=secret.secret_value, store=self._secret_value_store + ) + # Each value is already saved in the store, so its row commits before the next is touched. + session.commit() + moved_to_store += 1 + count = sql.select(sql.func.count()).select_from(bts.Secret) + return { + "moved_to_store": moved_to_store, + "already_stored": session.scalar(count.where(~is_plaintext)) + - moved_to_store, + "still_plaintext": session.scalar(count.where(is_plaintext)), + } + def delete_secret( self, *, diff --git a/tests/test_secret_value_store.py b/tests/test_secret_value_store.py index 2c3503d..8ba2638 100644 --- a/tests/test_secret_value_store.py +++ b/tests/test_secret_value_store.py @@ -29,6 +29,7 @@ def __init__( engine: sqlalchemy.Engine | None = None, fail_delete: bool = False, fail_get: bool = False, + fail_save_value: str | None = None, ) -> None: self.type = type self.versions: dict[str, str] = {} @@ -38,11 +39,14 @@ def __init__( self.row_version_at_delete: list[str | None] = [] self._fail_delete = fail_delete self._fail_get = fail_get + self._fail_save_value = fail_save_value def save( self, *, user_id: str, secret_name: str, value: str ) -> tuple[str, dict[str, Any]]: - """Saves the value as the next version.""" + """Saves the value as the next version, or fails for `fail_save_value`.""" + if value == self._fail_save_value: + raise RuntimeError("store unavailable") version = str(len(self.versions) + 1) self.versions[version] = value return "", {"version": version} @@ -227,14 +231,16 @@ def test_a_failed_retire_is_logged_and_the_update_stands( assert "v1" not in caplog.text and "v2" not in caplog.text -def _client(*, engine: sqlalchemy.Engine, store: _FakeStore | None) -> TestClient: +def _client( + *, engine: sqlalchemy.Engine, store: _FakeStore | None, admin: bool = False +) -> TestClient: """Returns an API client for alice, with the given store passed to setup_routes.""" app = fastapi.FastAPI() api_router.setup_routes( app=app, db_engine=engine, user_details_getter=lambda: api_router.UserDetails( - name="alice", permissions={"read": True, "write": True, "admin": False} + name="alice", permissions={"read": True, "write": True, "admin": admin} ), secret_value_store=store, ) @@ -373,3 +379,176 @@ def test_launch_fails_when_the_store_cannot_read(tmp_path: pathlib.Path): ) launch.assert_not_called() assert "store unavailable" in _system_error(execution=execution) + + +def _plain_rows( + *, engine: sqlalchemy.Engine, names: list[str], user_id: str = "alice" +) -> None: + """Creates the owner's secrets as plain-text rows, each valued as its name in lower case.""" + service = api_server_sql.SecretsApiService() + with orm.Session(bind=engine) as session: + for name in names: + service.create_secret( + session=session, + user_id=user_id, + secret_name=name, + secret_value=name.lower(), + ) + + +def _backfill( + *, + engine: sqlalchemy.Engine, + store: _FakeStore | None, + limit: int = 50, + **filters: str, +) -> dict[str, int]: + """Runs one backfill call through the service, narrowed by any `user_id` or `secret_name` given.""" + service = api_server_sql.SecretsApiService(secret_value_store=store) + with orm.Session(bind=engine) as session: + return service.backfill_secret_storage(session=session, limit=limit, **filters) + + +def _rows(*, engine: sqlalchemy.Engine) -> dict[str, tuple[str, Any]]: + """Reads every committed row as name -> (secret_value, extra_data).""" + with orm.Session(bind=engine) as session: + return { + row.secret_name: (row.secret_value, row.extra_data) + for row in session.scalars(sqlalchemy.select(bts.Secret)) + } + + +def _stored(version: str, type: str = "fake") -> tuple[str, Any]: + """Returns how a row looks once its value is in the store.""" + return "", {"storage": {"type": type, "details": {"version": version}}} + + +def test_backfill_moves_plain_rows_into_the_store(tmp_path: pathlib.Path): + engine = _engine(tmp_path=tmp_path) + _plain_rows(engine=engine, names=["A", "B"]) + store = _FakeStore() + result = _backfill(engine=engine, store=store) + assert result == {"moved_to_store": 2, "already_stored": 0, "still_plaintext": 0} + assert _rows(engine=engine) == {"A": _stored("1"), "B": _stored("2")} + assert store.versions == {"1": "a", "2": "b"} + + +def test_backfill_rerun_skips_moved_rows(tmp_path: pathlib.Path): + engine = _engine(tmp_path=tmp_path) + _plain_rows(engine=engine, names=["A", "B"]) + store = _FakeStore() + _backfill(engine=engine, store=store) + result = _backfill(engine=engine, store=store) + assert result == {"moved_to_store": 0, "already_stored": 2, "still_plaintext": 0} + assert store.versions == {"1": "a", "2": "b"} + + +def test_backfill_stops_at_the_limit(tmp_path: pathlib.Path): + engine = _engine(tmp_path=tmp_path) + _plain_rows(engine=engine, names=["A", "B", "C"]) + store = _FakeStore() + first = _backfill(engine=engine, store=store, limit=2) + assert first == {"moved_to_store": 2, "already_stored": 0, "still_plaintext": 1} + second = _backfill(engine=engine, store=store, limit=2) + assert second == {"moved_to_store": 1, "already_stored": 2, "still_plaintext": 0} + + +def test_a_failed_save_stops_the_backfill_and_keeps_earlier_rows( + tmp_path: pathlib.Path, +): + engine = _engine(tmp_path=tmp_path) + _plain_rows(engine=engine, names=["A", "B", "C"]) + with pytest.raises(RuntimeError, match="store unavailable"): + _backfill(engine=engine, store=_FakeStore(fail_save_value="b")) + assert _rows(engine=engine) == { + "A": _stored("1"), + "B": ("b", None), + "C": ("c", None), + } + + +def test_backfill_leaves_rows_in_another_store_alone(tmp_path: pathlib.Path): + engine = _engine(tmp_path=tmp_path) + _save(engine=engine, store=_FakeStore(type="other"), value="s3cret") + _plain_rows(engine=engine, names=["A"]) + result = _backfill(engine=engine, store=_FakeStore()) + assert result == {"moved_to_store": 1, "already_stored": 1, "still_plaintext": 0} + assert _rows(engine=engine) == { + "A": _stored("1"), + "TOKEN": _stored("1", type="other"), + } + + +def test_backfill_without_a_store_is_refused(tmp_path: pathlib.Path): + engine = _engine(tmp_path=tmp_path) + _plain_rows(engine=engine, names=["A"]) + with pytest.raises(errors.ApiValidationError, match="No secret value store"): + _backfill(engine=engine, store=None) + assert _rows(engine=engine) == {"A": ("a", None)} + + +@pytest.mark.parametrize("admin", [False, True], ids=["user", "admin"]) +def test_the_backfill_route_is_for_admins_only(tmp_path: pathlib.Path, admin: bool): + engine = _engine(tmp_path=tmp_path) + _plain_rows(engine=engine, names=["A"]) + response = _client(engine=engine, store=_FakeStore(), admin=admin).post( + "/api/admin/secrets/backfill_storage", params={"limit": 10} + ) + if admin: + assert response.status_code == 200, response.text + assert response.json() == { + "moved_to_store": 1, + "already_stored": 0, + "still_plaintext": 0, + } + assert _rows(engine=engine) == {"A": _stored("1")} + else: + assert response.status_code == 403, response.text + assert _rows(engine=engine) == {"A": ("a", None)} + + +def _stored_keys(*, engine: sqlalchemy.Engine) -> set[tuple[str, str]]: + """Returns (user_id, secret_name) of every committed row whose value is in a store.""" + with orm.Session(bind=engine) as session: + return { + (row.user_id, row.secret_name) + for row in session.scalars(sqlalchemy.select(bts.Secret)) + if row.extra_data + } + + +@pytest.mark.parametrize( + ("filters", "moved"), + [ + ({"user_id": "alice", "secret_name": "A"}, {("alice", "A")}), + ({"user_id": "alice"}, {("alice", "A"), ("alice", "B")}), + ({"secret_name": "A"}, {("alice", "A"), ("bob", "A")}), + ], + ids=["one-secret", "one-owner", "one-name"], +) +def test_backfill_moves_only_the_matching_rows( + tmp_path: pathlib.Path, filters: dict[str, str], moved: set[tuple[str, str]] +): + engine = _engine(tmp_path=tmp_path) + _plain_rows(engine=engine, names=["A", "B"]) + _plain_rows(engine=engine, names=["A"], user_id="bob") + result = _backfill(engine=engine, store=_FakeStore(), **filters) + assert result["moved_to_store"] == len(moved) + assert _stored_keys(engine=engine) == moved + + +def test_the_backfill_route_passes_the_filters(tmp_path: pathlib.Path): + engine = _engine(tmp_path=tmp_path) + _plain_rows(engine=engine, names=["A", "B"]) + _plain_rows(engine=engine, names=["A"], user_id="bob") + response = _client(engine=engine, store=_FakeStore(), admin=True).post( + "/api/admin/secrets/backfill_storage", + params={"user_id": "alice", "secret_name": "A"}, + ) + assert response.status_code == 200, response.text + assert response.json() == { + "moved_to_store": 1, + "already_stored": 0, + "still_plaintext": 2, + } + assert _stored_keys(engine=engine) == {("alice", "A")} From 5733e465f3f92606fe275985022c1c666fa3684b Mon Sep 17 00:00:00 2001 From: Yue Chao Qin Date: Wed, 18 Nov 2026 20:46:41 -0800 Subject: [PATCH 2/3] feat: GSM Secrets - Backfill moves table rows into the store; skips invalid rows, stops on errors Returns moved, skipped, left_in_table and stopped_on; skipped rows do not count toward limit. --- cloud_pipelines_backend/api_router.py | 2 +- cloud_pipelines_backend/api_server_sql.py | 73 ++++++++-------- tests/test_secret_store.py | 101 +++++++++++++++++++++- 3 files changed, 138 insertions(+), 38 deletions(-) diff --git a/cloud_pipelines_backend/api_router.py b/cloud_pipelines_backend/api_router.py index f1473a5..1c4333b 100644 --- a/cloud_pipelines_backend/api_router.py +++ b/cloud_pipelines_backend/api_router.py @@ -634,7 +634,7 @@ def admin_backfill_secret_storage( limit: typing.Annotated[int, fastapi.Query(ge=1, le=1000)] = 50, user_id: str | None = None, secret_name: str | None = None, - ) -> dict[str, int]: + ) -> dict[str, typing.Any]: return secrets_service.backfill_secret_storage( session=session, limit=limit, user_id=user_id, secret_name=secret_name ) diff --git a/cloud_pipelines_backend/api_server_sql.py b/cloud_pipelines_backend/api_server_sql.py index f40161b..b4ae8a3 100644 --- a/cloud_pipelines_backend/api_server_sql.py +++ b/cloud_pipelines_backend/api_server_sql.py @@ -1383,52 +1383,57 @@ def backfill_secret_storage( limit: int, user_id: str | None = None, secret_name: str | None = None, - ) -> dict[str, int]: - """Moves up to `limit` plain-text secrets into the store, committing each row, and reports how many are left. + ) -> dict[str, typing.Any]: + """Moves up to `limit` rows from the `secret` table into the store, committing each, and reports what is left. - `user_id` and `secret_name`, when given, move only matching rows; both together pick one secret. + `user_id` and `secret_name`, when given, move only matching rows. A row the store refuses as invalid (for + example, a value too large) is skipped and listed by name; any other error stops the run, which can then + be repeated. """ - if not self._secret_value_store: - raise errors.ApiValidationError("No secret value store is configured.") - is_plaintext = sql.func.json_extract( - bts.Secret.extra_data, f"$.{secret_value_store.EXTRA_DATA_STORAGE_KEY}" - ).is_(None) - secrets_for_backfill = session.execute( + if not self._secret_store: + raise errors.ApiValidationError("No secret store is configured.") + rows = session.execute( sql.select(bts.Secret.user_id, bts.Secret.secret_name) - .where(is_plaintext) .filter_by( **({"user_id": user_id} if user_id is not None else {}), **({"secret_name": secret_name} if secret_name is not None else {}), ) .order_by(bts.Secret.user_id, bts.Secret.secret_name) - .limit(limit) ).all() - moved_to_store = 0 - for user_id, secret_name in secrets_for_backfill: - # Re-read under a row lock: an update or another backfill may have moved or deleted it since. - secret = session.scalars( - sql.select(bts.Secret) - .where( - bts.Secret.user_id == user_id, - bts.Secret.secret_name == secret_name, - is_plaintext, + moved = 0 + skipped = [] + stopped_on = None + for row_user_id, row_secret_name in rows: + if moved == limit: + break + try: + secret_store.move_rows( + session=session, + secret_store=self._secret_store, + user_id=row_user_id, + secret_name=row_secret_name, ) - .with_for_update() - ).one_or_none() - if secret is None: + except errors.ApiValidationError: + # Releases the row lock `move_rows` took, so the skipped row is usable during the run. + session.rollback() + skipped.append({"user_id": row_user_id, "secret_name": row_secret_name}) continue - secret_value_store.store_secret_value( - secret=secret, value=secret.secret_value, store=self._secret_value_store - ) - # Each value is already saved in the store, so its row commits before the next is touched. - session.commit() - moved_to_store += 1 - count = sql.select(sql.func.count()).select_from(bts.Secret) + except Exception as e: + session.rollback() + stopped_on = { + "user_id": row_user_id, + "secret_name": row_secret_name, + "error": f"{type(e).__name__}: {e}", + } + break + moved += 1 return { - "moved_to_store": moved_to_store, - "already_stored": session.scalar(count.where(~is_plaintext)) - - moved_to_store, - "still_plaintext": session.scalar(count.where(is_plaintext)), + "moved": moved, + "skipped": skipped, + "left_in_table": session.scalar( + sql.select(sql.func.count()).select_from(bts.Secret) + ), + "stopped_on": stopped_on, } def delete_secret( diff --git a/tests/test_secret_store.py b/tests/test_secret_store.py index a43c299..7b26dad 100644 --- a/tests/test_secret_store.py +++ b/tests/test_secret_store.py @@ -22,14 +22,23 @@ class _FakeStore(secret_store.SecretStore): """An in-memory `SecretStore` that follows the base class's contract, keyed by user and name.""" - def __init__(self, *, fail_create: bool = False, fail_get: bool = False) -> None: + def __init__( + self, + *, + fail_create: bool = False, + fail_get: bool = False, + invalid_names: frozenset[str] = frozenset(), + ) -> None: self.secrets: dict[tuple[str, str], secret_store.StoredSecret] = {} self._fail_create = fail_create self._fail_get = fail_get + self._invalid_names = invalid_names def create(self, *, secret: secret_store.StoredSecret) -> secret_store.StoredSecret: if self._fail_create: raise RuntimeError("store unavailable") + if secret.secret_name in self._invalid_names: + raise errors.ApiValidationError("value too large") key = (secret.user_id, secret.secret_name) if key in self.secrets: raise errors.ItemAlreadyExistsError("exists") @@ -322,14 +331,16 @@ def test_a_move_that_failed_before_deleting_the_row_finishes_next_time( assert _rows(engine=engine) == [] -def _client(*, engine: sqlalchemy.Engine, store: _FakeStore | None) -> TestClient: +def _client( + *, engine: sqlalchemy.Engine, store: _FakeStore | None, admin: bool = False +) -> TestClient: """Returns an API client for alice, with the given store passed to setup_routes.""" app = fastapi.FastAPI() api_router.setup_routes( app=app, db_engine=engine, user_details_getter=lambda: api_router.UserDetails( - name="alice", permissions={"read": True, "write": True, "admin": False} + name="alice", permissions={"read": True, "write": True, "admin": admin} ), secret_store=store, ) @@ -462,3 +473,87 @@ def test_launch_fails_when_the_store_cannot_read(tmp_path: pathlib.Path) -> None launch, execution = _launch_token(engine=engine, store=_FakeStore(fail_get=True)) launch.assert_not_called() assert "store unavailable" in _system_error(execution=execution) + + +def _backfill( + *, engine: sqlalchemy.Engine, store: _FakeStore | None, **params: str | int +) -> dict: + """Calls the backfill route as an admin and returns its JSON.""" + response = _client(engine=engine, store=store, admin=True).post( + "/api/admin/secrets/backfill_storage", params=params + ) + assert response.status_code == 200, response.text + return response.json() + + +def _seed(*, engine: sqlalchemy.Engine, names: list[tuple[str, str]]) -> None: + for user_id, secret_name in names: + _create(engine=engine, store=None, user_id=user_id, secret_name=secret_name) + + +def test_backfill_without_a_store_is_a_422(tmp_path: pathlib.Path) -> None: + response = _client(engine=_engine(tmp_path=tmp_path), store=None, admin=True).post( + "/api/admin/secrets/backfill_storage" + ) + assert response.status_code == 422, response.text + + +def test_backfill_moves_up_to_the_limit(tmp_path: pathlib.Path) -> None: + engine = _engine(tmp_path=tmp_path) + _seed(engine=engine, names=[("alice", "A"), ("alice", "B"), ("bob", "C")]) + store = _FakeStore() + result = _backfill(engine=engine, store=store, limit=2) + assert result == {"moved": 2, "skipped": [], "left_in_table": 1, "stopped_on": None} + assert sorted(store.secrets) == [("alice", "A"), ("alice", "B")] + assert _backfill(engine=engine, store=store)["left_in_table"] == 0 + + +def test_backfill_filters_by_user_and_name(tmp_path: pathlib.Path) -> None: + engine = _engine(tmp_path=tmp_path) + _seed(engine=engine, names=[("alice", "A"), ("alice", "B"), ("bob", "A")]) + store = _FakeStore() + assert _backfill(engine=engine, store=store, user_id="bob")["moved"] == 1 + assert _backfill(engine=engine, store=store, secret_name="B")["moved"] == 1 + assert sorted(store.secrets) == [("alice", "B"), ("bob", "A")] + + +def test_backfill_skips_an_invalid_row_without_counting_it( + tmp_path: pathlib.Path, +) -> None: + engine = _engine(tmp_path=tmp_path) + _seed(engine=engine, names=[("alice", "A"), ("alice", "BIG"), ("alice", "C")]) + store = _FakeStore(invalid_names=frozenset({"BIG"})) + result = _backfill(engine=engine, store=store, limit=2) + assert result == { + "moved": 2, + "skipped": [{"user_id": "alice", "secret_name": "BIG"}], + "left_in_table": 1, + "stopped_on": None, + } + assert [row.secret_name for row in _rows(engine=engine)] == ["BIG"] + + +def test_backfill_stops_on_an_error_and_can_be_repeated( + tmp_path: pathlib.Path, +) -> None: + engine = _engine(tmp_path=tmp_path) + _seed(engine=engine, names=[("alice", "A"), ("alice", "B")]) + result = _backfill(engine=engine, store=_FakeStore(fail_create=True)) + assert result == { + "moved": 0, + "skipped": [], + "left_in_table": 2, + "stopped_on": { + "user_id": "alice", + "secret_name": "A", + "error": "RuntimeError: store unavailable", + }, + } + assert _backfill(engine=engine, store=_FakeStore())["left_in_table"] == 0 + + +def test_backfill_needs_an_admin(tmp_path: pathlib.Path) -> None: + response = _client(engine=_engine(tmp_path=tmp_path), store=_FakeStore()).post( + "/api/admin/secrets/backfill_storage" + ) + assert response.status_code == 403, response.text From e52c54bfc29c145232511aa3db693196a6ca5034 Mon Sep 17 00:00:00 2001 From: Yue Chao Qin Date: Wed, 18 Nov 2026 22:24:47 -0800 Subject: [PATCH 3/3] feat: GSM Secrets - Backfill moves each row itself: lock, copy to the store, delete The routes no longer move rows, so the backfill owns the move. A copy left by a stopped run is replaced by the row. --- cloud_pipelines_backend/api_server_sql.py | 33 ++++++++++--- tests/test_secret_store.py | 57 +++++++++++++++++++++++ 2 files changed, 84 insertions(+), 6 deletions(-) diff --git a/cloud_pipelines_backend/api_server_sql.py b/cloud_pipelines_backend/api_server_sql.py index afc9116..8d555f3 100644 --- a/cloud_pipelines_backend/api_server_sql.py +++ b/cloud_pipelines_backend/api_server_sql.py @@ -1412,14 +1412,11 @@ def backfill_secret_storage( if moved == limit: break try: - secret_store.move_rows( - session=session, - secret_store=self._secret_store, - user_id=row_user_id, - secret_name=row_secret_name, + self._move_row( + session=session, user_id=row_user_id, secret_name=row_secret_name ) except errors.ApiValidationError: - # Releases the row lock `move_rows` took, so the skipped row is usable during the run. + # Releases the row lock `_move_row` took, so the skipped row is usable during the run. session.rollback() skipped.append({"user_id": row_user_id, "secret_name": row_secret_name}) continue @@ -1441,6 +1438,30 @@ def backfill_secret_storage( "stopped_on": stopped_on, } + def _move_row( + self, *, session: orm.Session, user_id: str, secret_name: str + ) -> None: + """Copies one row into the store and deletes it, in one commit; a PUT or DELETE of it waits on the lock.""" + row = session.get(bts.Secret, (user_id, secret_name), with_for_update=True) + if not row: + return + secret = secret_store.StoredSecret( + user_id=row.user_id, + secret_name=row.secret_name, + value=row.secret_value, + description=row.description, + created_at=row.created_at, + updated_at=row.updated_at, + extra_data=row.extra_data, + ) + try: + self._secret_store.create(secret=secret) + except errors.ItemAlreadyExistsError: + # An earlier run stopped after copying the row; the row is still the truth, so it replaces the copy. + self._secret_store.update(secret=secret) + session.delete(row) + session.commit() + def delete_secret( self, *, diff --git a/tests/test_secret_store.py b/tests/test_secret_store.py index 4fcd024..981cea1 100644 --- a/tests/test_secret_store.py +++ b/tests/test_secret_store.py @@ -517,3 +517,60 @@ def test_backfill_needs_an_admin(tmp_path: pathlib.Path) -> None: ) assert response.status_code == 403, response.text + +def test_backfill_carries_every_column(tmp_path: pathlib.Path) -> None: + engine = _engine(tmp_path=tmp_path) + now = api_server_sql._get_current_time() + with orm.Session(bind=engine) as session: + session.add( + bts.Secret( + user_id="Alice", + secret_name="Token", + secret_value="v1", + description="d", + extra_data={"k": [1]}, + created_at=now, + updated_at=now, + ) + ) + session.commit() + store = _FakeStore() + assert _backfill(engine=engine, store=store, user_id="Alice")["moved"] == 1 + assert store.secrets[("Alice", "Token")] == secret_store.StoredSecret( + user_id="Alice", + secret_name="Token", + value="v1", + description="d", + extra_data={"k": [1]}, + created_at=now, + updated_at=now, + ) + + +def test_backfill_replaces_a_copy_left_by_a_stopped_run( + tmp_path: pathlib.Path, +) -> None: + engine = _engine(tmp_path=tmp_path) + _seed(engine=engine, names=[("alice", "TOKEN")]) + _update(engine=engine, store=None, value="newer") + store = _FakeStore() + store.create(secret=_stored(value="stale copy")) + assert _backfill(engine=engine, store=store)["moved"] == 1 + assert store.get(user_id="alice", secret_name="TOKEN") == "newer" + assert _rows(engine=engine) == [] + + +def test_backfill_locks_the_row_it_moves(tmp_path: pathlib.Path) -> None: + """A PUT or DELETE of the same row then waits for the move to commit.""" + engine = _engine(tmp_path=tmp_path) + _seed(engine=engine, names=[("alice", "TOKEN")]) + locked = [] + with orm.Session(bind=engine) as session: + sqlalchemy.event.listen( + session, + "do_orm_execute", + lambda state: state.statement._for_update_arg is not None + and locked.append(True), + ) + _service(store=_FakeStore()).backfill_secret_storage(session=session, limit=10) + assert locked == [True]