diff --git a/cloud_pipelines_backend/api_router.py b/cloud_pipelines_backend/api_router.py index ab77087..1c4333b 100644 --- a/cloud_pipelines_backend/api_router.py +++ b/cloud_pipelines_backend/api_router.py @@ -623,6 +623,22 @@ def admin_set_execution_node_status( execution_node = session.get(backend_types_sql.ExecutionNode, id) execution_node.container_execution_status = status + @router.post( + "/api/admin/secrets/backfill_storage", + tags=["admin"], + dependencies=[ensure_admin_user_dependency], + **default_config, + ) + def admin_backfill_secret_storage( + session: typing.Annotated[orm.Session, fastapi.Depends(get_session)], + limit: typing.Annotated[int, fastapi.Query(ge=1, le=1000)] = 50, + user_id: str | None = None, + secret_name: str | None = None, + ) -> dict[str, typing.Any]: + return secrets_service.backfill_secret_storage( + session=session, limit=limit, user_id=user_id, secret_name=secret_name + ) + @router.get("/api/admin/sql_engine_connection_pool_status") async def get_sql_engine_connection_pool_status( session: typing.Annotated[orm.Session, fastapi.Depends(get_session)], diff --git a/cloud_pipelines_backend/api_server_sql.py b/cloud_pipelines_backend/api_server_sql.py index b87609d..8d555f3 100644 --- a/cloud_pipelines_backend/api_server_sql.py +++ b/cloud_pipelines_backend/api_server_sql.py @@ -1381,6 +1381,87 @@ def _create_or_update_secret( session.commit() return response + def backfill_secret_storage( + self, + *, + session: orm.Session, + limit: int, + user_id: str | None = None, + secret_name: str | None = None, + ) -> dict[str, typing.Any]: + """Moves up to `limit` rows from the `secret` table into the store, committing each, and reports what is left. + + `user_id` and `secret_name`, when given, move only matching rows. A row the store refuses as invalid (for + example, a value too large) is skipped and listed by name; any other error stops the run, which can then + be repeated. + """ + if not self._secret_store: + raise errors.ApiValidationError("No secret store is configured.") + rows = session.execute( + sql.select(bts.Secret.user_id, bts.Secret.secret_name) + .filter_by( + **({"user_id": user_id} if user_id is not None else {}), + **({"secret_name": secret_name} if secret_name is not None else {}), + ) + .order_by(bts.Secret.user_id, bts.Secret.secret_name) + ).all() + moved = 0 + skipped = [] + stopped_on = None + for row_user_id, row_secret_name in rows: + if moved == limit: + break + try: + self._move_row( + session=session, user_id=row_user_id, secret_name=row_secret_name + ) + except errors.ApiValidationError: + # Releases the row lock `_move_row` took, so the skipped row is usable during the run. + session.rollback() + skipped.append({"user_id": row_user_id, "secret_name": row_secret_name}) + continue + except Exception as e: + session.rollback() + stopped_on = { + "user_id": row_user_id, + "secret_name": row_secret_name, + "error": f"{type(e).__name__}: {e}", + } + break + moved += 1 + return { + "moved": moved, + "skipped": skipped, + "left_in_table": session.scalar( + sql.select(sql.func.count()).select_from(bts.Secret) + ), + "stopped_on": stopped_on, + } + + def _move_row( + self, *, session: orm.Session, user_id: str, secret_name: str + ) -> None: + """Copies one row into the store and deletes it, in one commit; a PUT or DELETE of it waits on the lock.""" + row = session.get(bts.Secret, (user_id, secret_name), with_for_update=True) + if not row: + return + secret = secret_store.StoredSecret( + user_id=row.user_id, + secret_name=row.secret_name, + value=row.secret_value, + description=row.description, + created_at=row.created_at, + updated_at=row.updated_at, + extra_data=row.extra_data, + ) + try: + self._secret_store.create(secret=secret) + except errors.ItemAlreadyExistsError: + # An earlier run stopped after copying the row; the row is still the truth, so it replaces the copy. + self._secret_store.update(secret=secret) + session.delete(row) + session.commit() + def delete_secret( self, *, diff --git a/tests/test_secret_store.py b/tests/test_secret_store.py index e88b4f8..981cea1 100644 --- a/tests/test_secret_store.py +++ b/tests/test_secret_store.py @@ -22,11 +22,23 @@ class _FakeStore(secret_store.SecretStore): """An in-memory `SecretStore` that follows its contract, keyed by user and name.""" - def __init__(self, *, fail_get: bool = False) -> None: + def __init__( + self, + *, + fail_create: bool = False, + fail_get: bool = False, + invalid_names: frozenset[str] = frozenset(), + ) -> None: self.secrets: dict[tuple[str, str], secret_store.StoredSecret] = {} + self._fail_create = fail_create self._fail_get = fail_get + self._invalid_names = invalid_names def create(self, *, secret: secret_store.StoredSecret) -> secret_store.StoredSecret: + if self._fail_create: + raise RuntimeError("store unavailable") + if secret.secret_name in self._invalid_names: + raise errors.ApiValidationError("value too large") key = (secret.user_id, secret.secret_name) if key in self.secrets: raise errors.ItemAlreadyExistsError("exists") @@ -278,14 +290,16 @@ def test_stored_secret_never_shows_its_value() -> None: assert "hunter2" not in f"{secret}" -def _client(*, engine: sqlalchemy.Engine, store: _FakeStore | None) -> TestClient: +def _client( + *, engine: sqlalchemy.Engine, store: _FakeStore | None, admin: bool = False +) -> TestClient: """Returns an API client for alice, with the given store passed to setup_routes.""" app = fastapi.FastAPI() api_router.setup_routes( app=app, db_engine=engine, user_details_getter=lambda: api_router.UserDetails( - name="alice", permissions={"read": True, "write": True, "admin": False} + name="alice", permissions={"read": True, "write": True, "admin": admin} ), secret_store=store, ) @@ -418,3 +432,145 @@ def test_launch_fails_when_the_store_cannot_read(tmp_path: pathlib.Path) -> None launch, execution = _launch_token(engine=engine, store=_FakeStore(fail_get=True)) launch.assert_not_called() assert "store unavailable" in _system_error(execution=execution) + + +def _backfill( + *, engine: sqlalchemy.Engine, store: _FakeStore | None, **params: str | int +) -> dict: + """Calls the backfill route as an admin and returns its JSON.""" + response = _client(engine=engine, store=store, admin=True).post( + "/api/admin/secrets/backfill_storage", params=params + ) + assert response.status_code == 200, response.text + return response.json() + + +def _seed(*, engine: sqlalchemy.Engine, names: list[tuple[str, str]]) -> None: + for user_id, secret_name in names: + _create(engine=engine, store=None, user_id=user_id, secret_name=secret_name) + + +def test_backfill_without_a_store_is_a_422(tmp_path: pathlib.Path) -> None: + response = _client(engine=_engine(tmp_path=tmp_path), store=None, admin=True).post( + "/api/admin/secrets/backfill_storage" + ) + assert response.status_code == 422, response.text + + +def test_backfill_moves_up_to_the_limit(tmp_path: pathlib.Path) -> None: + engine = _engine(tmp_path=tmp_path) + _seed(engine=engine, names=[("alice", "A"), ("alice", "B"), ("bob", "C")]) + store = _FakeStore() + result = _backfill(engine=engine, store=store, limit=2) + assert result == {"moved": 2, "skipped": [], "left_in_table": 1, "stopped_on": None} + assert sorted(store.secrets) == [("alice", "A"), ("alice", "B")] + assert _backfill(engine=engine, store=store)["left_in_table"] == 0 + + +def test_backfill_filters_by_user_and_name(tmp_path: pathlib.Path) -> None: + engine = _engine(tmp_path=tmp_path) + _seed(engine=engine, names=[("alice", "A"), ("alice", "B"), ("bob", "A")]) + store = _FakeStore() + assert _backfill(engine=engine, store=store, user_id="bob")["moved"] == 1 + assert _backfill(engine=engine, store=store, secret_name="B")["moved"] == 1 + assert sorted(store.secrets) == [("alice", "B"), ("bob", "A")] + + +def test_backfill_skips_an_invalid_row_without_counting_it( + tmp_path: pathlib.Path, +) -> None: + engine = _engine(tmp_path=tmp_path) + _seed(engine=engine, names=[("alice", "A"), ("alice", "BIG"), ("alice", "C")]) + store = _FakeStore(invalid_names=frozenset({"BIG"})) + result = _backfill(engine=engine, store=store, limit=2) + assert result == { + "moved": 2, + "skipped": [{"user_id": "alice", "secret_name": "BIG"}], + "left_in_table": 1, + "stopped_on": None, + } + assert [row.secret_name for row in _rows(engine=engine)] == ["BIG"] + + +def test_backfill_stops_on_an_error_and_can_be_repeated( + tmp_path: pathlib.Path, +) -> None: + engine = _engine(tmp_path=tmp_path) + _seed(engine=engine, names=[("alice", "A"), ("alice", "B")]) + result = _backfill(engine=engine, store=_FakeStore(fail_create=True)) + assert result == { + "moved": 0, + "skipped": [], + "left_in_table": 2, + "stopped_on": { + "user_id": "alice", + "secret_name": "A", + "error": "RuntimeError: store unavailable", + }, + } + assert _backfill(engine=engine, store=_FakeStore())["left_in_table"] == 0 + + +def test_backfill_needs_an_admin(tmp_path: pathlib.Path) -> None: + response = _client(engine=_engine(tmp_path=tmp_path), store=_FakeStore()).post( + "/api/admin/secrets/backfill_storage" + ) + assert response.status_code == 403, response.text + + +def test_backfill_carries_every_column(tmp_path: pathlib.Path) -> None: + engine = _engine(tmp_path=tmp_path) + now = api_server_sql._get_current_time() + with orm.Session(bind=engine) as session: + session.add( + bts.Secret( + user_id="Alice", + secret_name="Token", + secret_value="v1", + description="d", + extra_data={"k": [1]}, + created_at=now, + updated_at=now, + ) + ) + session.commit() + store = _FakeStore() + assert _backfill(engine=engine, store=store, user_id="Alice")["moved"] == 1 + assert store.secrets[("Alice", "Token")] == secret_store.StoredSecret( + user_id="Alice", + secret_name="Token", + value="v1", + description="d", + extra_data={"k": [1]}, + created_at=now, + updated_at=now, + ) + + +def test_backfill_replaces_a_copy_left_by_a_stopped_run( + tmp_path: pathlib.Path, +) -> None: + engine = _engine(tmp_path=tmp_path) + _seed(engine=engine, names=[("alice", "TOKEN")]) + _update(engine=engine, store=None, value="newer") + store = _FakeStore() + store.create(secret=_stored(value="stale copy")) + assert _backfill(engine=engine, store=store)["moved"] == 1 + assert store.get(user_id="alice", secret_name="TOKEN") == "newer" + assert _rows(engine=engine) == [] + + +def test_backfill_locks_the_row_it_moves(tmp_path: pathlib.Path) -> None: + """A PUT or DELETE of the same row then waits for the move to commit.""" + engine = _engine(tmp_path=tmp_path) + _seed(engine=engine, names=[("alice", "TOKEN")]) + locked = [] + with orm.Session(bind=engine) as session: + sqlalchemy.event.listen( + session, + "do_orm_execute", + lambda state: state.statement._for_update_arg is not None + and locked.append(True), + ) + _service(store=_FakeStore()).backfill_secret_storage(session=session, limit=10) + assert locked == [True]