diff --git a/cloud_pipelines_backend/orchestrator_sql.py b/cloud_pipelines_backend/orchestrator_sql.py index 33f22f7..9c8c81e 100644 --- a/cloud_pipelines_backend/orchestrator_sql.py +++ b/cloud_pipelines_backend/orchestrator_sql.py @@ -22,6 +22,7 @@ from . import backend_types_sql as bts from . import component_structures as structures from . import filter_query_sql +from . import secret_interfaces from .launchers import common_annotations from .launchers import interfaces as launcher_interfaces from .instrumentation import bugsnag_instrumentation @@ -71,6 +72,7 @@ def __init__( output_data_purge_duration: datetime.timedelta = None, *, # Internal/experimental: + _secret_store: secret_interfaces.SecretStore | None = None, _max_container_execution_refresh_error_retries: int = 3, _max_queue_batch_size: int = 1, _max_queue_batch_duration: datetime.timedelta = datetime.timedelta(), @@ -94,6 +96,7 @@ def __init__( self._max_queue_batch_size = _max_queue_batch_size self._max_queue_batch_duration = _max_queue_batch_duration self._queued_execution_interceptor = queued_execution_interceptor + self._secret_store = _secret_store def run_loop(self): while True: @@ -709,11 +712,17 @@ def generate_execution_log_uri( user_id = pipeline_run.created_by secret_name = secret_parameters[DYNAMIC_DATA_SECRET_NAME_KEY] secret = session.get(bts.Secret, (user_id, secret_name)) - if not secret: + secret_value = secret.secret_value if secret else None + if secret_value is None and self._secret_store: + # The row first, then the store: a row not yet moved holds the same value as any copy + # the store has, and a launch only reads. + secret_value = self._secret_store.get( + user_id=user_id, secret_name=secret_name + ) + if secret_value is None: raise OrchestratorError( f"{execution.id=}: User error: Error resolving a secret argument for {input_name=}: User {user_id} does not have secret {secret_name}." ) - secret_value = secret.secret_value input_artifact_data[input_name] = bts.ArtifactData( total_size=len(secret_value.encode("utf-8")), is_dir=False, diff --git a/tests/test_secret_store.py b/tests/test_secret_store.py index 5fa1941..ae1c49f 100644 --- a/tests/test_secret_store.py +++ b/tests/test_secret_store.py @@ -1,6 +1,7 @@ import builtins import dataclasses import pathlib +from unittest import mock import fastapi import pytest @@ -11,16 +12,20 @@ from cloud_pipelines_backend import api_router from cloud_pipelines_backend import api_server_sql from cloud_pipelines_backend import backend_types_sql as bts +from cloud_pipelines_backend import component_structures as structures from cloud_pipelines_backend import database_ops from cloud_pipelines_backend import errors +from cloud_pipelines_backend import orchestrator_sql from cloud_pipelines_backend import secret_interfaces +from cloud_pipelines_backend.launchers import interfaces as launcher_interfaces class _FakeStore(secret_interfaces.SecretStore): """An in-memory `SecretStore` that follows its contract, keyed by user and name.""" - def __init__(self) -> None: + def __init__(self, *, fail_get: bool = False) -> None: self.secrets: dict[tuple[str, str], secret_interfaces.StoredSecret] = {} + self._fail_get = fail_get def create( self, *, secret: secret_interfaces.StoredSecret @@ -49,6 +54,8 @@ def update( return dataclasses.replace(self.secrets[key], value=None) def get(self, *, user_id: str, secret_name: str) -> str | None: + if self._fail_get: + raise RuntimeError("store unavailable") secret = self.secrets.get((user_id, secret_name)) return secret.value if secret else None @@ -306,3 +313,113 @@ def test_setup_routes_passes_the_store_to_secret_routes(tmp_path: pathlib.Path) assert [s["secret_name"] for s in listed.json()["secrets"]] == ["TOKEN"] assert store.get(user_id="alice", secret_name="TOKEN") == "v1" assert _rows(engine=engine) == [] + + +def _launch_token( + *, engine: sqlalchemy.Engine, store: _FakeStore | None +) -> tuple[mock.MagicMock, bts.ExecutionNode]: + """Runs a one-task pipeline that takes alice's TOKEN as an input, with `store` as the orchestrator's store.""" + session_factory = lambda: orm.Session(bind=engine) + component = structures.ComponentSpec( + inputs=[structures.InputSpec(name="token")], + implementation=structures.ContainerImplementation( + container=structures.ContainerSpec(image="python") + ), + ) + task = structures.TaskSpec( + component_ref=structures.ComponentReference(spec=component), + arguments={ + "token": structures.DynamicDataArgument( + dynamic_data={"secret": {"name": "TOKEN"}} + ) + }, + ) + pipeline = structures.ComponentSpec( + implementation=structures.GraphImplementation( + graph=structures.GraphSpec(tasks={"task": task}) + ) + ) + api_server_sql.PipelineRunsApiService_Sql().create( + session=session_factory(), + root_task=structures.TaskSpec( + component_ref=structures.ComponentReference(spec=pipeline) + ), + created_by="alice", + ) + launch = mock.MagicMock( + return_value=mock.MagicMock( + status=launcher_interfaces.ContainerStatus.PENDING, to_dict=lambda: {} + ) + ) + orchestrator_sql.OrchestratorService_Sql( + session_factory=session_factory, + launcher=mock.MagicMock(launch_container_task=launch), + storage_provider=mock.MagicMock(), + data_root_uri="file:///tmp/artifacts", + logs_root_uri="file:///tmp/logs", + _secret_store=store, + ).process_each_queue_once() + execution = ( + session_factory() + .query(bts.ExecutionNode) + .filter(bts.ExecutionNode.container_execution_status.is_not(None)) + .one() + ) + return launch, execution + + +def _launched_value(*, launch: mock.MagicMock) -> str: + """The value the launcher was given for the `token` input.""" + return launch.call_args.kwargs["input_arguments"]["token"].value + + +def _system_error(*, execution: bts.ExecutionNode) -> str: + """Asserts the launch failed as a system error, and returns its message.""" + assert ( + execution.container_execution_status + == bts.ContainerExecutionStatus.SYSTEM_ERROR + ) + return execution.extra_data[ + bts.EXECUTION_NODE_EXTRA_DATA_SYSTEM_ERROR_EXCEPTION_MESSAGE_KEY + ] + + +@pytest.mark.parametrize("with_store", [False, True], ids=["no-store", "store"]) +def test_launch_reads_a_row_and_leaves_it_in_the_table( + tmp_path: pathlib.Path, with_store: bool +) -> None: + engine = _engine(tmp_path=tmp_path) + _create(engine=engine, store=None, value="s3cret") + store = _FakeStore() if with_store else None + launch, _ = _launch_token(engine=engine, store=store) + assert _launched_value(launch=launch) == "s3cret" + assert len(_rows(engine=engine)) == 1 + if store: + assert store.secrets == {} + + +def test_launch_reads_a_moved_secret_from_the_store(tmp_path: pathlib.Path) -> None: + engine = _engine(tmp_path=tmp_path) + store = _FakeStore() + _create(engine=engine, store=store, value="s3cret") + launch, _ = _launch_token(engine=engine, store=store) + assert _launched_value(launch=launch) == "s3cret" + + +@pytest.mark.parametrize("with_store", [False, True], ids=["no-store", "store"]) +def test_launch_of_a_missing_secret_is_a_user_error( + tmp_path: pathlib.Path, with_store: bool +) -> None: + engine = _engine(tmp_path=tmp_path) + launch, execution = _launch_token( + engine=engine, store=_FakeStore() if with_store else None + ) + launch.assert_not_called() + assert "User alice does not have secret TOKEN" in _system_error(execution=execution) + + +def test_launch_fails_when_the_store_cannot_read(tmp_path: pathlib.Path) -> None: + engine = _engine(tmp_path=tmp_path) + launch, execution = _launch_token(engine=engine, store=_FakeStore(fail_get=True)) + launch.assert_not_called() + assert "store unavailable" in _system_error(execution=execution)