From a8dd9a969021594429389e0cef21ce3087030c0a Mon Sep 17 00:00:00 2001 From: Yue Chao Qin Date: Tue, 17 Nov 2026 21:11:59 -0800 Subject: [PATCH 1/2] feat: GSM Secrets - Read secret inputs through the store at launch --- cloud_pipelines_backend/orchestrator_sql.py | 7 +- cloud_pipelines_backend/secret_value_store.py | 22 ++++ tests/test_secret_value_store.py | 121 +++++++++++++++++- 3 files changed, 148 insertions(+), 2 deletions(-) diff --git a/cloud_pipelines_backend/orchestrator_sql.py b/cloud_pipelines_backend/orchestrator_sql.py index 33f22f7..be82459 100644 --- a/cloud_pipelines_backend/orchestrator_sql.py +++ b/cloud_pipelines_backend/orchestrator_sql.py @@ -22,6 +22,7 @@ from . import backend_types_sql as bts from . import component_structures as structures from . import filter_query_sql +from . import secret_value_store from .launchers import common_annotations from .launchers import interfaces as launcher_interfaces from .instrumentation import bugsnag_instrumentation @@ -70,6 +71,7 @@ def __init__( sleep_seconds_between_queue_sweeps: float = 1.0, output_data_purge_duration: datetime.timedelta = None, *, + secret_value_store: secret_value_store.SecretValueStore | None = None, # Internal/experimental: _max_container_execution_refresh_error_retries: int = 3, _max_queue_batch_size: int = 1, @@ -94,6 +96,7 @@ def __init__( self._max_queue_batch_size = _max_queue_batch_size self._max_queue_batch_duration = _max_queue_batch_duration self._queued_execution_interceptor = queued_execution_interceptor + self._secret_value_store = secret_value_store def run_loop(self): while True: @@ -713,7 +716,9 @@ def generate_execution_log_uri( raise OrchestratorError( f"{execution.id=}: User error: Error resolving a secret argument for {input_name=}: User {user_id} does not have secret {secret_name}." ) - secret_value = secret.secret_value + secret_value = secret_value_store.read_secret_value( + secret=secret, store=self._secret_value_store + ) input_artifact_data[input_name] = bts.ArtifactData( total_size=len(secret_value.encode("utf-8")), is_dir=False, diff --git a/cloud_pipelines_backend/secret_value_store.py b/cloud_pipelines_backend/secret_value_store.py index 254ef45..a1a3ab0 100644 --- a/cloud_pipelines_backend/secret_value_store.py +++ b/cloud_pipelines_backend/secret_value_store.py @@ -104,6 +104,28 @@ def raise_if_storage_type_mismatch( ) +def read_secret_value(*, secret: bts.Secret, store: SecretValueStore | None) -> str: + """Returns the secret's value for a launch: from the row, or through the store that holds it. + + Raises when the value is in a store other than this server's, so the launch fails instead of + passing the row's placeholder to the task as if it were the value. + """ + storage_type = get_storage_type(secret=secret) + if storage_type is None: + return secret.secret_value + configured = store.type if store else None + if storage_type != configured: + raise errors.SecretStorageTypeMismatchError( + f"Secret '{secret.secret_name}' is kept in the {storage_type!r} store, " + f"but this server's secret store is {configured!r}; it cannot be read for this launch." + ) + return store.get( + storage_details=get_storage_details(secret=secret), + user_id=secret.user_id, + secret_name=secret.secret_name, + ) + + def retire_secret_value( *, store: SecretValueStore, diff --git a/tests/test_secret_value_store.py b/tests/test_secret_value_store.py index 9ff1c6f..2c3503d 100644 --- a/tests/test_secret_value_store.py +++ b/tests/test_secret_value_store.py @@ -1,6 +1,7 @@ import logging import pathlib from typing import Any +from unittest import mock import fastapi import pytest @@ -11,8 +12,11 @@ from cloud_pipelines_backend import api_router from cloud_pipelines_backend import api_server_sql from cloud_pipelines_backend import backend_types_sql as bts +from cloud_pipelines_backend import component_structures as structures from cloud_pipelines_backend import database_ops from cloud_pipelines_backend import errors +from cloud_pipelines_backend import orchestrator_sql +from cloud_pipelines_backend.launchers import interfaces as launcher_interfaces class _FakeStore: @@ -24,6 +28,7 @@ def __init__( type: str = "fake", engine: sqlalchemy.Engine | None = None, fail_delete: bool = False, + fail_get: bool = False, ) -> None: self.type = type self.versions: dict[str, str] = {} @@ -32,6 +37,7 @@ def __init__( self._engine = engine self.row_version_at_delete: list[str | None] = [] self._fail_delete = fail_delete + self._fail_get = fail_get def save( self, *, user_id: str, secret_name: str, value: str @@ -44,7 +50,9 @@ def save( def get( self, *, storage_details: dict[str, Any], user_id: str, secret_name: str ) -> str: - """Returns the saved version.""" + """Returns the saved version, or fails when told to.""" + if self._fail_get: + raise RuntimeError("store unavailable") return self.versions[storage_details["version"]] def delete( @@ -254,3 +262,114 @@ def test_a_refusal_is_a_503_naming_both_stores(tmp_path: pathlib.Path): assert "'fake' store" in response.json()["message"] assert "None" in response.json()["message"] assert _row(engine=engine) is not None + + +def _launch_token( + *, tmp_path: pathlib.Path, read_store: _FakeStore | None +) -> tuple[mock.MagicMock, bts.ExecutionNode]: + """Runs a one-task pipeline that takes alice's TOKEN as an input, with `read_store` as the orchestrator's store.""" + engine = _engine(tmp_path=tmp_path) + session_factory = lambda: orm.Session(bind=engine) + component = structures.ComponentSpec( + inputs=[structures.InputSpec(name="token")], + implementation=structures.ContainerImplementation( + container=structures.ContainerSpec(image="python") + ), + ) + task = structures.TaskSpec( + component_ref=structures.ComponentReference(spec=component), + arguments={ + "token": structures.DynamicDataArgument( + dynamic_data={"secret": {"name": "TOKEN"}} + ) + }, + ) + pipeline = structures.ComponentSpec( + implementation=structures.GraphImplementation( + graph=structures.GraphSpec(tasks={"task": task}) + ) + ) + api_server_sql.PipelineRunsApiService_Sql().create( + session=session_factory(), + root_task=structures.TaskSpec( + component_ref=structures.ComponentReference(spec=pipeline) + ), + created_by="alice", + ) + launch = mock.MagicMock( + return_value=mock.MagicMock( + status=launcher_interfaces.ContainerStatus.PENDING, to_dict=lambda: {} + ) + ) + orchestrator_sql.OrchestratorService_Sql( + session_factory=session_factory, + launcher=mock.MagicMock(launch_container_task=launch), + storage_provider=mock.MagicMock(), + data_root_uri="file:///tmp/artifacts", + logs_root_uri="file:///tmp/logs", + secret_value_store=read_store, + ).process_each_queue_once() + execution = ( + session_factory() + .query(bts.ExecutionNode) + .filter(bts.ExecutionNode.container_execution_status.is_not(None)) + .one() + ) + return launch, execution + + +def _launched_value(*, launch: mock.MagicMock) -> str: + """The value the launcher was given for the `token` input.""" + return launch.call_args.kwargs["input_arguments"]["token"].value + + +def _system_error(*, execution: bts.ExecutionNode) -> str: + """Asserts the launch failed as a system error, and returns its message.""" + assert ( + execution.container_execution_status + == bts.ContainerExecutionStatus.SYSTEM_ERROR + ) + return execution.extra_data[ + bts.EXECUTION_NODE_EXTRA_DATA_SYSTEM_ERROR_EXCEPTION_MESSAGE_KEY + ] + + +@pytest.mark.parametrize("read_store", [None, _FakeStore()], ids=["no-store", "store"]) +def test_launch_passes_a_plain_row_as_is( + tmp_path: pathlib.Path, read_store: _FakeStore | None +): + _save(engine=_engine(tmp_path=tmp_path), store=None, value="s3cret") + launch, _ = _launch_token(tmp_path=tmp_path, read_store=read_store) + assert _launched_value(launch=launch) == "s3cret" + + +def test_launch_reads_a_stored_row_through_the_store(tmp_path: pathlib.Path): + store = _FakeStore() + _save(engine=_engine(tmp_path=tmp_path), store=store, value="s3cret") + launch, _ = _launch_token(tmp_path=tmp_path, read_store=store) + assert _launched_value(launch=launch) == "s3cret" + + +@pytest.mark.parametrize( + "read_store, configured", + [(None, "None"), (_FakeStore(type="other"), "'other'")], + ids=["no-store", "other-store"], +) +def test_launch_fails_when_the_store_does_not_match( + tmp_path: pathlib.Path, read_store: _FakeStore | None, configured: str +): + _save(engine=_engine(tmp_path=tmp_path), store=_FakeStore(), value="s3cret") + launch, execution = _launch_token(tmp_path=tmp_path, read_store=read_store) + launch.assert_not_called() + message = _system_error(execution=execution) + assert "kept in the 'fake' store" in message + assert f"this server's secret store is {configured}" in message + + +def test_launch_fails_when_the_store_cannot_read(tmp_path: pathlib.Path): + _save(engine=_engine(tmp_path=tmp_path), store=_FakeStore(), value="s3cret") + launch, execution = _launch_token( + tmp_path=tmp_path, read_store=_FakeStore(fail_get=True) + ) + launch.assert_not_called() + assert "store unavailable" in _system_error(execution=execution) From 666662f5880760f6e450de5038e602bf433ac7ea Mon Sep 17 00:00:00 2001 From: Yue Chao Qin Date: Wed, 18 Nov 2026 20:27:18 -0800 Subject: [PATCH 2/2] feat: GSM Secrets - Read secret inputs from the database, then the store, at launch A launch only reads: get takes no session, so nothing moves or commits during a launch. --- cloud_pipelines_backend/orchestrator_sql.py | 18 +-- tests/test_secret_store.py | 119 +++++++++++++++++++- 2 files changed, 129 insertions(+), 8 deletions(-) diff --git a/cloud_pipelines_backend/orchestrator_sql.py b/cloud_pipelines_backend/orchestrator_sql.py index be82459..2cff2ec 100644 --- a/cloud_pipelines_backend/orchestrator_sql.py +++ b/cloud_pipelines_backend/orchestrator_sql.py @@ -22,7 +22,7 @@ from . import backend_types_sql as bts from . import component_structures as structures from . import filter_query_sql -from . import secret_value_store +from . import secret_store from .launchers import common_annotations from .launchers import interfaces as launcher_interfaces from .instrumentation import bugsnag_instrumentation @@ -71,7 +71,7 @@ def __init__( sleep_seconds_between_queue_sweeps: float = 1.0, output_data_purge_duration: datetime.timedelta = None, *, - secret_value_store: secret_value_store.SecretValueStore | None = None, + secret_store: secret_store.SecretStore | None = None, # Internal/experimental: _max_container_execution_refresh_error_retries: int = 3, _max_queue_batch_size: int = 1, @@ -96,7 +96,7 @@ def __init__( self._max_queue_batch_size = _max_queue_batch_size self._max_queue_batch_duration = _max_queue_batch_duration self._queued_execution_interceptor = queued_execution_interceptor - self._secret_value_store = secret_value_store + self._secret_store = secret_store def run_loop(self): while True: @@ -712,13 +712,17 @@ def generate_execution_log_uri( user_id = pipeline_run.created_by secret_name = secret_parameters[DYNAMIC_DATA_SECRET_NAME_KEY] secret = session.get(bts.Secret, (user_id, secret_name)) - if not secret: + secret_value = secret.secret_value if secret else None + if secret_value is None and self._secret_store: + # The row first, then the store: a row not yet moved holds the same value as any copy + # the store has, and a launch only reads. + secret_value = self._secret_store.get( + user_id=user_id, secret_name=secret_name + ) + if secret_value is None: raise OrchestratorError( f"{execution.id=}: User error: Error resolving a secret argument for {input_name=}: User {user_id} does not have secret {secret_name}." ) - secret_value = secret_value_store.read_secret_value( - secret=secret, store=self._secret_value_store - ) input_artifact_data[input_name] = bts.ArtifactData( total_size=len(secret_value.encode("utf-8")), is_dir=False, diff --git a/tests/test_secret_store.py b/tests/test_secret_store.py index 98bb44d..a43c299 100644 --- a/tests/test_secret_store.py +++ b/tests/test_secret_store.py @@ -1,5 +1,6 @@ import dataclasses import pathlib +from unittest import mock import fastapi import pytest @@ -10,17 +11,21 @@ from cloud_pipelines_backend import api_router from cloud_pipelines_backend import api_server_sql from cloud_pipelines_backend import backend_types_sql as bts +from cloud_pipelines_backend import component_structures as structures from cloud_pipelines_backend import database_ops from cloud_pipelines_backend import errors +from cloud_pipelines_backend import orchestrator_sql from cloud_pipelines_backend import secret_store +from cloud_pipelines_backend.launchers import interfaces as launcher_interfaces class _FakeStore(secret_store.SecretStore): """An in-memory `SecretStore` that follows the base class's contract, keyed by user and name.""" - def __init__(self, *, fail_create: bool = False) -> None: + def __init__(self, *, fail_create: bool = False, fail_get: bool = False) -> None: self.secrets: dict[tuple[str, str], secret_store.StoredSecret] = {} self._fail_create = fail_create + self._fail_get = fail_get def create(self, *, secret: secret_store.StoredSecret) -> secret_store.StoredSecret: if self._fail_create: @@ -47,6 +52,8 @@ def update(self, *, secret: secret_store.StoredSecret) -> secret_store.StoredSec return dataclasses.replace(self.secrets[key], value=None) def get(self, *, user_id: str, secret_name: str) -> str | None: + if self._fail_get: + raise RuntimeError("store unavailable") secret = self.secrets.get((user_id, secret_name)) return secret.value if secret else None @@ -345,3 +352,113 @@ def test_setup_routes_passes_the_store_to_secret_routes(tmp_path: pathlib.Path) assert [s["secret_name"] for s in listed.json()["secrets"]] == ["TOKEN"] assert store.get(user_id="alice", secret_name="TOKEN") == "v1" assert _rows(engine=engine) == [] + + +def _launch_token( + *, engine: sqlalchemy.Engine, store: _FakeStore | None +) -> tuple[mock.MagicMock, bts.ExecutionNode]: + """Runs a one-task pipeline that takes alice's TOKEN as an input, with `store` as the orchestrator's store.""" + session_factory = lambda: orm.Session(bind=engine) + component = structures.ComponentSpec( + inputs=[structures.InputSpec(name="token")], + implementation=structures.ContainerImplementation( + container=structures.ContainerSpec(image="python") + ), + ) + task = structures.TaskSpec( + component_ref=structures.ComponentReference(spec=component), + arguments={ + "token": structures.DynamicDataArgument( + dynamic_data={"secret": {"name": "TOKEN"}} + ) + }, + ) + pipeline = structures.ComponentSpec( + implementation=structures.GraphImplementation( + graph=structures.GraphSpec(tasks={"task": task}) + ) + ) + api_server_sql.PipelineRunsApiService_Sql().create( + session=session_factory(), + root_task=structures.TaskSpec( + component_ref=structures.ComponentReference(spec=pipeline) + ), + created_by="alice", + ) + launch = mock.MagicMock( + return_value=mock.MagicMock( + status=launcher_interfaces.ContainerStatus.PENDING, to_dict=lambda: {} + ) + ) + orchestrator_sql.OrchestratorService_Sql( + session_factory=session_factory, + launcher=mock.MagicMock(launch_container_task=launch), + storage_provider=mock.MagicMock(), + data_root_uri="file:///tmp/artifacts", + logs_root_uri="file:///tmp/logs", + secret_store=store, + ).process_each_queue_once() + execution = ( + session_factory() + .query(bts.ExecutionNode) + .filter(bts.ExecutionNode.container_execution_status.is_not(None)) + .one() + ) + return launch, execution + + +def _launched_value(*, launch: mock.MagicMock) -> str: + """The value the launcher was given for the `token` input.""" + return launch.call_args.kwargs["input_arguments"]["token"].value + + +def _system_error(*, execution: bts.ExecutionNode) -> str: + """Asserts the launch failed as a system error, and returns its message.""" + assert ( + execution.container_execution_status + == bts.ContainerExecutionStatus.SYSTEM_ERROR + ) + return execution.extra_data[ + bts.EXECUTION_NODE_EXTRA_DATA_SYSTEM_ERROR_EXCEPTION_MESSAGE_KEY + ] + + +@pytest.mark.parametrize("with_store", [False, True], ids=["no-store", "store"]) +def test_launch_reads_a_row_and_leaves_it_in_the_table( + tmp_path: pathlib.Path, with_store: bool +) -> None: + engine = _engine(tmp_path=tmp_path) + _create(engine=engine, store=None, value="s3cret") + store = _FakeStore() if with_store else None + launch, _ = _launch_token(engine=engine, store=store) + assert _launched_value(launch=launch) == "s3cret" + assert len(_rows(engine=engine)) == 1 + if store: + assert store.secrets == {} + + +def test_launch_reads_a_moved_secret_from_the_store(tmp_path: pathlib.Path) -> None: + engine = _engine(tmp_path=tmp_path) + store = _FakeStore() + _create(engine=engine, store=store, value="s3cret") + launch, _ = _launch_token(engine=engine, store=store) + assert _launched_value(launch=launch) == "s3cret" + + +@pytest.mark.parametrize("with_store", [False, True], ids=["no-store", "store"]) +def test_launch_of_a_missing_secret_is_a_user_error( + tmp_path: pathlib.Path, with_store: bool +) -> None: + engine = _engine(tmp_path=tmp_path) + launch, execution = _launch_token( + engine=engine, store=_FakeStore() if with_store else None + ) + launch.assert_not_called() + assert "User alice does not have secret TOKEN" in _system_error(execution=execution) + + +def test_launch_fails_when_the_store_cannot_read(tmp_path: pathlib.Path) -> None: + engine = _engine(tmp_path=tmp_path) + launch, execution = _launch_token(engine=engine, store=_FakeStore(fail_get=True)) + launch.assert_not_called() + assert "store unavailable" in _system_error(execution=execution)