From 9b3934a5ef675319564be5975c48c76d9a7581a1 Mon Sep 17 00:00:00 2001 From: Lily Ma Date: Fri, 6 Nov 2026 02:23:06 +0000 Subject: [PATCH 1/3] fix(verifiers): grade a container verifier on the run's own LiteLLM key A run launched with a caller's key (user_overrides litellm_api_key / litellm_base_url, as deploy_agent and deploy_env already honor) now puts that key and URL in the verifier container's LITELLM_* / ANTHROPIC_* env. Before, _merged_env always used agent-env's platform key, so LLM-judge verifiers never ran on, or billed, the launcher's key, and every one failed while the platform key was rejected by the proxy. --- .../run_container_unit_tests_verifier.py | 26 ++++-- .../run_container_unit_tests_verifier_test.py | 85 +++++++++++++++++++ 2 files changed, 103 insertions(+), 8 deletions(-) diff --git a/src/agent_env/task_step/task_steps/verifiers/run_container_unit_tests_verifier.py b/src/agent_env/task_step/task_steps/verifiers/run_container_unit_tests_verifier.py index ab8cb2e96..9c326ec8e 100644 --- a/src/agent_env/task_step/task_steps/verifiers/run_container_unit_tests_verifier.py +++ b/src/agent_env/task_step/task_steps/verifiers/run_container_unit_tests_verifier.py @@ -235,13 +235,14 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext: config = get_config() command, extra_env = self._resolve_command(context) + user_overrides = context.metadata.get("user_overrides") or {} if self.env_id is not None: target = self._computer(context) - target.env = self._merged_env(config, target.deployed.sandbox_type, extra_env) + target.env = self._merged_env(config, target.deployed.sandbox_type, extra_env, user_overrides) exit_code, stdout, stderr, extracted_files = await self._run_in_computer(target, command) else: sandbox = await self._container_sandbox(context) - merged_env = self._merged_env(config, sandbox.type, extra_env) + merged_env = self._merged_env(config, sandbox.type, extra_env, user_overrides) exit_code, stdout, stderr, extracted_files = await self._run_in_container(sandbox, merged_env, command) timed_out = exit_code in (124, 137) logger.info( @@ -343,19 +344,28 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext: return context - def _merged_env(self, config, sandbox_type: Optional[str], extra_env: dict[str, str]) -> dict[str, str]: - """The commands' environment: the model endpoint's key and URL from agent-env's config, every sandbox - provider's container env, ``env_vars`` and this run's ``extra_env``, then the endpoint's headers, its URL as - ``sandbox_type``'s containers reach it.""" + def _merged_env( + self, config, sandbox_type: Optional[str], extra_env: dict[str, str], + user_overrides: Optional[dict[str, Any]] = None, + ) -> dict[str, str]: + """The commands' environment: the model endpoint's key and URL, every sandbox provider's container env, + ``env_vars`` and this run's ``extra_env``, then the endpoint's headers, its URL as ``sandbox_type``'s + containers reach it. The key and URL are the run's own (``user_overrides`` ``litellm_api_key`` / + ``litellm_base_url``, as ``deploy_agent`` takes them; ``judge_litellm_api_key`` first, as ``rubrics_verifier``) + when it names them, else agent-env's config: a run + launched on a caller's key grades on that key and bills it, not the platform's.""" + overrides = user_overrides or {} merged_env: dict[str, str] = {} try: - litellm_key = config.get_litellm_api_key() + # as rubrics_verifier: a judge key, when the run names one, then the run's own key + run_key = overrides.get("judge_litellm_api_key") or overrides.get("litellm_api_key") + litellm_key = config.resolve_reference(run_key) if run_key else config.get_litellm_api_key() merged_env["LITELLM_API_KEY"] = litellm_key merged_env["ANTHROPIC_API_KEY"] = litellm_key except (KeyError, ConfigError): logger.warning("LITELLM_API_KEY unavailable; verifier won't have LLM creds") try: - litellm_url = config.get_litellm_base_url() + litellm_url = overrides.get("litellm_base_url") or config.get_litellm_base_url() merged_env["LITELLM_BASE_URL"] = litellm_url merged_env["ANTHROPIC_BASE_URL"] = litellm_url except (KeyError, ConfigError): diff --git a/tst/unit/task_step/verifiers/run_container_unit_tests_verifier_test.py b/tst/unit/task_step/verifiers/run_container_unit_tests_verifier_test.py index e93918f9d..2bb13d3d9 100644 --- a/tst/unit/task_step/verifiers/run_container_unit_tests_verifier_test.py +++ b/tst/unit/task_step/verifiers/run_container_unit_tests_verifier_test.py @@ -453,3 +453,88 @@ def test_a_verifier_for_a_computer_that_cant_clear_the_reward_file_grades_nothin def test_a_verifier_naming_its_target_wrong_is_refused(fields, problem): with pytest.raises(ValueError, match=problem): Step(id="scrape", version=None, command="true", **fields) + + +class _Config: + """The two config reads ``_merged_env`` makes for the model endpoint, and no headers.""" + + def get_litellm_api_key(self): + return "sk-platform" + + def get_litellm_base_url(self): + return "https://proxy.example" + + def resolve_reference(self, value): + return value.removeprefix("env:RESOLVED_") if isinstance(value, str) else value + + def get_model_headers(self, base_url): + return {} + + +@pytest.fixture +def no_provider_env(monkeypatch): + monkeypatch.setattr(verifier_module, "all_sandbox_container_env", lambda: {}) + + +def test_a_run_launched_on_a_callers_key_grades_on_that_key(no_provider_env): + env = _step()._merged_env(_Config(), "modal", {}, {"litellm_api_key": "sk-caller"}) + assert env["LITELLM_API_KEY"] == env["ANTHROPIC_API_KEY"] == "sk-caller" + + +def test_a_runs_judge_key_wins_over_its_agent_key_as_in_the_rubrics_verifier(no_provider_env): + env = _step()._merged_env(_Config(), "modal", {}, + {"litellm_api_key": "sk-agent", "judge_litellm_api_key": "sk-judge"}) + assert env["LITELLM_API_KEY"] == env["ANTHROPIC_API_KEY"] == "sk-judge" + + +def test_a_run_key_given_as_a_reference_is_resolved(no_provider_env): + env = _step()._merged_env(_Config(), "modal", {}, {"litellm_api_key": "env:RESOLVED_sk-ref"}) + assert env["LITELLM_API_KEY"] == "sk-ref" + + +def test_a_run_without_its_own_key_or_url_keeps_the_platforms(no_provider_env): + env = _step()._merged_env(_Config(), "modal", {}, {"step_params": {}}) + assert env["LITELLM_API_KEY"] == "sk-platform" + assert env["LITELLM_BASE_URL"] == env["ANTHROPIC_BASE_URL"] == "https://proxy.example" + + +def test_a_run_endpoint_override_moves_the_url_too(no_provider_env): + env = _step()._merged_env(_Config(), "modal", {}, {"litellm_base_url": "https://other.example"}) + assert env["LITELLM_BASE_URL"] == "https://other.example" + + +def test_a_key_the_step_itself_sets_still_wins_over_the_runs(no_provider_env): + step = _step(env_vars={"LITELLM_API_KEY": "sk-step"}) + env = step._merged_env(_Config(), "modal", {}, {"litellm_api_key": "sk-caller"}) + assert env["LITELLM_API_KEY"] == "sk-step" + + +def test_the_executed_verifier_hands_the_runs_key_to_its_container(local_stores, monkeypatch): + """Through ``execute``: the hub's worker puts the launcher's decrypted key in ``user_overrides``.""" + monkeypatch.setenv("LITELLM_API_KEY", "sk-platform") + configure() + commands: list[str] = [] + + class _Recording(_Sandbox): + async def exec_script(self, script: str) -> str: + commands.append(script) + return "" + + async def exec_with_output(self, *args) -> tuple[int, str, str]: + commands.extend(a for a in args if isinstance(a, str)) + return 0, "ok", "" + + provider = SimpleNamespace(get_sandbox=lambda sandbox_id: asyncio.sleep(0, result=_Recording())) + monkeypatch.setattr(sandbox_provider, "get_sandbox_provider", lambda: provider) + monkeypatch.setattr(Step, "_upload_text_artifact", staticmethod( + lambda text, artifact_id, description, object_url: SimpleNamespace( + id=artifact_id, version=1, object_url=object_url))) + ctx = TaskStepContext() + ctx.metadata["deployed_docker_containers"] = [{"container_name": "c", "sandbox_name": "h"}] + ctx.deployed_sandboxes.append(DeployedSandbox(sandbox_name="h", sandbox_id="sb-1", sandbox_mode="vm")) + ctx.metadata["user_overrides"] = {"litellm_api_key": "sk-caller"} + + asyncio.run(_step(command="true").execute(ctx)) + + sent = " ".join(commands) + assert "LITELLM_API_KEY=sk-caller" in sent and "sk-platform" not in sent From 0c0a18125c5340dc6260187598e3bdb8cd4931e9 Mon Sep 17 00:00:00 2001 From: Lily Ma Date: Wed, 18 Nov 2026 17:29:01 +0000 Subject: [PATCH 2/3] fix(verifiers): a run's LiteLLM URL only with its own key; say when its key cannot be resolved agent-env's key never goes to an endpoint a run chose: user_overrides.litellm_base_url applies only when the run names its own key. A run key that cannot be resolved is logged as such and never swapped for agent-env's key. --- .../run_container_unit_tests_verifier.py | 15 ++++++++----- .../run_container_unit_tests_verifier_test.py | 21 +++++++++++++++++-- 2 files changed, 29 insertions(+), 7 deletions(-) diff --git a/src/agent_env/task_step/task_steps/verifiers/run_container_unit_tests_verifier.py b/src/agent_env/task_step/task_steps/verifiers/run_container_unit_tests_verifier.py index 9c326ec8e..4ac19d594 100644 --- a/src/agent_env/task_step/task_steps/verifiers/run_container_unit_tests_verifier.py +++ b/src/agent_env/task_step/task_steps/verifiers/run_container_unit_tests_verifier.py @@ -353,19 +353,24 @@ def _merged_env( containers reach it. The key and URL are the run's own (``user_overrides`` ``litellm_api_key`` / ``litellm_base_url``, as ``deploy_agent`` takes them; ``judge_litellm_api_key`` first, as ``rubrics_verifier``) when it names them, else agent-env's config: a run - launched on a caller's key grades on that key and bills it, not the platform's.""" + launched on a caller's key grades on that key and bills it, not the platform's. The run's URL is taken only + with the run's own key, so agent-env's key never goes to an endpoint a run chose.""" overrides = user_overrides or {} merged_env: dict[str, str] = {} + # as rubrics_verifier: a judge key, when the run names one, then the run's own key + run_key = overrides.get("judge_litellm_api_key") or overrides.get("litellm_api_key") try: - # as rubrics_verifier: a judge key, when the run names one, then the run's own key - run_key = overrides.get("judge_litellm_api_key") or overrides.get("litellm_api_key") litellm_key = config.resolve_reference(run_key) if run_key else config.get_litellm_api_key() merged_env["LITELLM_API_KEY"] = litellm_key merged_env["ANTHROPIC_API_KEY"] = litellm_key except (KeyError, ConfigError): - logger.warning("LITELLM_API_KEY unavailable; verifier won't have LLM creds") + if run_key: + # never agent-env's key instead: the run named its own, and would be billed for none of it + logger.warning("this run's LiteLLM key could not be resolved; verifier won't have LLM creds") + else: + logger.warning("LITELLM_API_KEY unavailable; verifier won't have LLM creds") try: - litellm_url = overrides.get("litellm_base_url") or config.get_litellm_base_url() + litellm_url = (overrides.get("litellm_base_url") if run_key else None) or config.get_litellm_base_url() merged_env["LITELLM_BASE_URL"] = litellm_url merged_env["ANTHROPIC_BASE_URL"] = litellm_url except (KeyError, ConfigError): diff --git a/tst/unit/task_step/verifiers/run_container_unit_tests_verifier_test.py b/tst/unit/task_step/verifiers/run_container_unit_tests_verifier_test.py index 2bb13d3d9..656eeeb5b 100644 --- a/tst/unit/task_step/verifiers/run_container_unit_tests_verifier_test.py +++ b/tst/unit/task_step/verifiers/run_container_unit_tests_verifier_test.py @@ -498,9 +498,26 @@ def test_a_run_without_its_own_key_or_url_keeps_the_platforms(no_provider_env): assert env["LITELLM_BASE_URL"] == env["ANTHROPIC_BASE_URL"] == "https://proxy.example" -def test_a_run_endpoint_override_moves_the_url_too(no_provider_env): +def test_a_run_endpoint_override_moves_the_url_with_its_own_key(no_provider_env): + env = _step()._merged_env(_Config(), "modal", {}, + {"litellm_api_key": "sk-caller", "litellm_base_url": "https://other.example"}) + assert (env["LITELLM_API_KEY"], env["LITELLM_BASE_URL"]) == ("sk-caller", "https://other.example") + + +def test_the_platforms_key_never_goes_to_a_url_the_run_chose(no_provider_env): env = _step()._merged_env(_Config(), "modal", {}, {"litellm_base_url": "https://other.example"}) - assert env["LITELLM_BASE_URL"] == "https://other.example" + assert (env["LITELLM_API_KEY"], env["LITELLM_BASE_URL"]) == ("sk-platform", "https://proxy.example") + + +def test_a_run_key_that_cannot_be_resolved_is_said_and_never_swapped_for_the_platforms(no_provider_env, caplog): + class _Unresolvable(_Config): + def resolve_reference(self, value): + raise KeyError(value) + + with caplog.at_level("WARNING"): + env = _step()._merged_env(_Unresolvable(), "modal", {}, {"litellm_api_key": "env:MISSING"}) + assert "LITELLM_API_KEY" not in env and "ANTHROPIC_API_KEY" not in env + assert "this run's LiteLLM key could not be resolved" in caplog.text and "MISSING" not in caplog.text def test_a_key_the_step_itself_sets_still_wins_over_the_runs(no_provider_env): From 0ba781fab7e0fd28c06fec08303022947ef0016f Mon Sep 17 00:00:00 2001 From: Lily Ma Date: Wed, 18 Nov 2026 18:04:18 +0000 Subject: [PATCH 3/3] fix(verifiers): a run's judge key goes to its judge URL, as in rubrics_verifier --- .../run_container_unit_tests_verifier.py | 15 +++++++++++---- .../run_container_unit_tests_verifier_test.py | 17 +++++++++++++++++ 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/src/agent_env/task_step/task_steps/verifiers/run_container_unit_tests_verifier.py b/src/agent_env/task_step/task_steps/verifiers/run_container_unit_tests_verifier.py index 4ac19d594..34212f09d 100644 --- a/src/agent_env/task_step/task_steps/verifiers/run_container_unit_tests_verifier.py +++ b/src/agent_env/task_step/task_steps/verifiers/run_container_unit_tests_verifier.py @@ -351,14 +351,21 @@ def _merged_env( """The commands' environment: the model endpoint's key and URL, every sandbox provider's container env, ``env_vars`` and this run's ``extra_env``, then the endpoint's headers, its URL as ``sandbox_type``'s containers reach it. The key and URL are the run's own (``user_overrides`` ``litellm_api_key`` / - ``litellm_base_url``, as ``deploy_agent`` takes them; ``judge_litellm_api_key`` first, as ``rubrics_verifier``) + ``litellm_base_url``, as ``deploy_agent`` takes them; ``judge_litellm_api_key`` / ``judge_litellm_base_url`` + first, as ``rubrics_verifier``) when it names them, else agent-env's config: a run launched on a caller's key grades on that key and bills it, not the platform's. The run's URL is taken only with the run's own key, so agent-env's key never goes to an endpoint a run chose.""" overrides = user_overrides or {} merged_env: dict[str, str] = {} - # as rubrics_verifier: a judge key, when the run names one, then the run's own key - run_key = overrides.get("judge_litellm_api_key") or overrides.get("litellm_api_key") + # as rubrics_verifier: a judge key and its URL, when the run names one, then the run's own key and URL + judge_key = overrides.get("judge_litellm_api_key") + run_key = judge_key or overrides.get("litellm_api_key") + run_url = ( + (overrides.get("judge_litellm_base_url") or overrides.get("litellm_base_url")) + if judge_key + else overrides.get("litellm_base_url") + ) try: litellm_key = config.resolve_reference(run_key) if run_key else config.get_litellm_api_key() merged_env["LITELLM_API_KEY"] = litellm_key @@ -370,7 +377,7 @@ def _merged_env( else: logger.warning("LITELLM_API_KEY unavailable; verifier won't have LLM creds") try: - litellm_url = (overrides.get("litellm_base_url") if run_key else None) or config.get_litellm_base_url() + litellm_url = (run_url if run_key else None) or config.get_litellm_base_url() merged_env["LITELLM_BASE_URL"] = litellm_url merged_env["ANTHROPIC_BASE_URL"] = litellm_url except (KeyError, ConfigError): diff --git a/tst/unit/task_step/verifiers/run_container_unit_tests_verifier_test.py b/tst/unit/task_step/verifiers/run_container_unit_tests_verifier_test.py index 656eeeb5b..7d44d9340 100644 --- a/tst/unit/task_step/verifiers/run_container_unit_tests_verifier_test.py +++ b/tst/unit/task_step/verifiers/run_container_unit_tests_verifier_test.py @@ -487,6 +487,23 @@ def test_a_runs_judge_key_wins_over_its_agent_key_as_in_the_rubrics_verifier(no_ assert env["LITELLM_API_KEY"] == env["ANTHROPIC_API_KEY"] == "sk-judge" +def test_a_runs_judge_key_goes_to_the_judge_endpoint_not_the_agents(no_provider_env): + env = _step()._merged_env(_Config(), "modal", {}, { + "litellm_api_key": "sk-agent", "litellm_base_url": "https://agent.example", + "judge_litellm_api_key": "sk-judge", "judge_litellm_base_url": "https://judge.example"}) + assert (env["LITELLM_API_KEY"], env["LITELLM_BASE_URL"]) == ("sk-judge", "https://judge.example") + # as in the rubrics verifier: a judge key with no judge URL goes to the run's URL + env = _step()._merged_env(_Config(), "modal", {}, { + "litellm_api_key": "sk-agent", "litellm_base_url": "https://agent.example", "judge_litellm_api_key": "sk-judge"}) + assert (env["LITELLM_API_KEY"], env["LITELLM_BASE_URL"]) == ("sk-judge", "https://agent.example") + + +def test_a_judge_url_without_a_judge_key_never_moves_the_runs_key(no_provider_env): + env = _step()._merged_env(_Config(), "modal", {}, { + "litellm_api_key": "sk-agent", "judge_litellm_base_url": "https://judge.example"}) + assert (env["LITELLM_API_KEY"], env["LITELLM_BASE_URL"]) == ("sk-agent", "https://proxy.example") + + def test_a_run_key_given_as_a_reference_is_resolved(no_provider_env): env = _step()._merged_env(_Config(), "modal", {}, {"litellm_api_key": "env:RESOLVED_sk-ref"}) assert env["LITELLM_API_KEY"] == "sk-ref"