diff --git a/src/agent_env/task_step/task_steps/verifiers/run_container_unit_tests_verifier.py b/src/agent_env/task_step/task_steps/verifiers/run_container_unit_tests_verifier.py index ab8cb2e96..34212f09d 100644 --- a/src/agent_env/task_step/task_steps/verifiers/run_container_unit_tests_verifier.py +++ b/src/agent_env/task_step/task_steps/verifiers/run_container_unit_tests_verifier.py @@ -235,13 +235,14 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext: config = get_config() command, extra_env = self._resolve_command(context) + user_overrides = context.metadata.get("user_overrides") or {} if self.env_id is not None: target = self._computer(context) - target.env = self._merged_env(config, target.deployed.sandbox_type, extra_env) + target.env = self._merged_env(config, target.deployed.sandbox_type, extra_env, user_overrides) exit_code, stdout, stderr, extracted_files = await self._run_in_computer(target, command) else: sandbox = await self._container_sandbox(context) - merged_env = self._merged_env(config, sandbox.type, extra_env) + merged_env = self._merged_env(config, sandbox.type, extra_env, user_overrides) exit_code, stdout, stderr, extracted_files = await self._run_in_container(sandbox, merged_env, command) timed_out = exit_code in (124, 137) logger.info( @@ -343,19 +344,40 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext: return context - def _merged_env(self, config, sandbox_type: Optional[str], extra_env: dict[str, str]) -> dict[str, str]: - """The commands' environment: the model endpoint's key and URL from agent-env's config, every sandbox - provider's container env, ``env_vars`` and this run's ``extra_env``, then the endpoint's headers, its URL as - ``sandbox_type``'s containers reach it.""" + def _merged_env( + self, config, sandbox_type: Optional[str], extra_env: dict[str, str], + user_overrides: Optional[dict[str, Any]] = None, + ) -> dict[str, str]: + """The commands' environment: the model endpoint's key and URL, every sandbox provider's container env, + ``env_vars`` and this run's ``extra_env``, then the endpoint's headers, its URL as ``sandbox_type``'s + containers reach it. The key and URL are the run's own (``user_overrides`` ``litellm_api_key`` / + ``litellm_base_url``, as ``deploy_agent`` takes them; ``judge_litellm_api_key`` / ``judge_litellm_base_url`` + first, as ``rubrics_verifier``) + when it names them, else agent-env's config: a run + launched on a caller's key grades on that key and bills it, not the platform's. The run's URL is taken only + with the run's own key, so agent-env's key never goes to an endpoint a run chose.""" + overrides = user_overrides or {} merged_env: dict[str, str] = {} + # as rubrics_verifier: a judge key and its URL, when the run names one, then the run's own key and URL + judge_key = overrides.get("judge_litellm_api_key") + run_key = judge_key or overrides.get("litellm_api_key") + run_url = ( + (overrides.get("judge_litellm_base_url") or overrides.get("litellm_base_url")) + if judge_key + else overrides.get("litellm_base_url") + ) try: - litellm_key = config.get_litellm_api_key() + litellm_key = config.resolve_reference(run_key) if run_key else config.get_litellm_api_key() merged_env["LITELLM_API_KEY"] = litellm_key merged_env["ANTHROPIC_API_KEY"] = litellm_key except (KeyError, ConfigError): - logger.warning("LITELLM_API_KEY unavailable; verifier won't have LLM creds") + if run_key: + # never agent-env's key instead: the run named its own, and would be billed for none of it + logger.warning("this run's LiteLLM key could not be resolved; verifier won't have LLM creds") + else: + logger.warning("LITELLM_API_KEY unavailable; verifier won't have LLM creds") try: - litellm_url = config.get_litellm_base_url() + litellm_url = (run_url if run_key else None) or config.get_litellm_base_url() merged_env["LITELLM_BASE_URL"] = litellm_url merged_env["ANTHROPIC_BASE_URL"] = litellm_url except (KeyError, ConfigError): diff --git a/tst/unit/task_step/verifiers/run_container_unit_tests_verifier_test.py b/tst/unit/task_step/verifiers/run_container_unit_tests_verifier_test.py index e93918f9d..7d44d9340 100644 --- a/tst/unit/task_step/verifiers/run_container_unit_tests_verifier_test.py +++ b/tst/unit/task_step/verifiers/run_container_unit_tests_verifier_test.py @@ -453,3 +453,122 @@ def test_a_verifier_for_a_computer_that_cant_clear_the_reward_file_grades_nothin def test_a_verifier_naming_its_target_wrong_is_refused(fields, problem): with pytest.raises(ValueError, match=problem): Step(id="scrape", version=None, command="true", **fields) + + +class _Config: + """The two config reads ``_merged_env`` makes for the model endpoint, and no headers.""" + + def get_litellm_api_key(self): + return "sk-platform" + + def get_litellm_base_url(self): + return "https://proxy.example" + + def resolve_reference(self, value): + return value.removeprefix("env:RESOLVED_") if isinstance(value, str) else value + + def get_model_headers(self, base_url): + return {} + + +@pytest.fixture +def no_provider_env(monkeypatch): + monkeypatch.setattr(verifier_module, "all_sandbox_container_env", lambda: {}) + + +def test_a_run_launched_on_a_callers_key_grades_on_that_key(no_provider_env): + env = _step()._merged_env(_Config(), "modal", {}, {"litellm_api_key": "sk-caller"}) + assert env["LITELLM_API_KEY"] == env["ANTHROPIC_API_KEY"] == "sk-caller" + + +def test_a_runs_judge_key_wins_over_its_agent_key_as_in_the_rubrics_verifier(no_provider_env): + env = _step()._merged_env(_Config(), "modal", {}, + {"litellm_api_key": "sk-agent", "judge_litellm_api_key": "sk-judge"}) + assert env["LITELLM_API_KEY"] == env["ANTHROPIC_API_KEY"] == "sk-judge" + + +def test_a_runs_judge_key_goes_to_the_judge_endpoint_not_the_agents(no_provider_env): + env = _step()._merged_env(_Config(), "modal", {}, { + "litellm_api_key": "sk-agent", "litellm_base_url": "https://agent.example", + "judge_litellm_api_key": "sk-judge", "judge_litellm_base_url": "https://judge.example"}) + assert (env["LITELLM_API_KEY"], env["LITELLM_BASE_URL"]) == ("sk-judge", "https://judge.example") + # as in the rubrics verifier: a judge key with no judge URL goes to the run's URL + env = _step()._merged_env(_Config(), "modal", {}, { + "litellm_api_key": "sk-agent", "litellm_base_url": "https://agent.example", "judge_litellm_api_key": "sk-judge"}) + assert (env["LITELLM_API_KEY"], env["LITELLM_BASE_URL"]) == ("sk-judge", "https://agent.example") + + +def test_a_judge_url_without_a_judge_key_never_moves_the_runs_key(no_provider_env): + env = _step()._merged_env(_Config(), "modal", {}, { + "litellm_api_key": "sk-agent", "judge_litellm_base_url": "https://judge.example"}) + assert (env["LITELLM_API_KEY"], env["LITELLM_BASE_URL"]) == ("sk-agent", "https://proxy.example") + + +def test_a_run_key_given_as_a_reference_is_resolved(no_provider_env): + env = _step()._merged_env(_Config(), "modal", {}, {"litellm_api_key": "env:RESOLVED_sk-ref"}) + assert env["LITELLM_API_KEY"] == "sk-ref" + + +def test_a_run_without_its_own_key_or_url_keeps_the_platforms(no_provider_env): + env = _step()._merged_env(_Config(), "modal", {}, {"step_params": {}}) + assert env["LITELLM_API_KEY"] == "sk-platform" + assert env["LITELLM_BASE_URL"] == env["ANTHROPIC_BASE_URL"] == "https://proxy.example" + + +def test_a_run_endpoint_override_moves_the_url_with_its_own_key(no_provider_env): + env = _step()._merged_env(_Config(), "modal", {}, + {"litellm_api_key": "sk-caller", "litellm_base_url": "https://other.example"}) + assert (env["LITELLM_API_KEY"], env["LITELLM_BASE_URL"]) == ("sk-caller", "https://other.example") + + +def test_the_platforms_key_never_goes_to_a_url_the_run_chose(no_provider_env): + env = _step()._merged_env(_Config(), "modal", {}, {"litellm_base_url": "https://other.example"}) + assert (env["LITELLM_API_KEY"], env["LITELLM_BASE_URL"]) == ("sk-platform", "https://proxy.example") + + +def test_a_run_key_that_cannot_be_resolved_is_said_and_never_swapped_for_the_platforms(no_provider_env, caplog): + class _Unresolvable(_Config): + def resolve_reference(self, value): + raise KeyError(value) + + with caplog.at_level("WARNING"): + env = _step()._merged_env(_Unresolvable(), "modal", {}, {"litellm_api_key": "env:MISSING"}) + assert "LITELLM_API_KEY" not in env and "ANTHROPIC_API_KEY" not in env + assert "this run's LiteLLM key could not be resolved" in caplog.text and "MISSING" not in caplog.text + + +def test_a_key_the_step_itself_sets_still_wins_over_the_runs(no_provider_env): + step = _step(env_vars={"LITELLM_API_KEY": "sk-step"}) + env = step._merged_env(_Config(), "modal", {}, {"litellm_api_key": "sk-caller"}) + assert env["LITELLM_API_KEY"] == "sk-step" + + +def test_the_executed_verifier_hands_the_runs_key_to_its_container(local_stores, monkeypatch): + """Through ``execute``: the hub's worker puts the launcher's decrypted key in ``user_overrides``.""" + monkeypatch.setenv("LITELLM_API_KEY", "sk-platform") + configure() + commands: list[str] = [] + + class _Recording(_Sandbox): + async def exec_script(self, script: str) -> str: + commands.append(script) + return "" + + async def exec_with_output(self, *args) -> tuple[int, str, str]: + commands.extend(a for a in args if isinstance(a, str)) + return 0, "ok", "" + + provider = SimpleNamespace(get_sandbox=lambda sandbox_id: asyncio.sleep(0, result=_Recording())) + monkeypatch.setattr(sandbox_provider, "get_sandbox_provider", lambda: provider) + monkeypatch.setattr(Step, "_upload_text_artifact", staticmethod( + lambda text, artifact_id, description, object_url: SimpleNamespace( + id=artifact_id, version=1, object_url=object_url))) + ctx = TaskStepContext() + ctx.metadata["deployed_docker_containers"] = [{"container_name": "c", "sandbox_name": "h"}] + ctx.deployed_sandboxes.append(DeployedSandbox(sandbox_name="h", sandbox_id="sb-1", sandbox_mode="vm")) + ctx.metadata["user_overrides"] = {"litellm_api_key": "sk-caller"} + + asyncio.run(_step(command="true").execute(ctx)) + + sent = " ".join(commands) + assert "LITELLM_API_KEY=sk-caller" in sent and "sk-platform" not in sent