From 0a52ea6f37715c4b1675e604f70097be7dad2a8a Mon Sep 17 00:00:00 2001 From: Sonike <1700162+Sonike@users.noreply.github.com> Date: Fri, 16 Oct 2026 10:28:40 +0800 Subject: [PATCH] fix: separate target credentials from mutation configuration --- .env.example | 13 +- README.md | 9 ++ .../mutiny_core/src/mutiny_core/llm/config.py | 34 +++-- tests/integration/test_llm_env.py | 105 ++++++++++++++ tests/unit/test_llm_config.py | 131 ++++++++++++++++++ 5 files changed, 279 insertions(+), 13 deletions(-) create mode 100644 tests/integration/test_llm_env.py create mode 100644 tests/unit/test_llm_config.py diff --git a/.env.example b/.env.example index f5dda9a..3969de3 100644 --- a/.env.example +++ b/.env.example @@ -1,11 +1,18 @@ # Copy to .env locally — never commit secrets. -# Featherless (OpenAI-compatible) mutation model +# Featherless (OpenAI-compatible) mutation model, separate from the target agent. +# OPENAI_API_KEY alone configures the target, not the Featherless mutator. +# Set a dedicated mutation key here (FEATHERLESS_API_KEY is also accepted). MUTINY_FEATHERLESS_API_KEY= -MUTINY_FEATHERLESS_BASE_URL=https://api.featherless.ai/v1 +# Optional endpoint override; the default is https://api.featherless.ai/v1. +# Setting a non-empty URL explicitly also allows OPENAI_API_KEY as a fallback +# mutation credential. Only opt in when that key belongs to the chosen endpoint. +# MUTINY_FEATHERLESS_BASE_URL=https://api.featherless.ai/v1 MUTINY_MUTATION_MODEL=meta-llama/Meta-Llama-3.1-8B-Instruct +# Timeout must be positive and finite; temperature non-negative and finite. +# Invalid numeric settings disable the mutator (template fallback; health stays up). MUTINY_LLM_TIMEOUT=30 MUTINY_LLM_TEMPERATURE=0.7 # Demo pin (see config/demo_pin.json). Smoke gate uses template fallback when -# MUTINY_FEATHERLESS_API_KEY is empty — that is the default Iris reliability path. +# no mutation key is configured — that is the default Iris reliability path. diff --git a/README.md b/README.md index 909cece..773857b 100644 --- a/README.md +++ b/README.md @@ -236,6 +236,15 @@ uv run mutiny test # replay saved regressions after a finding is saved The sample agent uses a scripted offline model when `OPENAI_API_KEY` is unset — good for local smoke and CI. More detail: [`examples/openai_support_agent/`](./examples/openai_support_agent/). +**Target credentials and mutation credentials are separate.** `OPENAI_API_KEY` +alone does not enable the Featherless mutator. To enable LLM mutations, set +`MUTINY_FEATHERLESS_API_KEY` (or `FEATHERLESS_API_KEY`). A non-empty +`MUTINY_FEATHERLESS_BASE_URL` explicitly opts into a compatible endpoint and +allows `OPENAI_API_KEY` as a fallback mutation key; use a key issued for that +endpoint. Otherwise mutations use templates. Invalid `MUTINY_LLM_TIMEOUT` or +`MUTINY_LLM_TEMPERATURE` values also select templates, and `/api/health` reports +`llm_configured=false` while remaining available. See [`.env.example`](./.env.example). + ### Your own OpenAI Agents SDK project ```bash diff --git a/packages/mutiny_core/src/mutiny_core/llm/config.py b/packages/mutiny_core/src/mutiny_core/llm/config.py index 4f0d9d5..52e5201 100644 --- a/packages/mutiny_core/src/mutiny_core/llm/config.py +++ b/packages/mutiny_core/src/mutiny_core/llm/config.py @@ -2,6 +2,7 @@ from __future__ import annotations +import math import os from dataclasses import dataclass @@ -29,23 +30,36 @@ def load_llm_config_from_env() -> LLMConfig: """Read Featherless / OpenAI-compatible settings from env. Env vars: - MUTINY_FEATHERLESS_API_KEY or FEATHERLESS_API_KEY or OPENAI_API_KEY - MUTINY_FEATHERLESS_BASE_URL (default Featherless) + MUTINY_FEATHERLESS_API_KEY or FEATHERLESS_API_KEY + MUTINY_FEATHERLESS_BASE_URL (default Featherless; an explicit non-empty + value opts into using OPENAI_API_KEY when no dedicated key is set) MUTINY_MUTATION_MODEL MUTINY_LLM_TIMEOUT MUTINY_LLM_TEMPERATURE + + Invalid numeric settings disable the optional mutator. CLI and health callers + can then use their existing template/unconfigured path without raising. """ + explicit_base_url = os.environ.get("MUTINY_FEATHERLESS_BASE_URL", "").strip() api_key = ( - os.environ.get("MUTINY_FEATHERLESS_API_KEY") - or os.environ.get("FEATHERLESS_API_KEY") - or os.environ.get("OPENAI_API_KEY") - ) - base_url = os.environ.get( - "MUTINY_FEATHERLESS_BASE_URL", DEFAULT_FEATHERLESS_BASE_URL + os.environ.get("MUTINY_FEATHERLESS_API_KEY", "").strip() + or os.environ.get("FEATHERLESS_API_KEY", "").strip() + or (os.environ.get("OPENAI_API_KEY", "").strip() if explicit_base_url else None) + or None ) + base_url = explicit_base_url or DEFAULT_FEATHERLESS_BASE_URL model = os.environ.get("MUTINY_MUTATION_MODEL", DEFAULT_MUTATION_MODEL) - timeout = float(os.environ.get("MUTINY_LLM_TIMEOUT", str(DEFAULT_TIMEOUT_SECONDS))) - temperature = float(os.environ.get("MUTINY_LLM_TEMPERATURE", "0.7")) + try: + timeout = float(os.environ.get("MUTINY_LLM_TIMEOUT", str(DEFAULT_TIMEOUT_SECONDS))) + temperature = float(os.environ.get("MUTINY_LLM_TEMPERATURE", "0.7")) + if not math.isfinite(timeout) or timeout <= 0: + raise ValueError("timeout must be positive and finite") + if not math.isfinite(temperature) or temperature < 0: + raise ValueError("temperature must be non-negative and finite") + except ValueError: + api_key = None + timeout = DEFAULT_TIMEOUT_SECONDS + temperature = 0.7 return LLMConfig( api_key=api_key, base_url=base_url.rstrip("/"), diff --git a/tests/integration/test_llm_env.py b/tests/integration/test_llm_env.py new file mode 100644 index 0000000..282671c --- /dev/null +++ b/tests/integration/test_llm_env.py @@ -0,0 +1,105 @@ +"""Bad mutation settings must not break liveness or an offline sample run.""" + +import shutil +from pathlib import Path + +import pytest +from agents import RunConfig, Runner +from fastapi.testclient import TestClient + +from mutiny_api.app import create_app +from mutiny_cli.main import main +from mutiny_core.llm import FeatherlessClient + + +@pytest.fixture(autouse=True) +def isolated_mutation_env(monkeypatch): + for name in ( + "MUTINY_FEATHERLESS_API_KEY", + "FEATHERLESS_API_KEY", + "OPENAI_API_KEY", + "MUTINY_FEATHERLESS_BASE_URL", + "MUTINY_MUTATION_MODEL", + "MUTINY_LLM_TIMEOUT", + "MUTINY_LLM_TEMPERATURE", + ): + monkeypatch.delenv(name, raising=False) + monkeypatch.setenv("MUTINY_SAMPLE_OFFLINE", "1") + + def unexpected_client(self): + pytest.fail("these tests must never create a remote LLM client") + + monkeypatch.setattr(FeatherlessClient, "_get_client", unexpected_client) + + +@pytest.mark.parametrize( + "settings,configured", + [ + ({"OPENAI_API_KEY": "target-key"}, False), + ({"MUTINY_FEATHERLESS_API_KEY": "mutation-key"}, True), + ( + { + "MUTINY_FEATHERLESS_API_KEY": "mutation-key", + "MUTINY_LLM_TIMEOUT": "not-a-float", + }, + False, + ), + ( + { + "MUTINY_FEATHERLESS_API_KEY": "mutation-key", + "MUTINY_LLM_TEMPERATURE": "not-a-float", + }, + False, + ), + ({"MUTINY_LLM_TIMEOUT": "not-a-float"}, False), + ], +) +def test_health_stays_available(tmp_path, monkeypatch, settings, configured): + for name, value in settings.items(): + monkeypatch.setenv(name, value) + with TestClient(create_app(tmp_path / "health.sqlite")) as client: + response = client.get("/api/health") + assert response.status_code == 200 + body = response.json() + assert body["status"] == "ok" + assert body["llm_configured"] is configured + assert body["mutator_mode"] == ("featherless" if configured else "template") + assert "target-key" not in response.text and "mutation-key" not in response.text + + +@pytest.mark.parametrize( + "settings", + [ + {"OPENAI_API_KEY": "target-key"}, + { + "MUTINY_FEATHERLESS_API_KEY": "mutation-key", + "MUTINY_LLM_TIMEOUT": "not-a-float", + }, + { + "MUTINY_FEATHERLESS_API_KEY": "mutation-key", + "MUTINY_LLM_TEMPERATURE": "not-a-float", + }, + ], +) +def test_sample_cli_falls_back_to_templates(tmp_path, monkeypatch, capsys, settings): + source = Path(__file__).resolve().parents[2] / "examples/openai_support_agent" + project = tmp_path / "sample" + shutil.copytree( + source, project, ignore=shutil.ignore_patterns("__pycache__", "tests", "runs") + ) + for name, value in settings.items(): + monkeypatch.setenv(name, value) + # Exercise the real SDK runner and scripted model without trace uploads. + run_sync = Runner.run_sync + + def run_without_tracing(*args, **kwargs): + kwargs["run_config"] = RunConfig(tracing_disabled=True) + return run_sync(*args, **kwargs) + + monkeypatch.setattr(Runner, "run_sync", run_without_tracing) + assert main(["run", "--path", str(project), "--no-hosted"]) == 0 + captured = capsys.readouterr() + assert "mutator: template" in captured.out + assert "Traceback" not in captured.out + captured.err + assert "target-key" not in captured.out + captured.err + assert "mutation-key" not in captured.out + captured.err diff --git a/tests/unit/test_llm_config.py b/tests/unit/test_llm_config.py new file mode 100644 index 0000000..9fbef6a --- /dev/null +++ b/tests/unit/test_llm_config.py @@ -0,0 +1,131 @@ +"""Mutation credentials and numeric configuration must fail closed (#58).""" + +import math +from pathlib import Path + +import pytest + +from mutiny_core.llm import load_llm_config_from_env, try_featherless_from_env +from mutiny_core.llm.config import DEFAULT_FEATHERLESS_BASE_URL + + +@pytest.fixture(autouse=True) +def clean_llm_env(monkeypatch): + for name in ( + "MUTINY_FEATHERLESS_API_KEY", + "FEATHERLESS_API_KEY", + "OPENAI_API_KEY", + "MUTINY_FEATHERLESS_BASE_URL", + "MUTINY_MUTATION_MODEL", + "MUTINY_LLM_TIMEOUT", + "MUTINY_LLM_TEMPERATURE", + ): + monkeypatch.delenv(name, raising=False) + + +def test_example_env_does_not_opt_target_key_into_mutations(monkeypatch): + example = Path(__file__).resolve().parents[2] / ".env.example" + for line in example.read_text().splitlines(): + line = line.strip() + if line and not line.startswith("#"): + name, value = line.split("=", 1) + monkeypatch.setenv(name, value) + monkeypatch.setenv("OPENAI_API_KEY", "target-key") + assert try_featherless_from_env() is None + + +@pytest.mark.parametrize("base_url", [None, "", " "]) +def test_target_key_alone_does_not_configure_mutator(monkeypatch, base_url): + monkeypatch.setenv("OPENAI_API_KEY", "target-key") + if base_url is not None: + monkeypatch.setenv("MUTINY_FEATHERLESS_BASE_URL", base_url) + config = load_llm_config_from_env() + assert not config.configured + assert config.api_key is None + assert try_featherless_from_env() is None + + +@pytest.mark.parametrize( + "key_name", ["MUTINY_FEATHERLESS_API_KEY", "FEATHERLESS_API_KEY"] +) +def test_dedicated_mutation_key_still_configures_featherless(monkeypatch, key_name): + monkeypatch.setenv(key_name, "mutation-key") + monkeypatch.setenv("OPENAI_API_KEY", "target-key") + client = try_featherless_from_env() + assert client is not None + assert client.config.api_key == "mutation-key" + assert client.config.base_url == DEFAULT_FEATHERLESS_BASE_URL + + +def test_dedicated_key_precedence(monkeypatch): + monkeypatch.setenv("MUTINY_FEATHERLESS_API_KEY", "mutiny-key") + monkeypatch.setenv("FEATHERLESS_API_KEY", "provider-key") + monkeypatch.setenv("OPENAI_API_KEY", "target-key") + monkeypatch.setenv("MUTINY_FEATHERLESS_BASE_URL", "https://example.invalid/v1/") + config = load_llm_config_from_env() + assert config.api_key == "mutiny-key" + assert config.base_url == "https://example.invalid/v1" + + +def test_explicit_endpoint_allows_compatible_key(monkeypatch): + monkeypatch.setenv("MUTINY_FEATHERLESS_BASE_URL", "https://example.invalid/v1/") + monkeypatch.setenv("OPENAI_API_KEY", "explicit-compatible-key") + client = try_featherless_from_env() + assert client is not None + assert client.config.api_key == "explicit-compatible-key" + assert client.config.base_url == "https://example.invalid/v1" + + +def test_endpoint_without_key_is_unconfigured(monkeypatch): + monkeypatch.setenv("MUTINY_FEATHERLESS_BASE_URL", "https://example.invalid/v1") + assert try_featherless_from_env() is None + + +@pytest.mark.parametrize( + "key_name", ["MUTINY_FEATHERLESS_API_KEY", "FEATHERLESS_API_KEY"] +) +def test_whitespace_key_does_not_enable_mutator(monkeypatch, key_name): + monkeypatch.setenv(key_name, " ") + monkeypatch.setenv("OPENAI_API_KEY", "target-key") + assert try_featherless_from_env() is None + + +@pytest.mark.parametrize( + "name,value", + [ + ("MUTINY_LLM_TIMEOUT", "not-a-float"), + ("MUTINY_LLM_TIMEOUT", ""), + ("MUTINY_LLM_TIMEOUT", "nan"), + ("MUTINY_LLM_TIMEOUT", "inf"), + ("MUTINY_LLM_TIMEOUT", "0"), + ("MUTINY_LLM_TIMEOUT", "-1"), + ("MUTINY_LLM_TEMPERATURE", "not-a-float"), + ("MUTINY_LLM_TEMPERATURE", ""), + ("MUTINY_LLM_TEMPERATURE", "nan"), + ("MUTINY_LLM_TEMPERATURE", "inf"), + ("MUTINY_LLM_TEMPERATURE", "-1"), + ], +) +@pytest.mark.parametrize("with_key", [False, True]) +def test_invalid_numbers_disable_mutation_client(monkeypatch, name, value, with_key): + if with_key: + monkeypatch.setenv("MUTINY_FEATHERLESS_API_KEY", "mutation-key") + monkeypatch.setenv(name, value) + config = load_llm_config_from_env() + assert not config.configured + assert try_featherless_from_env() is None + assert math.isfinite(config.timeout_seconds) and config.timeout_seconds > 0 + assert math.isfinite(config.temperature) and config.temperature >= 0 + + +@pytest.mark.parametrize("temperature", ["0", "0.7", "1.5"]) +def test_valid_numeric_settings_preserved(monkeypatch, temperature): + monkeypatch.setenv("MUTINY_FEATHERLESS_API_KEY", "mutation-key") + monkeypatch.setenv("MUTINY_MUTATION_MODEL", "configured-model") + monkeypatch.setenv("MUTINY_LLM_TIMEOUT", "2.5") + monkeypatch.setenv("MUTINY_LLM_TEMPERATURE", temperature) + config = load_llm_config_from_env() + assert config.configured + assert config.model == "configured-model" + assert config.timeout_seconds == 2.5 + assert config.temperature == float(temperature)