fix: separate target credentials from mutation configuration

.env.example

@@ -1,11 +1,18 @@
 # Copy to .env locally — never commit secrets.
-# Featherless (OpenAI-compatible) mutation model
+# Featherless (OpenAI-compatible) mutation model, separate from the target agent.
+# OPENAI_API_KEY alone configures the target, not the Featherless mutator.
+# Set a dedicated mutation key here (FEATHERLESS_API_KEY is also accepted).
 
 MUTINY_FEATHERLESS_API_KEY=
-MUTINY_FEATHERLESS_BASE_URL=https://api.featherless.ai/v1
+# Optional endpoint override; the default is https://api.featherless.ai/v1.
+# Setting a non-empty URL explicitly also allows OPENAI_API_KEY as a fallback
+# mutation credential. Only opt in when that key belongs to the chosen endpoint.
+# MUTINY_FEATHERLESS_BASE_URL=https://api.featherless.ai/v1
 MUTINY_MUTATION_MODEL=meta-llama/Meta-Llama-3.1-8B-Instruct
+# Timeout must be positive and finite; temperature non-negative and finite.
+# Invalid numeric settings disable the mutator (template fallback; health stays up).
 MUTINY_LLM_TIMEOUT=30
 MUTINY_LLM_TEMPERATURE=0.7
 
 # Demo pin (see config/demo_pin.json). Smoke gate uses template fallback when
-# MUTINY_FEATHERLESS_API_KEY is empty — that is the default Iris reliability path.
+# no mutation key is configured — that is the default Iris reliability path.

README.md

@@ -236,6 +236,15 @@ uv run mutiny test    # replay saved regressions after a finding is saved
 
 The sample agent uses a scripted offline model when `OPENAI_API_KEY` is unset — good for local smoke and CI. More detail: [`examples/openai_support_agent/`](./examples/openai_support_agent/).
 
+**Target credentials and mutation credentials are separate.** `OPENAI_API_KEY`
+alone does not enable the Featherless mutator. To enable LLM mutations, set
+`MUTINY_FEATHERLESS_API_KEY` (or `FEATHERLESS_API_KEY`). A non-empty
+`MUTINY_FEATHERLESS_BASE_URL` explicitly opts into a compatible endpoint and
+allows `OPENAI_API_KEY` as a fallback mutation key; use a key issued for that
+endpoint. Otherwise mutations use templates. Invalid `MUTINY_LLM_TIMEOUT` or
+`MUTINY_LLM_TEMPERATURE` values also select templates, and `/api/health` reports
+`llm_configured=false` while remaining available. See [`.env.example`](./.env.example).
+
 ### Your own OpenAI Agents SDK project
 
 ```bash

packages/mutiny_core/src/mutiny_core/llm/config.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+import math
 import os
 from dataclasses import dataclass
 
@@ -29,23 +30,36 @@ def load_llm_config_from_env() -> LLMConfig:
     """Read Featherless / OpenAI-compatible settings from env.
 
     Env vars:
-      MUTINY_FEATHERLESS_API_KEY or FEATHERLESS_API_KEY or OPENAI_API_KEY
-      MUTINY_FEATHERLESS_BASE_URL (default Featherless)
+      MUTINY_FEATHERLESS_API_KEY or FEATHERLESS_API_KEY
+      MUTINY_FEATHERLESS_BASE_URL (default Featherless; an explicit non-empty
+        value opts into using OPENAI_API_KEY when no dedicated key is set)
       MUTINY_MUTATION_MODEL
       MUTINY_LLM_TIMEOUT
       MUTINY_LLM_TEMPERATURE
+
+    Invalid numeric settings disable the optional mutator. CLI and health callers
+    can then use their existing template/unconfigured path without raising.
     """
+    explicit_base_url = os.environ.get("MUTINY_FEATHERLESS_BASE_URL", "").strip()
     api_key = (
-        os.environ.get("MUTINY_FEATHERLESS_API_KEY")
-        or os.environ.get("FEATHERLESS_API_KEY")
-        or os.environ.get("OPENAI_API_KEY")
-    )
-    base_url = os.environ.get(
-        "MUTINY_FEATHERLESS_BASE_URL", DEFAULT_FEATHERLESS_BASE_URL
+        os.environ.get("MUTINY_FEATHERLESS_API_KEY", "").strip()
+        or os.environ.get("FEATHERLESS_API_KEY", "").strip()
+        or (os.environ.get("OPENAI_API_KEY", "").strip() if explicit_base_url else None)
+        or None
     )
+    base_url = explicit_base_url or DEFAULT_FEATHERLESS_BASE_URL
     model = os.environ.get("MUTINY_MUTATION_MODEL", DEFAULT_MUTATION_MODEL)
-    timeout = float(os.environ.get("MUTINY_LLM_TIMEOUT", str(DEFAULT_TIMEOUT_SECONDS)))
-    temperature = float(os.environ.get("MUTINY_LLM_TEMPERATURE", "0.7"))
+    try:
+        timeout = float(os.environ.get("MUTINY_LLM_TIMEOUT", str(DEFAULT_TIMEOUT_SECONDS)))
+        temperature = float(os.environ.get("MUTINY_LLM_TEMPERATURE", "0.7"))
+        if not math.isfinite(timeout) or timeout <= 0:
+            raise ValueError("timeout must be positive and finite")
+        if not math.isfinite(temperature) or temperature < 0:
+            raise ValueError("temperature must be non-negative and finite")
+    except ValueError:
+        api_key = None
+        timeout = DEFAULT_TIMEOUT_SECONDS
+        temperature = 0.7
     return LLMConfig(
         api_key=api_key,
         base_url=base_url.rstrip("/"),

tests/integration/test_llm_env.py

@@ -0,0 +1,105 @@
+"""Bad mutation settings must not break liveness or an offline sample run."""
+
+import shutil
+from pathlib import Path
+
+import pytest
+from agents import RunConfig, Runner
+from fastapi.testclient import TestClient
+
+from mutiny_api.app import create_app
+from mutiny_cli.main import main
+from mutiny_core.llm import FeatherlessClient
+
+
+@pytest.fixture(autouse=True)
+def isolated_mutation_env(monkeypatch):
+    for name in (
+        "MUTINY_FEATHERLESS_API_KEY",
+        "FEATHERLESS_API_KEY",
+        "OPENAI_API_KEY",
+        "MUTINY_FEATHERLESS_BASE_URL",
+        "MUTINY_MUTATION_MODEL",
+        "MUTINY_LLM_TIMEOUT",
+        "MUTINY_LLM_TEMPERATURE",
+    ):
+        monkeypatch.delenv(name, raising=False)
+    monkeypatch.setenv("MUTINY_SAMPLE_OFFLINE", "1")
+
+    def unexpected_client(self):
+        pytest.fail("these tests must never create a remote LLM client")
+
+    monkeypatch.setattr(FeatherlessClient, "_get_client", unexpected_client)
+
+
+@pytest.mark.parametrize(
+    "settings,configured",
+    [
+        ({"OPENAI_API_KEY": "target-key"}, False),
+        ({"MUTINY_FEATHERLESS_API_KEY": "mutation-key"}, True),
+        (
+            {
+                "MUTINY_FEATHERLESS_API_KEY": "mutation-key",
+                "MUTINY_LLM_TIMEOUT": "not-a-float",
+            },
+            False,
+        ),
+        (
+            {
+                "MUTINY_FEATHERLESS_API_KEY": "mutation-key",
+                "MUTINY_LLM_TEMPERATURE": "not-a-float",
+            },
+            False,
+        ),
+        ({"MUTINY_LLM_TIMEOUT": "not-a-float"}, False),
+    ],
+)
+def test_health_stays_available(tmp_path, monkeypatch, settings, configured):
+    for name, value in settings.items():
+        monkeypatch.setenv(name, value)
+    with TestClient(create_app(tmp_path / "health.sqlite")) as client:
+        response = client.get("/api/health")
+    assert response.status_code == 200
+    body = response.json()
+    assert body["status"] == "ok"
+    assert body["llm_configured"] is configured
+    assert body["mutator_mode"] == ("featherless" if configured else "template")
+    assert "target-key" not in response.text and "mutation-key" not in response.text
+
+
+@pytest.mark.parametrize(
+    "settings",
+    [
+        {"OPENAI_API_KEY": "target-key"},
+        {
+            "MUTINY_FEATHERLESS_API_KEY": "mutation-key",
+            "MUTINY_LLM_TIMEOUT": "not-a-float",
+        },
+        {
+            "MUTINY_FEATHERLESS_API_KEY": "mutation-key",
+            "MUTINY_LLM_TEMPERATURE": "not-a-float",
+        },
+    ],
+)
+def test_sample_cli_falls_back_to_templates(tmp_path, monkeypatch, capsys, settings):
+    source = Path(__file__).resolve().parents[2] / "examples/openai_support_agent"
+    project = tmp_path / "sample"
+    shutil.copytree(
+        source, project, ignore=shutil.ignore_patterns("__pycache__", "tests", "runs")
+    )
+    for name, value in settings.items():
+        monkeypatch.setenv(name, value)
+    # Exercise the real SDK runner and scripted model without trace uploads.
+    run_sync = Runner.run_sync
+
+    def run_without_tracing(*args, **kwargs):
+        kwargs["run_config"] = RunConfig(tracing_disabled=True)
+        return run_sync(*args, **kwargs)
+
+    monkeypatch.setattr(Runner, "run_sync", run_without_tracing)
+    assert main(["run", "--path", str(project), "--no-hosted"]) == 0
+    captured = capsys.readouterr()
+    assert "mutator: template" in captured.out
+    assert "Traceback" not in captured.out + captured.err
+    assert "target-key" not in captured.out + captured.err
+    assert "mutation-key" not in captured.out + captured.err

tests/unit/test_llm_config.py

@@ -0,0 +1,131 @@
+"""Mutation credentials and numeric configuration must fail closed (#58)."""
+
+import math
+from pathlib import Path
+
+import pytest
+
+from mutiny_core.llm import load_llm_config_from_env, try_featherless_from_env
+from mutiny_core.llm.config import DEFAULT_FEATHERLESS_BASE_URL
+
+
+@pytest.fixture(autouse=True)
+def clean_llm_env(monkeypatch):
+    for name in (
+        "MUTINY_FEATHERLESS_API_KEY",
+        "FEATHERLESS_API_KEY",
+        "OPENAI_API_KEY",
+        "MUTINY_FEATHERLESS_BASE_URL",
+        "MUTINY_MUTATION_MODEL",
+        "MUTINY_LLM_TIMEOUT",
+        "MUTINY_LLM_TEMPERATURE",
+    ):
+        monkeypatch.delenv(name, raising=False)
+
+
+def test_example_env_does_not_opt_target_key_into_mutations(monkeypatch):
+    example = Path(__file__).resolve().parents[2] / ".env.example"
+    for line in example.read_text().splitlines():
+        line = line.strip()
+        if line and not line.startswith("#"):
+            name, value = line.split("=", 1)
+            monkeypatch.setenv(name, value)
+    monkeypatch.setenv("OPENAI_API_KEY", "target-key")
+    assert try_featherless_from_env() is None
+
+
+@pytest.mark.parametrize("base_url", [None, "", "   "])
+def test_target_key_alone_does_not_configure_mutator(monkeypatch, base_url):
+    monkeypatch.setenv("OPENAI_API_KEY", "target-key")
+    if base_url is not None:
+        monkeypatch.setenv("MUTINY_FEATHERLESS_BASE_URL", base_url)
+    config = load_llm_config_from_env()
+    assert not config.configured
+    assert config.api_key is None
+    assert try_featherless_from_env() is None
+
+
+@pytest.mark.parametrize(
+    "key_name", ["MUTINY_FEATHERLESS_API_KEY", "FEATHERLESS_API_KEY"]
+)
+def test_dedicated_mutation_key_still_configures_featherless(monkeypatch, key_name):
+    monkeypatch.setenv(key_name, "mutation-key")
+    monkeypatch.setenv("OPENAI_API_KEY", "target-key")
+    client = try_featherless_from_env()
+    assert client is not None
+    assert client.config.api_key == "mutation-key"
+    assert client.config.base_url == DEFAULT_FEATHERLESS_BASE_URL
+
+
+def test_dedicated_key_precedence(monkeypatch):
+    monkeypatch.setenv("MUTINY_FEATHERLESS_API_KEY", "mutiny-key")
+    monkeypatch.setenv("FEATHERLESS_API_KEY", "provider-key")
+    monkeypatch.setenv("OPENAI_API_KEY", "target-key")
+    monkeypatch.setenv("MUTINY_FEATHERLESS_BASE_URL", "https://example.invalid/v1/")
+    config = load_llm_config_from_env()
+    assert config.api_key == "mutiny-key"
+    assert config.base_url == "https://example.invalid/v1"
+
+
+def test_explicit_endpoint_allows_compatible_key(monkeypatch):
+    monkeypatch.setenv("MUTINY_FEATHERLESS_BASE_URL", "https://example.invalid/v1/")
+    monkeypatch.setenv("OPENAI_API_KEY", "explicit-compatible-key")
+    client = try_featherless_from_env()
+    assert client is not None
+    assert client.config.api_key == "explicit-compatible-key"
+    assert client.config.base_url == "https://example.invalid/v1"
+
+
+def test_endpoint_without_key_is_unconfigured(monkeypatch):
+    monkeypatch.setenv("MUTINY_FEATHERLESS_BASE_URL", "https://example.invalid/v1")
+    assert try_featherless_from_env() is None
+
+
+@pytest.mark.parametrize(
+    "key_name", ["MUTINY_FEATHERLESS_API_KEY", "FEATHERLESS_API_KEY"]
+)
+def test_whitespace_key_does_not_enable_mutator(monkeypatch, key_name):
+    monkeypatch.setenv(key_name, "   ")
+    monkeypatch.setenv("OPENAI_API_KEY", "target-key")
+    assert try_featherless_from_env() is None
+
+
+@pytest.mark.parametrize(
+    "name,value",
+    [
+        ("MUTINY_LLM_TIMEOUT", "not-a-float"),
+        ("MUTINY_LLM_TIMEOUT", ""),
+        ("MUTINY_LLM_TIMEOUT", "nan"),
+        ("MUTINY_LLM_TIMEOUT", "inf"),
+        ("MUTINY_LLM_TIMEOUT", "0"),
+        ("MUTINY_LLM_TIMEOUT", "-1"),
+        ("MUTINY_LLM_TEMPERATURE", "not-a-float"),
+        ("MUTINY_LLM_TEMPERATURE", ""),
+        ("MUTINY_LLM_TEMPERATURE", "nan"),
+        ("MUTINY_LLM_TEMPERATURE", "inf"),
+        ("MUTINY_LLM_TEMPERATURE", "-1"),
+    ],
+)
+@pytest.mark.parametrize("with_key", [False, True])
+def test_invalid_numbers_disable_mutation_client(monkeypatch, name, value, with_key):
+    if with_key:
+        monkeypatch.setenv("MUTINY_FEATHERLESS_API_KEY", "mutation-key")
+    monkeypatch.setenv(name, value)
+    config = load_llm_config_from_env()
+    assert not config.configured
+    assert try_featherless_from_env() is None
+    assert math.isfinite(config.timeout_seconds) and config.timeout_seconds > 0
+    assert math.isfinite(config.temperature) and config.temperature >= 0
+
+
+@pytest.mark.parametrize("temperature", ["0", "0.7", "1.5"])
+def test_valid_numeric_settings_preserved(monkeypatch, temperature):
+    monkeypatch.setenv("MUTINY_FEATHERLESS_API_KEY", "mutation-key")
+    monkeypatch.setenv("MUTINY_MUTATION_MODEL", "configured-model")
+    monkeypatch.setenv("MUTINY_LLM_TIMEOUT", "2.5")
+    monkeypatch.setenv("MUTINY_LLM_TEMPERATURE", temperature)
+    config = load_llm_config_from_env()
+    assert config.configured
+    assert config.model == "configured-model"
+    assert config.timeout_seconds == 2.5
+    assert config.temperature == float(temperature)