From b5f6b1aadc81b39757388fe881b43aca20defeb9 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 5 Apr 2026 07:32:10 +0000 Subject: [PATCH 1/3] Suppress noisy httpx/httpcore INFO logging Set httpx and httpcore loggers to WARNING level to prevent "HTTP Request: ..." lines from flooding validator/miner logs. https://claude.ai/code/session_01AhdgboEFFoijNdibDic3Ms --- miner/neuron.py | 2 ++ miner_template/deploy.py | 2 ++ validator/neuron.py | 2 ++ 3 files changed, 6 insertions(+) diff --git a/miner/neuron.py b/miner/neuron.py index 83d9039..43a86f6 100644 --- a/miner/neuron.py +++ b/miner/neuron.py @@ -109,6 +109,8 @@ def get_config() -> bt.Config: def main(): logging.basicConfig(level=logging.INFO, format="%(asctime)s | %(levelname)s | %(message)s") + logging.getLogger("httpx").setLevel(logging.WARNING) + logging.getLogger("httpcore").setLevel(logging.WARNING) config = get_config() miner = Miner(config) asyncio.run(miner.run()) diff --git a/miner_template/deploy.py b/miner_template/deploy.py index ddebc92..9c18e59 100644 --- a/miner_template/deploy.py +++ b/miner_template/deploy.py @@ -35,6 +35,8 @@ def deploy_trainer(image: str) -> str: def main(): logging.basicConfig(level=logging.INFO) + logging.getLogger("httpx").setLevel(logging.WARNING) + logging.getLogger("httpcore").setLevel(logging.WARNING) parser = argparse.ArgumentParser(description="Deploy Radar miner") parser.add_argument("--docker_image", type=str, required=True, help="Agent Docker image (e.g. myregistry/my-agent:v1)") diff --git a/validator/neuron.py b/validator/neuron.py index caf4454..f2b35b2 100644 --- a/validator/neuron.py +++ b/validator/neuron.py @@ -603,6 +603,8 @@ def get_config() -> bt.Config: def main(): logging.basicConfig(level=logging.INFO, format="%(asctime)s | %(levelname)s | %(message)s") + logging.getLogger("httpx").setLevel(logging.WARNING) + logging.getLogger("httpcore").setLevel(logging.WARNING) config = get_config() validator = Validator(config) asyncio.run(validator.run()) From 9bba42ab815a87e0de745fe02945ed399fa33c65 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 5 Apr 2026 07:45:32 +0000 Subject: [PATCH 2/3] Add LLM chat proxy endpoint for miner agents Miner agents were hitting /llm/chat on the DB server and getting 502s because the endpoint didn't exist. This adds: - validator/llm_proxy.py: rate-limited proxy that forwards chat requests to OpenAI-compatible or Anthropic APIs (follows desearch_proxy pattern) - Config entries: RADAR_LLM_ENABLED, RADAR_LLM_PROVIDER, RADAR_LLM_MODEL, RADAR_LLM_BASE_URL (also reads RADAR_CHUTES_API_URL), RADAR_LLM_API_KEY - Challenge.llm_url field so miners know where to call - Wired into validator neuron startup when LLM_ENABLED=true https://claude.ai/code/session_01AhdgboEFFoijNdibDic3Ms --- config.py | 6 + shared/protocol.py | 2 + tests/test_llm_proxy.py | 132 ++++++++++++++++++++++ validator/llm_proxy.py | 241 ++++++++++++++++++++++++++++++++++++++++ validator/neuron.py | 19 ++++ 5 files changed, 400 insertions(+) create mode 100644 tests/test_llm_proxy.py create mode 100644 validator/llm_proxy.py diff --git a/config.py b/config.py index 127c6f0..95fa218 100644 --- a/config.py +++ b/config.py @@ -21,6 +21,12 @@ class Config: # ── LLM Configuration ────────────────────────────────────────────── LLM_MODEL_STRONG: str = os.getenv("RADAR_LLM_MODEL_STRONG", "o3") # for planning + LLM_PROVIDER: str = os.getenv("RADAR_LLM_PROVIDER", "openai") + LLM_MODEL: str = os.getenv("RADAR_LLM_MODEL", "gpt-4.1") + LLM_BASE_URL: str = os.getenv("RADAR_LLM_BASE_URL", "") or os.getenv("RADAR_CHUTES_API_URL", "") + LLM_API_KEY: str = os.getenv("RADAR_LLM_API_KEY", "") + LLM_ENABLED: bool = os.getenv("RADAR_LLM_ENABLED", "false").lower() == "true" + LLM_MAX_REQUESTS: int = int(os.getenv("RADAR_LLM_MAX_REQUESTS", "50")) # ── Desearch Proxy ──────────────────────────────────────────────── # SN22 Desearch endpoint URL diff --git a/shared/protocol.py b/shared/protocol.py index 24dc896..f9d089a 100644 --- a/shared/protocol.py +++ b/shared/protocol.py @@ -24,6 +24,7 @@ class Challenge: task: dict = field(default_factory=dict) db_url: str = "" desearch_url: str = "" + llm_url: str = "" # Pareto front filtered to this round's size bucket # Each entry: {code, metric, objectives} for frontier points in range @@ -40,6 +41,7 @@ def to_json(self) -> str: "task": self.task, "db_url": self.db_url, "desearch_url": self.desearch_url, + "llm_url": self.llm_url, "feasible_frontier": self.feasible_frontier, }) diff --git a/tests/test_llm_proxy.py b/tests/test_llm_proxy.py new file mode 100644 index 0000000..1b35bf9 --- /dev/null +++ b/tests/test_llm_proxy.py @@ -0,0 +1,132 @@ +"""Tests for validator.llm_proxy — rate-limited LLM chat proxy.""" + +import time + +import pytest +from fastapi import FastAPI, HTTPException +from fastapi.testclient import TestClient + +from validator.llm_proxy import ( + ChatRequest, + ChatMessage, + LLMProxy, + register_routes, + set_proxy, +) + + +# ── Unit tests for LLMProxy ──────────────────────────────────────────── + + +class TestLLMProxy: + def test_initial_quota(self): + proxy = LLMProxy(max_requests=50) + assert proxy.remaining_requests(0) == 50 + assert proxy.remaining_requests(1) == 50 + + def test_rate_limiting(self): + proxy = LLMProxy(max_requests=3) + proxy._record_request(0) + assert proxy.remaining_requests(0) == 2 + proxy._record_request(0) + assert proxy.remaining_requests(0) == 1 + proxy._record_request(0) + assert proxy.remaining_requests(0) == 0 + + def test_rate_limit_per_miner(self): + proxy = LLMProxy(max_requests=2) + proxy._record_request(0) + proxy._record_request(0) + assert proxy.remaining_requests(0) == 0 + assert proxy.remaining_requests(1) == 2 + + def test_reset_limits(self): + proxy = LLMProxy(max_requests=2) + proxy._record_request(0) + proxy._record_request(0) + assert proxy.remaining_requests(0) == 0 + proxy.reset_limits() + assert proxy.remaining_requests(0) == 2 + + def test_old_requests_pruned(self): + proxy = LLMProxy(max_requests=2, tempo_seconds=10) + proxy._request_counts[0].append(time.time() - 20) + assert proxy.remaining_requests(0) == 2 + + @pytest.mark.asyncio + async def test_chat_rate_limit_exceeded(self): + proxy = LLMProxy(max_requests=1) + proxy._record_request(0) + req = ChatRequest(messages=[ChatMessage(role="user", content="hello")]) + with pytest.raises(HTTPException) as exc_info: + await proxy.chat(0, req) + assert exc_info.value.status_code == 429 + + def test_default_api_key_openai(self): + proxy = LLMProxy(provider="openai") + # Falls back to OPENAI_API_KEY env var (may be empty) + assert isinstance(proxy.api_key, str) + + def test_default_base_url_openai(self): + proxy = LLMProxy(provider="openai") + assert "openai" in proxy.base_url + + def test_default_base_url_anthropic(self): + proxy = LLMProxy(provider="anthropic") + assert "anthropic" in proxy.base_url + + +# ── Integration tests with FastAPI ────────────────────────────────────────── + + +def _make_app() -> tuple[FastAPI, LLMProxy]: + app = FastAPI() + proxy = LLMProxy(max_requests=5) + set_proxy(proxy) + register_routes(app) + return app, proxy + + +class TestLLMRoutes: + def test_health(self): + app, _ = _make_app() + client = TestClient(app) + r = client.get("/llm/health") + assert r.status_code == 200 + assert r.json()["status"] == "ok" + + def test_quota(self): + app, _ = _make_app() + client = TestClient(app) + r = client.get("/llm/quota", headers={"X-Miner-UID": "0"}) + assert r.status_code == 200 + assert r.json()["remaining_requests"] == 5 + assert r.json()["miner_uid"] == 0 + + def test_missing_miner_uid(self): + app, _ = _make_app() + client = TestClient(app) + r = client.get("/llm/quota") + assert r.status_code == 400 + + def test_invalid_miner_uid(self): + app, _ = _make_app() + client = TestClient(app) + r = client.get("/llm/quota", headers={"X-Miner-UID": "abc"}) + assert r.status_code == 400 + + def test_negative_miner_uid(self): + app, _ = _make_app() + client = TestClient(app) + r = client.get("/llm/quota", headers={"X-Miner-UID": "-1"}) + assert r.status_code == 400 + + def test_chat_validation_empty_messages(self): + app, _ = _make_app() + client = TestClient(app) + r = client.post( + "/llm/chat", + json={"messages": []}, + headers={"X-Miner-UID": "0"}, + ) + assert r.status_code == 422 diff --git a/validator/llm_proxy.py b/validator/llm_proxy.py new file mode 100644 index 0000000..56c1e69 --- /dev/null +++ b/validator/llm_proxy.py @@ -0,0 +1,241 @@ +""" +LLM proxy — HTTP proxy for miners to use a shared LLM endpoint. + +Rate-limited per miner per tempo. Runs as additional routes +on the existing FastAPI DB server. +""" + +from __future__ import annotations + +import logging +import os +import time +from collections import defaultdict +from typing import Optional + +import httpx +from fastapi import FastAPI, HTTPException, Request +from pydantic import BaseModel, Field + +logger = logging.getLogger(__name__) + +# Tempo duration in seconds (~72 min) +TEMPO_DURATION_SECONDS = 360 * 12 + + +class ChatMessage(BaseModel): + role: str = "user" + content: str = "" + + +class ChatRequest(BaseModel): + """Chat completion request from a miner.""" + + messages: list[ChatMessage] = Field(..., min_length=1) + model: str = "" + temperature: float = Field(default=0.7, ge=0.0, le=2.0) + max_tokens: int = Field(default=4096, ge=1, le=16384) + + +class ChatResponse(BaseModel): + """Chat completion response returned to miners.""" + + content: str = "" + model: str = "" + remaining_requests: int = 0 + + +class LLMProxy: + """ + Rate-limited proxy for LLM chat completions. + + Each miner gets max_requests queries per tempo window. + Supports OpenAI-compatible and Anthropic APIs. + """ + + def __init__( + self, + provider: str = "openai", + model: str = "gpt-4.1", + api_key: str = "", + base_url: str = "", + max_requests: int = 50, + tempo_seconds: int = TEMPO_DURATION_SECONDS, + ): + self.provider = provider + self.model = model + self.api_key = api_key or self._default_api_key() + self.base_url = base_url or self._default_base_url() + self.max_requests = max_requests + self.tempo_seconds = tempo_seconds + + self._request_counts: dict[int, list[float]] = defaultdict(list) + self._client: Optional[httpx.AsyncClient] = None + + def _default_api_key(self) -> str: + if self.provider == "anthropic": + return os.getenv("ANTHROPIC_API_KEY", "") + return os.getenv("OPENAI_API_KEY", "") + + def _default_base_url(self) -> str: + if self.provider == "anthropic": + return "https://api.anthropic.com" + return "https://api.openai.com/v1" + + async def _get_client(self) -> httpx.AsyncClient: + if self._client is None or self._client.is_closed: + self._client = httpx.AsyncClient(timeout=60.0) + return self._client + + def _prune_old_requests(self, miner_uid: int) -> int: + cutoff = time.time() - self.tempo_seconds + timestamps = self._request_counts[miner_uid] + self._request_counts[miner_uid] = [t for t in timestamps if t > cutoff] + return len(self._request_counts[miner_uid]) + + def remaining_requests(self, miner_uid: int) -> int: + used = self._prune_old_requests(miner_uid) + return max(0, self.max_requests - used) + + def _record_request(self, miner_uid: int): + self._request_counts[miner_uid].append(time.time()) + + async def chat(self, miner_uid: int, req: ChatRequest) -> ChatResponse: + """Forward a chat completion request to the LLM provider.""" + remaining = self.remaining_requests(miner_uid) + if remaining <= 0: + raise HTTPException( + status_code=429, + detail=f"Rate limit exceeded. Max {self.max_requests} requests per tempo.", + ) + + self._record_request(miner_uid) + remaining -= 1 + model = req.model or self.model + + try: + if self.provider == "anthropic": + result = await self._call_anthropic(req, model) + else: + result = await self._call_openai(req, model) + except httpx.HTTPStatusError as e: + logger.warning("LLM upstream error: %s", e.response.status_code) + raise HTTPException(status_code=502, detail="LLM upstream error") + except (httpx.RequestError, httpx.TimeoutException) as e: + logger.warning("LLM request failed: %s", e) + raise HTTPException(status_code=502, detail="LLM upstream unreachable") + + return ChatResponse( + content=result, model=model, remaining_requests=remaining, + ) + + async def _call_openai(self, req: ChatRequest, model: str) -> str: + client = await self._get_client() + resp = await client.post( + f"{self.base_url}/chat/completions", + headers={ + "Authorization": f"Bearer {self.api_key}", + "Content-Type": "application/json", + }, + json={ + "model": model, + "messages": [{"role": m.role, "content": m.content} for m in req.messages], + "temperature": req.temperature, + "max_tokens": req.max_tokens, + }, + ) + resp.raise_for_status() + data = resp.json() + return data["choices"][0]["message"]["content"] + + async def _call_anthropic(self, req: ChatRequest, model: str) -> str: + client = await self._get_client() + # Extract system message if present + system = "" + messages = [] + for m in req.messages: + if m.role == "system": + system = m.content + else: + messages.append({"role": m.role, "content": m.content}) + body: dict = { + "model": model, + "messages": messages, + "max_tokens": req.max_tokens, + "temperature": req.temperature, + } + if system: + body["system"] = system + resp = await client.post( + f"{self.base_url}/v1/messages", + headers={ + "x-api-key": self.api_key, + "anthropic-version": "2023-06-01", + "Content-Type": "application/json", + }, + json=body, + ) + resp.raise_for_status() + data = resp.json() + return data["content"][0]["text"] + + async def close(self): + if self._client and not self._client.is_closed: + await self._client.aclose() + + def reset_limits(self): + self._request_counts.clear() + + +# ── FastAPI routes ────────────────────────────────────────────────────────── + +_proxy: Optional[LLMProxy] = None + + +def set_proxy(proxy: LLMProxy): + global _proxy + _proxy = proxy + + +def get_proxy() -> LLMProxy: + if _proxy is None: + raise HTTPException(status_code=503, detail="LLM proxy not initialized") + return _proxy + + +def register_routes(app: FastAPI): + """Register LLM proxy routes on the given FastAPI app.""" + + @app.post("/llm/chat") + async def llm_chat(req: ChatRequest, request: Request): + """Chat completion proxy. Requires X-Miner-UID header.""" + proxy = get_proxy() + miner_uid = _extract_miner_uid(request) + return await proxy.chat(miner_uid, req) + + @app.get("/llm/quota") + def llm_quota(request: Request): + """Check remaining LLM request quota for this miner.""" + proxy = get_proxy() + miner_uid = _extract_miner_uid(request) + return { + "miner_uid": miner_uid, + "remaining_requests": proxy.remaining_requests(miner_uid), + } + + @app.get("/llm/health") + def llm_health(): + return {"status": "ok", "proxy_initialized": _proxy is not None} + + +def _extract_miner_uid(request: Request) -> int: + uid_str = request.headers.get("X-Miner-UID", "") + if not uid_str: + raise HTTPException(status_code=400, detail="Missing X-Miner-UID header") + try: + uid = int(uid_str) + except ValueError: + raise HTTPException(status_code=400, detail="Invalid X-Miner-UID header") + if uid < 0: + raise HTTPException(status_code=400, detail="Invalid miner UID") + return uid diff --git a/validator/neuron.py b/validator/neuron.py index f2b35b2..a8035bb 100644 --- a/validator/neuron.py +++ b/validator/neuron.py @@ -137,6 +137,22 @@ def __init__(self, config: bt.Config): set_proxy(self.desearch_proxy) register_routes(db_app) + # LLM proxy + self.llm_proxy = None + if Config.LLM_ENABLED: + from validator.llm_proxy import LLMProxy + from validator.llm_proxy import set_proxy as set_llm_proxy + from validator.llm_proxy import register_routes as register_llm_routes + self.llm_proxy = LLMProxy( + provider=Config.LLM_PROVIDER, + model=Config.LLM_MODEL, + api_key=Config.LLM_API_KEY, + base_url=Config.LLM_BASE_URL, + max_requests=Config.LLM_MAX_REQUESTS, + ) + set_llm_proxy(self.llm_proxy) + register_llm_routes(db_app) + # Access logger — shares the same SQLite connection self.access_logger = AccessLogger(conn=self.db.conn) set_access_logger(self.access_logger) @@ -260,6 +276,9 @@ async def run_round(self): challenge.desearch_url = ( f"http://localhost:{self.db_port}/desearch" if self.desearch_proxy else "" ) + challenge.llm_url = ( + f"http://localhost:{self.db_port}/llm" if self.llm_proxy else "" + ) # Set challenge on DB server for miners to query set_challenge(challenge.to_json()) From 036b3c52296ed35d318ddb2677b11367b4e0f645 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 5 Apr 2026 07:49:49 +0000 Subject: [PATCH 3/3] Use external DB URL for challenge URLs sent to remote miners Miner agents run on remote Basilica pods and can't reach localhost. When RADAR_DB_API_URL is set, use it as the base URL for db_url, desearch_url, and llm_url in the Challenge sent to miners. https://claude.ai/code/session_01AhdgboEFFoijNdibDic3Ms --- config.py | 4 ++++ validator/neuron.py | 7 ++++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/config.py b/config.py index 95fa218..dff7e5d 100644 --- a/config.py +++ b/config.py @@ -70,5 +70,9 @@ class Config: EMA_ALPHA: float = float(os.getenv("RADAR_EMA_ALPHA", "0.3")) SOFTMAX_TEMPERATURE: float = float(os.getenv("RADAR_SOFTMAX_TEMP", "0.1")) + # ── External URLs (for miner agents on remote pods) ──────── + DB_API_URL: str = os.getenv("RADAR_DB_API_URL", "") + VALIDATOR_EXTERNAL_URL: str = os.getenv("RADAR_VALIDATOR_EXTERNAL_URL", "") + # ── Query API ──────────────────────────────────────────────── QUERY_RATE_LIMIT: int = int(os.getenv("RADAR_QUERY_RATE_LIMIT", "10")) diff --git a/validator/neuron.py b/validator/neuron.py index a8035bb..e9ca208 100644 --- a/validator/neuron.py +++ b/validator/neuron.py @@ -272,12 +272,13 @@ async def run_round(self): } set_hotkey_map(hotkey_map) - challenge.db_url = f"http://localhost:{self.db_port}" + base_url = Config.DB_API_URL or f"http://localhost:{self.db_port}" + challenge.db_url = base_url challenge.desearch_url = ( - f"http://localhost:{self.db_port}/desearch" if self.desearch_proxy else "" + f"{base_url}/desearch" if self.desearch_proxy else "" ) challenge.llm_url = ( - f"http://localhost:{self.db_port}/llm" if self.llm_proxy else "" + f"{base_url}/llm" if self.llm_proxy else "" ) # Set challenge on DB server for miners to query