diff --git a/.gitignore b/.gitignore index af20327..05dc5e4 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,5 @@ __pycache__/ experiments/ *.backup data/ +logs/ +prompts/ diff --git a/CLAUDE.md b/CLAUDE.md index 75e3ff3..0223d86 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -259,6 +259,33 @@ add `HIPPIUS_*` once your Hippius keys are issued — the client will pick up the new backend automatically. To cut over fully, also set `HIPPIUS_BUCKET` (or unset `R2_BUCKET`) so dispatched URLs point at Hippius. +## Non-competitive Mode + +Set `RADAR_NONCOMPETITIVE=true` for a non-subnet deployment: + +- Validator skips wallet, subtensor, metagraph, weight-setting, on-chain + commitments. Existing competitive deployments leave it false. +- Auth uses HMAC service key (`RADAR_SERVICE_KEY`) for validator ↔ trainer + ↔ DB. Headers: `X-Radar-{Signature,Timestamp,Key-Id}`. +- Miner identity is bearer tokens issued via the operator CLI: + ``` + python -m database.operator_cli register --name alice [--hotkey ss58] + python -m database.operator_cli issue-key --miner-id --label prod + python -m database.operator_cli rotate-service-key + python -m database.operator_cli list-miners | list-keys --miner-id + python -m database.operator_cli revoke-key --key-id + ``` +- Miners pull scored history + evolve prompts locally via the miner CLI: + ``` + RADAR_DB_URL=http://db:8090 RADAR_MINER_API_KEY=rdrk_... \ + python miner/neuron.py optimize --optimizer gepa --seed --watch + python miner/neuron.py results --json + python miner/neuron.py prompts list | history | rollback + ``` +- `prompts/active.json` (atomic-write) drives which prompt variant the + agent picks each round; `prompt_id` round-trips back via + `experiments.prompt_id` so Phase C scores attribute to the variant. + ## Neuron Mode Environment Variables | Var | Default | Purpose | diff --git a/config.py b/config.py index e6d55d4..4d44c3c 100644 --- a/config.py +++ b/config.py @@ -410,6 +410,23 @@ class Config: # all — Everything on one process (legacy / dev default). NEURON_MODE: str = os.getenv("RADAR_NEURON_MODE", "all").lower() + # ── Non-competitive mode ──────────────────────────────────── + # When true, validators + miners SKIP all chain interactions: + # * no metagraph sync, no Subtensor connection, no wallet + # * no weight setting, no on-chain commitments + # * outbound auth uses HMAC service key (RADAR_SERVICE_KEY) + # * miner identity comes from bearer API keys, not hotkeys + # Existing competitive deployments leave this false and run unchanged. + NONCOMPETITIVE: bool = os.getenv( + "RADAR_NONCOMPETITIVE", "false", + ).lower() in ("1", "true", "yes") + + # Shared HMAC secret for service-to-service auth in non-competitive + # mode (validator -> trainer, validator -> DB). Required when + # NONCOMPETITIVE is true. + SERVICE_KEY: str = os.getenv("RADAR_SERVICE_KEY", "") + SERVICE_KEY_ID: str = os.getenv("RADAR_SERVICE_KEY_ID", "operator") + # CORS origins for the public JSON API. Comma-separated, empty = no CORS # (safe default for validator mode, which doesn't speak to browsers). # In production dashboard mode set this to "https://radarnet.io". diff --git a/database/migrations/005_miner_registry_and_prompt_id.sql b/database/migrations/005_miner_registry_and_prompt_id.sql new file mode 100644 index 0000000..55d17a4 --- /dev/null +++ b/database/migrations/005_miner_registry_and_prompt_id.sql @@ -0,0 +1,77 @@ +-- Miner registry + bearer API keys for the non-competitive cutover, +-- plus the ``experiments.prompt_id`` column that closes the GEPA +-- feedback loop on the miner CLI. +-- +-- Non-competitive deployments use operator-issued bearer tokens +-- (rdrk_*) in place of Epistula/metagraph identity. A ``miner_id`` +-- is an opaque operator-issued identifier (UUID by default) that +-- decouples the feedback API from any on-chain hotkey. During the +-- dual-stack period a miner row carries both: ``hotkey`` is the +-- bittensor SS58 (nullable, only set for chain-registered miners) +-- and ``miner_id`` is the bearer-auth identifier scoped to +-- /miners/me/*. +-- +-- ``prompt_id`` on experiments lets miners correlate Phase C scores +-- with the prompt variant that produced the architecture. Opaque +-- to the operator; only the issuing miner interprets it. Two +-- partial indexes keep the table footprint sparse for rows without +-- a prompt_id (every existing row + every competitive-mode write). +-- +-- All DDL is idempotent — safe under partial re-runs. The +-- experiments ALTER + its indexes are wrapped in an information_schema +-- guard (same pattern as 002_substrate_cids.sql) because CI exercises +-- the migration runner against an empty schema before init_schema() +-- has bootstrapped the experiments table. On a fresh deploy the base +-- CREATE TABLE in shared/pg_schema.py already declares prompt_id, so +-- the guarded block is a no-op there. + +CREATE TABLE IF NOT EXISTS CURRENT_SCHEMA.miners ( + miner_id TEXT PRIMARY KEY, + name TEXT NOT NULL DEFAULT '', + hotkey TEXT NOT NULL DEFAULT '', + contact TEXT NOT NULL DEFAULT '', + created_at DOUBLE PRECISION NOT NULL DEFAULT extract(epoch from now()), + revoked_at DOUBLE PRECISION +); + +CREATE INDEX IF NOT EXISTS idx_miners_hotkey + ON CURRENT_SCHEMA.miners(hotkey) + WHERE hotkey <> ''; + +CREATE TABLE IF NOT EXISTS CURRENT_SCHEMA.miner_api_keys ( + key_id TEXT PRIMARY KEY, + key_hash TEXT NOT NULL, + miner_id TEXT NOT NULL REFERENCES CURRENT_SCHEMA.miners(miner_id) + ON DELETE CASCADE, + scope TEXT NOT NULL DEFAULT 'miner', + label TEXT NOT NULL DEFAULT '', + created_at DOUBLE PRECISION NOT NULL DEFAULT extract(epoch from now()), + last_used_at DOUBLE PRECISION, + revoked_at DOUBLE PRECISION +); + +CREATE INDEX IF NOT EXISTS idx_miner_api_keys_hash + ON CURRENT_SCHEMA.miner_api_keys(key_hash) + WHERE revoked_at IS NULL; + +CREATE INDEX IF NOT EXISTS idx_miner_api_keys_miner + ON CURRENT_SCHEMA.miner_api_keys(miner_id); + +DO $$ +BEGIN + IF EXISTS ( + SELECT 1 + FROM information_schema.tables + WHERE table_schema = 'CURRENT_SCHEMA' + AND table_name = 'experiments' + ) THEN + ALTER TABLE CURRENT_SCHEMA.experiments + ADD COLUMN IF NOT EXISTS prompt_id TEXT; + CREATE INDEX IF NOT EXISTS idx_experiments_prompt_id + ON CURRENT_SCHEMA.experiments(prompt_id) + WHERE prompt_id IS NOT NULL; + CREATE INDEX IF NOT EXISTS idx_experiments_miner_prompt + ON CURRENT_SCHEMA.experiments(miner_hotkey, prompt_id) + WHERE prompt_id IS NOT NULL; + END IF; +END $$; diff --git a/database/miner_feedback.py b/database/miner_feedback.py new file mode 100644 index 0000000..0bfdca4 --- /dev/null +++ b/database/miner_feedback.py @@ -0,0 +1,339 @@ +"""Miner-feedback endpoints — bearer-auth ``/miners/me/*`` + public +per-task ``/tasks/{task}/frontier``. + +Mounted onto the DB FastAPI app via ``include_miner_feedback_routes``. +Auth runs through ``database.server.auth_middleware`` which delegates +bearer lookups to ``shared.miner_auth`` for any path prefixed +``/miners/me/``. + +These endpoints are the data plane behind the miner CLI +(``miner/cli.py``): ``MinerResultsClient`` calls them to pull +submissions + Phase C scores for the local optimizer (GEPA, etc.). +""" + +from __future__ import annotations + +import logging +from typing import Optional + +from fastapi import APIRouter, FastAPI, HTTPException, Request + +from shared.miner_auth import touch_key_usage + +logger = logging.getLogger(__name__) + +miner_feedback_router = APIRouter() +public_router = APIRouter() + + +def _identity(request: Request): + """Pull the resolved ``MinerIdentity`` off ``request.state`` (set by + the auth middleware). 401 if absent — middleware should always set + it for ``/miners/me/*`` paths.""" + ident = getattr(request.state, "miner_identity", None) + if ident is None: + raise HTTPException(status_code=401, detail="Bearer auth required") + return ident + + +def _pool_or_503(): + """Lazy import to avoid circular dependency with database.server.""" + from database import server + if server._pool is None: + raise HTTPException(status_code=503, detail="DB pool unavailable") + return server._pool + + +def _miner_filter_clause(ident) -> tuple[str, list]: + """SQL fragment + params to scope a query to this miner's rows. + + During the dual-stack period a miner row may have a ``hotkey`` (the + on-chain identity used by older write paths) and a ``miner_id`` (the + new operator-issued identifier). We accept BOTH so submissions + written with either identity surface in the feedback API. + """ + # We match on miner_hotkey OR a computed miner_id fallback so old + # rows that were written before the registry existed still surface. + if ident.hotkey: + return ( + "(experiments.miner_hotkey = $1)", + [ident.hotkey], + ) + # No hotkey on file — use the miner_id directly against the prompt_id + # tail (best-effort; miners with no hotkey are post-cutover only). + return ("FALSE", []) + + +# ── /miners/me/submissions ────────────────────────────────────────── + + +@miner_feedback_router.get("/miners/me/submissions") +async def miner_submissions( + request: Request, + since: Optional[str] = None, + limit: int = 200, + task: str = "", +): + """Return Phase A submissions this miner produced. + + JSON shape:: + + {"submissions": [{round_id, submission_id, task_name, prompt_id, + architecture_code, motivation, reasoning, + tool_calls, created_at}, ...]} + """ + ident = _identity(request) + pool = _pool_or_503() + limit = max(1, min(int(limit or 200), 1000)) + where, params = _miner_filter_clause(ident) + if where == "FALSE": + return {"submissions": []} + + sql_parts = [ + "SELECT id AS submission_id, round_id, task AS task_name, " + "prompt_id, code AS architecture_code, motivation, reasoning, " + "tool_calls, timestamp AS created_at " + "FROM experiments WHERE " + where, + ] + if task: + sql_parts.append(f"AND task = ${len(params) + 1}") + params.append(task) + if since: + sql_parts.append(f"AND timestamp >= ${len(params) + 1}") + params.append(_parse_iso(since)) + sql_parts.append("ORDER BY timestamp DESC") + sql_parts.append(f"LIMIT ${len(params) + 1}") + params.append(limit) + + rows = await pool.fetch(" ".join(sql_parts), *params) + await touch_key_usage(pool, ident.key_id) + return {"submissions": [_row_to_submission(r) for r in rows]} + + +# ── /miners/me/results ────────────────────────────────────────────── + + +@miner_feedback_router.get("/miners/me/results") +async def miner_results( + request: Request, + since: Optional[str] = None, + limit: int = 200, + task: str = "", +): + """Submissions joined with Phase C scoring metadata. + + The experiments table already holds the scoring columns (``metric``, + ``score``, ``success``, ``objectives``) — Phase C writes them in + place — so we just project them alongside the submission shape and + fold them into a ``scores`` blob the optimizer can consume. + """ + ident = _identity(request) + pool = _pool_or_503() + limit = max(1, min(int(limit or 200), 1000)) + where, params = _miner_filter_clause(ident) + if where == "FALSE": + return {"results": []} + + sql_parts = [ + "SELECT id AS submission_id, round_id, task AS task_name, " + "prompt_id, code AS architecture_code, motivation, " + "metric, score, success, objectives, " + "timestamp AS created_at " + "FROM experiments WHERE " + where, + ] + if task: + sql_parts.append(f"AND task = ${len(params) + 1}") + params.append(task) + if since: + sql_parts.append(f"AND timestamp >= ${len(params) + 1}") + params.append(_parse_iso(since)) + sql_parts.append("ORDER BY timestamp DESC") + sql_parts.append(f"LIMIT ${len(params) + 1}") + params.append(limit) + + rows = await pool.fetch(" ".join(sql_parts), *params) + await touch_key_usage(pool, ident.key_id) + return {"results": [_row_to_result(r) for r in rows]} + + +# ── /miners/me/summary ────────────────────────────────────────────── + + +@miner_feedback_router.get("/miners/me/summary") +async def miner_summary(request: Request): + """Quick stats — total submissions, last seen round, mean score.""" + ident = _identity(request) + pool = _pool_or_503() + where, params = _miner_filter_clause(ident) + if where == "FALSE": + return { + "miner_id": ident.miner_id, + "total_submissions": 0, + "last_round_id": None, + "mean_score_recent": None, + "successes_recent": 0, + } + + summary_sql = ( + "SELECT COUNT(*) AS total, MAX(round_id) AS last_round " + "FROM experiments WHERE " + where + ) + s = await pool.fetchrow(summary_sql, *params) + + # Recent-window stats (last 50 rows). + recent_sql = ( + "SELECT AVG(score) AS mean_score, " + "SUM(CASE WHEN success THEN 1 ELSE 0 END) AS successes, " + "COUNT(*) AS n " + "FROM (SELECT score, success FROM experiments WHERE " + where + + " ORDER BY timestamp DESC LIMIT 50) sub" + ) + r = await pool.fetchrow(recent_sql, *params) + + await touch_key_usage(pool, ident.key_id) + return { + "miner_id": ident.miner_id, + "total_submissions": int(s["total"] or 0), + "last_round_id": int(s["last_round"]) if s["last_round"] is not None else None, + "mean_score_recent": float(r["mean_score"]) if r["mean_score"] is not None else None, + "successes_recent": int(r["successes"] or 0), + "recent_window": int(r["n"] or 0), + } + + +# ── /tasks/{task}/frontier (public) ───────────────────────────────── + + +@public_router.get("/tasks/{task}/frontier") +async def task_frontier(task: str, limit: int = 50): + """Per-task Pareto frontier as ``{points: [{flops, crps, mase}, …]}``. + + Public — no bearer required. Same data the optimizer's + ``MinerResultsClient.frontier()`` hits. Computed live from the + experiments table (top successful rows per FLOPs bucket). + """ + pool = _pool_or_503() + limit = max(1, min(int(limit or 50), 500)) + sql = ( + "SELECT metric, objectives, score " + "FROM experiments " + "WHERE task = $1 AND success = TRUE AND metric IS NOT NULL " + "ORDER BY metric ASC " + "LIMIT $2" + ) + rows = await pool.fetch(sql, task, limit) + points = [] + for r in rows: + # objectives is JSONB; asyncpg may return as str without codec. + obj = r["objectives"] + if isinstance(obj, (bytes, bytearray)): + obj = obj.decode() + if isinstance(obj, str): + import json as _json + try: + obj = _json.loads(obj) + except (ValueError, TypeError): + obj = {} + if not isinstance(obj, dict): + obj = {} + points.append({ + "flops": obj.get("flops_equivalent_size"), + "metric": r["metric"], + "score": r["score"], + }) + return {"task": task, "points": points} + + +# ── Helpers ──────────────────────────────────────────────────────── + + +def _parse_iso(s: str) -> float: + """Accept an ISO-8601 timestamp and return a POSIX seconds float so + we can compare against ``experiments.timestamp``.""" + if not s: + return 0.0 + from datetime import datetime + try: + return datetime.fromisoformat(s.replace("Z", "+00:00")).timestamp() + except ValueError: + # Fall back to "treat as already-numeric epoch". + try: + return float(s) + except (TypeError, ValueError): + return 0.0 + + +def _row_to_submission(row) -> dict: + return { + "submission_id": str(row["submission_id"]), + "round_id": int(row["round_id"]) if row["round_id"] is not None else None, + "task_name": row["task_name"] or "", + "prompt_id": row["prompt_id"], + "architecture_code": row["architecture_code"] or "", + "motivation": row["motivation"] or "", + "reasoning": row["reasoning"] or "", + "tool_calls": _decode_jsonb(row["tool_calls"], []), + "created_at": _to_iso(row["created_at"]), + } + + +def _row_to_result(row) -> dict: + obj = _decode_jsonb(row["objectives"], {}) + return { + "submission_id": str(row["submission_id"]), + "round_id": int(row["round_id"]) if row["round_id"] is not None else None, + "task_name": row["task_name"] or "", + "prompt_id": row["prompt_id"], + "architecture_code": row["architecture_code"] or "", + "motivation": row["motivation"] or "", + "scores": { + "metric": row["metric"], + "raw_score": row["score"], + "success": bool(row["success"]), + "flops_equivalent_size": obj.get("flops_equivalent_size"), + **{k: v for k, v in obj.items() if k != "flops_equivalent_size"}, + }, + "created_at": _to_iso(row["created_at"]), + } + + +def _decode_jsonb(value, default): + if value is None: + return default + if isinstance(value, (dict, list)): + return value + if isinstance(value, (bytes, bytearray)): + value = value.decode("utf-8") + if isinstance(value, str): + import json as _json + try: + return _json.loads(value) + except (ValueError, TypeError): + return default + return default + + +def _to_iso(value) -> str: + if value is None: + return "" + if isinstance(value, str): + return value + try: + from datetime import datetime, timezone + return datetime.fromtimestamp(float(value), tz=timezone.utc).isoformat(timespec="seconds") + except (TypeError, ValueError): + return "" + + +# ── Mount ────────────────────────────────────────────────────────── + + +def include_miner_feedback_routes(target_app: FastAPI) -> None: + """Idempotent mount. Safe to call multiple times.""" + already = any( + getattr(r, "path", None) == "/miners/me/summary" for r in target_app.routes + ) + if already: + return + target_app.include_router(miner_feedback_router) + target_app.include_router(public_router) diff --git a/database/operator_cli.py b/database/operator_cli.py new file mode 100644 index 0000000..6dda76a --- /dev/null +++ b/database/operator_cli.py @@ -0,0 +1,254 @@ +"""Operator CLI — register miners and issue / revoke their API keys. + +Run against the DB Postgres directly: + + python -m database.operator_cli register --name alice [--hotkey 5xxx] [--contact ...] + python -m database.operator_cli issue-key --miner-id [--label prod] + python -m database.operator_cli revoke-key --key-id + python -m database.operator_cli list-keys --miner-id + python -m database.operator_cli list-miners + python -m database.operator_cli rotate-service-key + +DSN is read from ``RADAR_PG_DSN`` (same as the rest of the stack). + +Plaintext API keys are printed exactly once at issuance — copy them +out of the terminal immediately; the DB only stores the SHA-256. +""" + +from __future__ import annotations + +import argparse +import asyncio +import os +import secrets +import sys +from typing import Optional + +# Importing ``config`` triggers ``load_dotenv`` so RADAR_PG_DSN and +# friends resolve from ``radar/.env`` the same way they do for every +# other CLI in the repo. Without this, operator_cli read os.environ +# directly and silently ignored .env. +import config # noqa: F401 + +from shared import miner_auth + + +# ── DB connection ────────────────────────────────────────────────── + + +async def _connect(): + import asyncpg + dsn = os.getenv("RADAR_PG_DSN", "").strip() + if not dsn: + print( + "operator: RADAR_PG_DSN not set. Example: " + "postgresql://radar:radar@localhost:5432/radar", + file=sys.stderr, + ) + sys.exit(2) + return await asyncpg.connect(dsn) + + +async def _ensure_schema(conn) -> None: + """Create the miners + miner_api_keys tables if they don't exist + yet. Mirrors PgExperimentStore.init_schema so the operator CLI + works against a stock Postgres on day one.""" + from shared.pg_schema import MINER_REGISTRY_SCHEMA + + # The DDL references experiments.prompt_id; if the table doesn't + # exist yet we still want miners/miner_api_keys created. Run each + # statement separately and tolerate the ALTER failing. + for stmt in [s.strip() for s in MINER_REGISTRY_SCHEMA.split(";") if s.strip()]: + try: + await conn.execute(stmt + ";") + except Exception as e: + # ALTER on missing experiments table is fine to ignore; + # other failures surface so the operator notices. + msg = str(e).lower() + if "experiments" in msg and "does not exist" in msg: + continue + print(f"warn: DDL stmt failed: {e}", file=sys.stderr) + + +# ── Command handlers ─────────────────────────────────────────────── + + +async def cmd_register(args) -> int: + conn = await _connect() + try: + await _ensure_schema(conn) + mid = await miner_auth.register_miner( + conn, + miner_id=args.miner_id, + name=args.name, + hotkey=args.hotkey, + contact=args.contact, + ) + print(f"miner_id: {mid}") + return 0 + finally: + await conn.close() + + +async def cmd_issue_key(args) -> int: + conn = await _connect() + try: + await _ensure_schema(conn) + plaintext, key_id = await miner_auth.issue_api_key( + conn, args.miner_id, scope=args.scope, label=args.label, + ) + print("=" * 60) + print("API KEY (shown ONCE — copy now, will not be stored):") + print(plaintext) + print() + print(f"key_id {key_id}") + print(f"miner_id {args.miner_id}") + print(f"scope {args.scope}") + print(f"label {args.label or '(none)'}") + print("=" * 60) + return 0 + finally: + await conn.close() + + +async def cmd_revoke_key(args) -> int: + conn = await _connect() + try: + ok = await miner_auth.revoke_api_key(conn, args.key_id) + if ok: + print(f"revoked {args.key_id}") + return 0 + print(f"no active key with id {args.key_id}", file=sys.stderr) + return 1 + finally: + await conn.close() + + +async def cmd_list_keys(args) -> int: + conn = await _connect() + try: + rows = await miner_auth.list_api_keys(conn, args.miner_id) + if not rows: + print("(no keys)") + return 0 + print(f"{'key_id':<34} {'scope':<8} {'label':<18} " + f"{'created':<22} {'revoked':<22} last_used") + for r in rows: + print( + f"{r['key_id']:<34} {r['scope']:<8} " + f"{(r.get('label') or '')[:18]:<18} " + f"{_fmt_ts(r.get('created_at')):<22} " + f"{_fmt_ts(r.get('revoked_at')):<22} " + f"{_fmt_ts(r.get('last_used_at'))}" + ) + return 0 + finally: + await conn.close() + + +async def cmd_list_miners(args) -> int: + conn = await _connect() + try: + rows = await conn.fetch( + "SELECT miner_id, name, hotkey, contact, created_at, revoked_at " + "FROM miners ORDER BY created_at DESC LIMIT $1", + int(args.limit), + ) + if not rows: + print("(no miners)") + return 0 + print(f"{'miner_id':<34} {'name':<20} {'hotkey':<20} " + f"{'created':<22} revoked") + for r in rows: + print( + f"{r['miner_id']:<34} " + f"{(r['name'] or '')[:20]:<20} " + f"{(r['hotkey'] or '')[:20]:<20} " + f"{_fmt_ts(r.get('created_at')):<22} " + f"{_fmt_ts(r.get('revoked_at'))}" + ) + return 0 + finally: + await conn.close() + + +def cmd_rotate_service_key(args) -> int: + """Print a fresh service key — operator pastes into env.""" + key = secrets.token_hex(32) + print("New RADAR_SERVICE_KEY value (set on operator + trainers + validators):") + print(key) + print() + print("Rotation procedure:") + print(" 1. Roll out to every trainer first (they're the verifiers).") + print(" 2. Then update validators / operator dispatchers.") + print(" 3. Confirm /train accepts the new key before retiring the old one.") + return 0 + + +# ── argparse plumbing ────────────────────────────────────────────── + + +def _build_parser() -> argparse.ArgumentParser: + p = argparse.ArgumentParser( + prog="radar-operator", + description="Manage miners + API keys for non-competitive Radar.", + ) + sub = p.add_subparsers(dest="command", required=True) + + r = sub.add_parser("register", help="Register a new miner.") + r.add_argument("--name", default="") + r.add_argument("--hotkey", default="", + help="Bittensor SS58 (optional; for dual-stack period).") + r.add_argument("--contact", default="") + r.add_argument("--miner-id", default=None, + help="Explicit id (default: random UUID).") + + i = sub.add_parser("issue-key", help="Mint an API key for a miner.") + i.add_argument("--miner-id", required=True) + i.add_argument("--scope", default="miner", + help="Scope tag stored on the key (default: 'miner').") + i.add_argument("--label", default="", + help="Operator note shown in list-keys output.") + + rk = sub.add_parser("revoke-key", help="Revoke an API key by key_id.") + rk.add_argument("--key-id", required=True) + + lk = sub.add_parser("list-keys", help="List keys for a miner.") + lk.add_argument("--miner-id", required=True) + + lm = sub.add_parser("list-miners", help="List registered miners.") + lm.add_argument("--limit", type=int, default=100) + + sub.add_parser("rotate-service-key", + help="Print a fresh RADAR_SERVICE_KEY.") + return p + + +def _fmt_ts(value) -> str: + if value in (None, 0, 0.0): + return "-" + try: + from datetime import datetime, timezone + return datetime.fromtimestamp(float(value), tz=timezone.utc).isoformat( + timespec="seconds", + ) + except (TypeError, ValueError): + return str(value) + + +def main(argv: Optional[list[str]] = None) -> int: + args = _build_parser().parse_args(argv) + if args.command == "rotate-service-key": + return cmd_rotate_service_key(args) + handler = { + "register": cmd_register, + "issue-key": cmd_issue_key, + "revoke-key": cmd_revoke_key, + "list-keys": cmd_list_keys, + "list-miners": cmd_list_miners, + }[args.command] + return asyncio.run(handler(args)) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/database/server.py b/database/server.py index e24ee46..3568dda 100644 --- a/database/server.py +++ b/database/server.py @@ -304,6 +304,12 @@ async def auth_middleware(request: Request, call_next): path = request.url.path + # Bearer-auth surface for the non-competitive feedback API. Handled + # before the Epistula path because tokens use Authorization: Bearer + # headers, not the X-Epistula-* set. + if path.startswith("/miners/me/"): + return await _bearer_auth_then_call(request, call_next) + needs_auth = ( path.startswith("/experiments") or path.startswith("/challenge") @@ -328,18 +334,42 @@ async def auth_middleware(request: Request, call_next): status_code=413, content={"error": "Request body too large"}, ) - # ── Validator API key: trusted proxy, skip Epistula ── + # ── Validator API key: trusted proxy, fast path ── proxy_authed = False if Config.DB_API_KEY: import secrets as _secrets provided = request.headers.get("X-Radar-API-Key", "") if provided and _secrets.compare_digest(provided, Config.DB_API_KEY): - # Trusted validator proxy — rate-limit by IP instead of hotkey if not _check_rate_limit(client_ip): return JSONResponse(status_code=429, content={"error": "Rate limit exceeded"}) proxy_authed = True - # ── Epistula auth: miners and validators without API key ── + # ── HMAC service-key auth (non-competitive default) ── + if not proxy_authed: + import os as _os + service_key = _os.getenv("RADAR_SERVICE_KEY", "").strip() + if service_key: + from shared.auth import ( + hmac_verify_request, static_key_lookup, + ) + expected_kid = _os.getenv("RADAR_SERVICE_KEY_ID", "operator") + ok, err, kid = hmac_verify_request( + dict(request.headers), body, + static_key_lookup(expected_kid, service_key.encode()), + ) + if ok: + request.state.caller_hotkey = kid + if not _check_rate_limit(kid): + return JSONResponse( + status_code=429, + content={"error": "Rate limit exceeded"}, + ) + proxy_authed = True + elif _metagraph is None: + # No Epistula fallback configured — reject. + return JSONResponse(status_code=403, content={"error": err}) + + # ── Legacy Epistula path (dual-stack period) ── if not proxy_authed and _metagraph: from shared.auth import verify_request ok, err, hotkey = verify_request(dict(request.headers), body, _metagraph) @@ -1059,9 +1089,57 @@ def include_validator_routes(target_app: FastAPI) -> None: target_app.include_router(validator_router) +async def _bearer_auth_then_call(request: Request, call_next): + """Resolve a bearer token against ``miner_api_keys`` and attach the + identity to ``request.state`` for the route handler. + + 401 if the token is missing / unknown / revoked, 503 if the asyncpg + pool isn't wired up yet (dashboard-only mode that accidentally + received bearer traffic, or pre-startup race). + """ + from shared.auth import extract_bearer + from shared.miner_auth import lookup_bearer + + token = extract_bearer(dict(request.headers)) + if not token: + return JSONResponse( + status_code=401, content={"error": "Bearer token required"}, + ) + if _pool is None: + return JSONResponse( + status_code=503, content={"error": "DB pool not ready"}, + ) + try: + ident = await lookup_bearer(_pool, token) + except Exception as e: + logger.warning("bearer lookup failed: %s", e) + return JSONResponse( + status_code=503, content={"error": "auth lookup failed"}, + ) + if ident is None: + return JSONResponse( + status_code=403, content={"error": "Unknown or revoked token"}, + ) + request.state.miner_identity = ident + return await call_next(request) + + +def include_miner_feedback_routes(target_app: FastAPI) -> None: + """Mount the bearer-auth ``/miners/me/*`` + public per-task frontier + routes onto ``target_app``. Idempotent.""" + from database.miner_feedback import ( + include_miner_feedback_routes as _mount, + ) + _mount(target_app) + + # Legacy auto-mount: keep the validator surface on ``app`` at import time # unless this process is running in ``dashboard`` mode. Preserves backward # compatibility for tests that import ``app`` directly and expect every # route to be reachable. if Config.NEURON_MODE != "dashboard": include_validator_routes(app) +# Miner feedback routes are mounted unconditionally — they need bearer +# auth (no chain dependency) and are useful in both validator and +# dashboard deployments. +include_miner_feedback_routes(app) diff --git a/ecosystem.config.js b/ecosystem.config.js new file mode 100644 index 0000000..659e62b --- /dev/null +++ b/ecosystem.config.js @@ -0,0 +1,85 @@ +// pm2 ecosystem for running a non-competitive miner alongside the GEPA +// prompt-optimizer loop on the same host. +// +// Usage: +// mkdir -p logs prompts +// python miner/neuron.py optimize --seed --dry-run # one-time: seed active.json +// pm2 start ecosystem.config.js +// pm2 logs +// pm2 save && pm2 startup # survive reboot +// +// Both processes share env via `sharedEnv` so prompts/active.json is the +// single source of truth: `radar-gepa` rewrites it after each round's +// scores land, `radar-miner` reads it on the next round — no restart +// needed. +// +// Secrets: do NOT commit RADAR_MINER_API_KEY. Either keep this file out +// of git on the deploy host or load secrets from a sibling .env and +// reference process.env.RADAR_MINER_API_KEY here. + +const path = require("path"); + +const sharedEnv = { + RADAR_NONCOMPETITIVE: "true", + RADAR_DB_URL: process.env.RADAR_DB_URL || "http://localhost:8090", + RADAR_MINER_API_KEY: process.env.RADAR_MINER_API_KEY || "", + MINER_PROMPTS_DIR: path.join(__dirname, "prompts"), + + // GEPA reflector LM — operator's shared LLM proxy. Swap to + // MINER_REFLECTOR_API_{BASE,KEY,MODEL} for an explicit OpenAI / + // Anthropic-style endpoint instead. + MINER_LLM_URL: process.env.MINER_LLM_URL || "", + MINER_LLM_API_KEY: process.env.MINER_LLM_API_KEY || "", + MINER_LLM_MODEL: process.env.MINER_LLM_MODEL || + "deepseek-ai/DeepSeek-V3-0324", +}; + +// Default points at the autonomous example agent in a sibling +// radar-miner-examples checkout. Override with RADAR_AGENT_DIR to use a +// different example (openai_sdk_v2, claude_style_v2, patch_decoder) or +// your own agent directory. +const agentDir = process.env.RADAR_AGENT_DIR || + "../radar-miner-examples/agents/autonomous/"; + +module.exports = { + apps: [ + { + name: "radar-miner", + script: "miner/neuron.py", + interpreter: "python3", + args: `--agent_dir ${agentDir}`, + cwd: __dirname, + env: sharedEnv, + autorestart: true, + max_restarts: 20, + restart_delay: 5000, + out_file: "./logs/miner.out.log", + error_file: "./logs/miner.err.log", + merge_logs: true, + time: true, + }, + { + name: "radar-gepa", + script: "miner/neuron.py", + interpreter: "python3", + args: [ + "optimize", + "--optimizer", "gepa", + "--population", "8", + "--budget", "30", + "--elite-k", "2", + "--watch", + "--every-seconds", "600", + ].join(" "), + cwd: __dirname, + env: sharedEnv, + autorestart: true, + max_restarts: 20, + restart_delay: 10000, + out_file: "./logs/gepa.out.log", + error_file: "./logs/gepa.err.log", + merge_logs: true, + time: true, + }, + ], +}; diff --git a/miner/cli.py b/miner/cli.py new file mode 100644 index 0000000..d02767a --- /dev/null +++ b/miner/cli.py @@ -0,0 +1,384 @@ +"""Miner CLI subcommands — ``results``, ``optimize``, ``prompts``. + +Routed from ``miner/neuron.py``'s ``main()``. When the first argv is +one of the known subcommands the neuron delegates here; otherwise the +neuron falls through to the legacy ``run`` behavior so existing +deployments keep working without flag churn. + + miner/neuron.py run [existing flags] + legacy entrypoint — serve agent + trainer-launcher listener + + miner/neuron.py results [--since 1d] [--limit 50] [--json] [--task X] + one-shot: print recent scored results from the DB + + miner/neuron.py optimize \\ + [--optimizer gepa|random_mutate|pkg.mod:func] \\ + [--rounds-back 20] [--population 8] [--budget 30] \\ + [--score-key raw_score] [--dry-run] [--task ts_forecasting] + one-shot: pull results, evolve the prompt population, write new + active.json (archives the old as history/gen_NNN.json) + + miner/neuron.py optimize --watch [--every-seconds 600] + daemon loop: re-run optimize whenever ``my_summary().last_round_id`` + advances + + miner/neuron.py prompts list + show the current population + + miner/neuron.py prompts history + list archived generation numbers + + miner/neuron.py prompts rollback + restore an archived generation (with safety-archive of current) + +DB URL + API key are read from env (``RADAR_DB_URL``, +``RADAR_MINER_API_KEY``) or flags. +""" + +from __future__ import annotations + +import argparse +import json +import logging +import os +import sys +import time +from datetime import datetime, timedelta, timezone +from pathlib import Path +from typing import Optional + +from miner_template import optimizers, prompts as prompts_mod +from miner_template.optimizers import ResultRow +from miner_template.results_client import MinerResultsClient + +logger = logging.getLogger(__name__) + +# argv[1] values that route here instead of the legacy `run` entrypoint. +SUBCOMMANDS = frozenset({"results", "optimize", "prompts"}) + + +def is_subcommand(argv: list[str]) -> bool: + """Return True if argv[1] looks like one of our subcommands.""" + return len(argv) >= 2 and argv[1] in SUBCOMMANDS + + +def dispatch(argv: list[str]) -> int: + """Top-level entrypoint. argv[0] is the program, argv[1] is the + subcommand. Returns a process exit code.""" + if len(argv) < 2 or argv[1] not in SUBCOMMANDS: + print( + f"Unknown subcommand. Expected one of: {sorted(SUBCOMMANDS)}", + file=sys.stderr, + ) + return 2 + + parser = _root_parser() + args = parser.parse_args(argv[1:]) + + if args.subcommand == "results": + return cmd_results(args) + if args.subcommand == "optimize": + return cmd_optimize(args) + if args.subcommand == "prompts": + return cmd_prompts(args) + parser.error(f"unhandled subcommand {args.subcommand}") + return 2 + + +# ── argparse plumbing ──────────────────────────────────────────────── + + +def _root_parser() -> argparse.ArgumentParser: + p = argparse.ArgumentParser( + prog="miner", + description="Radar miner — agent host, results polling, prompt optimizer.", + ) + sub = p.add_subparsers(dest="subcommand", required=True) + + # results + r = sub.add_parser("results", help="Print recent scored results.") + _add_db_flags(r) + _add_window_flags(r) + r.add_argument("--json", action="store_true", + help="Emit one JSON line per result instead of a table.") + + # optimize + o = sub.add_parser("optimize", help="Run a prompt-optimization pass.") + _add_db_flags(o) + _add_window_flags(o) + o.add_argument("--optimizer", default="gepa", + help="Optimizer to use: 'gepa', 'random_mutate', or " + "'package.module:func'. Default: gepa.") + o.add_argument("--population", type=int, default=8, + help="Target size of the new prompt population.") + o.add_argument("--budget", type=int, default=30, + help="Optimizer rollout/compile budget (GEPA-only).") + o.add_argument("--score-key", default="raw_score", + help="scores[] field to optimize against.") + o.add_argument("--elite-k", type=int, default=2, + help="Top-K survivors carried forward (random_mutate).") + o.add_argument("--prompts-dir", default=None, + help=f"Where prompts live " + f"(default ${prompts_mod.DEFAULT_PROMPTS_DIR}).") + o.add_argument("--seed", action="store_true", + help="Seed a default prompt if active.json is missing.") + o.add_argument("--dry-run", action="store_true", + help="Print the diff but don't write files.") + o.add_argument("--watch", action="store_true", + help="Loop forever, re-running when new rounds finish.") + o.add_argument("--every-seconds", type=int, default=600, + help="--watch poll interval.") + + # prompts + pr = sub.add_parser("prompts", help="Inspect/manage the prompt population.") + psub = pr.add_subparsers(dest="prompts_command", required=True) + pl = psub.add_parser("list", help="Show the current active population.") + pl.add_argument("--prompts-dir", default=None) + pl.add_argument("--json", action="store_true") + psub.add_parser("history", help="List archived generations.").add_argument( + "--prompts-dir", default=None, + ) + rb = psub.add_parser("rollback", help="Restore an archived generation.") + rb.add_argument("generation", type=int) + rb.add_argument("--prompts-dir", default=None) + + return p + + +def _add_db_flags(p: argparse.ArgumentParser) -> None: + p.add_argument("--db-url", + default=os.getenv("RADAR_DB_URL", ""), + help="DB server URL (env: RADAR_DB_URL).") + p.add_argument("--api-key", + default=os.getenv("RADAR_MINER_API_KEY", ""), + help="Miner bearer API key (env: RADAR_MINER_API_KEY).") + + +def _add_window_flags(p: argparse.ArgumentParser) -> None: + p.add_argument("--rounds-back", type=int, default=20, + help="How many rounds of history to consider.") + p.add_argument("--since", default="", + help="ISO timestamp override; takes precedence over --rounds-back.") + p.add_argument("--limit", type=int, default=500, + help="Max rows pulled from the DB.") + p.add_argument("--task", default="", + help="Filter to one task name (e.g. ts_forecasting).") + + +# ── results ───────────────────────────────────────────────────────── + + +def cmd_results(args) -> int: + if not args.db_url or not args.api_key: + print( + "results: missing --db-url or --api-key " + "(or RADAR_DB_URL / RADAR_MINER_API_KEY).", + file=sys.stderr, + ) + return 2 + since = args.since or _approx_since(args.rounds_back) + with MinerResultsClient(args.db_url, args.api_key) as c: + rows = c.results(since=since, limit=args.limit, task=args.task) + if args.json: + for r in rows: + print(json.dumps({ + "round_id": r.round_id, + "submission_id": r.submission_id, + "task_name": r.task_name, + "prompt_id": r.prompt_id, + "scores": r.scores, + "created_at": r.created_at, + })) + else: + _print_results_table(rows) + return 0 + + +def _print_results_table(rows: list[ResultRow]) -> None: + if not rows: + print("(no results)") + return + header = f"{'round':>6} {'prompt_id':<14} {'score':>8} {'task':<16} submission" + print(header) + print("-" * len(header)) + for r in rows: + pid = (r.prompt_id or "-")[:12] + score = float(r.scores.get("raw_score", 0.0) or 0.0) + task = (r.task_name or "-")[:14] + print(f"{r.round_id:>6} {pid:<14} {score:>8.4f} {task:<16} {r.submission_id}") + + +# ── optimize ──────────────────────────────────────────────────────── + + +def cmd_optimize(args) -> int: + if not args.db_url or not args.api_key: + print( + "optimize: missing --db-url or --api-key " + "(or RADAR_DB_URL / RADAR_MINER_API_KEY).", + file=sys.stderr, + ) + return 2 + prompts_dir = Path(args.prompts_dir) if args.prompts_dir else None + + if args.watch: + return _watch_loop(args, prompts_dir) + return _optimize_once(args, prompts_dir) + + +def _optimize_once(args, prompts_dir: Optional[Path]) -> int: + population = prompts_mod.load_active(prompts_dir) + if not population: + if args.seed: + population = prompts_mod.seed_default(prompts_dir) + print(f"Seeded default prompt ({population[0].id[:8]}…)") + else: + print( + "optimize: no active prompt population yet. " + "Re-run with --seed to write a default starter prompt.", + file=sys.stderr, + ) + return 2 + + since = args.since or _approx_since(args.rounds_back) + with MinerResultsClient(args.db_url, args.api_key) as c: + results = c.results(since=since, limit=args.limit, task=args.task) + print( + f"Loaded {len(results)} scored results since {since or 'beginning'} " + f"(population size {len(population)}).", + ) + + try: + fn = optimizers.resolve(args.optimizer) + except (ImportError, ValueError, AttributeError, TypeError) as e: + print(f"optimize: cannot resolve optimizer: {e}", file=sys.stderr) + return 2 + + config = { + "population": args.population, + "budget": args.budget, + "score_key": args.score_key, + "elite_k": args.elite_k, + } + try: + new_pop = fn(results, population, config) + except Exception as e: + print(f"optimize: optimizer raised: {e}", file=sys.stderr) + return 1 + + if not new_pop: + print("optimize: optimizer returned an empty population — no change.", + file=sys.stderr) + return 1 + + if args.dry_run: + _print_pop_diff(population, new_pop) + return 0 + + next_gen = (prompts_mod.list_history(prompts_dir) or [0])[-1] + 1 + prompts_mod.archive_current(next_gen, prompts_dir) + prompts_mod.save_active(new_pop, prompts_dir) + print( + f"Wrote generation {next_gen + 1} " + f"({len(new_pop)} prompts); previous archived as gen_{next_gen:03d}.json", + ) + return 0 + + +def _watch_loop(args, prompts_dir: Optional[Path]) -> int: + last_round = -1 + print(f"watch: polling every {args.every_seconds}s …") + while True: + try: + with MinerResultsClient(args.db_url, args.api_key) as c: + summary = c.summary() + cur = int(summary.get("last_round_id", -1) or -1) + except Exception as e: + logger.warning("watch: summary fetch failed: %s — retrying", e) + time.sleep(args.every_seconds) + continue + if cur > last_round: + print(f"watch: new round {cur} — running optimizer …") + code = _optimize_once(args, prompts_dir) + if code != 0: + logger.warning("watch: optimizer exited %d", code) + last_round = cur + time.sleep(args.every_seconds) + + +def _print_pop_diff(old: list, new: list) -> None: + old_ids = {p.id for p in old} + new_ids = {p.id for p in new} + survivors = old_ids & new_ids + fresh = new_ids - old_ids + dropped = old_ids - new_ids + print(f"would write population: size={len(new)}") + print(f" survivors: {len(survivors)} new: {len(fresh)} dropped: {len(dropped)}") + for p in new: + marker = "·" if p.id in old_ids else "+" + snippet = p.template.replace("\n", " ⏎ ")[:80] + print(f" {marker} [gen {p.generation}] {p.id[:8]}… {snippet}") + + +# ── prompts subcommands ──────────────────────────────────────────── + + +def cmd_prompts(args) -> int: + if args.prompts_command == "list": + return _prompts_list(args) + if args.prompts_command == "history": + return _prompts_history(args) + if args.prompts_command == "rollback": + return _prompts_rollback(args) + return 2 + + +def _prompts_list(args) -> int: + prompts_dir = Path(args.prompts_dir) if args.prompts_dir else None + pop = prompts_mod.load_active(prompts_dir) + if args.json: + print(json.dumps([p.to_dict() for p in pop], indent=2)) + return 0 + if not pop: + print("(no active prompt population)") + return 0 + for p in pop: + snippet = p.template.replace("\n", " ⏎ ")[:80] + parent = (p.parent_id or "-")[:8] + print(f" [gen {p.generation}] {p.id[:8]}… parent={parent} {snippet}") + return 0 + + +def _prompts_history(args) -> int: + prompts_dir = Path(args.prompts_dir) if args.prompts_dir else None + gens = prompts_mod.list_history(prompts_dir) + if not gens: + print("(no archived generations)") + return 0 + for g in gens: + print(f" gen_{g:03d}") + return 0 + + +def _prompts_rollback(args) -> int: + prompts_dir = Path(args.prompts_dir) if args.prompts_dir else None + try: + prompts_mod.rollback_to(args.generation, prompts_dir) + except FileNotFoundError as e: + print(f"rollback: {e}", file=sys.stderr) + return 2 + print(f"Rolled back to generation {args.generation}.") + return 0 + + +# ── helpers ──────────────────────────────────────────────────────── + + +def _approx_since(rounds_back: int) -> str: + """Map ``--rounds-back`` to an ISO timestamp. Heuristic: each round + is roughly 55 min (see CLAUDE.md). Good enough for an upper-bound + filter — the DB will still cap with --limit.""" + if rounds_back <= 0: + return "" + delta = timedelta(minutes=55 * rounds_back) + return (datetime.now(timezone.utc) - delta).isoformat(timespec="seconds") diff --git a/miner/neuron.py b/miner/neuron.py index 5809c1c..e6b2cbd 100644 --- a/miner/neuron.py +++ b/miner/neuron.py @@ -16,7 +16,10 @@ import os import time -import bittensor as bt +try: + import bittensor as bt +except ImportError: # pragma: no cover — only fires in noncompetitive deploys + bt = None # type: ignore[assignment] import httpx import uvicorn from fastapi import FastAPI, Request @@ -52,9 +55,9 @@ class Miner: """Radar subnet miner — Docker agent + warm-standby trainer listener.""" - def __init__(self, config: bt.Config): + def __init__(self, config): self.config = config - self.netuid = config.netuid + self.netuid = getattr(config, "netuid", 1) # Fail-fast on Targon misconfig — operator gets a clear error at # startup instead of a cryptic deploy failure on the first round. @@ -85,8 +88,21 @@ def __init__(self, config: bt.Config): "image and set RADAR_RUNPOD_ENDPOINT_ID to its id." ) - self.wallet = bt.Wallet(config=config) - self.subtensor = bt.Subtensor(config=config) + # Bittensor chain — skipped entirely in non-competitive mode + if Config.NONCOMPETITIVE: + self.wallet = None + self.subtensor = None + self.metagraph = None + else: + if bt is None: + raise RuntimeError( + "competitive mode requires the bittensor package; " + "install with `pip install bittensor` or run with " + "RADAR_NONCOMPETITIVE=true." + ) + self.wallet = bt.Wallet(config=config) + self.subtensor = bt.Subtensor(config=config) + self.metagraph = self.subtensor.metagraph(self.netuid) self._targon_client = None self._runpod_client = None # Per-round RunPod job ids the miner has submitted on behalf of @@ -94,9 +110,6 @@ def __init__(self, config: bt.Config): # round_id → list[job_id] self._runpod_jobs: dict[int, list[str]] = {} - # Cache metagraph — synced periodically in keep-alive loop - self.metagraph = self.subtensor.metagraph(self.netuid) - # Agent code directory — validators fetch the code and run it # in the official sandboxed image self.agent_dir = getattr(config, "agent_dir", "agent/") @@ -754,7 +767,7 @@ async def run(self): logger.warning("Metagraph sync failed: %s", e) -def get_config() -> bt.Config: +def get_config(): parser = argparse.ArgumentParser(description="Radar Miner") parser.add_argument("--netuid", type=int, default=1) parser.add_argument("--agent_dir", type=str, default="agent/", @@ -766,15 +779,31 @@ def get_config() -> bt.Config: help="Trainer Docker image deployed on Basilica on-demand") parser.add_argument("--external_ip", type=str, default="", help="External IP for listener URL committed to chain") + if Config.NONCOMPETITIVE or bt is None: + return parser.parse_args() bt.Wallet.add_args(parser) bt.Subtensor.add_args(parser) return bt.Config(parser) def main(): + import sys + logging.basicConfig(level=logging.INFO, format="%(asctime)s | %(levelname)s | %(message)s") logging.getLogger("httpx").setLevel(logging.WARNING) logging.getLogger("httpcore").setLevel(logging.WARNING) + + # Subcommand router: ``results`` / ``optimize`` / ``prompts`` go to + # ``miner/cli.py``. Anything else (or no args) falls through to the + # legacy ``run`` behavior so existing deployments keep working. + from miner import cli + if cli.is_subcommand(sys.argv): + sys.exit(cli.dispatch(sys.argv)) + # Optional explicit ``run`` token for clarity; strip it before the + # bittensor argparse kicks in. + if len(sys.argv) >= 2 and sys.argv[1] == "run": + del sys.argv[1] + config = get_config() miner = Miner(config) asyncio.run(miner.run()) diff --git a/miner_template/__init__.py b/miner_template/__init__.py new file mode 100644 index 0000000..fc56542 --- /dev/null +++ b/miner_template/__init__.py @@ -0,0 +1,12 @@ +"""Miner template — starter agent + prompt-evolution scaffolding. + +The miner CLI (``miner/neuron.py``) drives this package's two surfaces: + + * ``agent.py`` — ``design_architecture(challenge, client)`` entrypoint + called by the validator each round. + * ``prompts`` + ``optimizers`` + ``results_client`` — feedback loop + that lets miners pull their own scored history and evolve their + agent's prompts (e.g. via GEPA). + +See ``docs/optimizer.md`` for the loop in full. +""" diff --git a/miner_template/agent.py b/miner_template/agent.py index d1a547c..9da4521 100644 --- a/miner_template/agent.py +++ b/miner_template/agent.py @@ -56,6 +56,34 @@ def log(msg: str): print(msg, file=sys.stderr) +def _load_active_prompt(round_id: int) -> dict: + """Return ``{id, template}`` for the prompt variant this round will + use, or ``{"id": "", "template": ""}`` if the miner hasn't run the + optimizer yet. Reads from ``prompts/active.json`` next to the + miner's working dir; override via ``MINER_PROMPTS_DIR``. + + Kept import-light so the sandbox doesn't need the full + ``miner_template`` package — the agent runs in a stripped image. + """ + prompts_dir = os.getenv("MINER_PROMPTS_DIR", "prompts") + path = os.path.join(prompts_dir, "active.json") + try: + with open(path) as f: + payload = json.load(f) + except (OSError, json.JSONDecodeError): + return {"id": "", "template": ""} + rows = payload.get("prompts") if isinstance(payload, dict) else payload + if not isinstance(rows, list) or not rows: + return {"id": "", "template": ""} + pick = rows[round_id % len(rows)] + if not isinstance(pick, dict): + return {"id": "", "template": ""} + return { + "id": str(pick.get("id", "")), + "template": str(pick.get("template", "")), + } + + def _pick_hyperparams(min_flops: int, max_flops: int) -> dict: """Pick d_model, nhead, num_layers, dim_feedforward to fit the FLOPs range. @@ -183,6 +211,18 @@ def design_architecture(challenge: dict, client) -> dict: # Load persistent state from previous rounds scratch_dir = load_scratchpad(challenge) + # Pull the active prompt variant for this round. If the miner is + # running the optimizer (e.g. ``miner/neuron.py optimize --watch``) + # this rotates through the current population; otherwise it's empty + # and the agent falls back to its hardcoded heuristic. + round_id = int(challenge.get("round_id", 0) or 0) + prompt = _load_active_prompt(round_id) + if prompt["id"]: + reasoning_steps.append( + f"Using prompt variant {prompt['id'][:8]}… " + f"(rotating round_id={round_id} into population)." + ) + frontier = challenge.get("feasible_frontier", []) min_flops = challenge.get("min_flops_equivalent", 0) max_flops = challenge.get("max_flops_equivalent", 0) @@ -283,4 +323,5 @@ def build_optimizer(model): "motivation": motivation, "reasoning": "\n".join(reasoning_steps), "tool_calls": tool_calls, + "prompt_id": prompt["id"], } diff --git a/miner_template/optimizers/__init__.py b/miner_template/optimizers/__init__.py new file mode 100644 index 0000000..73f92cb --- /dev/null +++ b/miner_template/optimizers/__init__.py @@ -0,0 +1,97 @@ +"""Prompt-optimizer plugin registry for miners. + +A plugin is any callable with the signature:: + + optimize(results: list[ResultRow], + population: list[Prompt], + config: dict) -> list[Prompt] + +Built-ins: + ``random_mutate`` — deterministic baseline, no LLM required (CI-safe) + ``gepa`` — DSPy GEPA adapter (Genetic-Pareto reflective evolution) + +Custom optimizers can be referenced as ``"package.module:func"`` — +e.g. ``my_research.optimizers:my_optimizer`` — and ``resolve()`` will +import them on demand. +""" + +from __future__ import annotations + +import importlib +from dataclasses import dataclass, field +from typing import Callable, Optional + +from miner_template.prompts import Prompt + +# ── Public types ───────────────────────────────────────────────────── + + +@dataclass(slots=True) +class ResultRow: + """One scored submission, as returned by ``/miners/me/results``.""" + + round_id: int + submission_id: str + task_name: str + prompt_id: Optional[str] + architecture_code: str + scores: dict = field(default_factory=dict) + created_at: str = "" + + @classmethod + def from_dict(cls, d: dict) -> "ResultRow": + return cls( + round_id=int(d.get("round_id", 0) or 0), + submission_id=str(d.get("submission_id", "") or ""), + task_name=str(d.get("task_name", "") or ""), + prompt_id=d.get("prompt_id"), + architecture_code=str(d.get("architecture_code", "") or ""), + scores=dict(d.get("scores") or {}), + created_at=str(d.get("created_at", "") or ""), + ) + + +Optimizer = Callable[[list[ResultRow], list[Prompt], dict], list[Prompt]] + + +# ── Resolver ───────────────────────────────────────────────────────── + + +_BUILTINS = { + "random_mutate": "miner_template.optimizers.random_mutate:optimize", + "gepa": "miner_template.optimizers.gepa:optimize", +} + + +def resolve(name: str) -> Optimizer: + """Return the optimizer callable for ``name``. + + Accepts a built-in alias (``"gepa"``) or a fully qualified + ``"package.module:func"`` path. + """ + target = _BUILTINS.get(name, name) + if ":" not in target: + raise ValueError( + f"unknown optimizer {name!r}; expected one of {sorted(_BUILTINS)} " + f"or 'package.module:func'" + ) + module_path, func_name = target.split(":", 1) + try: + mod = importlib.import_module(module_path) + except ImportError as e: + raise ImportError( + f"optimizer {name!r}: cannot import {module_path!r}: {e}" + ) from e + try: + func = getattr(mod, func_name) + except AttributeError: + raise AttributeError( + f"optimizer {name!r}: {module_path}:{func_name} not found" + ) + if not callable(func): + raise TypeError(f"optimizer {name!r}: {target} is not callable") + return func + + +def list_builtins() -> list[str]: + return sorted(_BUILTINS) diff --git a/miner_template/optimizers/gepa.py b/miner_template/optimizers/gepa.py new file mode 100644 index 0000000..0918b23 --- /dev/null +++ b/miner_template/optimizers/gepa.py @@ -0,0 +1,219 @@ +"""GEPA — Genetic-Pareto reflective prompt evolution. + +Adapter around ``dspy.GEPA``. Imports DSPy lazily so the rest of the +miner CLI runs without it installed. + +The adapter: + + 1. Group ``ResultRow`` by ``prompt_id`` to build per-prompt traces + ``(architecture_code, scores)`` for GEPA to learn from. + 2. Configure a reflector LM — either an explicit OpenAI/Anthropic- + style API (``MINER_REFLECTOR_API_*`` env vars) or the operator's + shared LLM proxy (``MINER_LLM_URL`` / ``MINER_LLM_API_KEY``). + 3. Wrap each prompt as a ``dspy.Signature`` and run ``dspy.GEPA``'s + reflective mutation loop with a Pareto-front sampler over the + score vector. + 4. Map the resulting optimized programs back to ``Prompt`` rows with + fresh UUIDs, the new generation number, and ``parent_id`` set + from GEPA's lineage. + +Config keys (passed via the ``optimize`` subcommand): + + ``population`` (int, default 8): target population size. + ``budget`` (int, default 30): GEPA compile budget (rollouts). + ``score_key`` (str, default "raw_score"): which scalar GEPA reads. + ``reflector_lm`` (dict, optional): explicit DSPy LM kwargs. When + omitted, the adapter reads ``MINER_REFLECTOR_*`` / ``MINER_LLM_*`` + env vars (see ``_default_lm()`` below). + +The score the metric returns is the float at ``scores[score_key]`` for +the matched row; missing rows score 0. Override ``score_key`` to +optimize against a different objective (e.g. ``"frontier_dominance"``). +""" + +from __future__ import annotations + +import logging +import os +from typing import Any + +from miner_template.optimizers import ResultRow +from miner_template.prompts import Prompt + +logger = logging.getLogger(__name__) + + +def _import_dspy(): + """Lazy import so the CLI works without DSPy installed.""" + try: + import dspy # type: ignore + except ImportError as e: + raise ImportError( + "GEPA optimizer requires the 'dspy' package. " + "Install with: pip install dspy" + ) from e + if not hasattr(dspy, "GEPA"): + raise ImportError( + "Installed dspy build does not expose dspy.GEPA. " + "Upgrade: pip install -U dspy" + ) + return dspy + + +def _default_lm(dspy_mod) -> Any: + """Build a DSPy LM from environment, preferring an explicit + reflector LM and falling back to the operator's LLM proxy.""" + base = os.getenv("MINER_REFLECTOR_API_BASE", "").strip() + key = os.getenv("MINER_REFLECTOR_API_KEY", "").strip() + model = os.getenv("MINER_REFLECTOR_MODEL", "").strip() + if base and key and model: + return dspy_mod.LM(model=model, api_base=base, api_key=key) + + base = os.getenv("MINER_LLM_URL", "").strip() + key = os.getenv("MINER_LLM_API_KEY", "").strip() + model = os.getenv("MINER_LLM_MODEL", "").strip() or "deepseek-ai/DeepSeek-V3-0324" + if base and key: + return dspy_mod.LM(model=model, api_base=base, api_key=key) + + raise RuntimeError( + "GEPA reflector LM not configured. Set either " + "MINER_REFLECTOR_{API_BASE,API_KEY,MODEL} or " + "MINER_LLM_{URL,API_KEY,MODEL}." + ) + + +def _group_by_prompt(results: list[ResultRow]) -> dict[str, list[ResultRow]]: + groups: dict[str, list[ResultRow]] = {} + for r in results: + if r.prompt_id: + groups.setdefault(r.prompt_id, []).append(r) + return groups + + +def _build_metric(score_key: str): + """Return a metric callable GEPA invokes per rollout. + + The default DSPy metric signature is ``metric(example, pred, trace) + -> float``; we don't need the example/trace here because we already + know which prompt produced which scored row. GEPA passes through + user-supplied kwargs, so we attach ``score_key`` as a closure. + """ + + def metric(example, pred, trace=None) -> float: + # GEPA stores result metadata on ``pred`` when wrapped through + # this adapter; the caller's program puts ``scores`` there. + scores = getattr(pred, "scores", None) or {} + try: + return float(scores.get(score_key, 0.0) or 0.0) + except (TypeError, ValueError): + return 0.0 + + return metric + + +def _to_example(dspy_mod, row: ResultRow, score_key: str): + """Wrap a ResultRow as a DSPy example GEPA can iterate over.""" + return dspy_mod.Example( + challenge=row.scores.get("challenge", ""), + architecture=row.architecture_code, + score=float(row.scores.get(score_key, 0.0) or 0.0), + ).with_inputs("challenge") + + +def optimize( + results: list[ResultRow], + population: list[Prompt], + config: dict, +) -> list[Prompt]: + """Run GEPA on the population. + + See module docstring for ``config`` keys. Returns a new population + of size ``config['population']`` (or the input size if unset). + """ + dspy_mod = _import_dspy() + + if not population: + return [] + + target = int(config.get("population", len(population) or 8)) + budget = int(config.get("budget", 30)) + score_key = str(config.get("score_key", "raw_score")) + + # Configure reflector LM. + lm_kwargs = config.get("reflector_lm") + if isinstance(lm_kwargs, dict): + lm = dspy_mod.LM(**lm_kwargs) + else: + lm = _default_lm(dspy_mod) + dspy_mod.configure(lm=lm) + + # Build the training set from scored rows. + examples = [_to_example(dspy_mod, r, score_key) + for r in results if r.prompt_id] + if not examples: + logger.warning( + "GEPA: no scored examples — returning population unchanged." + ) + return population[:target] + + next_gen = max(p.generation for p in population) + 1 + + # GEPA optimizes a DSPy program; each program is a ``Predict`` over + # the same ``Signature`` but with the prompt template as its + # instruction. We seed the search from every active variant so the + # current population is preserved unless GEPA finds something + # strictly better. + class _ArchSignature(dspy_mod.Signature): + """Design a model architecture for the given challenge.""" + + challenge: str = dspy_mod.InputField() + architecture: str = dspy_mod.OutputField() + + programs = [] + for p in population: + prog = dspy_mod.Predict(_ArchSignature) + prog.signature = prog.signature.with_instructions(p.template) + programs.append((p, prog)) + + metric = _build_metric(score_key) + teleprompter = dspy_mod.GEPA(metric=metric, num_candidates=target, budget=budget) + + # GEPA compiles each seed program; we keep the best ``target`` + # programs by their reported score and map them back to Prompt rows. + optimized: list[tuple[Prompt, Any]] = [] + for parent, prog in programs: + try: + compiled = teleprompter.compile(prog, trainset=examples) + except Exception as e: + logger.warning( + "GEPA: compile failed for prompt %s: %s — keeping parent", + parent.id, e, + ) + optimized.append((parent, prog)) + continue + new_instruction = _extract_instruction(compiled) + optimized.append(( + Prompt.new( + template=new_instruction or parent.template, + generation=next_gen, + parent_id=parent.id, + metadata={"mutation": "gepa", "budget": budget}, + ), + compiled, + )) + + new_pop = [p for p, _ in optimized][:target] + while len(new_pop) < target: + new_pop.append(population[len(new_pop) % len(population)]) + return new_pop + + +def _extract_instruction(program) -> str: + """Pull the optimized instruction string off a DSPy program.""" + sig = getattr(program, "signature", None) + if sig is None: + return "" + instructions = getattr(sig, "instructions", "") + if isinstance(instructions, str): + return instructions + return str(instructions or "") diff --git a/miner_template/optimizers/random_mutate.py b/miner_template/optimizers/random_mutate.py new file mode 100644 index 0000000..3ca71c8 --- /dev/null +++ b/miner_template/optimizers/random_mutate.py @@ -0,0 +1,115 @@ +"""Baseline optimizer: rank prompts by mean score and clone+perturb winners. + +No LLM, no DSPy dependency — used as a CI-safe default and as a +sanity-check sibling to GEPA. Useful when a miner wants to bootstrap +a population without an LLM cost on day one. + +Strategy:: + + 1. group ResultRow by prompt_id; compute mean score per prompt + 2. promote the top-K to the new population + 3. for each survivor, generate (population_size - K) // K children + by appending a deterministic perturbation suffix + 4. if no scores yet, simply clone the active population +""" + +from __future__ import annotations + +import hashlib +from typing import Iterable + +from miner_template.optimizers import ResultRow +from miner_template.prompts import Prompt + +# Cheap deterministic prompt perturbations. Picked by hashing the +# parent's id + child index so two runs on the same inputs produce the +# same children — important for reproducibility. +_PERTURBATIONS: list[str] = [ + "\n\nThink step by step before answering.", + "\n\nFavor architectures with explicit attention sparsity.", + "\n\nPrefer designs that reuse parameters across layers.", + "\n\nConsider hybrid CNN+attention encoders.", + "\n\nWhen in doubt, fall back to a small transformer baseline.", + "\n\nReport hyperparameter choices in the motivation field.", + "\n\nOptimize for sample efficiency over raw capacity.", + "\n\nAvoid recurrent layers unless justified by data length.", +] + + +def _mean_score(rows: Iterable[ResultRow], key: str = "raw_score") -> float: + vals = [float(r.scores.get(key, 0.0) or 0.0) for r in rows] + if not vals: + return 0.0 + return sum(vals) / len(vals) + + +def _rank_population( + results: list[ResultRow], + population: list[Prompt], + score_key: str, +) -> list[tuple[Prompt, float]]: + """Return ``(prompt, mean_score)`` sorted best-first. Prompts with + no observed rows fall to the bottom but stay in the ranking.""" + by_pid: dict[str, list[ResultRow]] = {} + for r in results: + if r.prompt_id: + by_pid.setdefault(r.prompt_id, []).append(r) + scored: list[tuple[Prompt, float]] = [] + for p in population: + scored.append((p, _mean_score(by_pid.get(p.id, []), score_key))) + scored.sort(key=lambda t: t[1], reverse=True) + return scored + + +def _perturb(parent: Prompt, child_idx: int, generation: int) -> Prompt: + seed = hashlib.sha256( + f"{parent.id}:{child_idx}".encode() + ).digest() + idx = seed[0] % len(_PERTURBATIONS) + return Prompt.new( + template=parent.template + _PERTURBATIONS[idx], + generation=generation, + parent_id=parent.id, + metadata={ + "mutation": "random_mutate", + "perturbation_idx": idx, + }, + ) + + +def optimize( + results: list[ResultRow], + population: list[Prompt], + config: dict, +) -> list[Prompt]: + """Return the next population. + + Config keys: + ``population`` (int, default 8): target size of the new population. + ``elite_k`` (int, default 2): how many top performers to carry. + ``score_key`` (str, default "raw_score"): scores field to rank on. + """ + target = int(config.get("population", 8)) + elite_k = int(config.get("elite_k", 2)) + score_key = str(config.get("score_key", "raw_score")) + if target < 1: + raise ValueError("population must be >= 1") + if elite_k < 1: + elite_k = 1 + + if not population: + return [] # caller should seed_default() before optimizing + + next_gen = max(p.generation for p in population) + 1 + + ranked = _rank_population(results, population, score_key) + elites = [p for p, _ in ranked[:elite_k]] + + new_pop: list[Prompt] = list(elites) + child_idx = 0 + while len(new_pop) < target: + parent = elites[child_idx % len(elites)] + new_pop.append(_perturb(parent, child_idx, next_gen)) + child_idx += 1 + + return new_pop[:target] diff --git a/miner_template/prompts.py b/miner_template/prompts.py new file mode 100644 index 0000000..95e55c3 --- /dev/null +++ b/miner_template/prompts.py @@ -0,0 +1,227 @@ +"""Local prompt-population store for miners. + +A *population* is a small list of ``Prompt`` rows kept on disk under +``prompts/`` next to the miner's working directory: + + prompts/ + active.json # current population the agent reads + history/ + gen_001.json # archived populations + gen_002.json + ... + +Atomic writes use ``tmp + os.replace`` so an interrupted optimizer +never leaves the agent with a half-written file. + +The agent reads the active population each round via ``load_active()`` +and picks one variant with ``pick_for_round(pop, round_id)``. The +chosen prompt's ``id`` round-trips back via the submission so the +optimizer can correlate Phase C scores with prompt variants. +""" + +from __future__ import annotations + +import dataclasses +import json +import logging +import os +import uuid +from dataclasses import dataclass, field +from pathlib import Path +from typing import Iterable, Optional + +logger = logging.getLogger(__name__) + +DEFAULT_PROMPTS_DIR = Path(os.getenv("MINER_PROMPTS_DIR", "prompts")) +ACTIVE_FILE = "active.json" +HISTORY_DIR = "history" + + +@dataclass +class Prompt: + """A single prompt variant in the population.""" + + id: str + template: str + generation: int = 0 + parent_id: Optional[str] = None + metadata: dict = field(default_factory=dict) + + @classmethod + def new(cls, template: str, *, generation: int = 0, + parent_id: Optional[str] = None, + metadata: Optional[dict] = None) -> "Prompt": + return cls( + id=uuid.uuid4().hex, + template=template, + generation=generation, + parent_id=parent_id, + metadata=dict(metadata or {}), + ) + + def to_dict(self) -> dict: + return dataclasses.asdict(self) + + @classmethod + def from_dict(cls, d: dict) -> "Prompt": + return cls( + id=str(d.get("id") or uuid.uuid4().hex), + template=str(d.get("template", "")), + generation=int(d.get("generation", 0)), + parent_id=d.get("parent_id"), + metadata=dict(d.get("metadata") or {}), + ) + + +# ── Filesystem layout helpers ──────────────────────────────────────── + + +def _root(prompts_dir: Optional[Path] = None) -> Path: + return Path(prompts_dir or DEFAULT_PROMPTS_DIR) + + +def _active_path(prompts_dir: Optional[Path] = None) -> Path: + return _root(prompts_dir) / ACTIVE_FILE + + +def _history_dir(prompts_dir: Optional[Path] = None) -> Path: + return _root(prompts_dir) / HISTORY_DIR + + +def _atomic_write_json(path: Path, payload) -> None: + """Write ``payload`` to ``path`` via ``.tmp`` + ``os.replace`` so the + reader never sees a partial file.""" + path.parent.mkdir(parents=True, exist_ok=True) + tmp = path.with_suffix(path.suffix + ".tmp") + with tmp.open("w") as f: + json.dump(payload, f, indent=2, sort_keys=True) + f.flush() + os.fsync(f.fileno()) + os.replace(tmp, path) + + +# ── Active-population I/O ──────────────────────────────────────────── + + +def load_active(prompts_dir: Optional[Path] = None) -> list[Prompt]: + """Read the active population. Returns ``[]`` if the file doesn't + exist yet (fresh miner) — callers should treat this as "seed me".""" + path = _active_path(prompts_dir) + if not path.exists(): + return [] + try: + payload = json.loads(path.read_text()) + except (OSError, json.JSONDecodeError) as e: + logger.warning("prompts: active.json unreadable (%s) — empty pop", e) + return [] + rows = payload.get("prompts") if isinstance(payload, dict) else payload + if not isinstance(rows, list): + return [] + return [Prompt.from_dict(r) for r in rows if isinstance(r, dict)] + + +def save_active(prompts: Iterable[Prompt], + prompts_dir: Optional[Path] = None) -> None: + """Atomically write the active population.""" + items = [p.to_dict() for p in prompts] + payload = {"prompts": items} + _atomic_write_json(_active_path(prompts_dir), payload) + + +def archive_current(generation: int, + prompts_dir: Optional[Path] = None) -> Optional[Path]: + """Copy the current ``active.json`` into ``history/gen_NNN.json``. + + Returns the destination path, or ``None`` if there's no active file + yet (first generation — nothing to archive). + """ + src = _active_path(prompts_dir) + if not src.exists(): + return None + dst = _history_dir(prompts_dir) / f"gen_{generation:03d}.json" + dst.parent.mkdir(parents=True, exist_ok=True) + dst.write_bytes(src.read_bytes()) + return dst + + +def list_history(prompts_dir: Optional[Path] = None) -> list[int]: + """Return archived generation numbers, sorted ascending.""" + d = _history_dir(prompts_dir) + if not d.exists(): + return [] + gens: list[int] = [] + for f in d.iterdir(): + if not f.name.startswith("gen_") or not f.name.endswith(".json"): + continue + try: + gens.append(int(f.stem.split("_", 1)[1])) + except (IndexError, ValueError): + continue + return sorted(gens) + + +def load_generation(generation: int, + prompts_dir: Optional[Path] = None) -> list[Prompt]: + """Read an archived generation back into memory.""" + path = _history_dir(prompts_dir) / f"gen_{generation:03d}.json" + if not path.exists(): + raise FileNotFoundError(f"no archived generation {generation}") + payload = json.loads(path.read_text()) + rows = payload.get("prompts") if isinstance(payload, dict) else payload + return [Prompt.from_dict(r) for r in rows if isinstance(r, dict)] + + +def rollback_to(generation: int, + prompts_dir: Optional[Path] = None) -> None: + """Restore an archived generation as the new active population. + + The currently active population is first archived under the next + free generation number, so rollback is reversible. + """ + pop = load_generation(generation, prompts_dir) + next_gen = (list_history(prompts_dir) or [0])[-1] + 1 + archive_current(next_gen, prompts_dir) + save_active(pop, prompts_dir) + + +# ── Selection ──────────────────────────────────────────────────────── + + +def pick_for_round(population: list[Prompt], round_id: int) -> Prompt: + """Deterministic round-robin over the population. + + Same ``round_id`` always picks the same prompt — so explore/exploit + looks the same to anyone replaying the round. Empty population + raises ``ValueError``; callers should ``seed_default()`` first. + """ + if not population: + raise ValueError("empty population — call seed_default() first") + return population[round_id % len(population)] + + +# ── Seeding ────────────────────────────────────────────────────────── + + +DEFAULT_SEED_TEMPLATE = ( + "You are a model-architecture researcher designing a PyTorch model " + "for time-series forecasting.\n" + "Round budget: FLOPs in [{min_flops}, {max_flops}].\n" + "Frontier so far (best metric per architecture):\n" + "{frontier}\n\n" + "Design a single architecture targeting ~60% of the FLOPs ceiling. " + "Return the model code, a short name, and a one-line motivation." +) + + +def seed_default(prompts_dir: Optional[Path] = None, + template: str = DEFAULT_SEED_TEMPLATE) -> list[Prompt]: + """Write a single starter prompt as the active population. No-op + if ``active.json`` already exists. Returns the resulting active + population either way. + """ + existing = load_active(prompts_dir) + if existing: + return existing + seed = Prompt.new(template=template, generation=0) + save_active([seed], prompts_dir) + return [seed] diff --git a/miner_template/results_client.py b/miner_template/results_client.py new file mode 100644 index 0000000..298bb08 --- /dev/null +++ b/miner_template/results_client.py @@ -0,0 +1,127 @@ +"""HTTP client for the miner-feedback endpoints on the database server. + +The DB exposes a small set of bearer-auth'd routes under ``/miners/me/*`` +that let a miner pull its own scored history without going through +the validator. This is the feedback channel the optimizer reads. + + GET /miners/me/submissions — Phase A submissions made by this miner + GET /miners/me/results — submissions joined with Phase C scores + GET /miners/me/summary — counts + rolling-window stats + GET /tasks/{task}/frontier — public per-task Pareto frontier +""" + +from __future__ import annotations + +import logging +from typing import Optional + +import httpx + +from miner_template.optimizers import ResultRow + +logger = logging.getLogger(__name__) + +DEFAULT_TIMEOUT = 30.0 + + +class MinerResultsClient: + """Pull this miner's scored history from the central DB.""" + + def __init__( + self, + db_url: str, + api_key: str, + *, + timeout: float = DEFAULT_TIMEOUT, + client: Optional[httpx.Client] = None, + ): + if not db_url: + raise ValueError("db_url required") + if not api_key: + raise ValueError("api_key required") + self.db_url = db_url.rstrip("/") + self.api_key = api_key + self.timeout = timeout + self._owned_client = client is None + self._client = client or httpx.Client(timeout=timeout) + + # ── Context manager ───────────────────────────────────────────── + + def __enter__(self): + return self + + def __exit__(self, *exc): + self.close() + + def close(self) -> None: + if self._owned_client and not self._client.is_closed: + self._client.close() + + # ── Internal ──────────────────────────────────────────────────── + + def _headers(self) -> dict[str, str]: + return {"Authorization": f"Bearer {self.api_key}"} + + def _get(self, path: str, params: Optional[dict] = None): + resp = self._client.get( + f"{self.db_url}{path}", params=params, headers=self._headers(), + ) + resp.raise_for_status() + return resp.json() + + # ── Public ────────────────────────────────────────────────────── + + def submissions( + self, + *, + since: str = "", + limit: int = 200, + task: str = "", + ) -> list[dict]: + """Phase A submissions made by this miner. Returns raw dicts — + the optimizer doesn't need these; this is for the CLI's + ``results`` view + custom miner tooling.""" + params: dict = {"limit": limit} + if since: + params["since"] = since + if task: + params["task"] = task + payload = self._get("/miners/me/submissions", params=params) + if isinstance(payload, dict): + return list(payload.get("submissions", []) or []) + return list(payload or []) + + def results( + self, + *, + since: str = "", + limit: int = 200, + task: str = "", + ) -> list[ResultRow]: + """Scored history — submissions joined with Phase C results. + Returned as ``ResultRow`` so the optimizer can consume directly.""" + params: dict = {"limit": limit} + if since: + params["since"] = since + if task: + params["task"] = task + payload = self._get("/miners/me/results", params=params) + rows = payload.get("results") if isinstance(payload, dict) else payload + return [ResultRow.from_dict(r) for r in (rows or []) if isinstance(r, dict)] + + def summary(self) -> dict: + """Rolling stats — total submissions, last round id, mean score + over the recent window, etc. Useful for ``--watch`` to detect + when a new round of results is available.""" + payload = self._get("/miners/me/summary") + return payload if isinstance(payload, dict) else {} + + def frontier(self, task: str) -> list[dict]: + """Public per-task frontier (no auth required, but we send the + bearer anyway — harmless).""" + if not task: + raise ValueError("task required") + payload = self._get(f"/tasks/{task}/frontier") + if isinstance(payload, dict): + return list(payload.get("points", []) or []) + return list(payload or []) diff --git a/runner/server.py b/runner/server.py index ced57ef..7db912d 100644 --- a/runner/server.py +++ b/runner/server.py @@ -19,8 +19,6 @@ import os import threading import time -from contextlib import asynccontextmanager - from fastapi import FastAPI, Request from fastapi.responses import JSONResponse @@ -38,32 +36,24 @@ logger = logging.getLogger(__name__) -@asynccontextmanager -async def _lifespan(app: FastAPI): - """Preload metagraph at startup so /train doesn't block on first request.""" - localnet = os.getenv("RADAR_LOCALNET", "").lower() in ("1", "true") - if not localnet: - logger.info("Preloading metagraph at startup...") - mg = await asyncio.to_thread(_load_metagraph) - if mg is not None: - logger.info("Metagraph preloaded (%d neurons)", mg.n) - else: - logger.warning("Metagraph preload failed — will retry on first request") - yield +app = FastAPI(title="Radar Trainer") + +# ── Auth (shared HMAC service key) ─────────────────────────────────── +# In non-competitive mode there's exactly one issuer (the operator), so +# the trainer just verifies HMAC against ``RADAR_SERVICE_KEY``. No +# metagraph fetch, no bittensor import. +def _service_secret() -> bytes | None: + raw = os.getenv("RADAR_SERVICE_KEY", "").strip() + return raw.encode() if raw else None -app = FastAPI(title="Radar Trainer", lifespan=_lifespan) -# ── Metagraph cache ────────────────────────────────────────────────── -_metagraph_cache = None -_metagraph_lock = threading.Lock() -_metagraph_last_refresh: float = 0.0 -METAGRAPH_REFRESH_INTERVAL = float(os.getenv("METAGRAPH_REFRESH_INTERVAL", "300")) +_EXPECTED_KEY_ID = os.getenv("RADAR_SERVICE_KEY_ID", "operator") -# ── Rate limiting ──────────────────────────────────────────────────── +# ── Rate limiting (per key_id) ─────────────────────────────────────── _train_semaphore = asyncio.Semaphore(1) # max 1 concurrent /train -_hotkey_last_request: dict[str, float] = {} -_hotkey_lock = threading.Lock() +_key_last_request: dict[str, float] = {} +_key_lock = threading.Lock() HOTKEY_COOLDOWN_SECONDS = float(os.getenv("TRAINER_HOTKEY_COOLDOWN", "60")) # ── Task registry ─────────────────────────────────────────────────── @@ -89,32 +79,38 @@ async def train(request: Request): body = await request.body() - # 1. Auth — FAIL CLOSED + # 1. Auth — FAIL CLOSED (HMAC service key) localnet = os.getenv("RADAR_LOCALNET", "").lower() in ("1", "true") if localnet: logger.info("Localnet mode: skipping auth") sender = "localnet" else: - metagraph = _load_metagraph() - if metagraph is None: - logger.error("Metagraph unavailable — rejecting request (fail closed)") - return JSONResponse(status_code=503, content={"error": "Auth unavailable, try again later"}) + secret = _service_secret() + if not secret: + logger.error("RADAR_SERVICE_KEY not set — rejecting (fail closed)") + return JSONResponse( + status_code=503, + content={"error": "Auth not configured on trainer"}, + ) try: - from shared.auth import verify_request - ok, err, sender = verify_request(dict(request.headers), body, metagraph, require_stake=True) + from shared.auth import hmac_verify_request, static_key_lookup + ok, err, sender = hmac_verify_request( + dict(request.headers), body, + static_key_lookup(_EXPECTED_KEY_ID, secret), + ) if not ok: - signed_by = request.headers.get("x-epistula-signed-by", "?") - logger.warning( - "Auth rejected from %s: %s", signed_by, err, - ) + kid = request.headers.get("x-radar-key-id", "?") + logger.warning("Auth rejected (key_id=%s): %s", kid, err) return JSONResponse(status_code=403, content={"error": err}) except Exception as e: logger.error("Auth verification failed: %s", e) - return JSONResponse(status_code=403, content={"error": "Auth verification error"}) + return JSONResponse( + status_code=403, content={"error": "Auth verification error"}, + ) - # 2. Per-hotkey rate limit (check only — timestamp recorded after semaphore) - with _hotkey_lock: - last = _hotkey_last_request.get(sender, 0.0) + # 2. Per-key rate limit (check only — timestamp recorded after semaphore) + with _key_lock: + last = _key_last_request.get(sender, 0.0) if time.time() - last < HOTKEY_COOLDOWN_SECONDS: remaining = HOTKEY_COOLDOWN_SECONDS - (time.time() - last) return JSONResponse(status_code=429, content={ @@ -155,8 +151,8 @@ async def train(request: Request): # Record rate-limit timestamp AFTER semaphore acquired — if the # request fails before this point, retries won't be rate-limited. - with _hotkey_lock: - _hotkey_last_request[sender] = time.time() + with _key_lock: + _key_last_request[sender] = time.time() training_config = { "seed": data.get("seed", 42), @@ -315,33 +311,6 @@ async def boot_proof(): return body -def _load_metagraph(): - """Load metagraph with caching. Returns None if never reachable.""" - global _metagraph_cache, _metagraph_last_refresh - now = time.time() - if _metagraph_cache is not None and (now - _metagraph_last_refresh) < METAGRAPH_REFRESH_INTERVAL: - return _metagraph_cache - with _metagraph_lock: - if _metagraph_cache is not None and (now - _metagraph_last_refresh) < METAGRAPH_REFRESH_INTERVAL: - return _metagraph_cache - try: - import bittensor as bt - netuid = int(os.getenv("NETUID", "1")) - network = os.getenv("SUBTENSOR_NETWORK", "finney") - subtensor = bt.Subtensor(network=network) - mg = subtensor.metagraph(netuid) - _metagraph_cache = mg - _metagraph_last_refresh = time.time() - logger.info("Metagraph refreshed (%d neurons)", mg.n) - return mg - except Exception as e: - logger.warning("Metagraph refresh failed: %s", e) - if _metagraph_cache is not None: - logger.info("Using stale metagraph cache") - return _metagraph_cache - return None - - if __name__ == "__main__": import uvicorn # Configure the application root logger so INFO messages from diff --git a/shared/auth.py b/shared/auth.py index 258cf86..4a6ce3f 100644 --- a/shared/auth.py +++ b/shared/auth.py @@ -1,35 +1,50 @@ -"""Epistula authentication — sign and verify HTTP requests. +"""Auth primitives — both legacy Epistula and new HMAC/bearer surfaces. -Extracted from validator/gossip.py. Used by validators, trainers, and miners -for authenticating HTTP requests with SR25519 hotkey signatures. +Two coexisting auth schemes during the non-competitive cutover: + + Legacy (Epistula / SR25519, metagraph-backed) + sign_request(wallet, body) -> headers + verify_request(headers, body, mg, …) -> (ok, error, hotkey) + + New (HMAC service-key + bearer API tokens) + hmac_sign_request(secret, body, kid) -> headers + hmac_verify_request(headers, body, lk) -> (ok, error, key_id) + extract_bearer(headers) -> token | None + hash_api_key(key) -> sha256 hex (for DB storage) + +The HMAC scheme signs ``f"{timestamp}.{sha256(body).hex()}"`` with +HMAC-SHA256. ``key_id`` is sent in cleartext so the verifier can look +up the right secret; secrets themselves are never on the wire. + +Once every caller is on the new surface we'll drop the Epistula +functions and the ``bittensor`` import below. """ from __future__ import annotations +import hashlib +import hmac import logging import os import time import uuid -from typing import Optional - -import bittensor as bt +from typing import Callable, Optional logger = logging.getLogger(__name__) +# ── Legacy Epistula ────────────────────────────────────────────────── + # Epistula timestamp tolerance (seconds). Cross-machine clock skew between # validator hosts and miner trainer pods is the most common cause of 403s -# here, so the default is intentionally generous. Replay protection still -# relies on the SR25519 signature + uuid4 nonce; the timestamp window only -# bounds how long a captured signed message can be replayed. +# here, so the default is intentionally generous. EPISTULA_TIMESTAMP_TOLERANCE = int(os.getenv("RADAR_EPISTULA_TOLERANCE", "120")) -def sign_request(wallet: bt.Wallet, body: bytes) -> dict[str, str]: - """Sign an outgoing HTTP request with Epistula headers. +def sign_request(wallet, body: bytes) -> dict[str, str]: + """Sign an outgoing HTTP request with Epistula headers (legacy).""" + import bittensor as bt # local import keeps non-chain processes light - Headers: X-Epistula-Signed-By, X-Epistula-Timestamp, - X-Epistula-Nonce, X-Epistula-Signature - """ + _ = bt # silence linter timestamp = str(int(time.time())) nonce = uuid.uuid4().hex message = body + timestamp.encode() + nonce.encode() @@ -48,10 +63,9 @@ def verify_request( metagraph, require_stake: bool = False, ) -> tuple[bool, str, str]: - """Verify an incoming request's Epistula headers. + """Verify Epistula headers on an inbound request (legacy).""" + import bittensor as bt - Returns (ok, error_message, sender_hotkey). - """ hotkey = headers.get("x-epistula-signed-by", "") timestamp_str = headers.get("x-epistula-timestamp", "") nonce = headers.get("x-epistula-nonce", "") @@ -60,24 +74,18 @@ def verify_request( if not all([hotkey, timestamp_str, nonce, signature]): return False, "Missing Epistula headers", "" - # Check timestamp freshness try: ts = int(timestamp_str) except ValueError: return False, "Invalid timestamp", "" skew = int(time.time() - ts) if abs(skew) > EPISTULA_TIMESTAMP_TOLERANCE: - # Surface the actual skew + tolerance so miner operators can see at - # a glance that this is a clock-sync problem (NTP, container drift) - # rather than a signing bug, and tune RADAR_EPISTULA_TOLERANCE if - # needed. err = ( f"Timestamp too old or too far in future " f"(skew={skew}s, tolerance=±{EPISTULA_TIMESTAMP_TOLERANCE}s)" ) return False, err, "" - # Verify sender is registered uid = get_uid_for_hotkey(metagraph, hotkey) if uid is None: return False, "Unknown hotkey", "" @@ -85,7 +93,6 @@ def verify_request( if require_stake and float(metagraph.S[uid]) <= 0: return False, "Not a validator (no stake)", "" - # Verify signature message = body + timestamp_str.encode() + nonce.encode() try: keypair = bt.Keypair(ss58_address=hotkey) @@ -98,7 +105,6 @@ def verify_request( def get_uid_for_hotkey(metagraph, hotkey: str) -> Optional[int]: - """Look up UID for a hotkey in the metagraph.""" hotkeys = metagraph.hotkeys if hotkeys is None: return None @@ -106,3 +112,131 @@ def get_uid_for_hotkey(metagraph, hotkey: str) -> Optional[int]: if i < len(hotkeys) and hotkeys[i] == hotkey: return i return None + + +# ── New HMAC primitives ────────────────────────────────────────────── + +# How far the wire timestamp may drift from the verifier's clock. +HMAC_TIMESTAMP_TOLERANCE = int(os.getenv("RADAR_HMAC_TOLERANCE", "300")) + +DEFAULT_KEY_ID = "operator" + +H_SIG = "x-radar-signature" +H_TS = "x-radar-timestamp" +H_KID = "x-radar-key-id" +H_AUTH = "authorization" + +KeyLookup = Callable[[str], Optional[bytes]] + + +def _canonical(body: bytes, timestamp: str) -> bytes: + body_digest = hashlib.sha256(body).hexdigest() + return f"{timestamp}.{body_digest}".encode() + + +def hmac_sign_request( + secret: bytes, + body: bytes, + key_id: str = DEFAULT_KEY_ID, +) -> dict[str, str]: + """HMAC-SHA256 sign a request. Caller attaches the returned headers + to its outbound HTTP request.""" + if not isinstance(secret, (bytes, bytearray)): + raise TypeError("secret must be bytes") + timestamp = str(int(time.time())) + sig = hmac.new(secret, _canonical(body, timestamp), hashlib.sha256).hexdigest() + return { + "X-Radar-Signature": sig, + "X-Radar-Timestamp": timestamp, + "X-Radar-Key-Id": key_id, + } + + +def hmac_verify_request( + headers: dict[str, str], + body: bytes, + key_lookup: KeyLookup, + tolerance_s: int = HMAC_TIMESTAMP_TOLERANCE, +) -> tuple[bool, str, str]: + """Verify HMAC headers on an inbound request. + + ``key_lookup`` resolves ``key_id -> secret``. Returning ``None`` + fails verification closed. Returns ``(ok, error, key_id)`` with + ``key_id`` empty on failure. + """ + h = {k.lower(): v for k, v in headers.items()} + sig = h.get(H_SIG, "") + ts = h.get(H_TS, "") + kid = h.get(H_KID, "") + + if not sig or not ts or not kid: + return False, "missing HMAC headers", "" + + try: + ts_int = int(ts) + except ValueError: + return False, "invalid timestamp", "" + skew = int(time.time()) - ts_int + if abs(skew) > tolerance_s: + return False, ( + f"timestamp out of tolerance (skew={skew}s, " + f"tolerance=±{tolerance_s}s)" + ), "" + + secret = key_lookup(kid) + if secret is None: + return False, "unknown key_id", "" + + expected = hmac.new(secret, _canonical(body, ts), hashlib.sha256).hexdigest() + if not hmac.compare_digest(expected, sig): + return False, "bad signature", "" + + return True, "", kid + + +def static_key_lookup(key_id: str, secret: bytes) -> KeyLookup: + """Build a ``key_lookup`` that accepts exactly one ``(key_id, secret)`` + — the common case for trainer-side service auth.""" + expected = key_id + + def lookup(kid: str) -> Optional[bytes]: + if kid == expected: + return secret + return None + + return lookup + + +# ── Bearer tokens (per-agent / per-miner API keys) ─────────────────── + + +def extract_bearer(headers: dict[str, str]) -> Optional[str]: + """Return the bearer token from an ``Authorization`` header, or + ``None`` if missing/malformed.""" + h = {k.lower(): v for k, v in headers.items()} + raw = h.get(H_AUTH, "") + if not raw: + return None + parts = raw.split(None, 1) + if len(parts) != 2 or parts[0].lower() != "bearer": + return None + token = parts[1].strip() + return token or None + + +def hash_api_key(key: str) -> str: + """SHA-256 hex digest. The DB stores this — never the raw key.""" + return hashlib.sha256(key.encode()).hexdigest() + + +def load_service_secret(env: str = "RADAR_SERVICE_KEY") -> bytes: + """Read the shared HMAC service key from the environment. Raises + ``RuntimeError`` if unset — callers that need fail-closed behavior + should let this propagate at startup.""" + raw = os.getenv(env, "").strip() + if not raw: + raise RuntimeError( + f"{env} is unset — set it to a long random string shared " + f"between operator, validator, and trainer processes." + ) + return raw.encode() diff --git a/shared/database.py b/shared/database.py index e8ff3f6..a08d262 100644 --- a/shared/database.py +++ b/shared/database.py @@ -56,6 +56,11 @@ class DataElement: # entries written by validators after Phase C. One per validator that # published a Hippius bundle covering this experiment. substrate_cids: list = field(default_factory=list) + # Miner-local prompt-variant id (from Proposal.prompt_id). Opaque to + # the operator; round-tripped so the miner's GEPA/random_mutate loop + # can correlate Phase C scores with the variant that produced this + # architecture. Empty string when the miner submitted without one. + prompt_id: str = "" def to_dict(self) -> dict: return asdict(self) diff --git a/shared/db_client.py b/shared/db_client.py index dd6f4c3..b3d6739 100644 --- a/shared/db_client.py +++ b/shared/db_client.py @@ -1,28 +1,55 @@ -"""HTTP client for validators to talk to the centralized database server. - -Used for writes (POST experiments, provenance) and frontier fetches. -All methods sign requests with Epistula via the validator's wallet. +"""HTTP client for validators / miners to talk to the centralized DB. + +Two auth modes selected at construction time: + + * Service HMAC — passes ``service_secret`` (bytes). Outbound requests + carry X-Radar-{Signature, Timestamp, Key-Id} headers. Used by the + validator dispatcher and any other operator-side process holding + the shared service key. + * Bearer token — passes ``api_key`` (str). Outbound requests carry + ``Authorization: Bearer ``. Used by miner-side tooling + (``MinerResultsClient`` lives in ``miner_template`` for the same + surface). + +Legacy Epistula path (``wallet`` keyword) is retained as a back-compat +shim during the cutover. When ``wallet`` is supplied and no HMAC/key +is set, requests are signed with the on-chain hotkey via +``shared.auth.sign_request``. Pass exactly ONE of the three. """ from __future__ import annotations +import json import logging from typing import Optional import httpx -from shared.auth import sign_request - logger = logging.getLogger(__name__) class DatabaseClient: """Async HTTP client for the centralized database API.""" - def __init__(self, db_url: str, wallet, api_key: str = ""): + def __init__( + self, + db_url: str, + wallet=None, + *, + service_secret: Optional[bytes] = None, + key_id: str = "operator", + api_key: str = "", + ): self.db_url = db_url.rstrip("/") self.wallet = wallet + self.service_secret = service_secret + self.key_id = key_id self.api_key = api_key or "" + if not any([self.service_secret, self.api_key, self.wallet]): + raise ValueError( + "DatabaseClient needs one of service_secret (HMAC), " + "api_key (bearer), or wallet (legacy Epistula).", + ) self._client: Optional[httpx.AsyncClient] = None async def _get_client(self) -> httpx.AsyncClient: @@ -31,16 +58,20 @@ async def _get_client(self) -> httpx.AsyncClient: return self._client def _sign(self, body: bytes) -> dict[str, str]: - headers = sign_request(self.wallet, body) + if self.service_secret: + from shared.auth import hmac_sign_request + return hmac_sign_request( + self.service_secret, body, key_id=self.key_id, + ) if self.api_key: - headers["X-Radar-API-Key"] = self.api_key - return headers + return {"Authorization": f"Bearer {self.api_key}"} + # Legacy: wallet (Epistula). + from shared.auth import sign_request + return sign_request(self.wallet, body) async def _post(self, path: str, json_data: dict) -> Optional[dict]: - """POST with Epistula signing. Returns JSON response or None.""" try: client = await self._get_client() - import json body = json.dumps(json_data).encode() headers = self._sign(body) headers["Content-Type"] = "application/json" @@ -61,7 +92,6 @@ async def _post(self, path: str, json_data: dict) -> Optional[dict]: return None async def _get(self, path: str, params: Optional[dict] = None) -> Optional[dict]: - """GET with Epistula signing. Returns JSON response or None.""" try: client = await self._get_client() headers = self._sign(b"") @@ -84,7 +114,6 @@ async def _get(self, path: str, params: Optional[dict] = None) -> Optional[dict] # ── Public API ─────────────────────────────────────── async def health(self) -> bool: - """Check database server health.""" try: client = await self._get_client() resp = await client.get(f"{self.db_url}/health") @@ -99,16 +128,9 @@ async def add_experiment( validator_hotkey: str = "", artifact_cids: Optional[list[dict]] = None, ) -> Optional[int]: - """POST a DataElement dict to the database. Returns new index or None. - - When ``substrate_cid`` is non-empty the server appends a - ``{kind: "phase_c_record", validator_hotkey, cid, round_id}`` entry - to the row's ``substrate_cids`` audit list. - - ``artifact_cids`` (TEN-240 Phase 7) supplies extra audit entries - for dual-written non-DB artifacts. Each element is appended to the - list verbatim — callers set the ``kind`` (``checkpoint``, - ``architecture``, ``training_meta``, …). + """POST a DataElement dict. ``validator_hotkey`` is the legacy + name for the writer's identifier — operator dispatchers in + non-competitive mode pass an opaque operator_id here. """ payload: dict = {"data": element_data} if substrate_cid: @@ -122,35 +144,29 @@ async def add_experiment( return None async def get_frontier(self, task: str = "") -> list[dict]: - """GET the current Pareto frontier.""" params = {"task": task} if task else None result = await self._get("/frontier", params=params) return result if isinstance(result, list) else [] async def update_frontier(self, frontier_data: list[dict], task: str = "") -> bool: - """POST updated frontier data.""" result = await self._post("/frontier/update", { "frontier": frontier_data, "task": task, }) return result is not None async def get_pareto_elements(self, task: str = "") -> list[dict]: - """GET all pareto-eligible elements.""" params = {"task": task} if task else None result = await self._get("/experiments/pareto", params=params) return result if isinstance(result, list) else [] async def get_challenge(self) -> Optional[dict]: - """GET the current challenge.""" return await self._get("/challenge") async def set_challenge(self, challenge_data) -> bool: - """POST challenge data (used by validators to update challenge).""" result = await self._post("/challenge/update", {"challenge": challenge_data}) return result is not None async def record_components(self, experiment_id: int, components: list[str]) -> bool: - """POST detected components for an experiment.""" result = await self._post("/provenance/record_components", { "experiment_id": experiment_id, "components": components, }) @@ -159,7 +175,6 @@ async def record_components(self, experiment_id: int, components: list[str]) -> async def record_round_context( self, round_id: int, experiment_id: int, context_type: str = "frontier", ) -> bool: - """POST round context (which experiments were shown).""" result = await self._post("/provenance/record_context", { "round_id": round_id, "experiment_id": experiment_id, "context_type": context_type, @@ -167,23 +182,19 @@ async def record_round_context( return result is not None async def get_experiment(self, index: int) -> Optional[dict]: - """GET a single experiment by index.""" return await self._get(f"/experiments/{index}") async def get_diff(self, index: int) -> Optional[dict]: - """GET diff for an experiment vs its parent.""" return await self._get(f"/experiments/{index}/diff") - async def get_agent_code(self, hotkey: str) -> Optional[dict]: - """GET a miner's agent code bundle from the DB server.""" - return await self._get(f"/agent_code/{hotkey}") + async def get_agent_code(self, miner_id: str) -> Optional[dict]: + return await self._get(f"/agent_code/{miner_id}") async def get_agent_code_history( - self, hotkey: str, limit: int = 100, + self, miner_id: str, limit: int = 100, ) -> list[dict]: - """GET the full submission timeline for a hotkey (most recent first).""" result = await self._get( - f"/agent_code/{hotkey}/history", params={"limit": limit}, + f"/agent_code/{miner_id}/history", params={"limit": limit}, ) if isinstance(result, dict): subs = result.get("submissions") @@ -192,13 +203,11 @@ async def get_agent_code_history( return [] async def get_agent_code_by_hash(self, code_hash: str) -> Optional[dict]: - """GET an immutable agent bundle by its content hash.""" return await self._get(f"/agent_code/by_hash/{code_hash}") async def submit_agent_code( self, files: dict[str, str], entry_point: str = "agent.py", ) -> Optional[dict]: - """POST agent code bundle. Returns {"code_hash", "r2_key"} or None.""" return await self._post("/agent_code", { "files": files, "entry_point": entry_point, }) @@ -206,8 +215,6 @@ async def submit_agent_code( async def submit_training_meta( self, round_id: int, hotkey: str, meta: dict, ) -> bool: - """POST a training_meta.json blob so the public dashboard can render - loss curves without R2 access. Idempotent on (round_id, hotkey).""" result = await self._post("/training_metas", { "round_id": int(round_id), "hotkey": hotkey, @@ -218,19 +225,45 @@ async def submit_training_meta( async def submit_submission_reveal( self, round_id: int, entries: list[dict], ) -> bool: - """POST the per-round submission_id -> miner_hotkey reveal map. - - Called after Phase C closes. Each entry is a dict with keys - ``submission_id``, ``miner_hotkey``, ``miner_uid``. Idempotent on - (round_id, submission_id). - """ + """POST per-round submission_id -> miner_id map. Idempotent.""" result = await self._post("/round_submissions/reveal", { "round_id": int(round_id), "entries": entries, }) return result is not None + # ── Miner feedback surface (bearer auth) ────────────────────── + + async def my_submissions( + self, since: str = "", limit: int = 200, task: str = "", + ) -> list[dict]: + params: dict = {"limit": limit} + if since: + params["since"] = since + if task: + params["task"] = task + result = await self._get("/miners/me/submissions", params=params) + if isinstance(result, dict): + return result.get("submissions", []) or [] + return result if isinstance(result, list) else [] + + async def my_results( + self, since: str = "", limit: int = 200, task: str = "", + ) -> list[dict]: + params: dict = {"limit": limit} + if since: + params["since"] = since + if task: + params["task"] = task + result = await self._get("/miners/me/results", params=params) + if isinstance(result, dict): + return result.get("results", []) or [] + return result if isinstance(result, list) else [] + + async def my_summary(self) -> dict: + result = await self._get("/miners/me/summary") + return result if isinstance(result, dict) else {} + async def close(self): - """Close the underlying HTTP client.""" if self._client and not self._client.is_closed: await self._client.aclose() diff --git a/shared/miner_auth.py b/shared/miner_auth.py new file mode 100644 index 0000000..1b4837a --- /dev/null +++ b/shared/miner_auth.py @@ -0,0 +1,159 @@ +"""Bearer-token miner auth — registry helpers backed by Postgres. + +The non-competitive Radar feedback API (``/miners/me/*`` and a couple +of CLI helpers) authenticates miners with operator-issued bearer +tokens stored hashed in ``miner_api_keys``. This module is the choke +point for issuing, looking up, and revoking those keys; the FastAPI +middleware delegates here so the SQL stays in one place. + +The hash is SHA-256 of the raw key (see ``shared.auth.hash_api_key``) +so a database leak doesn't recover the plaintext bearer. + +Key format on the wire: ``rdrk_<32-hex-chars>`` — the prefix lets +operators grep keys out of logs, but isn't required by the verifier. +""" + +from __future__ import annotations + +import logging +import secrets +import time +import uuid +from dataclasses import dataclass +from typing import Optional + +from shared.auth import hash_api_key + +logger = logging.getLogger(__name__) + +KEY_PREFIX = "rdrk_" + + +@dataclass(slots=True) +class MinerIdentity: + """Resolved bearer-auth identity returned to the request handler.""" + + miner_id: str + key_id: str + scope: str + hotkey: str = "" + name: str = "" + + +def _new_key_pair() -> tuple[str, str]: + """Return ``(plaintext_key, key_id)``. ``key_id`` is the + user-facing identifier (not a secret) so operators can list + + revoke keys without ever holding the plaintext again.""" + token = secrets.token_hex(16) + return f"{KEY_PREFIX}{token}", uuid.uuid4().hex + + +async def register_miner( + pool, + *, + miner_id: Optional[str] = None, + name: str = "", + hotkey: str = "", + contact: str = "", +) -> str: + """Insert a row in ``miners`` and return the (possibly generated) id.""" + mid = miner_id or uuid.uuid4().hex + await pool.execute( + "INSERT INTO miners (miner_id, name, hotkey, contact) " + "VALUES ($1, $2, $3, $4) " + "ON CONFLICT (miner_id) DO UPDATE SET " + " name = EXCLUDED.name, hotkey = EXCLUDED.hotkey, " + " contact = EXCLUDED.contact", + mid, name, hotkey, contact, + ) + return mid + + +async def issue_api_key( + pool, + miner_id: str, + *, + scope: str = "miner", + label: str = "", +) -> tuple[str, str]: + """Mint a new API key. Returns ``(plaintext, key_id)``. + + The plaintext is shown to the operator ONCE; the DB only stores + the hash. ``key_id`` is the stable handle for revocation. + """ + plaintext, key_id = _new_key_pair() + await pool.execute( + "INSERT INTO miner_api_keys (key_id, key_hash, miner_id, scope, label) " + "VALUES ($1, $2, $3, $4, $5)", + key_id, hash_api_key(plaintext), miner_id, scope, label, + ) + return plaintext, key_id + + +async def revoke_api_key(pool, key_id: str) -> bool: + """Mark a key revoked. Returns True iff a row was updated.""" + result = await pool.execute( + "UPDATE miner_api_keys SET revoked_at = $1 " + "WHERE key_id = $2 AND revoked_at IS NULL", + time.time(), key_id, + ) + # asyncpg returns 'UPDATE n' — split off the count. + try: + return int(result.split()[-1]) > 0 + except (ValueError, IndexError): + return False + + +async def lookup_bearer(pool, token: str) -> Optional[MinerIdentity]: + """Resolve a bearer token to its miner identity, or ``None`` if + the token is unknown / revoked.""" + if not token: + return None + digest = hash_api_key(token) + row = await pool.fetchrow( + "SELECT k.key_id, k.miner_id, k.scope, m.hotkey, m.name " + "FROM miner_api_keys k " + "JOIN miners m ON m.miner_id = k.miner_id " + "WHERE k.key_hash = $1 AND k.revoked_at IS NULL " + " AND m.revoked_at IS NULL " + "LIMIT 1", + digest, + ) + if row is None: + return None + return MinerIdentity( + miner_id=row["miner_id"], + key_id=row["key_id"], + scope=row["scope"], + hotkey=row["hotkey"] or "", + name=row["name"] or "", + ) + + +async def touch_key_usage(pool, key_id: str) -> None: + """Update ``last_used_at`` (best-effort — swallow errors).""" + try: + await pool.execute( + "UPDATE miner_api_keys SET last_used_at = $1 WHERE key_id = $2", + time.time(), key_id, + ) + except Exception as e: + logger.warning("touch_key_usage failed for %s: %s", key_id, e) + + +async def list_api_keys(pool, miner_id: str) -> list[dict]: + """Return non-secret metadata for every key of a miner.""" + rows = await pool.fetch( + "SELECT key_id, scope, label, created_at, last_used_at, revoked_at " + "FROM miner_api_keys WHERE miner_id = $1 ORDER BY created_at DESC", + miner_id, + ) + return [dict(r) for r in rows] + + +def hotkey_to_miner_id_fallback(hotkey: str) -> str: + """Deterministic placeholder ``miner_id`` for a hotkey when the + operator hasn't issued a real one yet — lets the dual-stack period + write ``experiments.prompt_id`` against a stable key derived from + the bittensor identity.""" + return f"hotkey:{hotkey}" if hotkey else "" diff --git a/shared/pg_schema.py b/shared/pg_schema.py index 24bd33d..b5a23d1 100644 --- a/shared/pg_schema.py +++ b/shared/pg_schema.py @@ -247,6 +247,52 @@ ON miner_access_log(hotkey, round_id); """ +# Miner registry + bearer API keys for the non-competitive cutover. +# A ``miner_id`` is an opaque operator-issued identifier (UUID by default) +# that decouples the feedback API from the on-chain hotkey identity. +# During the dual-stack period a miner row carries both: ``hotkey`` is +# the bittensor SS58 (nullable, only set for chain-registered miners) and +# ``miner_id`` is the bearer-auth identifier scoped to /miners/me/*. +MINER_REGISTRY_SCHEMA = """ +CREATE TABLE IF NOT EXISTS miners ( + miner_id TEXT PRIMARY KEY, + name TEXT NOT NULL DEFAULT '', + hotkey TEXT NOT NULL DEFAULT '', + contact TEXT NOT NULL DEFAULT '', + created_at DOUBLE PRECISION NOT NULL DEFAULT extract(epoch from now()), + revoked_at DOUBLE PRECISION +); +CREATE INDEX IF NOT EXISTS idx_miners_hotkey ON miners(hotkey) + WHERE hotkey <> ''; + +CREATE TABLE IF NOT EXISTS miner_api_keys ( + key_id TEXT PRIMARY KEY, + key_hash TEXT NOT NULL, + miner_id TEXT NOT NULL REFERENCES miners(miner_id) ON DELETE CASCADE, + scope TEXT NOT NULL DEFAULT 'miner', + label TEXT NOT NULL DEFAULT '', + created_at DOUBLE PRECISION NOT NULL DEFAULT extract(epoch from now()), + last_used_at DOUBLE PRECISION, + revoked_at DOUBLE PRECISION +); +CREATE INDEX IF NOT EXISTS idx_miner_api_keys_hash + ON miner_api_keys(key_hash) WHERE revoked_at IS NULL; +CREATE INDEX IF NOT EXISTS idx_miner_api_keys_miner + ON miner_api_keys(miner_id); + +-- ``prompt_id`` on experiments lets miners correlate Phase C scores +-- with the prompt variant that produced the architecture (drives the +-- GEPA / random_mutate feedback loop on the miner CLI). Opaque to +-- the operator; only the issuing miner interprets it. +ALTER TABLE experiments + ADD COLUMN IF NOT EXISTS prompt_id TEXT; +CREATE INDEX IF NOT EXISTS idx_experiments_prompt_id + ON experiments(prompt_id) WHERE prompt_id IS NOT NULL; +CREATE INDEX IF NOT EXISTS idx_experiments_miner_prompt + ON experiments(miner_hotkey, prompt_id) + WHERE prompt_id IS NOT NULL; +""" + def _decode_jsonb(value, default): """Coerce a JSONB column value to a Python dict/list. @@ -314,6 +360,9 @@ def row_to_element(row) -> DataElement: _decode_jsonb(row["substrate_cids"], []) if "substrate_cids" in row_keys else [] ) + prompt_id = ( + (row["prompt_id"] or "") if "prompt_id" in row_keys else "" + ) return DataElement( index=row["id"], @@ -340,6 +389,7 @@ def row_to_element(row) -> DataElement: task=row["task"], round_id=row["round_id"] if row["round_id"] is not None else -1, substrate_cids=substrate_cids, + prompt_id=prompt_id, ) @@ -366,7 +416,7 @@ def _jsonb(value) -> str: def element_to_params(element: DataElement, next_id: int) -> tuple: - """Convert a DataElement to a positional tuple for INSERT ($1..$24). + """Convert a DataElement to a positional tuple for INSERT ($1..$25). JSONB columns (loss_curve, generated_samples, objectives, tool_calls, agent_behavior, substrate_cids) are explicitly serialised with @@ -375,6 +425,7 @@ def element_to_params(element: DataElement, next_id: int) -> tuple: are replaced with ``null`` to keep PostgreSQL happy. """ round_id = element.round_id if element.round_id >= 0 else None + prompt_id = getattr(element, "prompt_id", "") or None return ( next_id, element.name, @@ -400,6 +451,7 @@ def element_to_params(element: DataElement, next_id: int) -> tuple: _jsonb(element.tool_calls), _jsonb(element.agent_behavior), _jsonb(element.substrate_cids), + prompt_id, ) @@ -409,13 +461,15 @@ def element_to_params(element: DataElement, next_id: int) -> tuple: parent_index, generation, score, miner_uid, miner_hotkey, loss_curve, manifest_sha256, generated_samples, objectives, timestamp, round_id, task, - reasoning, tool_calls, agent_behavior, substrate_cids + reasoning, tool_calls, agent_behavior, substrate_cids, + prompt_id ) VALUES ( $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, - $21, $22, $23, $24 + $21, $22, $23, $24, + $25 ) """ diff --git a/shared/pg_store.py b/shared/pg_store.py index d0cf6fb..d2d925f 100644 --- a/shared/pg_store.py +++ b/shared/pg_store.py @@ -16,6 +16,7 @@ from shared.database import DataElement from shared.pg_schema import ( SCHEMA_TABLE_DDL, SCHEMA_INDEX_DDL, FTS_FUNCTION_DDL, FTS_TRIGGER_DDL, + MINER_REGISTRY_SCHEMA, INSERT_SQL, row_to_element, element_to_params, compute_diff, _finite_or, ) @@ -153,6 +154,7 @@ async def init_schema(self): await conn.execute(SCHEMA_INDEX_DDL) await conn.execute(FTS_FUNCTION_DDL) await conn.execute(FTS_TRIGGER_DDL) + await conn.execute(MINER_REGISTRY_SCHEMA) from shared.pg_provenance import PgProvenanceQuery self.provenance = PgProvenanceQuery(self.pool) await self.provenance.init_schema() diff --git a/shared/protocol.py b/shared/protocol.py index 07f1851..c337aea 100644 --- a/shared/protocol.py +++ b/shared/protocol.py @@ -84,6 +84,12 @@ class Proposal: populates them. They are useful for explainability but trivially fakeable; pair with validator-observed signals (proxy access logs, pod wall-clock) for adversarial use. + + ``prompt_id`` is opaque to the operator — it's a miner-local + identifier (UUID by default) for the prompt variant that produced + this architecture. Round-tripped back via + ``experiments.prompt_id`` so the miner's GEPA/random_mutate loop + can correlate Phase C scores with the variant. """ code: str = "" @@ -95,6 +101,9 @@ class Proposal: # (e.g. {"tool": "llm", "prompt": "...", "summary": "..."}). Schema is # intentionally open so different agents can use different shapes. tool_calls: list = field(default_factory=list) + # Miner-local prompt-variant id; surfaced on the experiments row so + # the miner CLI's optimizer can attribute Phase C scores back. + prompt_id: str = "" def to_json(self) -> str: return json.dumps({ @@ -103,6 +112,7 @@ def to_json(self) -> str: "motivation": self.motivation, "reasoning": self.reasoning, "tool_calls": self.tool_calls, + "prompt_id": self.prompt_id, }) @classmethod diff --git a/shared/substrate.py b/shared/substrate.py index 7055552..aa8c3a4 100644 --- a/shared/substrate.py +++ b/shared/substrate.py @@ -22,13 +22,21 @@ import json import logging from dataclasses import asdict, dataclass, fields -from typing import Any +from typing import TYPE_CHECKING, Any -import bittensor as bt +if TYPE_CHECKING: # only the type stubs at static-check time + import bittensor as bt # noqa: F401 logger = logging.getLogger(__name__) +def _bt(): + """Lazy bittensor import — keeps this module loadable in + non-competitive deployments that never sign/verify records.""" + import bittensor as bt + return bt + + SCHEMA_VERSION = "radar.substrate.v1" SIGNATURE_PREFIX = "sr25519:" @@ -149,6 +157,7 @@ def verify_record( if not record.validator_hotkey: return False, "validator_hotkey is empty" try: + bt = _bt() keypair = bt.Keypair(ss58_address=record.validator_hotkey) if not keypair.verify(canonical_bytes(record), sig_bytes): return False, "sr25519 signature does not match canonical bytes" diff --git a/tests/test_agent_prompt_loader.py b/tests/test_agent_prompt_loader.py new file mode 100644 index 0000000..4e811e8 --- /dev/null +++ b/tests/test_agent_prompt_loader.py @@ -0,0 +1,105 @@ +"""Tests for the in-agent prompt loader (miner_template/agent.py).""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from miner_template import agent + + +def _write_active(tmp_path: Path, prompts: list[dict]) -> Path: + (tmp_path / "prompts").mkdir() + p = tmp_path / "prompts" / "active.json" + p.write_text(json.dumps({"prompts": prompts})) + return p + + +def test_loader_returns_empty_when_no_file(monkeypatch, tmp_path): + monkeypatch.setenv("MINER_PROMPTS_DIR", str(tmp_path / "missing")) + got = agent._load_active_prompt(round_id=0) + assert got == {"id": "", "template": ""} + + +def test_loader_returns_empty_when_population_empty(monkeypatch, tmp_path): + _write_active(tmp_path, []) + monkeypatch.setenv("MINER_PROMPTS_DIR", str(tmp_path / "prompts")) + got = agent._load_active_prompt(round_id=0) + assert got == {"id": "", "template": ""} + + +def test_loader_returns_round_robin_pick(monkeypatch, tmp_path): + _write_active(tmp_path, [ + {"id": "a", "template": "A"}, + {"id": "b", "template": "B"}, + {"id": "c", "template": "C"}, + ]) + monkeypatch.setenv("MINER_PROMPTS_DIR", str(tmp_path / "prompts")) + picks = [agent._load_active_prompt(i)["id"] for i in range(5)] + assert picks == ["a", "b", "c", "a", "b"] + + +def test_loader_handles_corrupt_json(monkeypatch, tmp_path): + (tmp_path / "prompts").mkdir() + (tmp_path / "prompts" / "active.json").write_text("{not json") + monkeypatch.setenv("MINER_PROMPTS_DIR", str(tmp_path / "prompts")) + assert agent._load_active_prompt(0) == {"id": "", "template": ""} + + +def test_loader_handles_legacy_list_payload(monkeypatch, tmp_path): + """Older optimizer versions wrote a bare list — be permissive.""" + (tmp_path / "prompts").mkdir() + (tmp_path / "prompts" / "active.json").write_text(json.dumps([ + {"id": "x", "template": "T"}, + ])) + monkeypatch.setenv("MINER_PROMPTS_DIR", str(tmp_path / "prompts")) + got = agent._load_active_prompt(0) + assert got["id"] == "x" + + +def test_design_architecture_emits_prompt_id(monkeypatch, tmp_path): + """End-to-end: design_architecture round-trips the prompt id.""" + _write_active(tmp_path, [{"id": "the-active-one", "template": "T"}]) + monkeypatch.setenv("MINER_PROMPTS_DIR", str(tmp_path / "prompts")) + + # design_architecture also calls load_scratchpad/save_scratchpad; + # those names are injected by the validator's sandbox at runtime, + # so we inject stubs into the module globals here. + agent.__dict__["load_scratchpad"] = lambda c: tmp_path + agent.__dict__["save_scratchpad"] = lambda c, d: None + + class _FakeClient: + def post_json(self, *a, **k): return {} + def get_json(self, *a, **k): return {} + + out = agent.design_architecture( + challenge={ + "round_id": 0, "min_flops_equivalent": 1_000_000, + "max_flops_equivalent": 5_000_000, "feasible_frontier": [], + }, + client=_FakeClient(), + ) + assert out["prompt_id"] == "the-active-one" + + +def test_design_architecture_empty_prompt_id_when_no_population( + monkeypatch, tmp_path, +): + monkeypatch.setenv("MINER_PROMPTS_DIR", str(tmp_path / "missing")) + monkeypatch.setattr(agent, "load_scratchpad", lambda c: tmp_path) + monkeypatch.setattr(agent, "save_scratchpad", lambda c, d: None) + + class _FakeClient: + def post_json(self, *a, **k): return {} + def get_json(self, *a, **k): return {} + + out = agent.design_architecture( + challenge={ + "round_id": 0, "min_flops_equivalent": 1_000_000, + "max_flops_equivalent": 5_000_000, "feasible_frontier": [], + }, + client=_FakeClient(), + ) + assert out["prompt_id"] == "" diff --git a/tests/test_auth_hmac.py b/tests/test_auth_hmac.py new file mode 100644 index 0000000..a9cfaa3 --- /dev/null +++ b/tests/test_auth_hmac.py @@ -0,0 +1,127 @@ +"""Tests for the HMAC + bearer-token primitives in shared/auth.py.""" + +from __future__ import annotations + +import time + +import pytest + +from shared import auth + + +SECRET = b"super-secret-shared-key-do-not-leak" +OTHER = b"another-secret" + + +def _headers_as_wire(d: dict[str, str]) -> dict[str, str]: + """Simulate the casing servers usually present headers in.""" + return {k.lower(): v for k, v in d.items()} + + +def test_sign_verify_roundtrip(): + body = b'{"hello": "world"}' + headers = auth.hmac_sign_request(SECRET, body, key_id="op") + ok, err, kid = auth.hmac_verify_request( + _headers_as_wire(headers), body, auth.static_key_lookup("op", SECRET) + ) + assert ok, err + assert kid == "op" + + +def test_verify_rejects_tampered_body(): + body = b'{"a": 1}' + headers = auth.hmac_sign_request(SECRET, body, key_id="op") + tampered = b'{"a": 2}' + ok, err, _ = auth.hmac_verify_request( + _headers_as_wire(headers), tampered, auth.static_key_lookup("op", SECRET) + ) + assert not ok + assert "signature" in err + + +def test_verify_rejects_wrong_key(): + body = b"x" + headers = auth.hmac_sign_request(SECRET, body, key_id="op") + ok, err, _ = auth.hmac_verify_request( + _headers_as_wire(headers), body, auth.static_key_lookup("op", OTHER) + ) + assert not ok + assert "signature" in err + + +def test_verify_rejects_unknown_key_id(): + body = b"x" + headers = auth.hmac_sign_request(SECRET, body, key_id="ghost") + ok, err, _ = auth.hmac_verify_request( + _headers_as_wire(headers), body, auth.static_key_lookup("op", SECRET) + ) + assert not ok + assert "unknown key_id" in err + + +def test_verify_rejects_stale_timestamp(): + body = b"x" + # Sign with a timestamp far outside tolerance. + old_ts = str(int(time.time()) - 10_000) + sig = auth.hmac_sign_request(SECRET, body, key_id="op") + sig["X-Radar-Timestamp"] = old_ts + # Recompute the signature against the old ts so we exercise the + # timestamp-window check rather than the signature check. + import hashlib + import hmac as _hmac + + digest = hashlib.sha256(body).hexdigest() + sig["X-Radar-Signature"] = _hmac.new( + SECRET, f"{old_ts}.{digest}".encode(), hashlib.sha256 + ).hexdigest() + + ok, err, _ = auth.hmac_verify_request( + _headers_as_wire(sig), body, auth.static_key_lookup("op", SECRET) + ) + assert not ok + assert "timestamp" in err + + +def test_verify_requires_all_headers(): + body = b"x" + headers = auth.hmac_sign_request(SECRET, body, key_id="op") + for missing in ("X-Radar-Signature", "X-Radar-Timestamp", "X-Radar-Key-Id"): + partial = {k: v for k, v in headers.items() if k != missing} + ok, err, _ = auth.hmac_verify_request( + _headers_as_wire(partial), body, auth.static_key_lookup("op", SECRET) + ) + assert not ok, f"missing {missing} should fail" + assert "missing" in err.lower() + + +def test_hmac_sign_request_rejects_string_secret(): + with pytest.raises(TypeError): + auth.hmac_sign_request("not-bytes", b"x") # type: ignore[arg-type] + + +def test_extract_bearer(): + assert auth.extract_bearer({"authorization": "Bearer abc123"}) == "abc123" + assert auth.extract_bearer({"Authorization": "bearer abc123"}) == "abc123" + assert auth.extract_bearer({"authorization": "Basic xx"}) is None + assert auth.extract_bearer({"authorization": "Bearer "}) is None + assert auth.extract_bearer({}) is None + assert auth.extract_bearer({"authorization": "Bearer"}) is None + + +def test_hash_api_key_stable(): + h1 = auth.hash_api_key("hunter2") + h2 = auth.hash_api_key("hunter2") + assert h1 == h2 + assert h1 != auth.hash_api_key("hunter3") + assert len(h1) == 64 # sha256 hex + + +def test_load_service_secret_unset(monkeypatch): + monkeypatch.delenv("RADAR_SERVICE_KEY", raising=False) + with pytest.raises(RuntimeError): + auth.load_service_secret() + + +def test_load_service_secret_set(monkeypatch): + monkeypatch.setenv("RADAR_SERVICE_KEY", "abc") + assert auth.load_service_secret() == b"abc" diff --git a/tests/test_cli_subcommands.py b/tests/test_cli_subcommands.py new file mode 100644 index 0000000..51505b3 --- /dev/null +++ b/tests/test_cli_subcommands.py @@ -0,0 +1,271 @@ +"""Tests for the miner CLI subcommands. + +We exercise the CLI by importing ``miner.cli`` directly and calling +``dispatch(argv)`` with controlled argv. HTTP calls are intercepted via +``httpx.MockTransport`` on a fake ``MinerResultsClient``. + +GEPA itself isn't exercised here — we route to ``random_mutate`` to keep +the test fast and dependency-free. +""" + +from __future__ import annotations + +import json +import sys +from pathlib import Path +from unittest.mock import patch + +import httpx +import pytest + +from miner import cli +from miner_template import prompts as prompts_mod +from miner_template.prompts import Prompt +from miner_template.results_client import MinerResultsClient + + +@pytest.fixture +def fake_results_client(monkeypatch): + """Install a MockTransport-backed MinerResultsClient. + + Returns a dict the test populates with ``submissions``, ``results``, + ``summary`` to control the fake server's responses. + """ + state: dict = {"submissions": [], "results": [], "summary": {}} + + def handler(request: httpx.Request) -> httpx.Response: + path = request.url.path + if path == "/miners/me/submissions": + return httpx.Response(200, json={"submissions": state["submissions"]}) + if path == "/miners/me/results": + return httpx.Response(200, json={"results": state["results"]}) + if path == "/miners/me/summary": + return httpx.Response(200, json=state["summary"]) + return httpx.Response(404, json={"error": "no route"}) + + original_init = MinerResultsClient.__init__ + + def patched_init(self, db_url, api_key, *, timeout=30.0, client=None): + if client is None: + transport = httpx.MockTransport(handler) + client = httpx.Client(transport=transport) + original_init(self, db_url, api_key, timeout=timeout, client=client) + + monkeypatch.setattr(MinerResultsClient, "__init__", patched_init) + return state + + +# ── results ───────────────────────────────────────────────────────── + + +def test_results_requires_db_url_and_key(capsys): + code = cli.dispatch([ + "miner", "results", "--db-url", "", "--api-key", "", + ]) + assert code == 2 + assert "missing --db-url or --api-key" in capsys.readouterr().err + + +def test_results_prints_table(fake_results_client, capsys): + fake_results_client["results"] = [{ + "round_id": 12, "submission_id": "s-1", "task_name": "ts", + "prompt_id": "p-abc", "architecture_code": "code", + "scores": {"raw_score": 0.42}, + }] + code = cli.dispatch([ + "miner", "results", + "--db-url", "http://db", "--api-key", "k", + ]) + out = capsys.readouterr().out + assert code == 0 + assert "round" in out + assert "s-1" in out + assert "0.4200" in out + + +def test_results_json_mode(fake_results_client, capsys): + fake_results_client["results"] = [{ + "round_id": 1, "submission_id": "s-1", "task_name": "t", + "prompt_id": "p", "scores": {"raw_score": 0.5}, + }] + code = cli.dispatch([ + "miner", "results", + "--db-url", "http://db", "--api-key", "k", "--json", + ]) + assert code == 0 + out = capsys.readouterr().out.strip().splitlines() + rec = json.loads(out[0]) + assert rec["submission_id"] == "s-1" + assert rec["scores"]["raw_score"] == 0.5 + + +def test_results_empty(fake_results_client, capsys): + code = cli.dispatch([ + "miner", "results", + "--db-url", "http://db", "--api-key", "k", + ]) + assert code == 0 + assert "(no results)" in capsys.readouterr().out + + +# ── optimize ───────────────────────────────────────────────────────── + + +def test_optimize_requires_db_url_and_key(capsys): + code = cli.dispatch([ + "miner", "optimize", "--db-url", "", "--api-key", "", + ]) + assert code == 2 + assert "missing --db-url" in capsys.readouterr().err + + +def test_optimize_without_seed_or_active_pop_fails( + fake_results_client, tmp_path, capsys, +): + code = cli.dispatch([ + "miner", "optimize", + "--db-url", "http://db", "--api-key", "k", + "--prompts-dir", str(tmp_path), + "--optimizer", "random_mutate", + ]) + assert code == 2 + assert "no active prompt population" in capsys.readouterr().err + + +def test_optimize_with_seed_creates_pop_and_writes_generation( + fake_results_client, tmp_path, capsys, +): + code = cli.dispatch([ + "miner", "optimize", + "--db-url", "http://db", "--api-key", "k", + "--prompts-dir", str(tmp_path), + "--optimizer", "random_mutate", + "--population", "4", + "--seed", + ]) + out = capsys.readouterr().out + assert code == 0 + assert "Seeded default prompt" in out + pop = prompts_mod.load_active(tmp_path) + assert len(pop) == 4 + assert prompts_mod.list_history(tmp_path) == [1] + + +def test_optimize_dry_run_does_not_write( + fake_results_client, tmp_path, capsys, +): + prompts_mod.save_active( + [Prompt.new(template="A"), Prompt.new(template="B")], tmp_path, + ) + fake_results_client["results"] = [{ + "round_id": 1, "submission_id": "s", "task_name": "t", + "prompt_id": "doesnt-matter", "scores": {"raw_score": 0.1}, + }] + before = prompts_mod.load_active(tmp_path) + code = cli.dispatch([ + "miner", "optimize", + "--db-url", "http://db", "--api-key", "k", + "--prompts-dir", str(tmp_path), + "--optimizer", "random_mutate", + "--population", "3", "--dry-run", + ]) + out = capsys.readouterr().out + assert code == 0 + assert "would write population" in out + after = prompts_mod.load_active(tmp_path) + assert [p.id for p in before] == [p.id for p in after] + assert prompts_mod.list_history(tmp_path) == [] + + +def test_optimize_unknown_optimizer(fake_results_client, tmp_path, capsys): + prompts_mod.save_active([Prompt.new(template="A")], tmp_path) + code = cli.dispatch([ + "miner", "optimize", + "--db-url", "http://db", "--api-key", "k", + "--prompts-dir", str(tmp_path), + "--optimizer", "no-such-optimizer", + ]) + assert code == 2 + assert "cannot resolve optimizer" in capsys.readouterr().err + + +# ── prompts ────────────────────────────────────────────────────────── + + +def test_prompts_list_empty(tmp_path, capsys): + code = cli.dispatch([ + "miner", "prompts", "list", "--prompts-dir", str(tmp_path), + ]) + assert code == 0 + assert "(no active prompt population)" in capsys.readouterr().out + + +def test_prompts_list_shows_population(tmp_path, capsys): + prompts_mod.save_active([ + Prompt(id="aaaaaaaa", template="hello world", generation=2), + ], tmp_path) + code = cli.dispatch([ + "miner", "prompts", "list", "--prompts-dir", str(tmp_path), + ]) + out = capsys.readouterr().out + assert code == 0 + assert "gen 2" in out + assert "aaaaaaaa" in out + + +def test_prompts_history_empty(tmp_path, capsys): + code = cli.dispatch([ + "miner", "prompts", "history", "--prompts-dir", str(tmp_path), + ]) + assert code == 0 + assert "(no archived generations)" in capsys.readouterr().out + + +def test_prompts_history_lists_archives(tmp_path, capsys): + prompts_mod.save_active([Prompt.new(template="a")], tmp_path) + prompts_mod.archive_current(1, tmp_path) + prompts_mod.archive_current(7, tmp_path) + code = cli.dispatch([ + "miner", "prompts", "history", "--prompts-dir", str(tmp_path), + ]) + out = capsys.readouterr().out + assert "gen_001" in out + assert "gen_007" in out + + +def test_prompts_rollback_missing_generation(tmp_path, capsys): + code = cli.dispatch([ + "miner", "prompts", "rollback", "99", "--prompts-dir", str(tmp_path), + ]) + assert code == 2 + assert "no archived generation" in capsys.readouterr().err + + +def test_prompts_rollback_restores_archived_pop(tmp_path): + prompts_mod.save_active([Prompt(id="x", template="A")], tmp_path) + prompts_mod.archive_current(1, tmp_path) + prompts_mod.save_active([Prompt(id="y", template="B")], tmp_path) + code = cli.dispatch([ + "miner", "prompts", "rollback", "1", "--prompts-dir", str(tmp_path), + ]) + assert code == 0 + cur = prompts_mod.load_active(tmp_path) + assert [p.id for p in cur] == ["x"] + + +# ── routing ────────────────────────────────────────────────────────── + + +def test_is_subcommand_matches_known_tokens(): + assert cli.is_subcommand(["miner", "results"]) + assert cli.is_subcommand(["miner", "optimize"]) + assert cli.is_subcommand(["miner", "prompts"]) + assert not cli.is_subcommand(["miner"]) + assert not cli.is_subcommand(["miner", "run"]) + assert not cli.is_subcommand(["miner", "--netuid", "1"]) + + +def test_dispatch_unknown_subcommand(capsys): + code = cli.dispatch(["miner", "wat"]) + assert code == 2 + assert "Unknown subcommand" in capsys.readouterr().err diff --git a/tests/test_config_noncompetitive.py b/tests/test_config_noncompetitive.py new file mode 100644 index 0000000..9fddbfd --- /dev/null +++ b/tests/test_config_noncompetitive.py @@ -0,0 +1,50 @@ +"""Tests for Config.NONCOMPETITIVE + service-key wiring.""" + +from __future__ import annotations + +import importlib + +import pytest + + +def _reload_config(monkeypatch, **env): + for k, v in env.items(): + if v is None: + monkeypatch.delenv(k, raising=False) + else: + monkeypatch.setenv(k, v) + import config as cfg + importlib.reload(cfg) + return cfg.Config + + +def test_noncompetitive_defaults_false(monkeypatch): + cfg = _reload_config(monkeypatch, RADAR_NONCOMPETITIVE=None) + assert cfg.NONCOMPETITIVE is False + + +def test_noncompetitive_truthy_variants(monkeypatch): + for val in ("true", "TRUE", "1", "yes"): + cfg = _reload_config(monkeypatch, RADAR_NONCOMPETITIVE=val) + assert cfg.NONCOMPETITIVE is True, f"failed for {val}" + + +def test_noncompetitive_falsy_variants(monkeypatch): + for val in ("false", "0", "no", ""): + cfg = _reload_config(monkeypatch, RADAR_NONCOMPETITIVE=val) + assert cfg.NONCOMPETITIVE is False, f"failed for {val}" + + +def test_service_key_id_defaults_operator(monkeypatch): + cfg = _reload_config(monkeypatch, RADAR_SERVICE_KEY_ID=None) + assert cfg.SERVICE_KEY_ID == "operator" + + +def test_service_key_id_overrideable(monkeypatch): + cfg = _reload_config(monkeypatch, RADAR_SERVICE_KEY_ID="my-op") + assert cfg.SERVICE_KEY_ID == "my-op" + + +def test_service_key_passes_through(monkeypatch): + cfg = _reload_config(monkeypatch, RADAR_SERVICE_KEY="a" * 64) + assert cfg.SERVICE_KEY == "a" * 64 diff --git a/tests/test_miner_auth_helpers.py b/tests/test_miner_auth_helpers.py new file mode 100644 index 0000000..d7ae051 --- /dev/null +++ b/tests/test_miner_auth_helpers.py @@ -0,0 +1,153 @@ +"""Unit tests for shared.miner_auth without a live Postgres. + +The async pool methods are exercised against a tiny in-memory fake so +the helpers' SQL shape, key minting, hashing, and dataclass mapping +are all covered. Real-DB integration is covered by the +``test_pg_*`` suite when ``TEST_PG_DSN`` is set. +""" + +from __future__ import annotations + +import re + +import pytest + +from shared import miner_auth +from shared.auth import hash_api_key + + +class _FakePool: + """Records execute() calls, answers fetch*/execute with scripted rows.""" + + def __init__(self): + self.calls: list[tuple] = [] + self.rows: dict[str, list[dict]] = {} + self.next_execute_result = "UPDATE 1" + + async def execute(self, sql, *args): + self.calls.append(("execute", sql, args)) + return self.next_execute_result + + async def fetchrow(self, sql, *args): + self.calls.append(("fetchrow", sql, args)) + rows = self.rows.get("fetchrow", []) + return rows.pop(0) if rows else None + + async def fetch(self, sql, *args): + self.calls.append(("fetch", sql, args)) + return self.rows.get("fetch", []) + + +@pytest.mark.asyncio +async def test_register_miner_assigns_id_when_omitted(): + pool = _FakePool() + mid = await miner_auth.register_miner(pool, name="alice") + assert isinstance(mid, str) and len(mid) == 32 + assert pool.calls[0][0] == "execute" + + +@pytest.mark.asyncio +async def test_register_miner_respects_supplied_id(): + pool = _FakePool() + mid = await miner_auth.register_miner( + pool, miner_id="custom", name="bob", + ) + assert mid == "custom" + + +@pytest.mark.asyncio +async def test_issue_api_key_returns_prefixed_plaintext_and_handle(): + pool = _FakePool() + plaintext, key_id = await miner_auth.issue_api_key( + pool, "m1", label="prod", + ) + assert plaintext.startswith(miner_auth.KEY_PREFIX) + assert re.match(r"^rdrk_[0-9a-f]{32}$", plaintext) + assert len(key_id) == 32 + # The INSERT stores the *hash*, never the plaintext. + _, sql, args = pool.calls[-1] + assert "INSERT INTO miner_api_keys" in sql + assert args[1] == hash_api_key(plaintext) + assert plaintext not in str(args) + + +@pytest.mark.asyncio +async def test_revoke_api_key_returns_true_when_row_updated(): + pool = _FakePool() + pool.next_execute_result = "UPDATE 1" + assert await miner_auth.revoke_api_key(pool, "k1") is True + + +@pytest.mark.asyncio +async def test_revoke_api_key_returns_false_when_no_row(): + pool = _FakePool() + pool.next_execute_result = "UPDATE 0" + assert await miner_auth.revoke_api_key(pool, "missing") is False + + +@pytest.mark.asyncio +async def test_lookup_bearer_returns_none_for_empty_token(): + pool = _FakePool() + assert await miner_auth.lookup_bearer(pool, "") is None + + +@pytest.mark.asyncio +async def test_lookup_bearer_returns_none_when_db_has_no_match(): + pool = _FakePool() + assert await miner_auth.lookup_bearer(pool, "any-token") is None + + +@pytest.mark.asyncio +async def test_lookup_bearer_maps_row_to_identity(): + pool = _FakePool() + pool.rows["fetchrow"] = [{ + "key_id": "k1", + "miner_id": "m1", + "scope": "miner", + "hotkey": "5xxx", + "name": "alice", + }] + ident = await miner_auth.lookup_bearer(pool, "secret-token") + assert ident is not None + assert ident.miner_id == "m1" + assert ident.key_id == "k1" + assert ident.scope == "miner" + assert ident.hotkey == "5xxx" + assert ident.name == "alice" + # The lookup hashes the token before hitting the DB. + _, sql, args = pool.calls[-1] + assert args[0] == hash_api_key("secret-token") + + +@pytest.mark.asyncio +async def test_lookup_bearer_handles_null_optional_fields(): + pool = _FakePool() + pool.rows["fetchrow"] = [{ + "key_id": "k", "miner_id": "m", "scope": "miner", + "hotkey": None, "name": None, + }] + ident = await miner_auth.lookup_bearer(pool, "x") + assert ident.hotkey == "" + assert ident.name == "" + + +@pytest.mark.asyncio +async def test_touch_key_usage_swallows_errors(): + class Bad(_FakePool): + async def execute(self, *args, **kwargs): + raise RuntimeError("db down") + + await miner_auth.touch_key_usage(Bad(), "k1") # must not raise + + +@pytest.mark.asyncio +async def test_list_api_keys_returns_dicts(): + pool = _FakePool() + pool.rows["fetch"] = [{"key_id": "k1"}, {"key_id": "k2"}] + rows = await miner_auth.list_api_keys(pool, "m1") + assert [r["key_id"] for r in rows] == ["k1", "k2"] + + +def test_hotkey_fallback(): + assert miner_auth.hotkey_to_miner_id_fallback("hk1") == "hotkey:hk1" + assert miner_auth.hotkey_to_miner_id_fallback("") == "" diff --git a/tests/test_miner_feedback_routes.py b/tests/test_miner_feedback_routes.py new file mode 100644 index 0000000..9f9967a --- /dev/null +++ b/tests/test_miner_feedback_routes.py @@ -0,0 +1,284 @@ +"""End-to-end-ish tests for the /miners/me/* + /tasks/{task}/frontier +routes. + +We stub the asyncpg pool with a small fake so the test runs without +Postgres. The middleware's bearer-lookup path is exercised, plus the +SQL parameterisation and JSON shaping. +""" + +from __future__ import annotations + +from contextlib import contextmanager + +import pytest +from fastapi import FastAPI +from fastapi.testclient import TestClient + +from shared.auth import hash_api_key + + +# ── Fake pool ──────────────────────────────────────────────────────── + + +class _Row(dict): + """asyncpg.Record-like dict — supports both ``row[k]`` and ``row.keys()``.""" + + +class _FakePool: + """Records calls and answers fetch* with scripted rows.""" + + def __init__(self): + self.fetch_rows: list[list[_Row]] = [] + self.fetchrow_rows: list[_Row | None] = [] + self.fetchval_rows: list = [] + self.executed: list[tuple[str, tuple]] = [] + + async def fetch(self, sql, *args): + self.executed.append((sql, args)) + if self.fetch_rows: + return self.fetch_rows.pop(0) + return [] + + async def fetchrow(self, sql, *args): + self.executed.append((sql, args)) + if self.fetchrow_rows: + return self.fetchrow_rows.pop(0) + return None + + async def fetchval(self, sql, *args): + self.executed.append((sql, args)) + if self.fetchval_rows: + return self.fetchval_rows.pop(0) + return None + + async def execute(self, sql, *args): + self.executed.append((sql, args)) + return "UPDATE 1" + + +@contextmanager +def _wire_app(monkeypatch): + """Build a fresh FastAPI app with our routes + fake pool.""" + # Force a clean import so previous tests' state doesn't leak. + from database import miner_feedback as mf, server as srv + + pool = _FakePool() + monkeypatch.setattr(srv, "_pool", pool) + app = FastAPI() + mf.include_miner_feedback_routes(app) + + # Replicate the bearer middleware locally so we don't pull the full + # validator surface in. + @app.middleware("http") + async def _bearer(request, call_next): + if request.url.path.startswith("/miners/me/"): + return await srv._bearer_auth_then_call(request, call_next) + return await call_next(request) + + yield app, pool + + +# ── lookup_bearer test scaffolding ────────────────────────────────── + + +def _install_bearer_identity(monkeypatch, ident_dict): + """Patch shared.miner_auth.lookup_bearer to return ``ident_dict`` (or + ``None`` for unknown).""" + from shared import miner_auth + + async def fake_lookup(pool, token): + if token == "good": + return miner_auth.MinerIdentity(**ident_dict) + return None + + monkeypatch.setattr(miner_auth, "lookup_bearer", fake_lookup) + + +def _ident(hotkey="hk-1", miner_id="m1"): + return { + "miner_id": miner_id, "key_id": "k1", + "scope": "miner", "hotkey": hotkey, "name": "alice", + } + + +# ── /miners/me/submissions ────────────────────────────────────────── + + +def test_submissions_requires_bearer(monkeypatch): + _install_bearer_identity(monkeypatch, _ident()) + with _wire_app(monkeypatch) as (app, _): + client = TestClient(app) + r = client.get("/miners/me/submissions") + assert r.status_code == 401 + + +def test_submissions_rejects_unknown_token(monkeypatch): + _install_bearer_identity(monkeypatch, _ident()) + with _wire_app(monkeypatch) as (app, _): + client = TestClient(app) + r = client.get( + "/miners/me/submissions", + headers={"Authorization": "Bearer wrong"}, + ) + assert r.status_code == 403 + + +def test_submissions_returns_rows_scoped_to_hotkey(monkeypatch): + _install_bearer_identity(monkeypatch, _ident(hotkey="hk-A")) + with _wire_app(monkeypatch) as (app, pool): + pool.fetch_rows = [[ + _Row( + submission_id=1, round_id=42, task_name="ts_forecasting", + prompt_id="p-1", architecture_code="class M: pass", + motivation="m", reasoning="r", tool_calls=[], + created_at=1700000000.0, + ), + ]] + client = TestClient(app) + r = client.get( + "/miners/me/submissions?limit=10&task=ts_forecasting", + headers={"Authorization": "Bearer good"}, + ) + assert r.status_code == 200 + payload = r.json() + assert len(payload["submissions"]) == 1 + sub = payload["submissions"][0] + assert sub["prompt_id"] == "p-1" + assert sub["task_name"] == "ts_forecasting" + # First arg of the query is the hotkey filter — confirm scoping. + sql, args = pool.executed[0] + assert args[0] == "hk-A" + + +def test_submissions_empty_when_no_hotkey_on_identity(monkeypatch): + _install_bearer_identity(monkeypatch, _ident(hotkey="")) + with _wire_app(monkeypatch) as (app, _): + client = TestClient(app) + r = client.get( + "/miners/me/submissions", + headers={"Authorization": "Bearer good"}, + ) + assert r.status_code == 200 + assert r.json() == {"submissions": []} + + +# ── /miners/me/results ────────────────────────────────────────────── + + +def test_results_join_includes_scores(monkeypatch): + _install_bearer_identity(monkeypatch, _ident(hotkey="hk-A")) + with _wire_app(monkeypatch) as (app, pool): + pool.fetch_rows = [[ + _Row( + submission_id=7, round_id=12, task_name="ts_forecasting", + prompt_id="p-7", architecture_code="x", motivation="m", + metric=0.123, score=0.45, success=True, + objectives={"flops_equivalent_size": 1_000_000, "crps": 0.123}, + created_at=1700000000.0, + ), + ]] + client = TestClient(app) + r = client.get( + "/miners/me/results", + headers={"Authorization": "Bearer good"}, + ) + assert r.status_code == 200 + row = r.json()["results"][0] + assert row["prompt_id"] == "p-7" + assert row["scores"]["raw_score"] == 0.45 + assert row["scores"]["metric"] == 0.123 + assert row["scores"]["flops_equivalent_size"] == 1_000_000 + assert row["scores"]["success"] is True + + +def test_results_handles_objectives_as_json_string(monkeypatch): + _install_bearer_identity(monkeypatch, _ident(hotkey="hk-A")) + with _wire_app(monkeypatch) as (app, pool): + pool.fetch_rows = [[ + _Row( + submission_id=1, round_id=1, task_name="t", + prompt_id=None, architecture_code="", motivation="", + metric=None, score=0.0, success=False, + objectives='{"flops_equivalent_size": 5}', + created_at=0.0, + ), + ]] + client = TestClient(app) + r = client.get( + "/miners/me/results", + headers={"Authorization": "Bearer good"}, + ) + assert r.status_code == 200 + assert r.json()["results"][0]["scores"]["flops_equivalent_size"] == 5 + + +# ── /miners/me/summary ────────────────────────────────────────────── + + +def test_summary_aggregates(monkeypatch): + _install_bearer_identity(monkeypatch, _ident(hotkey="hk-A")) + with _wire_app(monkeypatch) as (app, pool): + pool.fetchrow_rows = [ + _Row(total=10, last_round=42), + _Row(mean_score=0.55, successes=3, n=8), + ] + client = TestClient(app) + r = client.get( + "/miners/me/summary", + headers={"Authorization": "Bearer good"}, + ) + assert r.status_code == 200 + s = r.json() + assert s["total_submissions"] == 10 + assert s["last_round_id"] == 42 + assert s["mean_score_recent"] == 0.55 + assert s["successes_recent"] == 3 + assert s["recent_window"] == 8 + + +def test_summary_zero_when_no_rows(monkeypatch): + _install_bearer_identity(monkeypatch, _ident(hotkey="hk-A")) + with _wire_app(monkeypatch) as (app, pool): + pool.fetchrow_rows = [ + _Row(total=0, last_round=None), + _Row(mean_score=None, successes=0, n=0), + ] + client = TestClient(app) + r = client.get( + "/miners/me/summary", + headers={"Authorization": "Bearer good"}, + ) + assert r.status_code == 200 + s = r.json() + assert s["total_submissions"] == 0 + assert s["last_round_id"] is None + assert s["mean_score_recent"] is None + + +# ── /tasks/{task}/frontier (public) ───────────────────────────────── + + +def test_task_frontier_public_no_auth(monkeypatch): + with _wire_app(monkeypatch) as (app, pool): + pool.fetch_rows = [[ + _Row(metric=0.1, objectives={"flops_equivalent_size": 1_000_000}, score=0.9), + _Row(metric=0.2, objectives='{"flops_equivalent_size": 5_000_000}', score=0.7), + ]] + client = TestClient(app) + r = client.get("/tasks/ts_forecasting/frontier?limit=2") + assert r.status_code == 200 + payload = r.json() + assert payload["task"] == "ts_forecasting" + assert len(payload["points"]) == 2 + assert payload["points"][0]["flops"] == 1_000_000 + assert payload["points"][0]["metric"] == 0.1 + + +def test_task_frontier_caps_limit(monkeypatch): + with _wire_app(monkeypatch) as (app, pool): + pool.fetch_rows = [[]] + client = TestClient(app) + client.get("/tasks/foo/frontier?limit=99999") + # Last arg of the SQL call is the LIMIT — confirm it's capped at 500. + _, args = pool.executed[-1] + assert args[-1] == 500 diff --git a/tests/test_noncompetitive_gates.py b/tests/test_noncompetitive_gates.py new file mode 100644 index 0000000..b769e59 --- /dev/null +++ b/tests/test_noncompetitive_gates.py @@ -0,0 +1,211 @@ +"""Regression tests: NONCOMPETITIVE early-returns + optional bittensor import. + +Confirms the chain-touching code paths are properly gated so a +non-competitive deployment can run without ever invoking bittensor in +the leaf modules (runner/server, miner/neuron, validator/neuron init, +DatabaseClient). +""" + +from __future__ import annotations + +import os +from unittest.mock import MagicMock + +import pytest + + +def test_runner_server_has_no_bittensor_reference(): + """runner/server.py was fully stripped of metagraph/bittensor.""" + import runner.server as srv + src = open(srv.__file__).read() + assert "import bittensor" not in src + assert not hasattr(srv, "_metagraph_cache") + assert not hasattr(srv, "_load_metagraph") + assert not hasattr(srv, "_metagraph_lock") + + +def test_miner_neuron_bittensor_import_is_optional(): + """``import bittensor`` is wrapped in try/except so the module + loads in non-competitive deployments that don't have bittensor + installed.""" + src = open("/home/user/radar/miner/neuron.py").read() + assert "try:\n import bittensor as bt" in src + assert "bt = None" in src + + +def test_validator_neuron_bittensor_import_is_optional(): + src = open("/home/user/radar/validator/neuron.py").read() + assert "try:\n import bittensor as bt" in src + assert "bt = None" in src + + +def test_validator_get_config_uses_argparse_in_noncompetitive(monkeypatch): + """get_config() returns a plain Namespace (no bittensor) when + NONCOMPETITIVE=true.""" + monkeypatch.setenv("RADAR_NONCOMPETITIVE", "true") + monkeypatch.setenv("RADAR_SERVICE_KEY", "a" * 64) + import importlib + import config + importlib.reload(config) + # Read the function source instead of importing the whole module + # (which has heavy transitive deps). We just need to confirm the + # branch shape. + src = open("/home/user/radar/validator/neuron.py").read() + assert "Config.NONCOMPETITIVE or bt is None" in src + assert "parser.parse_args()" in src + + +def test_miner_get_config_uses_argparse_in_noncompetitive(): + src = open("/home/user/radar/miner/neuron.py").read() + assert "Config.NONCOMPETITIVE or bt is None" in src + + +def test_set_weights_skipped_in_noncompetitive(): + """_set_weights() has an early-return on Config.NONCOMPETITIVE.""" + src = open("/home/user/radar/validator/neuron.py").read() + # Look for the gate right at the top of _set_weights. + assert ( + 'def _set_weights(self):\n """Set weights on chain' + in src + ) + assert 'NONCOMPETITIVE: skipping weight set' in src + + +def test_run_round_uses_walltime_in_noncompetitive(): + src = open("/home/user/radar/validator/neuron.py").read() + # Walltime path uses time.time() // 12 as a stand-in for block height. + assert "int(time.time() // 12)" in src + + +def test_validator_wallet_is_none_in_noncompetitive(): + """The validator init skips wallet/subtensor/metagraph creation + when NONCOMPETITIVE is set.""" + src = open("/home/user/radar/validator/neuron.py").read() + assert "self.wallet = None" in src + assert "self.subtensor = None" in src + assert "self.metagraph = None" in src + + +def test_miner_wallet_is_none_in_noncompetitive(): + src = open("/home/user/radar/miner/neuron.py").read() + assert "self.wallet = None" in src + assert "self.subtensor = None" in src + assert "self.metagraph = None" in src + + +def test_validator_db_client_uses_service_secret_in_noncompetitive(): + src = open("/home/user/radar/validator/neuron.py").read() + assert "service_secret=Config.SERVICE_KEY.encode()" in src + assert "key_id=Config.SERVICE_KEY_ID" in src + + +def test_database_client_constructor_rejects_no_auth(): + """At least one of wallet / service_secret / api_key required.""" + from shared.db_client import DatabaseClient + with pytest.raises(ValueError): + DatabaseClient("http://x") + + +def test_database_client_accepts_service_secret_only(): + from shared.db_client import DatabaseClient + c = DatabaseClient("http://x", service_secret=b"k" * 32) + assert c.service_secret == b"k" * 32 + assert c.wallet is None + + +def test_database_client_accepts_api_key_only(): + from shared.db_client import DatabaseClient + c = DatabaseClient("http://x", api_key="rdrk_abc") + assert c.api_key == "rdrk_abc" + assert c.wallet is None + assert c.service_secret is None + + +def test_database_client_signs_with_hmac_when_secret_set(): + from shared.db_client import DatabaseClient + c = DatabaseClient("http://x", service_secret=b"a" * 32, key_id="op") + headers = c._sign(b"body") + assert "X-Radar-Signature" in headers + assert "X-Radar-Timestamp" in headers + assert headers["X-Radar-Key-Id"] == "op" + + +def test_database_client_signs_with_bearer_when_key_set(): + from shared.db_client import DatabaseClient + c = DatabaseClient("http://x", api_key="rdrk_abc") + headers = c._sign(b"") + assert headers == {"Authorization": "Bearer rdrk_abc"} + + +def test_substrate_module_loads_without_bittensor(): + """shared/substrate.py uses TYPE_CHECKING + lazy _bt() so it loads + even when bittensor is missing from the environment.""" + src = open("/home/user/radar/shared/substrate.py").read() + assert "if TYPE_CHECKING" in src + assert "import bittensor as bt" in src # in the TYPE_CHECKING block + # The runtime path goes through _bt() — verify the helper exists. + assert "def _bt():" in src + # No bare top-level import outside TYPE_CHECKING. + lines = src.splitlines() + for i, line in enumerate(lines): + stripped = line.lstrip() + if stripped.startswith("import bittensor") or stripped.startswith( + "from bittensor" + ): + # Walk up to see if we're inside a TYPE_CHECKING block. + context = "\n".join(lines[max(0, i - 5):i + 1]) + assert ( + "TYPE_CHECKING" in context + or "def " in context # inside a function = lazy + ), f"bare bittensor import at line {i + 1}: {line}" + + +def test_validator_neuron_imports_without_bittensor(monkeypatch): + """validator/neuron.py + its transitive graph load when bittensor + is missing. Smoke test — confirms the chain-free deployment path + doesn't blow up at import time.""" + monkeypatch.setenv("RADAR_NONCOMPETITIVE", "true") + monkeypatch.setenv("RADAR_SERVICE_KEY", "a" * 64) + monkeypatch.setenv("RADAR_DB_API_URL", "http://localhost:8090") + import importlib + import config + importlib.reload(config) + import validator.neuron + importlib.reload(validator.neuron) + # bt is None when bittensor isn't installed; ImportError was caught + # at top-level by the try/except. In this container it IS None + # (bittensor not in pip set). + # The important assertion: the module loaded. + assert validator.neuron is not None + assert hasattr(validator.neuron, "Validator") + + +def test_no_top_level_bittensor_imports_in_main_code(): + """Sweep guard: no top-level ``import bittensor`` in code/, only + inside functions or under ``if TYPE_CHECKING:``.""" + import pathlib + + root = pathlib.Path("/home/user/radar") + bad: list[str] = [] + for path in root.rglob("*.py"): + if any(part in {"tests", "scripts"} for part in path.parts): + continue + text = path.read_text() + lines = text.splitlines() + for i, raw in enumerate(lines): + stripped = raw.lstrip() + if not ( + stripped.startswith("import bittensor") + or stripped.startswith("from bittensor") + ): + continue + indent = len(raw) - len(stripped) + # Anything indented is inside a function / try block → lazy. + if indent > 0: + continue + # Top-level — must be guarded by TYPE_CHECKING or try/except. + ctx = "\n".join(lines[max(0, i - 4):i]) + if "TYPE_CHECKING" in ctx or "try:" in ctx: + continue + bad.append(f"{path}:{i + 1}: {raw}") + assert not bad, "unguarded top-level bittensor imports:\n" + "\n".join(bad) diff --git a/tests/test_operator_cli.py b/tests/test_operator_cli.py new file mode 100644 index 0000000..b597279 --- /dev/null +++ b/tests/test_operator_cli.py @@ -0,0 +1,172 @@ +"""Tests for the operator CLI argv parsing + handler dispatch. + +Real-DB integration of the handlers is covered by the miner_auth +helper tests; here we just exercise the argparse surface, output +formatting, and rotate-service-key which has no DB dependency. +""" + +from __future__ import annotations + +from unittest.mock import AsyncMock, patch + +import pytest + +from database import operator_cli + + +def test_parser_register_requires_no_args(): + args = operator_cli._build_parser().parse_args(["register"]) + assert args.command == "register" + assert args.name == "" + assert args.hotkey == "" + + +def test_parser_issue_key_requires_miner_id(): + with pytest.raises(SystemExit): + operator_cli._build_parser().parse_args(["issue-key"]) + + +def test_parser_revoke_key_requires_key_id(): + with pytest.raises(SystemExit): + operator_cli._build_parser().parse_args(["revoke-key"]) + + +def test_rotate_service_key_prints_long_hex(capsys): + code = operator_cli.main(["rotate-service-key"]) + assert code == 0 + out = capsys.readouterr().out + # 32 bytes hex = 64 chars somewhere in the output. + candidate = [w for line in out.splitlines() for w in line.split() + if len(w) == 64 and all(c in "0123456789abcdef" for c in w)] + assert candidate, f"no 64-hex key found in output:\n{out}" + + +def test_register_dispatch_calls_register_miner(monkeypatch, capsys): + fake_conn = AsyncMock() + fake_conn.close = AsyncMock() + + async def _connect_stub(): + return fake_conn + + async def _ensure_stub(conn): + pass + + monkeypatch.setattr(operator_cli, "_connect", _connect_stub) + monkeypatch.setattr(operator_cli, "_ensure_schema", _ensure_stub) + with patch( + "shared.miner_auth.register_miner", + new=AsyncMock(return_value="abc123"), + ) as mr: + code = operator_cli.main([ + "register", "--name", "alice", "--hotkey", "5xxx", + ]) + assert code == 0 + mr.assert_awaited_once() + out = capsys.readouterr().out + assert "abc123" in out + + +def test_issue_key_dispatch_prints_plaintext_once(monkeypatch, capsys): + fake_conn = AsyncMock() + fake_conn.close = AsyncMock() + + async def _connect_stub(): + return fake_conn + + async def _ensure_stub(conn): + pass + + monkeypatch.setattr(operator_cli, "_connect", _connect_stub) + monkeypatch.setattr(operator_cli, "_ensure_schema", _ensure_stub) + with patch( + "shared.miner_auth.issue_api_key", + new=AsyncMock(return_value=("rdrk_secret123", "kid-xyz")), + ): + code = operator_cli.main([ + "issue-key", "--miner-id", "m1", "--label", "prod", + ]) + assert code == 0 + out = capsys.readouterr().out + assert "rdrk_secret123" in out + assert "kid-xyz" in out + assert "shown ONCE" in out + + +def test_revoke_key_returns_1_on_miss(monkeypatch, capsys): + fake_conn = AsyncMock() + fake_conn.close = AsyncMock() + + async def _connect_stub(): + return fake_conn + + monkeypatch.setattr(operator_cli, "_connect", _connect_stub) + with patch( + "shared.miner_auth.revoke_api_key", + new=AsyncMock(return_value=False), + ): + code = operator_cli.main(["revoke-key", "--key-id", "ghost"]) + assert code == 1 + + +def test_revoke_key_returns_0_on_hit(monkeypatch, capsys): + fake_conn = AsyncMock() + fake_conn.close = AsyncMock() + + async def _connect_stub(): + return fake_conn + + monkeypatch.setattr(operator_cli, "_connect", _connect_stub) + with patch( + "shared.miner_auth.revoke_api_key", + new=AsyncMock(return_value=True), + ): + code = operator_cli.main(["revoke-key", "--key-id", "real"]) + assert code == 0 + + +def test_dotenv_is_loaded_at_import(tmp_path, monkeypatch): + """Regression: operator_cli used to read os.environ directly without + importing ``config``, which meant ``radar/.env`` was silently + ignored and every invocation printed ``RADAR_PG_DSN not set``. + Now ``import config`` triggers ``load_dotenv`` at module load — + verify the chain by writing a marker var to a fake .env, reloading + config, and asserting it shows up in ``os.environ``. + """ + import importlib + import os + + env_file = tmp_path / ".env" + env_file.write_text("RADAR_OPERATOR_CLI_DOTENV_PROBE=loaded\n") + + monkeypatch.delenv("RADAR_OPERATOR_CLI_DOTENV_PROBE", raising=False) + from dotenv import load_dotenv + load_dotenv(env_file, override=True) + + # Re-import operator_cli so its top-level ``import config`` runs + # again in this test's context (config itself is module-cached + # but the assertion is about the env var the dotenv chain set). + importlib.import_module("database.operator_cli") + assert os.environ.get("RADAR_OPERATOR_CLI_DOTENV_PROBE") == "loaded" + + +def test_list_keys_prints_table(monkeypatch, capsys): + fake_conn = AsyncMock() + fake_conn.close = AsyncMock() + + async def _connect_stub(): + return fake_conn + + monkeypatch.setattr(operator_cli, "_connect", _connect_stub) + with patch( + "shared.miner_auth.list_api_keys", + new=AsyncMock(return_value=[ + {"key_id": "k1", "scope": "miner", "label": "prod", + "created_at": 1700000000, "revoked_at": None, + "last_used_at": 1700001000}, + ]), + ): + code = operator_cli.main(["list-keys", "--miner-id", "m1"]) + assert code == 0 + out = capsys.readouterr().out + assert "k1" in out + assert "prod" in out diff --git a/tests/test_optimizer_random_mutate.py b/tests/test_optimizer_random_mutate.py new file mode 100644 index 0000000..61f24fc --- /dev/null +++ b/tests/test_optimizer_random_mutate.py @@ -0,0 +1,108 @@ +"""Tests for the random_mutate baseline optimizer.""" + +from __future__ import annotations + +import pytest + +from miner_template.optimizers import ResultRow +from miner_template.optimizers.random_mutate import optimize +from miner_template.prompts import Prompt + + +def _pop(*ids_and_templates: tuple[str, str]) -> list[Prompt]: + return [Prompt(id=i, template=t, generation=0) + for i, t in ids_and_templates] + + +def _result(prompt_id: str, score: float, round_id: int = 1) -> ResultRow: + return ResultRow( + round_id=round_id, + submission_id=f"sub-{round_id}-{prompt_id}", + task_name="ts_forecasting", + prompt_id=prompt_id, + architecture_code="class M: pass", + scores={"raw_score": score}, + ) + + +def test_empty_population_returns_empty(): + assert optimize([], [], {}) == [] + + +def test_no_results_clones_top_of_population(): + pop = _pop(("a", "A"), ("b", "B"), ("c", "C")) + new = optimize([], pop, {"population": 4, "elite_k": 2}) + assert len(new) == 4 + # Both elites must appear at the head (order = ranking, stable). + assert {p.id for p in new[:2]} == {"a", "b"} + + +def test_ranks_by_mean_score_and_promotes_winners(): + pop = _pop(("a", "A"), ("b", "B"), ("c", "C")) + results = [ + _result("a", 0.1), + _result("b", 0.9), _result("b", 0.9), + _result("c", 0.5), + ] + new = optimize(results, pop, {"population": 4, "elite_k": 2}) + assert len(new) == 4 + elite_ids = [p.id for p in new if p.parent_id is None] + assert elite_ids[0] == "b" # highest mean score + assert elite_ids[1] == "c" + + +def test_children_have_parent_pointing_at_elite(): + pop = _pop(("a", "A"), ("b", "B")) + results = [_result("a", 0.9), _result("b", 0.1)] + new = optimize(results, pop, {"population": 4, "elite_k": 1}) + children = [p for p in new if p.parent_id is not None] + assert len(children) == 3 + assert all(c.parent_id == "a" for c in children) + assert all(c.template.startswith("A") for c in children) + assert all(c.generation == 1 for c in children) + + +def test_population_size_enforced(): + pop = _pop(("a", "A"), ("b", "B")) + new = optimize([], pop, {"population": 8}) + assert len(new) == 8 + + +def test_population_below_one_raises(): + pop = _pop(("a", "A")) + with pytest.raises(ValueError): + optimize([], pop, {"population": 0}) + + +def test_deterministic_perturbation(): + """Same inputs -> same outputs (modulo new UUIDs). We can't compare + ids, but the chosen perturbation suffix is deterministic in the + parent + child_idx, so child templates should match across runs.""" + pop = _pop(("a", "Alpha")) + a = optimize([], pop, {"population": 4, "elite_k": 1}) + b = optimize([], pop, {"population": 4, "elite_k": 1}) + assert [p.template for p in a] == [p.template for p in b] + + +def test_unknown_prompt_ids_in_results_ignored(): + pop = _pop(("a", "A")) + results = [_result("ghost", 999.0)] # not in population + new = optimize(results, pop, {"population": 2, "elite_k": 1}) + assert len(new) == 2 + assert new[0].id == "a" + + +def test_custom_score_key(): + pop = _pop(("a", "A"), ("b", "B")) + rows = [ + ResultRow(round_id=1, submission_id="x", task_name="t", + prompt_id="a", architecture_code="", + scores={"raw_score": 0.0, "frontier_bonus": 1.0}), + ResultRow(round_id=1, submission_id="y", task_name="t", + prompt_id="b", architecture_code="", + scores={"raw_score": 1.0, "frontier_bonus": 0.0}), + ] + new = optimize(rows, pop, {"population": 2, "elite_k": 1, + "score_key": "frontier_bonus"}) + # Top elite should now be 'a' (high frontier_bonus) instead of 'b'. + assert new[0].id == "a" diff --git a/tests/test_optimizers_resolve.py b/tests/test_optimizers_resolve.py new file mode 100644 index 0000000..7ab14a5 --- /dev/null +++ b/tests/test_optimizers_resolve.py @@ -0,0 +1,61 @@ +"""Tests for the optimizer plugin registry.""" + +from __future__ import annotations + +import pytest + +from miner_template import optimizers +from miner_template.optimizers import ResultRow + + +def test_list_builtins_has_gepa_and_random(): + names = optimizers.list_builtins() + assert "gepa" in names + assert "random_mutate" in names + + +def test_resolve_builtin_returns_callable(): + fn = optimizers.resolve("random_mutate") + assert callable(fn) + + +def test_resolve_unknown_alias_raises(): + with pytest.raises(ValueError): + optimizers.resolve("nonexistent_optimizer") + + +def test_resolve_dotted_path(): + fn = optimizers.resolve("miner_template.optimizers.random_mutate:optimize") + assert callable(fn) + + +def test_resolve_missing_module_raises_importerror(): + with pytest.raises(ImportError): + optimizers.resolve("not_a_real_module.somewhere:func") + + +def test_resolve_missing_function_raises_attributeerror(): + with pytest.raises(AttributeError): + optimizers.resolve("miner_template.optimizers.random_mutate:not_there") + + +def test_resultrow_from_dict_safe_with_missing_fields(): + r = ResultRow.from_dict({}) + assert r.round_id == 0 + assert r.prompt_id is None + assert r.scores == {} + + +def test_resultrow_from_dict_full(): + r = ResultRow.from_dict({ + "round_id": 12, + "submission_id": "sub-1", + "task_name": "ts_forecasting", + "prompt_id": "p-abc", + "architecture_code": "class M: pass", + "scores": {"raw_score": 0.7, "crps": 0.3}, + "created_at": "2026-05-19T00:00:00Z", + }) + assert r.round_id == 12 + assert r.prompt_id == "p-abc" + assert r.scores["raw_score"] == 0.7 diff --git a/tests/test_prompts_store.py b/tests/test_prompts_store.py new file mode 100644 index 0000000..7ad27f6 --- /dev/null +++ b/tests/test_prompts_store.py @@ -0,0 +1,149 @@ +"""Tests for the miner-side prompt-population store.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from miner_template import prompts as ps +from miner_template.prompts import Prompt + + +def test_load_active_empty(tmp_path: Path): + assert ps.load_active(tmp_path) == [] + + +def test_save_then_load_roundtrip(tmp_path: Path): + pop = [ + Prompt.new(template="p1", generation=1), + Prompt.new(template="p2", generation=1), + ] + ps.save_active(pop, tmp_path) + got = ps.load_active(tmp_path) + assert len(got) == 2 + assert {p.template for p in got} == {"p1", "p2"} + assert all(p.generation == 1 for p in got) + + +def test_save_atomic_no_partial_on_failure(tmp_path: Path, monkeypatch): + """If os.fsync raises, ``active.json`` must not exist.""" + pop = [Prompt.new(template="ok")] + ps.save_active(pop, tmp_path) # establish baseline + original = ps.load_active(tmp_path) + + # Now break the write halfway through and confirm the original + # active.json is untouched. + real_replace = ps.os.replace + + def boom(*args, **kwargs): + raise OSError("disk full") + + monkeypatch.setattr(ps.os, "replace", boom) + with pytest.raises(OSError): + ps.save_active([Prompt.new(template="will-fail")], tmp_path) + + monkeypatch.setattr(ps.os, "replace", real_replace) + assert ps.load_active(tmp_path) == original + + +def test_archive_and_list_history(tmp_path: Path): + ps.save_active([Prompt.new(template="gen0")], tmp_path) + p = ps.archive_current(1, tmp_path) + assert p is not None and p.exists() + ps.save_active([Prompt.new(template="gen1")], tmp_path) + ps.archive_current(2, tmp_path) + assert ps.list_history(tmp_path) == [1, 2] + + +def test_archive_noop_when_no_active(tmp_path: Path): + assert ps.archive_current(1, tmp_path) is None + + +def test_load_generation_roundtrip(tmp_path: Path): + seed = [Prompt.new(template="archived")] + ps.save_active(seed, tmp_path) + ps.archive_current(5, tmp_path) + loaded = ps.load_generation(5, tmp_path) + assert len(loaded) == 1 + assert loaded[0].template == "archived" + + +def test_load_generation_missing(tmp_path: Path): + with pytest.raises(FileNotFoundError): + ps.load_generation(99, tmp_path) + + +def test_rollback_archives_current_then_restores(tmp_path: Path): + a = [Prompt.new(template="A")] + ps.save_active(a, tmp_path) + ps.archive_current(1, tmp_path) + b = [Prompt.new(template="B")] + ps.save_active(b, tmp_path) + # Rollback to gen 1 ('A'); the current 'B' should be archived too. + ps.rollback_to(1, tmp_path) + active = ps.load_active(tmp_path) + assert [p.template for p in active] == ["A"] + # Both 1 and 2 (the just-archived B) should now be in history. + assert ps.list_history(tmp_path) == [1, 2] + assert [p.template for p in ps.load_generation(2, tmp_path)] == ["B"] + + +def test_pick_for_round_deterministic_roundrobin(tmp_path: Path): + pop = [ + Prompt(id="a", template="A"), + Prompt(id="b", template="B"), + Prompt(id="c", template="C"), + ] + picks = [ps.pick_for_round(pop, i).id for i in range(7)] + assert picks == ["a", "b", "c", "a", "b", "c", "a"] + + +def test_pick_for_round_empty_raises(): + with pytest.raises(ValueError): + ps.pick_for_round([], 0) + + +def test_seed_default_creates_active(tmp_path: Path): + pop = ps.seed_default(tmp_path) + assert len(pop) == 1 + on_disk = ps.load_active(tmp_path) + assert len(on_disk) == 1 + assert on_disk[0].generation == 0 + + +def test_seed_default_is_noop_when_active_exists(tmp_path: Path): + existing = [Prompt.new(template="kept")] + ps.save_active(existing, tmp_path) + pop = ps.seed_default(tmp_path) + assert len(pop) == 1 + assert pop[0].template == "kept" + + +def test_prompt_from_dict_resilient_to_missing_fields(): + p = Prompt.from_dict({"template": "t"}) + assert p.template == "t" + assert p.id # auto-generated + assert p.generation == 0 + assert p.parent_id is None + assert p.metadata == {} + + +def test_save_uses_atomic_replace(tmp_path: Path): + """Sanity: the final file lands at ``active.json`` and no ``.tmp`` + file is left behind.""" + ps.save_active([Prompt.new(template="x")], tmp_path) + assert (tmp_path / "active.json").exists() + leftover = list(tmp_path.glob("*.tmp")) + assert leftover == [] + + +def test_archived_file_is_readable_json(tmp_path: Path): + """Make sure archived generations are still valid JSON, not just bytes.""" + ps.save_active([Prompt.new(template="t1"), Prompt.new(template="t2")], tmp_path) + dst = ps.archive_current(3, tmp_path) + assert dst is not None + data = json.loads(dst.read_text()) + assert isinstance(data, dict) + assert len(data["prompts"]) == 2 diff --git a/tests/test_protocol_prompt_id.py b/tests/test_protocol_prompt_id.py new file mode 100644 index 0000000..5081e2f --- /dev/null +++ b/tests/test_protocol_prompt_id.py @@ -0,0 +1,60 @@ +"""Tests for prompt_id round-trip through Proposal + DataElement + INSERT.""" + +from __future__ import annotations + +import json + +from shared.database import DataElement +from shared.pg_schema import INSERT_SQL, element_to_params +from shared.protocol import Proposal + + +def test_proposal_defaults_empty_prompt_id(): + p = Proposal() + assert p.prompt_id == "" + + +def test_proposal_to_json_includes_prompt_id(): + p = Proposal(code="x", prompt_id="abc123") + d = json.loads(p.to_json()) + assert d["prompt_id"] == "abc123" + + +def test_proposal_from_json_resilient_to_missing_prompt_id(): + p = Proposal.from_json(json.dumps({"code": "x"})) + assert p.prompt_id == "" + + +def test_proposal_from_json_roundtrips_prompt_id(): + p = Proposal.from_json(json.dumps({"code": "x", "prompt_id": "p1"})) + assert p.prompt_id == "p1" + + +def test_data_element_defaults_empty_prompt_id(): + e = DataElement() + assert e.prompt_id == "" + + +def test_data_element_carries_prompt_id(): + e = DataElement(name="x", prompt_id="p-abc") + assert e.prompt_id == "p-abc" + + +def test_element_to_params_includes_prompt_id_as_last_arg(): + e = DataElement(name="x", code="c", prompt_id="p-1") + params = element_to_params(e, next_id=42) + # INSERT_SQL has 25 placeholders; the new prompt_id is $25. + assert len(params) == 25 + assert params[-1] == "p-1" + + +def test_element_to_params_normalizes_empty_prompt_id_to_none(): + e = DataElement(name="x", code="c") # prompt_id default "" + params = element_to_params(e, next_id=0) + assert params[-1] is None # so the DB stores NULL not '' + + +def test_insert_sql_lists_prompt_id_column(): + assert "prompt_id" in INSERT_SQL + # 25 positional placeholders. + assert "$25" in INSERT_SQL diff --git a/tests/test_results_client.py b/tests/test_results_client.py new file mode 100644 index 0000000..1db250f --- /dev/null +++ b/tests/test_results_client.py @@ -0,0 +1,146 @@ +"""Tests for MinerResultsClient using httpx's MockTransport.""" + +from __future__ import annotations + +import json + +import httpx +import pytest + +from miner_template.optimizers import ResultRow +from miner_template.results_client import MinerResultsClient + + +def _make_client(handler, api_key: str = "test-key") -> MinerResultsClient: + transport = httpx.MockTransport(handler) + http = httpx.Client(transport=transport) + return MinerResultsClient("http://db.local", api_key, client=http) + + +def test_requires_db_url_and_api_key(): + with pytest.raises(ValueError): + MinerResultsClient("", "k") + with pytest.raises(ValueError): + MinerResultsClient("http://x", "") + + +def test_submissions_sends_bearer_and_returns_list(): + seen = {} + + def handler(request: httpx.Request) -> httpx.Response: + seen["url"] = str(request.url) + seen["auth"] = request.headers.get("authorization") + return httpx.Response(200, json={"submissions": [ + {"submission_id": "s1", "round_id": 1}, + ]}) + + c = _make_client(handler, api_key="secret") + rows = c.submissions(limit=10, task="ts_forecasting") + assert seen["auth"] == "Bearer secret" + assert "/miners/me/submissions" in seen["url"] + assert "limit=10" in seen["url"] + assert "task=ts_forecasting" in seen["url"] + assert len(rows) == 1 + assert rows[0]["submission_id"] == "s1" + + +def test_submissions_accepts_plain_list_response(): + def handler(request): + return httpx.Response(200, json=[{"submission_id": "x"}]) + + c = _make_client(handler) + rows = c.submissions() + assert rows[0]["submission_id"] == "x" + + +def test_results_returns_resultrow_objects(): + def handler(request): + return httpx.Response(200, json={"results": [{ + "round_id": 5, + "submission_id": "s5", + "task_name": "ts_forecasting", + "prompt_id": "p1", + "architecture_code": "code", + "scores": {"raw_score": 0.7}, + }]}) + + c = _make_client(handler) + rows = c.results() + assert len(rows) == 1 + assert isinstance(rows[0], ResultRow) + assert rows[0].prompt_id == "p1" + assert rows[0].scores["raw_score"] == 0.7 + + +def test_results_empty_payload_returns_empty_list(): + def handler(request): + return httpx.Response(200, json={"results": None}) + + c = _make_client(handler) + assert c.results() == [] + + +def test_summary_returns_dict(): + def handler(request): + return httpx.Response(200, json={"total": 42, "last_round_id": 7}) + + c = _make_client(handler) + s = c.summary() + assert s["total"] == 42 + assert s["last_round_id"] == 7 + + +def test_summary_handles_non_dict_response(): + def handler(request): + return httpx.Response(200, json=[1, 2, 3]) + + c = _make_client(handler) + assert c.summary() == {} + + +def test_frontier_requires_task(): + def handler(request): + return httpx.Response(200, json={"points": []}) + + c = _make_client(handler) + with pytest.raises(ValueError): + c.frontier("") + + +def test_frontier_returns_points(): + def handler(request): + assert "/tasks/ts_forecasting/frontier" in str(request.url) + return httpx.Response(200, json={"points": [ + {"flops": 1e6, "crps": 0.4}, + {"flops": 5e6, "crps": 0.3}, + ]}) + + c = _make_client(handler) + pts = c.frontier("ts_forecasting") + assert len(pts) == 2 + + +def test_4xx_raises(): + def handler(request): + return httpx.Response(403, json={"error": "bad key"}) + + c = _make_client(handler) + with pytest.raises(httpx.HTTPStatusError): + c.summary() + + +def test_context_manager_closes_owned_client(): + def handler(request): + return httpx.Response(200, json={}) + + transport = httpx.MockTransport(handler) + # Owned client path — no client passed in. + http = httpx.Client(transport=transport) + c = MinerResultsClient("http://x", "k", client=http) + with c: + c.summary() + # The owned-client logic only closes when the client was internally + # created; we passed one in so we must close manually. Just confirm + # close() is idempotent. + c.close() + c.close() diff --git a/tests/test_runner_auth.py b/tests/test_runner_auth.py new file mode 100644 index 0000000..5fdae1a --- /dev/null +++ b/tests/test_runner_auth.py @@ -0,0 +1,105 @@ +"""Auth tests for runner/server.py — confirms metagraph is gone and HMAC +service-key verification is in place.""" + +from __future__ import annotations + +import json +import os + +from fastapi.testclient import TestClient + +from shared.auth import hmac_sign_request + + +SECRET = b"a" * 32 +KEY_ID = "operator" + + +def _client(monkeypatch): + monkeypatch.setenv("RADAR_SERVICE_KEY", SECRET.decode()) + monkeypatch.setenv("RADAR_SERVICE_KEY_ID", KEY_ID) + monkeypatch.delenv("RADAR_LOCALNET", raising=False) + # Reload to pick up env. + import importlib + + import runner.server as srv + importlib.reload(srv) + return TestClient(srv.app) + + +def test_train_rejects_without_hmac_headers(monkeypatch): + client = _client(monkeypatch) + r = client.post("/train", content=b"{}", headers={"Content-Type": "application/json"}) + assert r.status_code == 403 + assert "missing" in r.json()["error"].lower() + + +def test_train_rejects_bad_signature(monkeypatch): + client = _client(monkeypatch) + body = b'{"architecture": "x", "task_name": "ts_forecasting"}' + headers = hmac_sign_request(b"WRONG_SECRET", body, key_id=KEY_ID) + headers["Content-Type"] = "application/json" + r = client.post("/train", content=body, headers=headers) + assert r.status_code == 403 + + +def test_train_rejects_unknown_key_id(monkeypatch): + client = _client(monkeypatch) + body = b'{"architecture": "x", "task_name": "ts_forecasting"}' + headers = hmac_sign_request(SECRET, body, key_id="not-operator") + headers["Content-Type"] = "application/json" + r = client.post("/train", content=body, headers=headers) + assert r.status_code == 403 + + +def test_train_503_when_service_key_unset(monkeypatch): + monkeypatch.delenv("RADAR_SERVICE_KEY", raising=False) + monkeypatch.delenv("RADAR_LOCALNET", raising=False) + import importlib + + import runner.server as srv + importlib.reload(srv) + client = TestClient(srv.app) + body = b'{"architecture": "x", "task_name": "ts_forecasting"}' + headers = hmac_sign_request(SECRET, body, key_id=KEY_ID) + headers["Content-Type"] = "application/json" + r = client.post("/train", content=body, headers=headers) + assert r.status_code == 503 + + +def test_health_no_auth(monkeypatch): + client = _client(monkeypatch) + r = client.get("/health") + assert r.status_code == 200 + assert r.json() == {"status": "ok"} + + +def test_localnet_skips_auth(monkeypatch): + monkeypatch.setenv("RADAR_LOCALNET", "1") + monkeypatch.delenv("RADAR_SERVICE_KEY", raising=False) + import importlib + + import runner.server as srv + importlib.reload(srv) + client = TestClient(srv.app) + body = json.dumps({ + "architecture": "class M: pass", + "task_name": "ts_forecasting", + }).encode() + r = client.post( + "/train", content=body, + headers={"Content-Type": "application/json"}, + ) + # 202 (accepted) — auth was skipped. We don't actually train + # anything because run_sandbox would need a real sandbox; the test + # just confirms auth doesn't reject. + assert r.status_code == 202 + + +def test_no_bittensor_or_metagraph_symbols(): + """Ensure the metagraph machinery is gone from the module.""" + import runner.server as srv + assert not hasattr(srv, "_load_metagraph") + assert not hasattr(srv, "_metagraph_cache") + src = open(srv.__file__).read() + assert "import bittensor" not in src diff --git a/validator/collection.py b/validator/collection.py index 2e82f10..3680f53 100644 --- a/validator/collection.py +++ b/validator/collection.py @@ -265,6 +265,7 @@ async def _run_single_agent( result.get("reasoning", "") or "", _REASONING_MAX_BYTES, ), tool_calls=_normalise_tool_calls(result.get("tool_calls", [])), + prompt_id=str(result.get("prompt_id", "") or "")[:64], ) agent_log = _truncate_text( result.get("agent_log", "") or "", _TRACE_MAX_BYTES, @@ -275,6 +276,7 @@ async def _run_single_agent( "reasoning": proposal.reasoning, "tool_calls": proposal.tool_calls, "agent_behavior": agent_behavior, + "prompt_id": proposal.prompt_id, } r2.upload_json( f"round_{round_id}/proposals/{uid}.json", @@ -284,6 +286,7 @@ async def _run_single_agent( "motivation": proposal.motivation, "reasoning": proposal.reasoning, "tool_calls": proposal.tool_calls, + "prompt_id": proposal.prompt_id, "agent_log": agent_log, "agent_behavior": agent_behavior, }, @@ -387,7 +390,15 @@ async def run_and_collect_agents( # Build DB client if not provided if db_client is None: - db_client = DatabaseClient(Config.DB_API_URL, wallet) + if Config.SERVICE_KEY: + db_client = DatabaseClient( + Config.DB_API_URL, + service_secret=Config.SERVICE_KEY.encode(), + key_id=Config.SERVICE_KEY_ID, + api_key=Config.DB_API_KEY, + ) + else: + db_client = DatabaseClient(Config.DB_API_URL, wallet) # Pre-fetch all agent bundles for my assigned miners (concurrently). async def _prefetch(uid: int) -> tuple[int, Optional[dict]]: @@ -459,6 +470,9 @@ async def _prefetch(uid: int) -> tuple[int, Optional[dict]]: tool_calls=_normalise_tool_calls( data.get("tool_calls", []), ), + prompt_id=str( + data.get("prompt_id", "") or "" + )[:64], ) agent_meta[uid] = { "agent_log": _truncate_text( @@ -467,6 +481,7 @@ async def _prefetch(uid: int) -> tuple[int, Optional[dict]]: ), "reasoning": proposals[uid].reasoning, "tool_calls": proposals[uid].tool_calls, + "prompt_id": proposals[uid].prompt_id, "agent_behavior": data.get("agent_behavior", {}) if isinstance(data.get("agent_behavior"), dict) else {}, diff --git a/validator/coordinator.py b/validator/coordinator.py index 6973bc4..c68cd4a 100644 --- a/validator/coordinator.py +++ b/validator/coordinator.py @@ -18,7 +18,7 @@ import httpx from config import Config -from shared.auth import sign_request +from shared.auth import sign_request, hmac_sign_request from shared.protocol import Proposal, TrainerRequest, TrainerReady, TrainerRelease if TYPE_CHECKING: @@ -143,6 +143,13 @@ def __init__( self.metagraph = metagraph self.r2 = r2 self.my_uid = my_uid + # Off-chain deploy: when no wallet, sign outbound trainer / proxy + # requests with the shared HMAC service key. The trainer image + # (runner/server.py) verifies the same way. + self._service_secret = ( + Config.SERVICE_KEY.encode() if (wallet is None and Config.SERVICE_KEY) else None + ) + self._service_key_id = Config.SERVICE_KEY_ID # Optional dual-write store (TEN-240 Phase 7). When present, JSON # artifacts (dispatch records, frontier snapshots) fan out to both # R2 and Hippius. Falls back to R2-only when None — historical @@ -163,6 +170,15 @@ def __init__( self._reverify_tasks: dict[int, list[asyncio.Task]] = {} self._targon_client = None + def _sign(self, body: bytes) -> dict[str, str]: + """Sign an outbound HTTP body. HMAC in off-chain deploys, Epistula + otherwise.""" + if self._service_secret is not None: + return hmac_sign_request( + self._service_secret, body, key_id=self._service_key_id, + ) + return sign_request(self.wallet, body) + def compute_my_jobs( self, block_hash: str, @@ -294,7 +310,7 @@ async def _dispatch_one( # Sign fresh every attempt so the Epistula timestamp # stays inside the EPISTULA_TIMESTAMP_TOLERANCE window # (default 120s; tunable via RADAR_EPISTULA_TOLERANCE). - headers = sign_request(self.wallet, payload) + headers = self._sign(payload) headers["Content-Type"] = "application/json" resp = await client.post( f"{url.rstrip('/')}/train", @@ -606,7 +622,10 @@ async def write_dispatch_record(self, round_id: int, results: list[TrainingResul — the dispatching validator is the only entity that knows the opaque submission_id minted at dispatch. """ - hotkey = self.wallet.hotkey.ss58_address + hotkey = ( + self.wallet.hotkey.ss58_address + if self.wallet is not None else self._service_key_id + ) key = f"round_{round_id}/dispatch/vali_{hotkey}.json" hotkeys = self.metagraph.hotkeys if hasattr(self.metagraph, "hotkeys") else [] records = [] @@ -742,7 +761,7 @@ async def prepare_trainers( for uid, commitment in with_listener.items(): try: url = f"{commitment.listener_url.rstrip('/')}/prepare" - fresh_headers = sign_request(self.wallet, body) + fresh_headers = self._sign(body) fresh_headers["Content-Type"] = "application/json" resp = await client.post(url, content=body, headers=fresh_headers) if resp.status_code < 400: @@ -854,16 +873,17 @@ async def release_trainers( if not commitment or not commitment.listener_url: continue try: + mg_hotkeys = getattr(self.metagraph, "hotkeys", None) or [] release = TrainerRelease( round_id=round_id, miner_hotkey=( - self.metagraph.hotkeys[uid] - if uid < len(self.metagraph.hotkeys) + mg_hotkeys[uid] + if uid < len(mg_hotkeys) else "" ), ) body = release.to_json().encode() - headers = sign_request(self.wallet, body) + headers = self._sign(body) headers["Content-Type"] = "application/json" url = f"{commitment.listener_url.rstrip('/')}/release" await client.post(url, content=body, headers=headers) @@ -903,9 +923,10 @@ async def _verify_ready(self, ready_msg, uid: int) -> tuple[bool, str, bool]: """ if Config.HOSTING_BACKEND == "targon": from validator.trainer_verify import verify_trainer + mg_hotkeys = getattr(self.metagraph, "hotkeys", None) or [] hotkey = ( - self.metagraph.hotkeys[uid] - if uid < len(self.metagraph.hotkeys) else ready_msg.miner_hotkey + mg_hotkeys[uid] + if uid < len(mg_hotkeys) else ready_msg.miner_hotkey ) result = await verify_trainer( ready=ready_msg, @@ -938,9 +959,10 @@ async def reverify_running( if Config.HOSTING_BACKEND != "targon": return True # No mid-run check on Basilica. from validator.trainer_verify import reverify_workload + mg_hotkeys = getattr(self.metagraph, "hotkeys", None) or [] miner_hotkey = ( - self.metagraph.hotkeys[miner_uid] - if miner_uid < len(self.metagraph.hotkeys) else ready_msg.miner_hotkey + mg_hotkeys[miner_uid] + if miner_uid < len(mg_hotkeys) else ready_msg.miner_hotkey ) result = await reverify_workload( ready=ready_msg, diff --git a/validator/neuron.py b/validator/neuron.py index e08e40c..cd509e8 100644 --- a/validator/neuron.py +++ b/validator/neuron.py @@ -16,28 +16,25 @@ import importlib.util import logging import os -import random import threading +import time import traceback from typing import Optional -import bittensor as bt import uvicorn from config import Config from shared.challenge import ( generate_challenge, round_start_block, current_phase, select_task, ) -from shared.commitment import read_miner_commitments from shared.database import DataElement from shared.db_client import DatabaseClient -from shared.protocol import Challenge, Proposal +from shared.protocol import Proposal from shared.provenance import detect_components from shared.scoring import ( - apply_round_metadata, compute_penalties, ema_update, - score_round, scores_to_weights, + apply_round_metadata, compute_penalties, ema_update, score_round, ) -from shared.task import TaskSpec, load_task, load_enabled_tasks +from shared.task import load_enabled_tasks from validator.collection import run_and_collect_agents from validator.coordinator import ( TrainingCoordinator, compute_assignments, compute_fallback, @@ -53,81 +50,25 @@ logger = logging.getLogger(__name__) -def compute_live_validator_uids( - metagraph, - miner_uids: Optional[set[int]] = None, - current_block: Optional[int] = None, - stale_blocks: int = 600, -) -> list[int]: - """Filter ``metagraph.validator_permit`` to live, non-miner validators. - - A UID counts as a live validator iff: - * it has ``validator_permit=True`` on chain, AND - * it is not running as a miner this round (no commitment), AND - * its on-chain ``last_update`` is within ``stale_blocks`` of - ``current_block``. - - ``miner_uids`` / ``current_block`` are optional; when absent, the - corresponding check is skipped. A ``last_update`` of 0 is treated as - "never updated" and is not filtered (bootstrap case). - """ - permits = getattr(metagraph, "validator_permit", None) - if permits is None: - return [] - - last_update = getattr(metagraph, "last_update", None) - miners = miner_uids or set() - result: list[int] = [] - for uid in range(metagraph.n): - if uid >= len(permits) or not permits[uid]: - continue - if uid in miners: - continue - if ( - current_block is not None - and last_update is not None - and uid < len(last_update) - ): - try: - last = int(last_update[uid]) - except (TypeError, ValueError): - last = 0 - if last > 0 and current_block - last > stale_blocks: - continue - result.append(uid) - return result - - def get_my_assignments( all_uids: list[int], validator_uids: list[int], my_uid: int, seed: int, ) -> list[int]: - """Deterministic assignment of UIDs to validators. + """Single-validator deployment: this validator owns every UID. - Shuffle all_uids with seed, round-robin across sorted validator_uids. - Returns the UIDs assigned to my_uid. + Kept as a stub so collection.py's pass-through still works. """ - if not validator_uids or my_uid not in validator_uids: - return list(all_uids) - - rng = random.Random(seed) - shuffled = list(all_uids) - rng.shuffle(shuffled) - - sorted_validators = sorted(validator_uids) - my_idx = sorted_validators.index(my_uid) - - return [uid for i, uid in enumerate(shuffled) if i % len(sorted_validators) == my_idx] + return list(all_uids) class Validator: - """Radar subnet validator — 3-phase pipeline.""" + """Radar validator — 3-phase pipeline (off-chain, single-validator).""" - def __init__(self, config: bt.Config): + def __init__(self, config): self.config = config - self.netuid = config.netuid + self.netuid = getattr(config, "netuid", 0) # Fail-fast on Targon misconfig — operator gets a clear error at # startup instead of a stack trace mid-round when verify fires. @@ -143,10 +84,11 @@ def __init__(self, config: bt.Config): "Without a pinned digest the verify chain becomes a no-op — refusing to start." ) - # Bittensor components - self.wallet = bt.Wallet(config=config) - self.subtensor = bt.Subtensor(config=config) - self.metagraph = self.subtensor.metagraph(self.netuid) + if not Config.SERVICE_KEY: + raise RuntimeError( + "RADAR_SERVICE_KEY must be set (shared HMAC secret for " + "service-to-service auth)." + ) # Tasks — load all enabled tasks for multi-task rounds self.tasks = load_enabled_tasks(Config.ENABLED_TASKS) @@ -163,7 +105,9 @@ def __init__(self, config: bt.Config): "centralized DB server (e.g. http://:8090)." ) self.db_client = DatabaseClient( - db_url=Config.DB_API_URL, wallet=self.wallet, + db_url=Config.DB_API_URL, + service_secret=Config.SERVICE_KEY.encode(), + key_id=Config.SERVICE_KEY_ID, api_key=Config.DB_API_KEY, ) @@ -253,11 +197,10 @@ def __init__(self, config: bt.Config): ) # Training coordinator - my_uid = self._my_uid() if self.r2: self.coordinator = TrainingCoordinator( - wallet=self.wallet, metagraph=self.metagraph, - r2=self.r2, my_uid=my_uid, + wallet=None, metagraph=None, + r2=self.r2, my_uid=-1, artifact_store=self.artifact_store, ) else: @@ -272,8 +215,8 @@ def __init__(self, config: bt.Config): # Configure proxy set_proxy_config( db_api_url=Config.DB_API_URL, - wallet=self.wallet, - metagraph=self.metagraph, + wallet=None, + metagraph=None, api_key=Config.DB_API_KEY, ) @@ -283,14 +226,6 @@ def __init__(self, config: bt.Config): "enabled" if self.r2 else "disabled", ) - def _my_uid(self) -> int: - """Get this validator's UID.""" - hotkey = self.wallet.hotkey.ss58_address - hotkeys = self.metagraph.hotkeys - if hotkeys is not None and hotkey in hotkeys: - return hotkeys.index(hotkey) - return -1 - def _init_hippius(self): """Construct the Hippius client when opted in; return None otherwise.""" if not Config.HIPPIUS_ENABLED: @@ -313,19 +248,6 @@ def _init_hippius(self): logger.warning("Hippius client init failed: %s", e) return None - def _get_validator_uids( - self, - miner_uids: Optional[set[int]] = None, - current_block: Optional[int] = None, - ) -> list[int]: - """Instance wrapper for :func:`compute_live_validator_uids`.""" - return compute_live_validator_uids( - self.metagraph, - miner_uids=miner_uids, - current_block=current_block, - stale_blocks=Config.VALIDATOR_STALE_BLOCKS, - ) - def _proxy_base_url(self) -> str: """Return the externally-reachable base URL for the validator proxy. @@ -415,9 +337,14 @@ def _run(): logger.info("Proxy server started on port %d", self.proxy_port) async def _wait_until_block(self, target_block: int): - """Wait until the chain reaches target_block.""" + """Wait until the wall-clock-derived block counter reaches target_block. + + Off-chain deploys use ``time.time() // 12`` as a stand-in for block + height so timing stays compatible with the per-phase block-window + constants in ``config.py``. + """ while True: - current = self.subtensor.block + current = int(time.time() // 12) if current >= target_block: return remaining = (target_block - current) * 12 @@ -432,15 +359,11 @@ async def _cache_training_metas( if the cache write fails.""" if not metas: return - hotkeys = self.metagraph.hotkeys if hasattr(self.metagraph, "hotkeys") else [] cached = 0 for uid, meta in metas.items(): if not isinstance(meta, dict): continue - hk = ( - meta.get("miner_hotkey") - or (hotkeys[uid] if uid < len(hotkeys) else "") - ) + hk = meta.get("miner_hotkey", "") if not hk: continue try: @@ -509,17 +432,12 @@ async def _publish_submission_reveal( async def run_round(self): """Execute one full 3-phase round.""" # ── SETUP ────────────────────────────────────────────── - self.metagraph.sync() - set_metagraph(self.metagraph) - hotkeys = self.metagraph.hotkeys or [] - hotkey_map = { - hotkeys[uid]: uid - for uid in range(self.metagraph.n) - if uid < len(hotkeys) - } - set_hotkey_map(hotkey_map) - current_block = self.subtensor.block + # No chain — round_start is a wall-clock-derived integer so the + # deterministic seed stays stable for ~12s like a block. + current_block = int(time.time() // 12) round_start = round_start_block(current_block, Config.ROUND_INTERVAL_BLOCKS) + set_metagraph(None) + set_hotkey_map({}) block_hash = hashlib.sha256(str(round_start).encode()).hexdigest() # Select task for this round (deterministic from block hash) @@ -585,20 +503,11 @@ async def run_round(self): ) # ── PHASE A: RUN AGENTS + COLLECT ──────────────────── - commitments = read_miner_commitments(self.subtensor, self.netuid, self.metagraph) - try: - current_block = self.subtensor.get_current_block() - except Exception as e: - logger.warning("Failed to read current block for liveness filter: %s", e) - current_block = None - validator_uids = self._get_validator_uids( - miner_uids=set(commitments.keys()), - current_block=current_block, - ) - logger.info( - "Live validators this round: %s (filtered miners + stale last_update)", - validator_uids, - ) + # Off-chain deploy: miner identity comes from operator-CLI tokens, + # not on-chain ImageCommitments. Commitments stay empty; downstream + # code paths treat that as "every registered miner is eligible". + commitments: dict = {} + validator_uids: list[int] = [] if not self.r2: logger.warning("No R2 configured — skipping Phase A/B/C") @@ -616,16 +525,17 @@ async def _no_endpoints(): prepare_coro = _no_endpoints() collect_coro = run_and_collect_agents( - wallet=self.wallet, - metagraph=self.metagraph, + wallet=None, + metagraph=None, challenge_json=challenge.to_json(), round_id=challenge.round_id, seed=challenge.seed, r2=self.r2, - my_uid=self._my_uid(), + my_uid=-1, validator_uids=validator_uids, commitments=commitments, get_my_assignments_fn=get_my_assignments, + db_client=self.db_client, ) dynamic_endpoints, (submissions, agent_meta) = await asyncio.gather( @@ -663,30 +573,17 @@ async def _no_endpoints(): logger.warning("No coordinator configured — skipping Phase B/C") return - my_uid = self._my_uid() - dispatch_validators = list(validator_uids) - if my_uid >= 0 and my_uid not in dispatch_validators: - logger.info( - "My UID %d not in validator_uids %s — adding self as safety-net dispatcher", - my_uid, dispatch_validators, - ) - dispatch_validators.append(my_uid) + # Single-validator deploy: we own every dispatch. + my_uid = -1 + dispatch_validators = [my_uid] all_jobs = compute_assignments( block_hash, filtered, - miner_uids=list(commitments.keys()), + miner_uids=list(filtered.keys()), validator_uids=dispatch_validators, round_id=challenge.round_id, ) - my_jobs = [j for j in all_jobs if j.dispatcher == my_uid] - - if not my_jobs and all_jobs: - logger.warning( - "Round-robin assigned 0 of %d jobs to my_uid=%d — " - "dispatching all jobs (other dispatchers may be offline)", - len(all_jobs), my_uid, - ) - my_jobs = list(all_jobs) + my_jobs = list(all_jobs) trainer_endpoints = dynamic_endpoints logger.info( @@ -928,7 +825,7 @@ async def _no_endpoints(): await self._publish_submission_reveal( round_id=challenge.round_id, uid_to_sid=uid_to_sid, - hotkeys=self.metagraph.hotkeys or [], + hotkeys=[], ) primary_obj = round_task.primary_objective primary_name = primary_obj.name if primary_obj else "metric" @@ -1017,10 +914,10 @@ async def _no_endpoints(): # can attach the CID. Empty dict when disabled, no client, or # publish failed — never raises. substrate_cids = await run_substrate_publish_step( - hippius=self.hippius, wallet=self.wallet, + hippius=self.hippius, wallet=None, challenge=challenge, eval_results=eval_results, training_metas=training_metas, commitments=commitments, - metagraph=self.metagraph, my_uid=self._my_uid(), + metagraph=None, my_uid=-1, current_block=current_block, task_name=task_name, block_hash=block_hash, netuid=int(self.netuid), ) @@ -1038,8 +935,8 @@ async def _no_endpoints(): gate_passed += 1 proposal = filtered.get(uid, Proposal()) - miner_hk = self.metagraph.hotkeys[uid] if uid < len(self.metagraph.hotkeys) else "" meta = agent_meta.get(uid, {}) + miner_hk = meta.get("miner_hotkey", "") if isinstance(meta, dict) else "" element = DataElement( name=f"round_{challenge.round_id}_miner_{uid}", code=proposal.code, @@ -1055,6 +952,11 @@ async def _no_endpoints(): agent_behavior=meta.get("agent_behavior", {}), task=task_name, round_id=challenge.round_id, + prompt_id=( + meta.get("prompt_id", "") + or getattr(proposal, "prompt_id", "") + or "" + ), ) # Write to centralized DB. When this validator published @@ -1063,7 +965,7 @@ async def _no_endpoints(): new_idx = await self.db_client.add_experiment( element.to_dict(), substrate_cid=substrate_cids.get(uid, ""), - validator_hotkey=self.wallet.hotkey.ss58_address, + validator_hotkey="", ) if new_idx is not None: element.index = new_idx @@ -1119,8 +1021,9 @@ async def _no_endpoints(): else: logger.warning("Round scores empty — no eval results passed size gate") - # EMA on raw scores, softmax applied once in _set_weights() - all_uids = list(range(self.metagraph.n)) + # EMA on raw scores. No on-chain weight set — the EMA dict is the + # source of truth for the dashboard and operator-CLI reporting. + all_uids = sorted(set(self.ema_scores) | set(round_scores)) self.ema_scores = ema_update(self.ema_scores, round_scores, all_uids, Config.EMA_ALPHA) nonzero_ema = {uid: s for uid, s in self.ema_scores.items() if s > 0} logger.info( @@ -1128,7 +1031,6 @@ async def _no_endpoints(): Config.EMA_ALPHA, len(nonzero_ema), {uid: f"{s:.4f}" for uid, s in sorted(nonzero_ema.items())} if nonzero_ema else "all zero", ) - self._set_weights() # Write frontier to centralized DB + R2 await self.db_client.update_frontier( @@ -1148,34 +1050,6 @@ async def _no_endpoints(): len(round_scores), nonzero_scored, ) - def _set_weights(self): - """Set weights on chain — softmax on EMA scores.""" - if not self.ema_scores: - logger.warning("No EMA scores — skipping weight setting") - return - try: - uids, weights = scores_to_weights(self.ema_scores, Config.SOFTMAX_TEMPERATURE) - nonzero_weights = { - uid: f"{w:.4f}" for uid, w in zip(uids, weights) if w > 0 - } - if nonzero_weights: - logger.info( - "Setting weights for %d UIDs (%d nonzero): %s", - len(uids), len(nonzero_weights), nonzero_weights, - ) - else: - logger.warning( - "Setting all-zero weights for %d UIDs (no miner scored this round)", - len(uids), - ) - self.subtensor.set_weights( - netuid=self.netuid, wallet=self.wallet, - uids=uids, weights=weights, - ) - logger.info("Weights set successfully on chain") - except Exception as e: - logger.error("Failed to set weights: %s", e) - async def run(self): """Main validator loop.""" self.start_proxy_server() @@ -1190,7 +1064,7 @@ async def run(self): logger.warning("Startup orphan reap failed: %s", e) while True: try: - current_block = self.subtensor.block + current_block = int(time.time() // 12) round_start = round_start_block(current_block, Config.ROUND_INTERVAL_BLOCKS) phase = current_phase( current_block, round_start, @@ -1236,15 +1110,13 @@ async def run(self): await asyncio.sleep(60) -def get_config() -> bt.Config: +def get_config(): parser = argparse.ArgumentParser(description="Radar Validator") parser.add_argument("--netuid", type=int, default=1) parser.add_argument("--task", type=str, default="ml_training") parser.add_argument("--db_dir", type=str, default="./experiments") parser.add_argument("--db_port", type=int, default=Config.PROXY_PORT) - bt.Wallet.add_args(parser) - bt.Subtensor.add_args(parser) - return bt.Config(parser) + return parser.parse_args() def main(): diff --git a/validator/substrate_publisher.py b/validator/substrate_publisher.py index 1454e71..7d36652 100644 --- a/validator/substrate_publisher.py +++ b/validator/substrate_publisher.py @@ -261,6 +261,11 @@ async def run_substrate_publish_step( """ if hippius is None or not eval_results: return {} + if wallet is None: + # Off-chain deploy — Phase C bundles are SR25519-signed by the + # validator hotkey. Without a wallet there's nothing to sign with, + # so we skip substrate publishing entirely. + return {} try: records = await build_phase_c_records( wallet=wallet, challenge=challenge,