diff --git a/.github/workflows/schema-migrations.yml b/.github/workflows/schema-migrations.yml index 875a338..4994490 100644 --- a/.github/workflows/schema-migrations.yml +++ b/.github/workflows/schema-migrations.yml @@ -79,6 +79,14 @@ jobs: with: fetch-depth: 0 - name: Scan new migrations for destructive patterns + env: + # Pass the PR body via env rather than ${{ }} interpolation — + # bodies routinely contain backticks, $vars, and quotes that + # break the runner's bash script when interpolated raw (the + # script source ends up containing PR-body text and bash treats + # backticks as command substitution). See PR #142 (TEN-240 + # Phase 7) which tripped exit code 2 from a backtick-rich body. + PR_BODY: ${{ github.event.pull_request.body }} run: | NEW=$(git diff --diff-filter=A origin/${{ github.base_ref }}...HEAD \ -- 'database/migrations/*.sql' || true) @@ -89,7 +97,7 @@ jobs: CONTENT=$(git diff --diff-filter=A origin/${{ github.base_ref }}...HEAD \ -- 'database/migrations/*.sql') if echo "$CONTENT" | grep -iE '^\+.*(DROP TABLE|DROP COLUMN|ALTER COLUMN [^,;]+ TYPE)' > /dev/null; then - if ! echo "${{ github.event.pull_request.body }}" | grep -q "ALLOW_DESTRUCTIVE"; then + if ! printf '%s' "$PR_BODY" | grep -q "ALLOW_DESTRUCTIVE"; then echo "::error::Destructive migration detected. Add 'ALLOW_DESTRUCTIVE' to the PR description to approve." exit 1 fi diff --git a/CLAUDE.md b/CLAUDE.md index e174356..7b41be7 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -82,7 +82,7 @@ scripts/ Localnet test script + Postgres startup | `shared/dedup.py` | Code similarity (provenance queries) | | `shared/scoring.py` | Size-gated Pareto frontier scoring (Phase C) | | `shared/commitment.py` | On-chain Docker image + endpoint commitment | -| `shared/r2_audit.py` | R2 storage for checkpoints, snapshots, dispatch records | +| `shared/r2_audit.py` | S3-compatible artifact storage. Hippius (Substrate) is the primary backend; Cloudflare R2 stays supported as a legacy fallback. Exports `HippiusStorage` (preferred) and `R2AuditLog` (alias). | | `database/server.py` | Centralized DB API (FastAPI, all experiment routes) | | `database/neuron.py` | Subnet owner process (Postgres + API server) | | `validator/neuron.py` | Main validator loop (3-phase: collect → train → evaluate) | @@ -235,6 +235,29 @@ frontier/latest.json # Current Pareto fro - [x] Validator reverse proxy for miners (`validator/db_proxy.py`) - [x] DatabaseClient for validator -> DB server communication (`shared/db_client.py`) - [x] Validator / dashboard deploy split (`RADAR_NEURON_MODE`) +- [x] Hippius (Substrate) artifact backend with R2 legacy fallback (`shared/r2_audit.py`) + +## Artifact Storage (Hippius / R2) + +Primary backend is **Hippius**, a Substrate-based decentralized object store +with an S3-compatible API at `https://s3.hippius.com`. Cloudflare R2 stays +supported as a legacy fallback so existing deployments keep working without +an env-var change. The client (`shared.r2_audit.HippiusStorage`, aliased as +`R2AuditLog`) reads `HIPPIUS_*` first, then falls back to `R2_*`: + +| Var | Default | Purpose | +|-----|---------|---------| +| `HIPPIUS_ACCESS_KEY_ID` | `R2_ACCESS_KEY_ID` | Hippius access key (begins with `hip_`). Falls back to the legacy R2 var. | +| `HIPPIUS_SECRET_ACCESS_KEY` | `R2_SECRET_ACCESS_KEY` | Hippius secret. Falls back to the legacy R2 var. | +| `HIPPIUS_BUCKET` | `R2_BUCKET` | Bucket name for experiment artifacts. | +| `HIPPIUS_REGION` | `decentralized` | Required by Hippius; leave as default. | +| `HIPPIUS_ENDPOINT_URL` | `https://s3.hippius.com` | Override for private gateways. The legacy `R2_ACCOUNT_ID`-derived endpoint is only used when no `HIPPIUS_*` vars are set. | +| `MOCK_S3_ENDPOINT` | unset | Localnet override (`scripts/mock_r2_server.py`). `MOCK_R2_ENDPOINT` remains an alias for back-compat. | + +Operators migrating from R2: leave the existing `R2_*` vars in place and +add `HIPPIUS_*` once your Hippius keys are issued — the client will pick up +the new backend automatically. To cut over fully, also set +`HIPPIUS_BUCKET` (or unset `R2_BUCKET`) so dispatched URLs point at Hippius. ## Neuron Mode Environment Variables diff --git a/README.md b/README.md index cf81c04..507b20b 100644 --- a/README.md +++ b/README.md @@ -166,7 +166,7 @@ python validator/neuron.py --netuid --subtensor.network --wallet.n ### Infrastructure Requirements -* **R2 storage** for checkpoint storage and artifact sharing. Set `R2_ACCOUNT_ID`, `R2_ACCESS_KEY_ID`, `R2_SECRET_ACCESS_KEY`, `R2_BUCKET` env vars. +* **Artifact storage** (S3-compatible) for checkpoints and cross-validator sharing. The default backend is **Hippius** (Substrate-based decentralized object store at `https://s3.hippius.com`); set `HIPPIUS_ACCESS_KEY_ID`, `HIPPIUS_SECRET_ACCESS_KEY`, `HIPPIUS_BUCKET`. Cloudflare R2 stays supported as a legacy fallback — if `HIPPIUS_*` is unset the client transparently reads `R2_ACCOUNT_ID`, `R2_ACCESS_KEY_ID`, `R2_SECRET_ACCESS_KEY`, `R2_BUCKET` instead, so existing deployments keep working without an env-var change. * **CPU for Phase C eval.** Checkpoint evaluation runs on CPU by default (override with `RADAR_EVAL_DEVICE`). Each eval takes seconds. No GPU required for validators. * **DB proxy.** The validator runs a reverse-proxy FastAPI on `RADAR_PROXY_PORT` (default 8080). It forwards `/experiments/*`, `/challenge`, `/frontier`, `/provenance/*` to the subnet owner's centralised DB server, and hosts `/desearch/*` and `/llm/v1/*` locally with per-miner rate limits. The DB itself (Postgres) is run by the subnet owner, not the validator. * **DB server deployable modes.** The subnet-owner `database/neuron.py` binary supports three modes via `RADAR_NEURON_MODE`: @@ -233,7 +233,7 @@ shared/ Core libraries (validator + miner) protocol.py Challenge/Proposal wire format (JSON) auth.py Epistula signing/verification (SR25519) challenge.py Deterministic challenge + size buckets - r2_audit.py R2 storage, upload/download/hash verification + r2_audit.py Hippius/R2 artifact storage, upload/download/hash verification task.py TaskSpec + Objective, pluggable task definitions pg_schema.py Postgres DDL + row conversion pg_store.py Async Postgres experiment store diff --git a/config.py b/config.py index 832634c..e2aebc5 100644 --- a/config.py +++ b/config.py @@ -54,13 +54,63 @@ class Config: # Comma-separated list of enabled task names. Empty or "all" = all built-in tasks. ENABLED_TASKS: str = os.getenv("RADAR_ENABLED_TASKS", "ts_forecasting") - # ── R2 Audit Log ──────────────────────────────────────────────── + # ── Artifact Storage (Hippius primary, R2 legacy fallback) ────── + # Primary backend is Hippius — the Substrate-based decentralized object + # store at https://s3.hippius.com. Cloudflare R2 stays supported as a + # legacy fallback during the migration: any unset HIPPIUS_* var falls + # back to its R2_* counterpart so existing deployments keep working + # without an env-var change. + HIPPIUS_ACCESS_KEY_ID: str = os.getenv("HIPPIUS_ACCESS_KEY_ID", "") \ + or os.getenv("R2_ACCESS_KEY_ID", "") + HIPPIUS_SECRET_ACCESS_KEY: str = os.getenv("HIPPIUS_SECRET_ACCESS_KEY", "") \ + or os.getenv("R2_SECRET_ACCESS_KEY", "") + HIPPIUS_BUCKET: str = os.getenv("HIPPIUS_BUCKET", "") \ + or os.getenv("R2_BUCKET", "") + HIPPIUS_ENDPOINT_URL: str = os.getenv("HIPPIUS_ENDPOINT_URL", "") + HIPPIUS_REGION: str = os.getenv("HIPPIUS_REGION", "decentralized") + PRESIGNED_TTL: int = int(os.getenv("RADAR_PRESIGNED_TTL", "5400")) + + # Legacy R2 names — read directly from env so an operator who has only + # set HIPPIUS_* vars sees these as empty (and code that gates on them + # behaves correctly). Code that needs "is artifact storage configured?" + # should consult HIPPIUS_BUCKET instead, since that's already the + # resolved value. R2_ACCOUNT_ID: str = os.getenv("R2_ACCOUNT_ID", "") R2_ACCESS_KEY_ID: str = os.getenv("R2_ACCESS_KEY_ID", "") R2_SECRET_ACCESS_KEY: str = os.getenv("R2_SECRET_ACCESS_KEY", "") - R2_BUCKET: str = os.getenv("R2_BUCKET", "") + R2_BUCKET: str = os.getenv("R2_BUCKET", "") or os.getenv("HIPPIUS_BUCKET", "") R2_PRESIGNED_TTL: int = int(os.getenv("RADAR_PRESIGNED_TTL", "5400")) + # ── Hippius / Substrate Phase C publishing ────────────────────── + # Phase 1 of the Hippius migration ships the signing/serialisation + # contract (`shared/substrate.py`) but leaves the publishing pipeline + # off by default — existing validators are not affected by merging + # this code. Operators opt in by setting HIPPIUS_ENABLED=true once the + # downstream phases (IPFS upload, on-chain extrinsic) are wired up. + HIPPIUS_ENABLED: bool = os.getenv("HIPPIUS_ENABLED", "false").lower() == "true" + HIPPIUS_IPFS_API_URL: str = os.getenv("HIPPIUS_IPFS_API_URL", "") + HIPPIUS_KEY: str = os.getenv("HIPPIUS_KEY", "") + HIPPIUS_SUBSTRATE_RPC: str = os.getenv("HIPPIUS_SUBSTRATE_RPC", "") + SUBSTRATE_APP_TAG: str = os.getenv("SUBSTRATE_APP_TAG", "radar") + SUBSTRATE_SCHEMA_VERSION: str = os.getenv( + "SUBSTRATE_SCHEMA_VERSION", "radar.substrate.v1", + ) + + # ── Hippius artifact dual-write (TEN-240 Phase 7) ─────────────── + # When enabled, validator-side artifact writes (dispatch records, + # frontier snapshots, training_meta cache) fan out to both R2 and + # Hippius. Defaults off so existing deployments are unaffected by + # merging — operators flip in staging first. See docs/HIPPIUS_STORAGE.md. + DUAL_WRITE_ARTIFACTS: bool = ( + os.getenv("RADAR_DUAL_WRITE_ARTIFACTS", "false").lower() == "true" + ) + # When enabled, ArtifactStore.get_bytes falls back to the other backend + # when the primary one fails. Off by default during rollout so failures + # stay loud and we don't accidentally paper over a misconfiguration. + HIPPIUS_ARTIFACT_FALLBACK: bool = ( + os.getenv("RADAR_HIPPIUS_ARTIFACT_FALLBACK", "false").lower() == "true" + ) + # ── Round Timing ─────────────────────────────────────────────── # # Timing is controlled at three distinct layers. Read this carefully — diff --git a/database/migrations/002_substrate_cids.sql b/database/migrations/002_substrate_cids.sql new file mode 100644 index 0000000..3268a2e --- /dev/null +++ b/database/migrations/002_substrate_cids.sql @@ -0,0 +1,37 @@ +-- TEN-240: Phase C substrate audit trail. +-- +-- Adds a per-experiment list of Hippius CIDs (one per validator that +-- published a Phase C bundle covering this row). The column is JSONB so +-- consumers can query into individual entries without a join, and the +-- GIN index makes "find every experiment a given validator published" +-- and "find every experiment backed by CID X" cheap. +-- +-- Each element shape: +-- {"kind": "phase_c_record", +-- "validator_hotkey": "5G...", +-- "cid": "bafy...", +-- "round_id": 123} +-- +-- Defaults to '[]' so legacy rows remain valid. +-- +-- Guarded on ``experiments`` existing because the migration runner is +-- exercised against an empty schema in CI before init_schema() runs +-- (see workflows/schema-migrations.yml). On a fresh deploy the base +-- CREATE TABLE in shared/pg_schema.py already declares the column, so +-- this migration is a no-op there. On existing deployments init_schema() +-- runs first and the table exists, so the ALTER applies normally. + +DO $$ +BEGIN + IF EXISTS ( + SELECT 1 + FROM information_schema.tables + WHERE table_schema = 'CURRENT_SCHEMA' + AND table_name = 'experiments' + ) THEN + ALTER TABLE CURRENT_SCHEMA.experiments + ADD COLUMN IF NOT EXISTS substrate_cids JSONB NOT NULL DEFAULT '[]'; + CREATE INDEX IF NOT EXISTS idx_substrate_cids + ON CURRENT_SCHEMA.experiments USING GIN (substrate_cids); + END IF; +END $$; diff --git a/database/neuron.py b/database/neuron.py index 8071de1..fa0c836 100644 --- a/database/neuron.py +++ b/database/neuron.py @@ -30,7 +30,7 @@ app, include_validator_routes, set_db, set_auth, set_challenge, set_frontier, set_access_logger, set_hotkey_map, set_rate_limit, set_ip_rate_limit, - set_r2, set_pool, + set_r2, set_hippius, set_pool, get_current_challenge, get_current_frontier, ) from shared.migrations import apply_migrations @@ -165,11 +165,45 @@ def __init__(self, config): except Exception as e: logger.warning("R2 audit log unavailable: %s", e) + # Hippius client — opt-in via Config.HIPPIUS_ENABLED. Powers the + # /experiments/{id}/verify endpoint on the validator surface; the + # rest of the DB server doesn't need it. Lazy import + tolerant + # construction mirrors the validator-side wiring (TEN-244). + self.hippius = self._init_hippius() + # Pool, store, logger set in async init self.pool = None self.store = None self.access_logger = None + def _init_hippius(self): + """Construct the Hippius client when opted in; return None otherwise. + + Mirrors `Validator._init_hippius`: lazy import so the module loads + without the TEN-242 wrapper present, tolerant constructor so a flap + at startup downgrades to disabled instead of crashing the DB server. + """ + if not Config.HIPPIUS_ENABLED: + return None + try: + from shared.hippius_client import HippiusClient # type: ignore + except ImportError: + logger.warning( + "HIPPIUS_ENABLED=true but shared.hippius_client is not " + "available yet (TEN-242). /experiments/{id}/verify will " + "respond 503 until the wrapper ships." + ) + return None + try: + return HippiusClient( + ipfs_api_url=Config.HIPPIUS_IPFS_API_URL, + hippius_key=Config.HIPPIUS_KEY, + substrate_rpc=Config.HIPPIUS_SUBSTRATE_RPC, + ) + except Exception as e: # noqa: BLE001 + logger.warning("Hippius client init failed: %s", e) + return None + def _build_task_bucket_resolver(self) -> Callable[[str], list[tuple[int, int]]]: """Closure that maps a task name to its FLOPs-equivalent bucket list. @@ -339,6 +373,8 @@ async def _apply_timeout(conn): set_pool(self.pool) if self.r2: set_r2(self.r2) + if self.hippius: + set_hippius(self.hippius) set_access_logger(self.access_logger) # ── Validator-surface wiring (Epistula auth + rate limit) ── diff --git a/database/server.py b/database/server.py index 10473c3..583e1d2 100644 --- a/database/server.py +++ b/database/server.py @@ -35,6 +35,7 @@ db = None # PgExperimentStore _r2 = None # R2AuditLog (for agent code storage) _pool = None # asyncpg pool (for agent_submissions table) +_hippius = None # HippiusClient — used by GET /experiments/{id}/verify # Auth middleware reference _metagraph = None @@ -93,6 +94,12 @@ def set_r2(r2): _r2 = r2 +def set_hippius(hippius): + """Inject a Hippius client (used by /experiments/{id}/verify).""" + global _hippius + _hippius = hippius + + def set_pool(pool): global _pool _pool = pool @@ -198,8 +205,21 @@ class SearchRequest(BaseModel): class AddExperimentRequest(BaseModel): - """Validator POSTs experiment data after Phase C.""" + """Validator POSTs experiment data after Phase C. + + ``substrate_cid`` and ``validator_hotkey`` are optional; when set, the + server appends a ``{kind: "phase_c_record", validator_hotkey, cid, + round_id}`` entry to the row's ``substrate_cids`` audit list. + + ``artifact_cids`` (TEN-240 Phase 7) carries Hippius CIDs for the + dual-written non-DB artifacts (checkpoint, architecture, + training_meta). Each element is appended to the audit list verbatim; + callers are responsible for the ``kind`` field. + """ data: dict + substrate_cid: str = "" + validator_hotkey: str = "" + artifact_cids: list[dict] = [] class RecordComponentsRequest(BaseModel): @@ -508,15 +528,67 @@ async def get_experiment(index: int): return elem.to_api_dict() +@validator_router.get("/experiments/{index}/verify") +async def verify_experiment_route(index: int): + """Independently verify the substrate audit trail for one experiment. + + For each CID in ``substrate_cids``: download the bundle from Hippius, + locate the (round_id, miner_uid) record, sr25519-verify the signature, + and compare the record's metric dict with what's in the DB. Returns + per-CID flags + discrepancy strings, never raises on per-CID errors. + + 503 when the server has no Hippius client wired up. + """ + d = _require_db() + elem = await d.get(index) + if elem is None: + raise HTTPException(status_code=404, detail=f"Experiment {index} not found") + if _hippius is None: + raise HTTPException( + status_code=503, + detail="Hippius not configured — substrate verification unavailable", + ) + from database.verify import verify_experiment + verifications = await verify_experiment(hippius=_hippius, element=elem) + return { + "experiment_id": index, + "substrate_cids": list(elem.substrate_cids or []), + "verifications": verifications, + } + + # ── Write endpoints (new — validators only) ────────────── @validator_router.post("/experiments/add") async def add_experiment(req: AddExperimentRequest): - """Validator writes a DataElement after Phase C.""" + """Validator writes a DataElement after Phase C. + + When ``req.substrate_cid`` is set, append a ``phase_c_record`` entry to + the element's ``substrate_cids`` audit list before insert. The entry + captures who published the bundle (validator_hotkey), where it lives + (cid), and which round it covers (taken from element.round_id). + """ d = _require_db() from shared.database import DataElement try: element = DataElement.from_dict(req.data) + # Defensive: from_dict may have produced something unexpected. + if not isinstance(element.substrate_cids, list): + element.substrate_cids = [] + cids = list(element.substrate_cids) + if req.substrate_cid: + cids.append({ + "kind": "phase_c_record", + "validator_hotkey": req.validator_hotkey, + "cid": req.substrate_cid, + "round_id": int(element.round_id), + }) + # Append artifact CIDs verbatim (TEN-240 Phase 7). Filter to dicts + # so a malformed caller can't poison the list with non-objects. + for entry in req.artifact_cids or []: + if isinstance(entry, dict) and entry.get("cid"): + cids.append(entry) + element.substrate_cids = cids idx = await d.add(element) except Exception as e: logger.error("add_experiment failed: %s", e, exc_info=True) diff --git a/database/verify.py b/database/verify.py new file mode 100644 index 0000000..36cc6d6 --- /dev/null +++ b/database/verify.py @@ -0,0 +1,191 @@ +"""Independent verification of Phase C records on Hippius. + +Powers the ``GET /experiments/{index}/verify`` endpoint: for each substrate +CID an experiment carries, fetch the bundle, find the record matching this +miner+round, sr25519-verify the signature, and compare the record's metric +dict against what's stored in the DB. Pure-async, no FastAPI dependency, so +this is straightforward to unit-test. + +The Hippius client is duck-typed: anything with +``await client.download_bundle(cid: str) -> bytes`` works, which keeps this +module decoupled from the (still in flight) TEN-242 wrapper. +""" + +from __future__ import annotations + +import logging +from typing import Any + +from shared.database import DataElement +from shared.substrate import ( + PhaseCRecord, + records_from_bundle, + verify_record, +) + +logger = logging.getLogger(__name__) + + +def _normalise_metrics(d: dict) -> dict: + """Filter to JSON-comparable scalar entries. + + The publisher already drops non-scalars from records before signing + (see validator/substrate_publisher._filter_metrics). Mirroring that + filter here means the comparison is "what was signed" vs. "what was + stored", not contaminated by columns the DB happened to add (e.g. + ``loss_curve`` which lives elsewhere on the row). + """ + out: dict[str, Any] = {} + if not isinstance(d, dict): + return out + for k, v in d.items(): + if isinstance(v, (bool, int, float, str)) or v is None: + out[k] = v + return out + + +def _diff_metrics(record: PhaseCRecord, element: DataElement) -> list[str]: + """Return a list of human-readable discrepancies between record + DB row. + + Empty list = clean match. Only keys present in the (signed) record are + compared; columns the DB carries that aren't in the bundle are ignored, + since the bundle is the authoritative claim and the DB is the cache. + """ + discrepancies: list[str] = [] + db_objectives = _normalise_metrics(element.objectives or {}) + rec_metrics = _normalise_metrics(record.metrics or {}) + for key, rec_val in rec_metrics.items(): + if key not in db_objectives: + discrepancies.append(f"metric {key!r} present in record, missing in DB") + continue + if db_objectives[key] != rec_val: + discrepancies.append( + f"metric {key!r}: record={rec_val!r} db={db_objectives[key]!r}", + ) + if record.miner_uid != element.miner_uid: + discrepancies.append( + f"miner_uid: record={record.miner_uid} db={element.miner_uid}", + ) + if record.miner_hotkey and record.miner_hotkey != element.miner_hotkey: + discrepancies.append( + f"miner_hotkey: record={record.miner_hotkey!r} " + f"db={element.miner_hotkey!r}", + ) + return discrepancies + + +def _find_record( + records: list[PhaseCRecord], element: DataElement, +) -> PhaseCRecord | None: + """Pick the record in the bundle matching this experiment's identity. + + Match key is ``(round_id, miner_uid)`` — a single bundle covers one + round, one validator, all miners, so that pair is unique within it. + """ + target_round = int(element.round_id) + target_uid = int(element.miner_uid) + for r in records: + if r.round_id == target_round and r.miner_uid == target_uid: + return r + return None + + +async def verify_experiment( + *, + hippius, + element: DataElement, +) -> list[dict]: + """Independently verify every CID an experiment claims to be backed by. + + Returns a list of per-CID verification dicts (shape declared in the + Phase 5 spec). Empty list if the experiment has no substrate_cids. + Raises nothing: each CID's failure modes (network, missing record, + bad signature, metric mismatch) appear as flags + a ``discrepancies`` + list on its own dict. + """ + if not element.substrate_cids: + return [] + if hippius is None: + # The endpoint guards on this, but defend in depth. + raise RuntimeError("Hippius client not configured") + + out: list[dict] = [] + for entry in element.substrate_cids: + if not isinstance(entry, dict): + continue + cid = entry.get("cid", "") + validator_hotkey = entry.get("validator_hotkey", "") + result: dict[str, Any] = { + "validator_hotkey": validator_hotkey, + "cid": cid, + "fetchable": False, + "signature_valid": False, + "matches_db": False, + "discrepancies": [], + } + if not cid: + result["discrepancies"].append("cid is empty") + out.append(result) + continue + + # 1. Fetch bundle. + try: + data = await hippius.download_bundle(cid) + result["fetchable"] = True + except Exception as e: # noqa: BLE001 — surface as flag, not exception + result["discrepancies"].append(f"fetch failed: {e}") + out.append(result) + continue + + # 1b. Tag verification (TEN-242). When the Hippius client supports + # ``head_bundle`` and we hit it via the S3 path, sanity-check that + # the object's tags claim the same validator we expect. Best-effort: + # IPFS-gateway-only deploys, public mirrors, and pre-Phase-2 audit + # entries all skip this without failing the row. + if validator_hotkey and hasattr(hippius, "head_bundle"): + try: + ref = await hippius.head_bundle(cid) + meta_hk = ref.metadata.get("validator_hotkey", "") if ref else "" + if meta_hk and meta_hk != validator_hotkey: + result["discrepancies"].append( + f"tag validator_hotkey mismatch: object={meta_hk!r} " + f"audit={validator_hotkey!r}", + ) + except Exception: # noqa: BLE001 — head is optional + pass + + # 2. Parse + locate the matching record. + try: + records = records_from_bundle(data) + except Exception as e: # noqa: BLE001 + result["discrepancies"].append(f"bundle parse failed: {e}") + out.append(result) + continue + record = _find_record(records, element) + if record is None: + result["discrepancies"].append( + f"no record matching round_id={element.round_id} " + f"miner_uid={element.miner_uid} in bundle" + ) + out.append(result) + continue + + # 3. Verify signature against the bundle's claimed validator hotkey + # (cross-checked against the audit-list entry's claimed hotkey). + ok, err = verify_record(record, expected_hotkey=validator_hotkey or None) + result["signature_valid"] = ok + if not ok: + result["discrepancies"].append(f"signature: {err}") + + # 4. Compare against DB. matches_db only goes true if signature was + # valid AND the metric/identity comparison was clean. + diffs = _diff_metrics(record, element) + if diffs: + result["discrepancies"].extend(diffs) + result["matches_db"] = ok and not diffs + + out.append(result) + return out + + +__all__ = ["verify_experiment"] diff --git a/docs/HIPPIUS_STORAGE.md b/docs/HIPPIUS_STORAGE.md new file mode 100644 index 0000000..4cd3fd1 --- /dev/null +++ b/docs/HIPPIUS_STORAGE.md @@ -0,0 +1,118 @@ +# Hippius artifact storage + +RADAR's non-DB artifacts (proposals, dispatch records, frontier snapshots, +checkpoints, architecture code, training metadata) live in S3-compatible +object storage. With Phase 7 of TEN-240 these artifacts can be **dual-written +to Hippius** (Substrate-based decentralized storage) alongside the existing +R2 (or Hippius-S3) backend. Hippius is the durable, censorship-resistant +record; R2 stays as the hot-path cache. + +> Dual-write is opt-in. The default deploy keeps the historical R2-only +> behaviour. Set `RADAR_DUAL_WRITE_ARTIFACTS=true` to turn it on. + +## Path → metadata mapping + +When an artifact is written through `shared.artifact_store.ArtifactStore` it +carries a metadata dict alongside the bytes. The metadata is the equivalent +of the R2 key path — search-discoverable on the Hippius side once the SDK +gains tag-based listing (TEN-242). Until then the metadata travels inside +the bundle/header so a fetcher can verify what they got. + +| R2 key | Hippius metadata | +|--------|------------------| +| `round_{id}/proposals/{uid}.json` | `app=radar`, `kind=proposal`, `round_id={id}`, `miner_uid={uid}` | +| `round_{id}/miner_{hk}/checkpoint.safetensors` | `app=radar`, `kind=checkpoint`, `round_id={id}`, `miner_hotkey={hk}` | +| `round_{id}/miner_{hk}/architecture.py` | `app=radar`, `kind=architecture`, `round_id={id}`, `miner_hotkey={hk}` | +| `round_{id}/miner_{hk}/training_meta.json` | `app=radar`, `kind=training_meta`, `round_id={id}`, `miner_hotkey={hk}` | +| `round_{id}/miner_{hk}/stdout.log` | `app=radar`, `kind=stdout`, `round_id={id}`, `miner_hotkey={hk}` | +| `round_{id}/dispatch/vali_{hk}.json` | `app=radar`, `kind=dispatch`, `round_id={id}`, `validator_hotkey={hk}` | +| `frontier/{task}/latest.json` | `app=radar`, `kind=frontier`, `task={task}` | + +`app=radar` is the global tag that scopes RADAR's writes against any other +tenant on the same Hippius account. + +## Substrate audit-trail integration + +When a validator writes an artifact through `ArtifactStore`, the returned +`hippius_cid` is appended to that experiment's `substrate_cids` JSONB +column (see Phase 5): + +```json +[ + {"kind": "phase_c_record", "validator_hotkey": "5G...", "cid": "bafy1..."}, + {"kind": "checkpoint", "cid": "bafy2..."}, + {"kind": "architecture", "cid": "bafy3..."}, + {"kind": "training_meta", "cid": "bafy4..."} +] +``` + +The verifier (`tools/verify_radar_experiment.py`) already iterates this +list and inspects each entry; new `kind` values surface as additional +verifications without a tool-side change. + +## Configuration + +```bash +# Enable dual-write end-to-end. Default false — historical R2-only behaviour. +export RADAR_DUAL_WRITE_ARTIFACTS=true + +# Allow `ArtifactStore.get_bytes` to fall back to the other backend when +# the primary one fails. Off by default to keep failure modes loud during +# initial rollout. +export RADAR_HIPPIUS_ARTIFACT_FALLBACK=true + +# Hippius credentials & gateway — same vars Phase 0 introduced. +export HIPPIUS_ENABLED=true +export HIPPIUS_IPFS_API_URL=https://get.hippius.network +export HIPPIUS_KEY=hip_xxx +``` + +## What's wired today + +Validator-side, dual-writeable as of this PR: + +* Dispatch records (`coordinator.write_dispatch_record`) +* Frontier snapshots (`coordinator.write_frontier`) +* Training metadata cached during Phase B (`coordinator._cache_training_meta`, + whenever the validator pulls a meta back from R2 to seed the DB cache) + +Validator-side, **deferred** until trainer-side credentials land: + +* **Checkpoints.** The trainer subprocess uploads to R2 via presigned URLs + generated by the validator. Routing checkpoints through Hippius requires + either trainer-side Hippius creds (operationally invasive) or a + validator-side post-fetch dual-write (extra bandwidth on every dispatcher). + Both are real options; the cost note below frames the decision. +* **Architecture code, stdout logs.** Same flow as checkpoints — uploaded + by the trainer via presigned URLs. + +The trainer-side flow can be migrated piecemeal: each call site that +currently does `r2.upload_text(...)` is one-line-replaceable with +`artifact_store.put_bytes(...)` once the trainer holds a `HippiusClient`. + +Agent scratchpad (miner-facing presigned URLs) stays R2-only by design — +that surface is short-lived and miner-controlled. + +## Cost ballpark + +Rough back-of-envelope: 100 miners × 26 rounds/day × ~50 MB average +checkpoint × 365 days ≈ 50 TB/year. At Hippius/Filecoin-tier rates +(~$5/TB/month) that's ~$3000/year just for checkpoint dual-writes. +Non-trivial. The JSON-tier artifacts (dispatch records, frontier, +training_meta) are <1 MB/round each — order-of-magnitude cheaper than +the checkpoint stream. + +## Rollout sequence + +1. Merge with `RADAR_DUAL_WRITE_ARTIFACTS=false` everywhere — no change. +2. Enable in staging for ~1 week. Watch the dual-write counters + (`artifact_store.dual_write.r2_ok`, `.hippius_ok`, `.both_failed`, + `.bytes_written`) and the Hippius bill. +3. Enable in production validators for a month. Same observation set. +4. **Separate decision**: flip read preference to Hippius-first for + specific kinds (frontier first — small, infrequent, auditable), + keep checkpoints as R2-primary with Hippius cold-storage. +5. Eventually deprecate R2 paths kind by kind as confidence grows. + +No deprecation in this PR. Both R2 paths and Hippius writes remain +available indefinitely; operators choose what to enable. diff --git a/runner/timeseries_forecast/env.py b/runner/timeseries_forecast/env.py index 8aedf8f..c3c2425 100644 --- a/runner/timeseries_forecast/env.py +++ b/runner/timeseries_forecast/env.py @@ -15,8 +15,16 @@ import zipfile -# Env vars to strip from subprocess (secrets that miners must not access) -_SECRET_KEYS = {"R2_PRESIGNED_URL", "R2_ACCESS_KEY", "R2_SECRET_KEY"} +# Env vars to strip from subprocess (secrets that miners must not access). +# Covers both the legacy R2 names and the new Hippius names that replaced +# them — the harness must not leak either backend's credentials into the +# user-supplied ``submission.py`` subprocess. +_SECRET_KEYS = { + "R2_PRESIGNED_URL", "R2_ACCESS_KEY", "R2_SECRET_KEY", + "R2_ACCESS_KEY_ID", "R2_SECRET_ACCESS_KEY", "R2_ACCOUNT_ID", + "HIPPIUS_ACCESS_KEY_ID", "HIPPIUS_SECRET_ACCESS_KEY", + "HIPPIUS_PRESIGNED_URL", +} class Actor: diff --git a/scripts/mock_r2_server.py b/scripts/mock_r2_server.py index 924504f..1a5e7f2 100644 --- a/scripts/mock_r2_server.py +++ b/scripts/mock_r2_server.py @@ -1,20 +1,28 @@ """Minimal S3-compatible server for localnet testing. -Accepts PUT/GET/HEAD/LIST on any key, stores files on the local filesystem. +Used as a stand-in for the production artifact backend (Hippius primary, +Cloudflare R2 legacy) so localnet runs without external network. Accepts +PUT/GET/HEAD/LIST on any key, stores files on the local filesystem. Generates "presigned" URLs that are just http://localhost:PORT/BUCKET/KEY. Usage: python scripts/mock_r2_server.py # port 9000 python scripts/mock_r2_server.py --port 9001 # custom port -Configure the validator/trainer to use this by setting: +Configure the validator/trainer to use this by setting EITHER the new +Hippius vars or the legacy R2 vars (the client reads both, Hippius first): + export HIPPIUS_ACCESS_KEY_ID=test + export HIPPIUS_SECRET_ACCESS_KEY=test + export HIPPIUS_BUCKET=radar-test + # ... or, equivalently for legacy deployments: export R2_ACCOUNT_ID=local export R2_ACCESS_KEY_ID=test export R2_SECRET_ACCESS_KEY=test export R2_BUCKET=radar-test -Then override the R2AuditLog endpoint to point here instead of Cloudflare. -The test scripts handle this via MOCK_R2_ENDPOINT env var. +Override the storage endpoint to point at this mock instead of Hippius/R2: + export MOCK_S3_ENDPOINT=http://127.0.0.1:9000 # preferred name + export MOCK_R2_ENDPOINT=http://127.0.0.1:9000 # legacy alias, also honoured """ import argparse diff --git a/scripts/test_localnet.sh b/scripts/test_localnet.sh index 877ce7c..9f87c5f 100755 --- a/scripts/test_localnet.sh +++ b/scripts/test_localnet.sh @@ -435,8 +435,15 @@ for attempt in $(seq 1 10); do sleep 1 done -# Configure R2 credentials — these are accepted by the mock server (no auth check) -export MOCK_R2_ENDPOINT="http://127.0.0.1:${MOCK_R2_PORT}" +# Configure storage credentials — accepted by the mock server (no auth check). +# Export both the new HIPPIUS_* names (primary) and the legacy R2_* names so +# any code path on either side of the migration finds what it expects. +export MOCK_S3_ENDPOINT="http://127.0.0.1:${MOCK_R2_PORT}" +export MOCK_R2_ENDPOINT="$MOCK_S3_ENDPOINT" # legacy alias +export HIPPIUS_ACCESS_KEY_ID="test" +export HIPPIUS_SECRET_ACCESS_KEY="test" +export HIPPIUS_BUCKET="$R2_BUCKET" +export HIPPIUS_REGION="decentralized" export R2_ACCOUNT_ID="local" export R2_ACCESS_KEY_ID="test" export R2_SECRET_ACCESS_KEY="test" @@ -444,7 +451,7 @@ export R2_BUCKET="$R2_BUCKET" # Create the bucket directory so ListObjects works mkdir -p "$MOCK_R2_STORAGE/$R2_BUCKET" -ok "R2 configured: endpoint=$MOCK_R2_ENDPOINT bucket=$R2_BUCKET" +ok "Artifact storage configured: endpoint=$MOCK_S3_ENDPOINT bucket=$R2_BUCKET" echo "" # ============================================================================= diff --git a/shared/artifact_store.py b/shared/artifact_store.py new file mode 100644 index 0000000..fe12fa7 --- /dev/null +++ b/shared/artifact_store.py @@ -0,0 +1,304 @@ +"""Unified dual-write artifact store: R2 (hot cache) + Hippius (durable). + +Phase 7 of TEN-240. R2 stays the hot-path cache because it's the existing +deploy and millisecond-scale; Hippius becomes the durable, censorship- +resistant copy. Both writes are best-effort — failure of either backend +logs but does not raise unless *both* fail. + +Reads default to R2-first (cache locality). When ``allow_fallback`` is set +(``Config.HIPPIUS_ARTIFACT_FALLBACK``), a missing/failing R2 read drops +through to Hippius. We deliberately keep this off by default during rollout +so misconfigurations stay loud. + +The store is duck-typed against: + * ``r2.upload_text(key, str)`` / ``download_text(key) -> str | None`` + * ``r2.upload_json(key, dict)`` / ``download_json(key) -> dict | None`` + * ``r2.upload_file_from_disk(local, key) -> bool`` + * ``hippius.upload_bundle(bytes, dict) -> UploadResult`` + * ``hippius.download_bundle(cid) -> bytes`` +so the existing ``shared.r2_audit.HippiusStorage`` (the S3-compatible +client) and the lazy ``shared.hippius_client.HippiusClient`` plug straight in. + +Observability counters on the instance — not Prometheus, not bittensor +metrics — let callers log a per-round summary without a separate metrics +backend. The validator's round-loop reads them and emits one INFO line. +""" + +from __future__ import annotations + +import json +import logging +from dataclasses import dataclass, field +from typing import Any, Optional + +logger = logging.getLogger(__name__) + + +@dataclass +class DualWriteResult: + """One round-trip's worth of dual-write outcome. + + ``r2_key`` and ``hippius_cid`` are only populated when their respective + side succeeded. ``both_failed`` is True iff neither side wrote anything, + in which case callers may want to retry or escalate (e.g. fail the + round, surface a metric). + """ + r2_key: str = "" + hippius_cid: str = "" + r2_ok: bool = False + hippius_ok: bool = False + both_failed: bool = False + + +@dataclass +class _Counters: + """Cumulative counters for one ArtifactStore lifetime.""" + r2_writes_ok: int = 0 + r2_writes_fail: int = 0 + hippius_writes_ok: int = 0 + hippius_writes_fail: int = 0 + both_failed: int = 0 + bytes_written: int = 0 + mismatches: int = 0 # bytes returned by R2 vs Hippius differ on read + + +class ArtifactStore: + """Dual-write wrapper over R2 (or any S3 client) + Hippius. + + Either ``r2`` or ``hippius`` may be ``None``: an absent backend simply + skips that side of the dual-write. With both set to ``None`` every + write returns ``both_failed=True`` and no bytes move. + """ + + def __init__( + self, + r2=None, + hippius=None, + *, + dual_write: bool = True, + allow_fallback: bool = False, + ): + self.r2 = r2 + self.hippius = hippius + self.dual_write = dual_write + self.allow_fallback = allow_fallback + self.counters = _Counters() + + # ── Internal helpers ───────────────────────────────────────────── + + async def _write_r2(self, key: str, data: bytes) -> bool: + """Best-effort R2 write. Returns True on success.""" + if self.r2 is None or not key: + return False + try: + ok = self.r2.upload_text(key, data.decode("utf-8")) \ + if self._looks_like_text(data) \ + else self._upload_binary_to_r2(key, data) + return bool(ok) + except Exception as e: + logger.warning("R2 write %s failed: %s", key, e) + return False + + @staticmethod + def _looks_like_text(data: bytes) -> bool: + """Heuristic: prefer upload_text for JSON/UTF-8, upload_file for blobs. + + We have no MIME hints from callers, so this falls back to a "decode + succeeds" check. Robust enough for the JSON-tier artifacts this + store currently fronts; large binary blobs (checkpoints) take the + binary path via ``put_file``. + """ + if not data: + return True + try: + data.decode("utf-8") + return True + except UnicodeDecodeError: + return False + + def _upload_binary_to_r2(self, key: str, data: bytes) -> bool: + """R2 binary write via a temp file (S3 client takes a path).""" + import tempfile + with tempfile.NamedTemporaryFile(delete=False) as f: + f.write(data) + path = f.name + try: + return bool(self.r2.upload_file_from_disk(path, key)) + finally: + import os as _os + try: + _os.unlink(path) + except OSError: + pass + + async def _write_hippius(self, data: bytes, metadata: dict) -> str: + """Best-effort Hippius write. Returns the S3 key on success, '' on failure. + + Translates the caller's free-form metadata dict into the structured + ``upload_bundle`` call shape Phase 2 settled on: + * ``app_tag`` → metadata["app"] (default "radar") + * ``phase`` → metadata["kind"] (e.g. "frontier", + "dispatch", "checkpoint") + * ``run_id`` → metadata["run_id"] / "round_id" + / metadata["task"] / "default" + * ``netuid`` → metadata["netuid"] (default 0) + Anything else in the dict goes through as ``extra_metadata``. + """ + if self.hippius is None: + return "" + meta = dict(metadata or {}) + app_tag = str(meta.pop("app", None) or "radar") + phase = str(meta.pop("kind", None) or meta.pop("phase", None) or "blob") + run_id = str( + meta.pop("run_id", None) or meta.pop("round_id", None) + or meta.pop("task", None) or "default", + ) + try: + netuid = int(meta.pop("netuid", 0) or 0) + except (TypeError, ValueError): + netuid = 0 + try: + result = await self.hippius.upload_bundle( + data, app_tag=app_tag, phase=phase, run_id=run_id, + netuid=netuid, extra_metadata=meta, + ) + except NotImplementedError as e: + # Defensive: a stub client (read-only fork) treated as disabled. + logger.debug("Hippius upload skipped (not yet implemented): %s", e) + return "" + except Exception as e: + logger.warning("Hippius upload failed: %s", e) + return "" + if result is None: + return "" + return getattr(result, "key", "") or getattr(result, "cid", "") or ( + result.get("key", "") or result.get("cid", "") + if isinstance(result, dict) else "" + ) + + # ── Public API ─────────────────────────────────────────────────── + + async def put_bytes( + self, + key: str, + data: bytes, + metadata: Optional[dict] = None, + ) -> DualWriteResult: + """Dual-write a byte payload. + + Both backends are attempted in parallel. The result captures which + side(s) succeeded. ``both_failed=True`` is the only outcome that + warrants caller intervention; partial failures are logged and + counted but treated as success. + """ + meta = dict(metadata or {}) + result = DualWriteResult(r2_key=key) + + if self.dual_write or self.r2 is not None: + result.r2_ok = await self._write_r2(key, data) + if self.dual_write or self.hippius is not None: + cid = await self._write_hippius(data, meta) + result.hippius_ok = bool(cid) + result.hippius_cid = cid + + # Counters + c = self.counters + if result.r2_ok: + c.r2_writes_ok += 1 + elif self.r2 is not None: + c.r2_writes_fail += 1 + if result.hippius_ok: + c.hippius_writes_ok += 1 + elif self.hippius is not None: + c.hippius_writes_fail += 1 + c.bytes_written += len(data) + + if not (result.r2_ok or result.hippius_ok): + result.both_failed = True + c.both_failed += 1 + logger.error( + "Both R2 and Hippius failed for key=%s metadata=%s", + key, sorted(meta.keys()), + ) + return result + + async def put_json( + self, + key: str, + data: dict, + metadata: Optional[dict] = None, + ) -> DualWriteResult: + """Convenience wrapper: dump dict to deterministic JSON bytes.""" + body = json.dumps(data, sort_keys=True, separators=(",", ":")).encode() + return await self.put_bytes(key, body, metadata) + + async def get_bytes( + self, + key: str = "", + cid: str = "", + ) -> bytes: + """Get by R2 key (preferred) or Hippius CID. Raises on no-data. + + With ``allow_fallback`` set, a failed R2 read falls through to + Hippius (and vice versa). Empty values for both ``key`` and + ``cid`` raises ``ValueError`` — the caller forgot to identify + the artifact. + """ + if not key and not cid: + raise ValueError("get_bytes requires either key or cid") + + # R2-first when we have a key. + if key and self.r2 is not None: + data = self._read_r2(key) + if data is not None: + return data + if not self.allow_fallback or not cid: + raise FileNotFoundError(f"R2 read failed for key={key}") + + # Hippius read (primary path when caller passed only cid, or + # fallback path when R2 missed). + if cid and self.hippius is not None: + try: + return await self.hippius.download_bundle(cid) + except Exception as e: + if not self.allow_fallback or not key: + raise + logger.warning( + "Hippius read for cid=%s failed: %s — falling back to R2", + cid, e, + ) + + # Last-resort R2 fallback (cid path tried first). + if key and self.r2 is not None: + data = self._read_r2(key) + if data is not None: + return data + + raise FileNotFoundError(f"both backends missed: key={key!r} cid={cid!r}") + + def _read_r2(self, key: str) -> Optional[bytes]: + """Try text then binary for parity with `_write_r2`'s heuristic.""" + try: + text = self.r2.download_text(key) + except Exception as e: + logger.warning("R2 read %s failed: %s", key, e) + return None + if text is not None: + return text.encode() + return None + + def snapshot_counters(self) -> dict: + """Return a JSON-friendly snapshot of cumulative counters.""" + c = self.counters + return { + "r2_writes_ok": c.r2_writes_ok, + "r2_writes_fail": c.r2_writes_fail, + "hippius_writes_ok": c.hippius_writes_ok, + "hippius_writes_fail": c.hippius_writes_fail, + "both_failed": c.both_failed, + "bytes_written": c.bytes_written, + "mismatches": c.mismatches, + } + + +__all__ = ["ArtifactStore", "DualWriteResult"] diff --git a/shared/database.py b/shared/database.py index 6d7d964..e8ff3f6 100644 --- a/shared/database.py +++ b/shared/database.py @@ -52,6 +52,10 @@ class DataElement: generated_samples: list = field(default_factory=list) task: str = "" round_id: int = -1 + # Substrate audit trail: list of {kind, validator_hotkey, cid, round_id} + # entries written by validators after Phase C. One per validator that + # published a Hippius bundle covering this experiment. + substrate_cids: list = field(default_factory=list) def to_dict(self) -> dict: return asdict(self) @@ -134,6 +138,15 @@ def _safe(v): "loss_curve": safe_lc, } + substrate_cids = self.substrate_cids + if isinstance(substrate_cids, str): + try: + substrate_cids = json.loads(substrate_cids) + except (ValueError, TypeError): + substrate_cids = [] + if not isinstance(substrate_cids, list): + substrate_cids = [] + return { "index": self.index, "timestamp": self.timestamp, @@ -149,6 +162,7 @@ def _safe(v): "analysis": self.analysis, "score": score, "round_id": self.round_id, + "substrate_cids": substrate_cids, "provenance": { "self_reported": { "motivation": self.motivation, diff --git a/shared/db_client.py b/shared/db_client.py index 16488ec..1c50277 100644 --- a/shared/db_client.py +++ b/shared/db_client.py @@ -92,9 +92,31 @@ async def health(self) -> bool: except Exception: return False - async def add_experiment(self, element_data: dict) -> Optional[int]: - """POST a DataElement dict to the database. Returns new index or None.""" - result = await self._post("/experiments/add", {"data": element_data}) + async def add_experiment( + self, + element_data: dict, + substrate_cid: str = "", + validator_hotkey: str = "", + artifact_cids: Optional[list[dict]] = None, + ) -> Optional[int]: + """POST a DataElement dict to the database. Returns new index or None. + + When ``substrate_cid`` is non-empty the server appends a + ``{kind: "phase_c_record", validator_hotkey, cid, round_id}`` entry + to the row's ``substrate_cids`` audit list. + + ``artifact_cids`` (TEN-240 Phase 7) supplies extra audit entries + for dual-written non-DB artifacts. Each element is appended to the + list verbatim — callers set the ``kind`` (``checkpoint``, + ``architecture``, ``training_meta``, …). + """ + payload: dict = {"data": element_data} + if substrate_cid: + payload["substrate_cid"] = substrate_cid + payload["validator_hotkey"] = validator_hotkey + if artifact_cids: + payload["artifact_cids"] = artifact_cids + result = await self._post("/experiments/add", payload) if result and "index" in result: return result["index"] return None diff --git a/shared/hippius_client.py b/shared/hippius_client.py new file mode 100644 index 0000000..900afea --- /dev/null +++ b/shared/hippius_client.py @@ -0,0 +1,298 @@ +"""Async client for Hippius S3 (https://s3.hippius.com). + +Per docs.hippius.com/cli/usage the legacy ``hippius`` PyPI SDK is deprecated; +all integrations target the S3 data plane, which supports object metadata, +tagging, and prefix-filtered listing — the surfaces Phase 2 needs. + +Constraints (docs.hippius.com/storage/s3/compatibility): + * Path-style addressing, region ``decentralized``, signature v4. + * No object versioning. Overwrites are silent — we use content-addressed + keys ``{app_tag}/{netuid}/{phase}/{run_id}/{sha256[:16]}.tar`` so a + write never collides with a different payload. + * Object tags = small queryable subset; user metadata (``x-amz-meta-*``) + carries the full per-bundle schema, ASCII-only, ~2 KiB total. + +Reads fall back to the IPFS gateway for clients without S3 creds (the +verify CLI). Without either, ``download_bundle`` raises ``FileNotFoundError``. +""" + +from __future__ import annotations + +import asyncio +import hashlib +import logging +import os +from dataclasses import dataclass +from typing import Optional +from urllib.parse import urlencode + +import httpx + +logger = logging.getLogger(__name__) + + +HIPPIUS_S3_DEFAULT_ENDPOINT = "https://s3.hippius.com" +HIPPIUS_S3_DEFAULT_REGION = "decentralized" +HIPPIUS_IPFS_DEFAULT_GATEWAY = "https://get.hippius.network" + + +@dataclass(frozen=True) +class BundleRef: + """Pointer + provenance for a bundle stored on Hippius S3.""" + key: str + metadata: dict + tags: dict + size: int + etag: str = "" + + @property + def cid(self) -> str: + """Backwards-compat alias used by older substrate audit entries.""" + return self.key + + +# Stable export name for callers that imported the placeholder UploadResult. +UploadResult = BundleRef + + +def _sanitize_metadata(meta: dict, *, max_bytes: int = 1900) -> dict: + """Keep only ASCII-safe keys/values, total under ~2 KiB (S3 header limit).""" + out: dict[str, str] = {} + used = 0 + for raw_k, raw_v in (meta or {}).items(): + try: + k = str(raw_k).encode("ascii").decode("ascii") + v = str(raw_v).encode("ascii").decode("ascii") + except UnicodeEncodeError: + continue + cost = len(k) + len(v) + 4 + if used + cost > max_bytes: + break + out[k] = v + used += cost + return out + + +def _build_prefix(app_tag, netuid, phase, run_id) -> str: + """Leftmost contiguous filter prefix for ListObjectsV2.""" + parts: list[str] = [] + for value in (app_tag, netuid, phase, run_id): + if value is None or value == "": + break + parts.append(str(value)) + return "/".join(parts) + ("/" if parts else "") + + +class HippiusClient: + """Async wrapper over boto3 S3 against Hippius. + + Constructor falls back through env in order: + * ``HIPPIUS_S3_ACCESS_KEY`` → ``HIPPIUS_ACCESS_KEY_ID`` + * ``HIPPIUS_S3_SECRET_KEY`` → ``HIPPIUS_SECRET_ACCESS_KEY`` + * ``HIPPIUS_S3_BUCKET`` → ``HIPPIUS_BUCKET`` + * ``HIPPIUS_S3_ENDPOINT`` (default https://s3.hippius.com) + * ``HIPPIUS_S3_REGION`` (default decentralized) + + Read-only callers can omit creds; download falls back to the IPFS + gateway. + """ + + def __init__( + self, + access_key: str = "", secret_key: str = "", bucket: str = "", + endpoint_url: str = "", region: str = "", + *, + ipfs_gateway_url: str = "", + # Legacy kwargs honoured by older lazy imports. + ipfs_api_url: str = "", hippius_key: str = "", substrate_rpc: str = "", + ): + env = os.getenv + self._access_key = access_key or env("HIPPIUS_S3_ACCESS_KEY", "") \ + or env("HIPPIUS_ACCESS_KEY_ID", "") + self._secret_key = secret_key or env("HIPPIUS_S3_SECRET_KEY", "") \ + or env("HIPPIUS_SECRET_ACCESS_KEY", "") + self.bucket = bucket or env("HIPPIUS_S3_BUCKET", "") or env("HIPPIUS_BUCKET", "") + self.endpoint_url = endpoint_url or env("HIPPIUS_S3_ENDPOINT", "") \ + or HIPPIUS_S3_DEFAULT_ENDPOINT + self.region = region or env("HIPPIUS_S3_REGION", "") or HIPPIUS_S3_DEFAULT_REGION + self._ipfs_gateway = ( + ipfs_gateway_url or ipfs_api_url or env("HIPPIUS_IPFS_GATEWAY", "") + or HIPPIUS_IPFS_DEFAULT_GATEWAY + ).rstrip("/") + self._http: Optional[httpx.AsyncClient] = None + self._s3 = self._build_s3() if self._access_key and self._secret_key else None + + def _build_s3(self): + import boto3 + from botocore.config import Config as BotoConfig + return boto3.client( + "s3", + endpoint_url=self.endpoint_url, + aws_access_key_id=self._access_key, + aws_secret_access_key=self._secret_key, + region_name=self.region, + config=BotoConfig( + signature_version="s3v4", + s3={"addressing_style": "path"}, + ), + ) + + @property + def has_s3(self) -> bool: + """True when S3 creds were supplied — required for writes.""" + return self._s3 is not None and bool(self.bucket) + + async def upload_bundle( + self, + data: bytes, + *, + app_tag: str = "radar", + phase: str, + run_id: str, + netuid: int = 0, + extra_metadata: Optional[dict] = None, + ) -> BundleRef: + """Content-addressed PUT to Hippius S3.""" + if not self.has_s3: + raise RuntimeError( + "HippiusClient: S3 credentials not configured " + "(set HIPPIUS_S3_ACCESS_KEY / _SECRET_KEY / _BUCKET)" + ) + sha = hashlib.sha256(data).hexdigest() + key = f"{app_tag}/{netuid}/{phase}/{run_id}/{sha[:16]}.tar" + tags = {"app_tag": app_tag, "phase": phase, "netuid": str(netuid)} + metadata = _sanitize_metadata({ + "app_tag": app_tag, "phase": phase, "run_id": run_id, + "netuid": str(netuid), "sha256": sha, + **(extra_metadata or {}), + }) + tagging = urlencode(tags) + + def _put(): + return self._s3.put_object( + Bucket=self.bucket, Key=key, Body=data, + Metadata=metadata, Tagging=tagging, + ) + + resp = await asyncio.to_thread(_put) + return BundleRef( + key=key, metadata=metadata, tags=tags, size=len(data), + etag=(resp.get("ETag") or "").strip('"'), + ) + + async def download_bundle(self, key_or_cid: str) -> bytes: + """Fetch by S3 key (preferred) or IPFS CID gateway fallback.""" + if not key_or_cid: + raise ValueError("download_bundle requires a key or cid") + if self.has_s3: + try: + return await self._download_from_s3(key_or_cid) + except FileNotFoundError: + logger.debug("S3 miss for %s — trying IPFS gateway", key_or_cid) + if self._ipfs_gateway: + return await self._download_from_ipfs(key_or_cid) + raise FileNotFoundError( + f"download_bundle: no backend able to resolve {key_or_cid!r}" + ) + + async def _download_from_s3(self, key: str) -> bytes: + from botocore.exceptions import ClientError + + def _get(): + return self._s3.get_object(Bucket=self.bucket, Key=key) + + try: + resp = await asyncio.to_thread(_get) + except ClientError as e: + code = e.response.get("Error", {}).get("Code", "") + if code in {"NoSuchKey", "404"}: + raise FileNotFoundError(key) from e + raise + return resp["Body"].read() + + async def _download_from_ipfs(self, cid: str) -> bytes: + if self._http is None or self._http.is_closed: + self._http = httpx.AsyncClient(timeout=30.0) + resp = await self._http.get(f"{self._ipfs_gateway}/ipfs/{cid}") + if resp.status_code == 404: + raise FileNotFoundError(cid) + resp.raise_for_status() + return resp.content + + async def head_bundle(self, key: str) -> BundleRef: + """``head_object`` + ``get_object_tagging``, returned as a BundleRef.""" + if not self.has_s3: + raise RuntimeError("HippiusClient: S3 credentials required for head_bundle") + from botocore.exceptions import ClientError + + try: + head = await asyncio.to_thread( + lambda: self._s3.head_object(Bucket=self.bucket, Key=key), + ) + except ClientError as e: + code = e.response.get("Error", {}).get("Code", "") + if code in {"NoSuchKey", "404", "NotFound"}: + raise FileNotFoundError(key) from e + raise + tag_resp = await asyncio.to_thread( + lambda: self._s3.get_object_tagging(Bucket=self.bucket, Key=key), + ) + tags = {t["Key"]: t["Value"] for t in tag_resp.get("TagSet", [])} + return BundleRef( + key=key, metadata=dict(head.get("Metadata") or {}), + tags=tags, size=int(head.get("ContentLength") or 0), + etag=(head.get("ETag") or "").strip('"'), + ) + + async def list_by_metadata( + self, + *, + app_tag: Optional[str] = None, + netuid: Optional[int] = None, + phase: Optional[str] = None, + run_id: Optional[str] = None, + match_tags: Optional[dict] = None, + max_keys: int = 1000, + ) -> list[BundleRef]: + """Server-side prefix filter + optional client-side tag match.""" + if not self.has_s3: + raise RuntimeError("HippiusClient: S3 credentials required for list_by_metadata") + prefix = _build_prefix(app_tag, netuid, phase, run_id) + resp = await asyncio.to_thread( + lambda: self._s3.list_objects_v2( + Bucket=self.bucket, Prefix=prefix, MaxKeys=max_keys, + ), + ) + out: list[BundleRef] = [] + for obj in (resp.get("Contents") or []): + key = obj["Key"] + ref = BundleRef( + key=key, metadata={}, tags={}, + size=int(obj.get("Size") or 0), + etag=(obj.get("ETag") or "").strip('"'), + ) + if match_tags: + tag_resp = await asyncio.to_thread( + lambda k=key: self._s3.get_object_tagging(Bucket=self.bucket, Key=k), + ) + tags = {t["Key"]: t["Value"] for t in tag_resp.get("TagSet", [])} + if not all(tags.get(k) == v for k, v in match_tags.items()): + continue + ref = BundleRef( + key=ref.key, metadata=ref.metadata, tags=tags, + size=ref.size, etag=ref.etag, + ) + out.append(ref) + return out + + async def close(self): + if self._http is not None and not self._http.is_closed: + await self._http.aclose() + self._http = None + + +__all__ = [ + "BundleRef", "HippiusClient", "UploadResult", + "HIPPIUS_S3_DEFAULT_ENDPOINT", "HIPPIUS_S3_DEFAULT_REGION", + "HIPPIUS_IPFS_DEFAULT_GATEWAY", +] diff --git a/shared/pg_schema.py b/shared/pg_schema.py index 19a903e..de4d58c 100644 --- a/shared/pg_schema.py +++ b/shared/pg_schema.py @@ -32,6 +32,9 @@ timestamp DOUBLE PRECISION NOT NULL DEFAULT 0.0, round_id BIGINT, task TEXT NOT NULL DEFAULT '', + -- Substrate audit trail (TEN-240). One element per validator that + -- published a Phase C Hippius bundle covering this experiment. + substrate_cids JSONB NOT NULL DEFAULT '[]', search_vector tsvector ); -- Migration: round_id is derived from ``seed_int % 2**32`` in @@ -60,6 +63,7 @@ CREATE INDEX IF NOT EXISTS idx_task_metric ON experiments(task, metric) WHERE metric IS NOT NULL; CREATE INDEX IF NOT EXISTS idx_search ON experiments USING GIN(search_vector); +CREATE INDEX IF NOT EXISTS idx_substrate_cids ON experiments USING GIN(substrate_cids); """ FTS_FUNCTION_DDL = """ @@ -282,6 +286,10 @@ def row_to_element(row) -> DataElement: _decode_jsonb(row["agent_behavior"], {}) if "agent_behavior" in row_keys else {} ) + substrate_cids = ( + _decode_jsonb(row["substrate_cids"], []) + if "substrate_cids" in row_keys else [] + ) return DataElement( index=row["id"], @@ -307,6 +315,7 @@ def row_to_element(row) -> DataElement: timestamp=row["timestamp"], task=row["task"], round_id=row["round_id"] if row["round_id"] is not None else -1, + substrate_cids=substrate_cids, ) @@ -333,13 +342,13 @@ def _jsonb(value) -> str: def element_to_params(element: DataElement, next_id: int) -> tuple: - """Convert a DataElement to a positional tuple for INSERT ($1..$23). + """Convert a DataElement to a positional tuple for INSERT ($1..$24). JSONB columns (loss_curve, generated_samples, objectives, tool_calls, - agent_behavior) are explicitly serialised with ``json.dumps`` so - asyncpg sends a valid JSON string regardless of statement-cache or - connection-pooler settings. Float ``inf``/``nan`` are replaced with - ``null`` to keep PostgreSQL happy. + agent_behavior, substrate_cids) are explicitly serialised with + ``json.dumps`` so asyncpg sends a valid JSON string regardless of + statement-cache or connection-pooler settings. Float ``inf``/``nan`` + are replaced with ``null`` to keep PostgreSQL happy. """ round_id = element.round_id if element.round_id >= 0 else None return ( @@ -366,6 +375,7 @@ def element_to_params(element: DataElement, next_id: int) -> tuple: element.reasoning, _jsonb(element.tool_calls), _jsonb(element.agent_behavior), + _jsonb(element.substrate_cids), ) @@ -375,13 +385,13 @@ def element_to_params(element: DataElement, next_id: int) -> tuple: parent_index, generation, score, miner_uid, miner_hotkey, loss_curve, manifest_sha256, generated_samples, objectives, timestamp, round_id, task, - reasoning, tool_calls, agent_behavior + reasoning, tool_calls, agent_behavior, substrate_cids ) VALUES ( $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, - $21, $22, $23 + $21, $22, $23, $24 ) """ diff --git a/shared/r2_audit.py b/shared/r2_audit.py index 384398b..3d59e84 100644 --- a/shared/r2_audit.py +++ b/shared/r2_audit.py @@ -1,7 +1,14 @@ -"""R2-compatible storage for experiment artifacts. +"""S3-compatible storage for experiment artifacts. -Stores experiment data (code, checkpoints, proposals, dispatch records) in -S3-compatible storage (Cloudflare R2) for cross-validator sharing. +Primary backend is **Hippius** (Substrate-based decentralized object storage, +S3-compatible at ``https://s3.hippius.com``). Cloudflare R2 is supported as a +legacy backend during the migration: if no ``HIPPIUS_*`` env vars are set the +client falls back to the historical ``R2_*`` env vars and the per-account R2 +endpoint URL. + +The class name ``R2AuditLog`` is preserved as a stable alias so the rest of the +codebase doesn't need a sweeping rename. New code should prefer +``HippiusStorage`` (same class, friendlier name). """ from __future__ import annotations @@ -16,43 +23,103 @@ logger = logging.getLogger(__name__) -class R2AuditLog: - """S3-compatible experiment storage (Cloudflare R2).""" +# Hippius S3 endpoint (Substrate-backed decentralized storage). +# See https://docs.hippius.com/storage/s3/integration for details. +HIPPIUS_DEFAULT_ENDPOINT = "https://s3.hippius.com" +HIPPIUS_DEFAULT_REGION = "decentralized" + + +def _first_set(*values: str) -> str: + """Return the first truthy value, or ``""``.""" + for v in values: + if v: + return v + return "" + + +class HippiusStorage: + """S3-compatible artifact storage (Hippius primary, R2 legacy fallback).""" def __init__( self, - account_id: str = "", access_key_id: str = "", secret_access_key: str = "", bucket: str = "", + endpoint_url: str = "", + region: str = "", + # Legacy: R2 used a per-account endpoint derived from account_id. + # Kept for backwards compatibility with callers that still pass it. + account_id: str = "", ): - self.bucket = bucket or os.getenv("R2_BUCKET", "radar-experiments") - account_id = account_id or os.getenv("R2_ACCOUNT_ID", "") - access_key_id = access_key_id or os.getenv("R2_ACCESS_KEY_ID", "") - secret_access_key = secret_access_key or os.getenv("R2_SECRET_ACCESS_KEY", "") - - # Allow overriding the endpoint for testing (e.g. local mock S3 server) - endpoint_override = os.getenv("MOCK_R2_ENDPOINT", "") - if endpoint_override: - endpoint_url = endpoint_override - elif account_id: - endpoint_url = f"https://{account_id}.r2.cloudflarestorage.com" + self.bucket = bucket or _first_set( + os.getenv("HIPPIUS_BUCKET", ""), + os.getenv("R2_BUCKET", ""), + ) or "radar-experiments" + + access_key_id = access_key_id or _first_set( + os.getenv("HIPPIUS_ACCESS_KEY_ID", ""), + os.getenv("R2_ACCESS_KEY_ID", ""), + ) + secret_access_key = secret_access_key or _first_set( + os.getenv("HIPPIUS_SECRET_ACCESS_KEY", ""), + os.getenv("R2_SECRET_ACCESS_KEY", ""), + ) + region = region or _first_set( + os.getenv("HIPPIUS_REGION", ""), + os.getenv("R2_REGION", ""), + HIPPIUS_DEFAULT_REGION, + ) + + # Endpoint resolution order: + # 1. Explicit constructor arg + # 2. MOCK_S3_ENDPOINT / MOCK_R2_ENDPOINT (test override) + # 3. HIPPIUS_ENDPOINT_URL (production override / private gateway) + # 4. Legacy R2 per-account endpoint (only if R2_ACCOUNT_ID is set + # and no Hippius endpoint is otherwise configured) + # 5. HIPPIUS_DEFAULT_ENDPOINT (https://s3.hippius.com) + mock_endpoint = _first_set( + os.getenv("MOCK_S3_ENDPOINT", ""), + os.getenv("MOCK_R2_ENDPOINT", ""), + ) + legacy_account_id = account_id or os.getenv("R2_ACCOUNT_ID", "") + if endpoint_url: + resolved_endpoint = endpoint_url + elif mock_endpoint: + resolved_endpoint = mock_endpoint + elif os.getenv("HIPPIUS_ENDPOINT_URL", ""): + resolved_endpoint = os.getenv("HIPPIUS_ENDPOINT_URL", "") + elif legacy_account_id and not ( + os.getenv("HIPPIUS_ACCESS_KEY_ID") + or os.getenv("HIPPIUS_BUCKET") + ): + # Only fall back to R2's per-account endpoint when the operator + # has actively configured R2 (set R2_ACCOUNT_ID) and not begun + # the Hippius migration. Removes the "I set HIPPIUS_* but it + # still hits R2" footgun. + resolved_endpoint = f"https://{legacy_account_id}.r2.cloudflarestorage.com" else: - endpoint_url = None + resolved_endpoint = HIPPIUS_DEFAULT_ENDPOINT import boto3 from botocore.config import Config as BotoConfig + # Hippius requires path-style addressing (it does not host buckets at + # virtual subdomains the way AWS does). R2 also accepts path-style, + # so this is safe for the legacy backend too. self._s3 = boto3.client( "s3", - endpoint_url=endpoint_url, + endpoint_url=resolved_endpoint, aws_access_key_id=access_key_id, aws_secret_access_key=secret_access_key, - config=BotoConfig(signature_version="s3v4"), + region_name=region, + config=BotoConfig( + signature_version="s3v4", + s3={"addressing_style": "path"}, + ), ) def upload_json(self, key: str, data: dict) -> bool: - """Upload a JSON object to R2.""" + """Upload a JSON object.""" try: body = json.dumps(data, indent=2).encode() self._s3.put_object(Bucket=self.bucket, Key=key, Body=body) @@ -62,13 +129,13 @@ def upload_json(self, key: str, data: dict) -> bool: return False def download_json(self, key: str) -> Optional[dict]: - """Download and parse a JSON object from R2.""" + """Download and parse a JSON object.""" try: resp = self._s3.get_object(Bucket=self.bucket, Key=key) return json.loads(resp["Body"].read()) except ClientError as e: if e.response["Error"]["Code"] == "NoSuchKey": - logger.debug("Key not found in R2: %s", key) + logger.debug("Key not found: %s", key) else: logger.error("Failed to download JSON %s: %s", key, e) return None @@ -81,7 +148,6 @@ def list_experiments(self, prefix: str = "") -> list[str]: try: resp = self._s3.list_objects_v2(Bucket=self.bucket, Prefix=prefix) keys = [obj["Key"] for obj in resp.get("Contents", [])] - # Extract experiment IDs from bundle paths experiments = set() for key in keys: parts = key.split("/") @@ -102,7 +168,7 @@ def list_keys(self, prefix: str = "") -> list[str]: return [] def upload_file_from_disk(self, local_path: str, key: str) -> bool: - """Upload a file from local disk to R2.""" + """Upload a file from local disk.""" try: self._s3.upload_file(local_path, self.bucket, key) logger.info("Uploaded file: %s", key) @@ -112,7 +178,7 @@ def upload_file_from_disk(self, local_path: str, key: str) -> bool: return False def download_file_to_disk(self, key: str, local_path: str) -> bool: - """Download a file from R2 to local disk.""" + """Download a file to local disk.""" try: resp = self._s3.get_object(Bucket=self.bucket, Key=key) with open(local_path, "wb") as f: @@ -123,7 +189,7 @@ def download_file_to_disk(self, key: str, local_path: str) -> bool: return False def upload_text(self, key: str, text: str) -> bool: - """Upload a text string to R2.""" + """Upload a text string.""" try: self._s3.put_object(Bucket=self.bucket, Key=key, Body=text.encode()) return True @@ -132,7 +198,7 @@ def upload_text(self, key: str, text: str) -> bool: return False def download_text(self, key: str) -> Optional[str]: - """Download a text string from R2.""" + """Download a text string.""" try: resp = self._s3.get_object(Bucket=self.bucket, Key=key) return resp["Body"].read().decode() @@ -146,17 +212,7 @@ def generate_presigned_put_url( ttl: int = 3600, max_content_length: int = 0, ) -> str: - """Generate a pre-signed PUT URL for uploading to a specific key. - - Args: - key: The S3 key to generate the URL for. - ttl: Time-to-live in seconds (default 1 hour). - max_content_length: If > 0, add Content-Length condition to limit - upload size. S3-compatible stores enforce this server-side. - - Returns: - Pre-signed URL string, or empty string on failure. - """ + """Generate a pre-signed PUT URL for uploading to a specific key.""" try: params: dict = {"Bucket": self.bucket, "Key": key} if max_content_length > 0: @@ -172,7 +228,7 @@ def generate_presigned_put_url( return "" def generate_presigned_get_url(self, key: str, ttl: int = 900) -> str: - """Generate a presigned GET URL for downloading from R2.""" + """Generate a presigned GET URL for downloading.""" try: return self._s3.generate_presigned_url( "get_object", @@ -190,3 +246,12 @@ def key_exists(self, key: str) -> bool: return True except Exception: return False + + +# Backwards-compatible alias. Existing callers import ``R2AuditLog``; we keep +# the name so the migration doesn't fan out across every module. New code +# should prefer ``HippiusStorage``. +R2AuditLog = HippiusStorage + + +__all__ = ["HippiusStorage", "R2AuditLog"] diff --git a/shared/substrate.py b/shared/substrate.py new file mode 100644 index 0000000..7055552 --- /dev/null +++ b/shared/substrate.py @@ -0,0 +1,233 @@ +"""Phase C Substrate record schema, signing, and verification. + +A pure-Python contract for what a validator publishes on-chain (or on Hippius) +at the end of Phase C: a `PhaseCRecord` is the structured, signed claim that a +particular checkpoint was evaluated by a particular validator with a particular +result. Every other piece of the substrate pipeline (bundling, IPFS upload, +chain extrinsic) binds to the canonical bytes computed here. + +Design constraints: + * Deterministic — same record always serialises to the same bytes so two + validators verifying the same record agree on what was signed. + * Forward compatible — `record_from_json` silently drops unknown fields so a + future schema can add columns without breaking older readers. + * Self-contained — no network calls, no Hippius, no chain. Just data + crypto. +""" + +from __future__ import annotations + +import dataclasses +import gzip +import hashlib +import json +import logging +from dataclasses import asdict, dataclass, fields +from typing import Any + +import bittensor as bt + +logger = logging.getLogger(__name__) + + +SCHEMA_VERSION = "radar.substrate.v1" +SIGNATURE_PREFIX = "sr25519:" + + +@dataclass(frozen=True) +class PhaseCRecord: + """A single Phase C evaluation claim, signed by the validator. + + `signature` is computed last by `sign_record` and intentionally excluded + from `canonical_bytes` so that signing and verification compare like with + like. Callers should treat instances as immutable: use + `dataclasses.replace(record, ...)` to derive a modified copy. + """ + + # Schema + schema_version: str + + # Round identity + round_id: int + block_hash: str + task: str + + # Miner + miner_uid: int + miner_hotkey: str + code_hash: str + architecture_sha256: str + checkpoint_sha256: str + + # Measurement + metrics: dict + passed_size_gate: bool + flops_verified: bool + eval_status: str + + # Validator + validator_uid: int + validator_hotkey: str + validator_block_height: int + timestamp: float + + # Signature (computed last; sr25519:) + signature: str = "" + + +_KNOWN_FIELDS = frozenset(f.name for f in fields(PhaseCRecord)) + + +def _filter_unknown(payload: dict) -> dict: + """Drop keys not declared on PhaseCRecord (forward-compat).""" + return {k: v for k, v in payload.items() if k in _KNOWN_FIELDS} + + +def canonical_bytes(record: PhaseCRecord) -> bytes: + """Return the deterministic byte string that gets signed and verified. + + Excludes the `signature` field so a signature can be embedded in the + record without changing the bytes used to verify it. JSON is emitted + with sorted keys and the most compact separator pair so two callers + (potentially on different platforms) produce byte-identical output. + """ + payload = asdict(record) + payload.pop("signature", None) + return json.dumps(payload, sort_keys=True, separators=(",", ":")).encode() + + +def sign_record(record: PhaseCRecord, wallet: "bt.Wallet") -> PhaseCRecord: + """Sign a record with the wallet hotkey and return a new frozen copy. + + Uses sr25519 via `wallet.hotkey.sign()`. The returned record carries + the signature in `sr25519:` form; the original is left untouched. + Raises ValueError if the record's `validator_hotkey` doesn't match the + wallet — silently signing under the wrong hotkey would let a record + pretend to come from another validator. + """ + if record.validator_hotkey and record.validator_hotkey != wallet.hotkey.ss58_address: + raise ValueError( + f"validator_hotkey mismatch: record claims " + f"{record.validator_hotkey!r}, wallet is " + f"{wallet.hotkey.ss58_address!r}" + ) + raw = wallet.hotkey.sign(canonical_bytes(record)) + return dataclasses.replace(record, signature=f"{SIGNATURE_PREFIX}{raw.hex()}") + + +def verify_record( + record: PhaseCRecord, + expected_hotkey: str | None = None, +) -> tuple[bool, str]: + """Verify a record's sr25519 signature against its canonical bytes. + + Returns `(ok, error_message)`. On success `error_message` is empty. If + `expected_hotkey` is supplied, the record's `validator_hotkey` must + match it (caller-supplied identity check, on top of the cryptographic + check). The function never raises on bad input — malformed records + return `(False, "...")` so callers can log and reject without a + try/except wrapper. + """ + if record.schema_version != SCHEMA_VERSION: + return False, ( + f"Unsupported schema_version {record.schema_version!r}; " + f"expected {SCHEMA_VERSION!r}" + ) + if not record.signature.startswith(SIGNATURE_PREFIX): + return False, ( + f"Signature missing required {SIGNATURE_PREFIX!r} prefix" + ) + if expected_hotkey and record.validator_hotkey != expected_hotkey: + return False, ( + f"validator_hotkey mismatch: record has " + f"{record.validator_hotkey!r}, expected {expected_hotkey!r}" + ) + hex_sig = record.signature[len(SIGNATURE_PREFIX):] + try: + sig_bytes = bytes.fromhex(hex_sig) + except ValueError: + return False, "Signature is not valid hex" + if not record.validator_hotkey: + return False, "validator_hotkey is empty" + try: + keypair = bt.Keypair(ss58_address=record.validator_hotkey) + if not keypair.verify(canonical_bytes(record), sig_bytes): + return False, "sr25519 signature does not match canonical bytes" + except Exception as e: # bt.Keypair raises various Substrate errors + return False, f"Signature verification failed: {e}" + return True, "" + + +def record_to_json(record: PhaseCRecord) -> str: + """Serialise a record (including signature) to deterministic JSON.""" + return json.dumps(asdict(record), sort_keys=True, separators=(",", ":")) + + +def record_from_json(s: str) -> PhaseCRecord: + """Parse a record from JSON, dropping unknown fields for forward compat.""" + payload = json.loads(s) + if not isinstance(payload, dict): + raise ValueError("PhaseCRecord JSON must be an object") + return PhaseCRecord(**_filter_unknown(payload)) + + +def records_to_bundle(records: list[PhaseCRecord]) -> bytes: + """Pack a list of records into a gzip-compressed JSON bundle. + + Bundle shape: ``{"schema_version", "count", "records"}``. The bundle + schema is the same as the per-record schema so a future bump applies + to both at once. Output is always gzipped — `records_from_bundle` + accepts plain JSON for human/debug use, but the canonical write path + is always compressed. + """ + payload: dict[str, Any] = { + "schema_version": SCHEMA_VERSION, + "count": len(records), + "records": [asdict(r) for r in records], + } + raw = json.dumps(payload, sort_keys=True, separators=(",", ":")).encode() + return gzip.compress(raw) + + +def records_from_bundle(data: bytes) -> list[PhaseCRecord]: + """Parse a bundle produced by `records_to_bundle`. + + Accepts either gzip-compressed bytes (the canonical form) or plain + JSON bytes (debug / hand-rolled). Unknown record fields are silently + dropped for forward compatibility. + """ + try: + raw = gzip.decompress(data) + except (OSError, gzip.BadGzipFile): + raw = data + payload = json.loads(raw) + if not isinstance(payload, dict): + raise ValueError("Bundle JSON must be an object") + if payload.get("schema_version") != SCHEMA_VERSION: + raise ValueError( + f"Unsupported bundle schema_version " + f"{payload.get('schema_version')!r}; expected {SCHEMA_VERSION!r}" + ) + raw_records = payload.get("records", []) + if not isinstance(raw_records, list): + raise ValueError("Bundle 'records' must be a list") + return [PhaseCRecord(**_filter_unknown(r)) for r in raw_records] + + +def bundle_sha256(data: bytes) -> str: + """Return ``sha256:`` digest of arbitrary bundle bytes.""" + return f"sha256:{hashlib.sha256(data).hexdigest()}" + + +__all__ = [ + "SCHEMA_VERSION", + "SIGNATURE_PREFIX", + "PhaseCRecord", + "canonical_bytes", + "sign_record", + "verify_record", + "record_to_json", + "record_from_json", + "records_to_bundle", + "records_from_bundle", + "bundle_sha256", +] diff --git a/tests/test_agent_harness.py b/tests/test_agent_harness.py index e4d7c84..de610fd 100644 --- a/tests/test_agent_harness.py +++ b/tests/test_agent_harness.py @@ -76,6 +76,15 @@ def test_no_r2_credentials(self): assert "R2_ACCESS_KEY_ID" not in env assert "R2_SECRET_ACCESS_KEY" not in env + def test_no_hippius_credentials(self): + with patch.dict(os.environ, { + "HIPPIUS_ACCESS_KEY_ID": "key", + "HIPPIUS_SECRET_ACCESS_KEY": "secret", + }): + env = _build_agent_env_vars() + assert "HIPPIUS_ACCESS_KEY_ID" not in env + assert "HIPPIUS_SECRET_ACCESS_KEY" not in env + def test_forwards_subtensor(self): with patch.dict(os.environ, { "SUBTENSOR_NETWORK": "finney", diff --git a/tests/test_artifact_store.py b/tests/test_artifact_store.py new file mode 100644 index 0000000..3a7557a --- /dev/null +++ b/tests/test_artifact_store.py @@ -0,0 +1,345 @@ +"""Tests for shared/artifact_store.py — dual-write R2 + Hippius.""" + +from __future__ import annotations + +import json +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +from shared.artifact_store import ArtifactStore, DualWriteResult + + +# ── Fakes ───────────────────────────────────────────────────────────── + + +class _FakeR2: + """In-memory stand-in for R2AuditLog covering the ArtifactStore surface.""" + + def __init__(self, *, fail: bool = False): + self._store: dict[str, bytes] = {} + self.fail = fail + + def upload_text(self, key: str, text: str) -> bool: + if self.fail: + return False + self._store[key] = text.encode() + return True + + def upload_json(self, key: str, data: dict) -> bool: + if self.fail: + return False + self._store[key] = json.dumps(data).encode() + return True + + def upload_file_from_disk(self, path: str, key: str) -> bool: + if self.fail: + return False + with open(path, "rb") as f: + self._store[key] = f.read() + return True + + def download_text(self, key: str): + if self.fail: + return None + if key not in self._store: + return None + try: + return self._store[key].decode() + except UnicodeDecodeError: + return None + + +class _FakeHippius: + """Async stand-in for HippiusClient.upload/download (Phase 2 structured).""" + + def __init__(self, *, fail: bool = False, return_none: bool = False): + self._store: dict[str, bytes] = {} + self._meta: dict[str, dict] = {} + self.fail = fail + self.return_none = return_none + self._next_id = 0 + + async def upload_bundle( + self, data: bytes, *, + app_tag: str = "radar", phase: str, run_id: str, + netuid: int = 0, extra_metadata=None, + ): + if self.fail: + raise RuntimeError("hippius down") + if self.return_none: + return None + self._next_id += 1 + # Mirror the real client's content-addressed key shape so tests can + # assert against it. Also stash everything for inspection. + import hashlib as _hl + sha = _hl.sha256(data).hexdigest() + key = f"{app_tag}/{netuid}/{phase}/{run_id}/{sha[:16]}.tar" + self._store[key] = data + self._meta[key] = { + "app_tag": app_tag, "phase": phase, "run_id": run_id, + "netuid": str(netuid), **(extra_metadata or {}), + } + return MagicMock( + key=key, cid=key, size=len(data), etag="fakeetag", + metadata=self._meta[key], + tags={"app_tag": app_tag, "phase": phase, "netuid": str(netuid)}, + ) + + async def download_bundle(self, key_or_cid: str) -> bytes: + if self.fail or key_or_cid not in self._store: + raise RuntimeError(f"missing key {key_or_cid}") + return self._store[key_or_cid] + + +# ── put_bytes / put_json ────────────────────────────────────────────── + + +@pytest.mark.asyncio +async def test_put_bytes_both_succeed(): + r2, hip = _FakeR2(), _FakeHippius() + store = ArtifactStore(r2=r2, hippius=hip) + res = await store.put_bytes("k.json", b'{"a":1}', {"kind": "test"}) + assert res.r2_ok and res.hippius_ok + assert res.r2_key == "k.json" + # Content-addressed key shape: {app_tag}/{netuid}/{phase}/{run_id}/... + assert "/" in res.hippius_cid and res.hippius_cid.endswith(".tar") + assert not res.both_failed + assert r2._store["k.json"] == b'{"a":1}' + counters = store.snapshot_counters() + assert counters["r2_writes_ok"] == 1 + assert counters["hippius_writes_ok"] == 1 + assert counters["bytes_written"] == len(b'{"a":1}') + + +@pytest.mark.asyncio +async def test_put_bytes_r2_fails_hippius_succeeds(): + """A flaky R2 must not block the Hippius copy.""" + r2 = _FakeR2(fail=True) + hip = _FakeHippius() + store = ArtifactStore(r2=r2, hippius=hip) + res = await store.put_bytes("k.json", b"hello", {}) + assert not res.r2_ok + assert res.hippius_ok + assert not res.both_failed + counters = store.snapshot_counters() + assert counters["r2_writes_fail"] == 1 + assert counters["hippius_writes_ok"] == 1 + + +@pytest.mark.asyncio +async def test_put_bytes_hippius_fails_r2_succeeds(): + r2 = _FakeR2() + hip = _FakeHippius(fail=True) + store = ArtifactStore(r2=r2, hippius=hip) + res = await store.put_bytes("k.json", b"hello", {}) + assert res.r2_ok + assert not res.hippius_ok + assert not res.both_failed + counters = store.snapshot_counters() + assert counters["hippius_writes_fail"] == 1 + + +@pytest.mark.asyncio +async def test_put_bytes_both_fail_flags_result(): + r2 = _FakeR2(fail=True) + hip = _FakeHippius(fail=True) + store = ArtifactStore(r2=r2, hippius=hip) + res = await store.put_bytes("k.json", b"hello", {}) + assert res.both_failed + assert not res.r2_ok and not res.hippius_ok + counters = store.snapshot_counters() + assert counters["both_failed"] == 1 + + +@pytest.mark.asyncio +async def test_put_bytes_no_backends_fails_loudly(): + """Constructed without either side, every write trips both_failed.""" + store = ArtifactStore(r2=None, hippius=None) + res = await store.put_bytes("k.json", b"hello", {}) + assert res.both_failed + + +@pytest.mark.asyncio +async def test_put_bytes_only_r2_configured(): + r2 = _FakeR2() + store = ArtifactStore(r2=r2, hippius=None) + res = await store.put_bytes("k.json", b"hello", {}) + assert res.r2_ok + assert not res.hippius_ok + assert not res.both_failed + + +@pytest.mark.asyncio +async def test_put_bytes_only_hippius_configured(): + hip = _FakeHippius() + store = ArtifactStore(r2=None, hippius=hip) + res = await store.put_bytes("k.json", b"hello", {}) + assert res.hippius_ok + assert not res.r2_ok + assert not res.both_failed + + +@pytest.mark.asyncio +async def test_put_bytes_handles_hippius_not_implemented(): + """Phase 2 stop: upload_bundle raises NotImplementedError → treated as + a disabled backend, not a failure that masks R2's success.""" + class _NotImplHippius: + async def upload_bundle(self, data, metadata): + raise NotImplementedError("see TEN-242") + + r2 = _FakeR2() + store = ArtifactStore(r2=r2, hippius=_NotImplHippius()) + res = await store.put_bytes("k.json", b"hello", {}) + assert res.r2_ok + assert not res.hippius_ok + # NotImplementedError isn't counted as a failure (it's a disabled + # backend), but R2's success keeps both_failed False. + assert not res.both_failed + + +@pytest.mark.asyncio +async def test_put_json_serialises_deterministically(): + r2, hip = _FakeR2(), _FakeHippius() + store = ArtifactStore(r2=r2, hippius=hip) + await store.put_json("k.json", {"b": 2, "a": 1}, {"kind": "test"}) + # Sorted keys → deterministic across runs. + assert r2._store["k.json"] == b'{"a":1,"b":2}' + + +# ── get_bytes ───────────────────────────────────────────────────────── + + +@pytest.mark.asyncio +async def test_get_bytes_r2_first(): + r2, hip = _FakeR2(), _FakeHippius() + store = ArtifactStore(r2=r2, hippius=hip) + await store.put_bytes("k.json", b"hello", {}) + data = await store.get_bytes(key="k.json") + assert data == b"hello" + + +@pytest.mark.asyncio +async def test_get_bytes_by_cid(): + r2, hip = _FakeR2(), _FakeHippius() + store = ArtifactStore(r2=r2, hippius=hip) + res = await store.put_bytes("k.json", b"hello", {}) + data = await store.get_bytes(cid=res.hippius_cid) + assert data == b"hello" + + +@pytest.mark.asyncio +async def test_get_bytes_fallback_off_r2_miss_raises(): + """Default behaviour: a missed R2 read raises rather than silently + falling through. Loud failures during rollout.""" + hip = _FakeHippius() + store = ArtifactStore( + r2=_FakeR2(fail=True), hippius=hip, allow_fallback=False, + ) + with pytest.raises(FileNotFoundError): + await store.get_bytes(key="k.json", cid="bafy1") + + +@pytest.mark.asyncio +async def test_get_bytes_fallback_on_falls_through_to_hippius(): + r2, hip = _FakeR2(), _FakeHippius() + store = ArtifactStore(r2=r2, hippius=hip, allow_fallback=True) + res = await store.put_bytes("k.json", b"hello", {}) + # Wipe R2's copy to force fallback. + r2._store.clear() + data = await store.get_bytes(key="k.json", cid=res.hippius_cid) + assert data == b"hello" + + +@pytest.mark.asyncio +async def test_get_bytes_requires_key_or_cid(): + store = ArtifactStore(r2=_FakeR2(), hippius=_FakeHippius()) + with pytest.raises(ValueError): + await store.get_bytes() + + +@pytest.mark.asyncio +async def test_get_bytes_both_miss_raises(): + r2, hip = _FakeR2(), _FakeHippius() + store = ArtifactStore(r2=r2, hippius=hip, allow_fallback=True) + with pytest.raises(FileNotFoundError): + await store.get_bytes(key="absent", cid="bafyabsent") + + +# ── Roundtrip integration ──────────────────────────────────────────── + + +@pytest.mark.asyncio +async def test_coordinator_routes_through_artifact_store_when_present(): + """TrainingCoordinator.write_dispatch_record honours artifact_store.""" + from validator.coordinator import TrainingCoordinator, TrainingResult + + r2, hip = _FakeR2(), _FakeHippius() + store = ArtifactStore(r2=r2, hippius=hip) + + # Build a coordinator without invoking __init__'s broader concerns. + coord = TrainingCoordinator.__new__(TrainingCoordinator) + coord.wallet = MagicMock() + coord.wallet.hotkey.ss58_address = "5Vali" + coord.metagraph = MagicMock() + coord.r2 = r2 + coord.my_uid = 0 + coord.artifact_store = store + coord._fallback_uids = {} + + results = [TrainingResult( + arch_owner=1, trainer_uid=2, status="success", + flops_equivalent_size=1_000_000, training_time_seconds=42.0, + checkpoint_key="round_7/miner_x/checkpoint.safetensors", + )] + await coord.write_dispatch_record(round_id=7, results=results) + + # Dual-written: R2 carries the JSON, Hippius carries an identical copy. + r2_payload = json.loads(r2._store["round_7/dispatch/vali_5Vali.json"]) + assert r2_payload["dispatcher"] == "5Vali" + assert r2_payload["round_id"] == 7 + assert len(hip._store) == 1 + key = next(iter(hip._store)) + # Translation: artifact_store metadata.kind -> Hippius phase tag. + assert hip._meta[key]["phase"] == "dispatch" + assert hip._meta[key]["validator_hotkey"] == "5Vali" + + +@pytest.mark.asyncio +async def test_coordinator_falls_back_to_r2_only_when_no_store(): + """Backward compat: artifact_store=None preserves the R2-only path.""" + from validator.coordinator import TrainingCoordinator, TrainingResult + + r2 = _FakeR2() + coord = TrainingCoordinator.__new__(TrainingCoordinator) + coord.wallet = MagicMock() + coord.wallet.hotkey.ss58_address = "5Vali" + coord.metagraph = MagicMock() + coord.r2 = r2 + coord.my_uid = 0 + coord.artifact_store = None + coord._fallback_uids = {} + + await coord.write_dispatch_record( + round_id=7, results=[TrainingResult( + arch_owner=1, trainer_uid=2, status="success", + flops_equivalent_size=1_000_000, training_time_seconds=1.0, + checkpoint_key="x", + )], + ) + assert "round_7/dispatch/vali_5Vali.json" in r2._store + + +@pytest.mark.asyncio +async def test_dual_write_then_fetch_via_cid_matches_bytes(): + r2, hip = _FakeR2(), _FakeHippius() + store = ArtifactStore(r2=r2, hippius=hip, allow_fallback=True) + payload = b'{"frontier": [1, 2, 3]}' + res = await store.put_bytes("frontier/ts/latest.json", payload, { + "app": "radar", "kind": "frontier", "task": "ts", + }) + assert res.r2_ok and res.hippius_ok + # Both backends carry identical bytes — the dual-write contract. + via_r2 = await store.get_bytes(key="frontier/ts/latest.json") + via_hippius = await store.get_bytes(cid=res.hippius_cid) + assert via_r2 == via_hippius == payload diff --git a/tests/test_hippius_client.py b/tests/test_hippius_client.py new file mode 100644 index 0000000..c82b24d --- /dev/null +++ b/tests/test_hippius_client.py @@ -0,0 +1,305 @@ +"""Tests for shared/hippius_client.py — boto3-backed Hippius S3 client. + +We mock the boto3 S3 client (no real network), exercise the four public +methods (``upload_bundle``, ``download_bundle``, ``head_bundle``, +``list_by_metadata``), and confirm: + * the content-addressed key shape is right; + * tags only carry the queryable subset (``app_tag``, ``phase``, ``netuid``); + * user metadata is ASCII-clamped to ~2 KiB; + * NoSuchKey maps to FileNotFoundError; + * IPFS-gateway fallback kicks in when no S3 creds are configured. +""" + +from __future__ import annotations + +import hashlib +from unittest.mock import MagicMock, patch + +import httpx +import pytest + +from shared.hippius_client import ( + BundleRef, + HippiusClient, + _build_prefix, + _sanitize_metadata, +) + + +# ── Helpers / fixtures ─────────────────────────────────────────────── + + +class _FakeS3: + """In-memory stand-in that records put/get/head/list/tagging calls. + + Only models the bits ``HippiusClient`` actually exercises. Real boto3 + would round-trip an S3 server; here we keep just enough state to + answer the same questions. + """ + + def __init__(self, *, raise_on_get: str = ""): + self._objects: dict[str, dict] = {} # key -> {"body", "metadata", "etag"} + self._tags: dict[str, dict] = {} # key -> {tag_key: tag_value} + self.raise_on_get = raise_on_get + self.calls: list = [] + + def put_object(self, **kwargs): + key = kwargs["Key"] + body = kwargs["Body"] + meta = kwargs.get("Metadata") or {} + tagging = kwargs.get("Tagging") or "" + # Tagging is x-www-form-urlencoded "k=v&k=v". + from urllib.parse import parse_qs + tags = {k: v[0] for k, v in parse_qs(tagging).items()} if tagging else {} + etag = '"' + hashlib.md5(body).hexdigest() + '"' + self._objects[key] = {"body": body, "metadata": dict(meta), "etag": etag} + self._tags[key] = tags + self.calls.append(("put_object", kwargs)) + return {"ETag": etag} + + def get_object(self, **kwargs): + key = kwargs["Key"] + if key == self.raise_on_get or key not in self._objects: + from botocore.exceptions import ClientError + raise ClientError( + {"Error": {"Code": "NoSuchKey", "Message": "missing"}}, + "GetObject", + ) + rec = self._objects[key] + body = MagicMock() + body.read = MagicMock(return_value=rec["body"]) + return {"Body": body, "ETag": rec["etag"], "Metadata": rec["metadata"]} + + def head_object(self, **kwargs): + key = kwargs["Key"] + if key not in self._objects: + from botocore.exceptions import ClientError + raise ClientError( + {"Error": {"Code": "NoSuchKey"}}, "HeadObject", + ) + rec = self._objects[key] + return { + "ETag": rec["etag"], + "Metadata": rec["metadata"], + "ContentLength": len(rec["body"]), + } + + def get_object_tagging(self, **kwargs): + key = kwargs["Key"] + return {"TagSet": [ + {"Key": k, "Value": v} for k, v in (self._tags.get(key) or {}).items() + ]} + + def list_objects_v2(self, **kwargs): + prefix = kwargs.get("Prefix", "") or "" + contents = [] + for key, rec in self._objects.items(): + if key.startswith(prefix): + contents.append({ + "Key": key, "Size": len(rec["body"]), "ETag": rec["etag"], + }) + return {"Contents": contents} + + +def _make_client(*, fake_s3: _FakeS3) -> HippiusClient: + """Build a HippiusClient with a pre-injected fake S3.""" + c = HippiusClient( + access_key="ak", secret_key="sk", bucket="radar-test", + ) + c._s3 = fake_s3 # bypass real boto3 + return c + + +# ── Helpers under test ─────────────────────────────────────────────── + + +def test_sanitize_metadata_drops_non_ascii(): + out = _sanitize_metadata({"ok": "hello", "bad": "héllo", "n": 7}) + assert "ok" in out and "bad" not in out + assert out["n"] == "7" # values get stringified + + +def test_sanitize_metadata_caps_total_size(): + huge = {f"k{i}": "x" * 200 for i in range(40)} + out = _sanitize_metadata(huge, max_bytes=1900) + # Total bytes must stay under the cap (with the +4 overhead per entry). + total = sum(len(k) + len(v) + 4 for k, v in out.items()) + assert total <= 1900 + assert len(out) < len(huge) # some were dropped + + +def test_build_prefix_is_leftmost_contiguous(): + assert _build_prefix("radar", 1, "phase_c", "42") == "radar/1/phase_c/42/" + assert _build_prefix("radar", 1, None, "42") == "radar/1/" # gap → stop + assert _build_prefix(None, 1, "phase_c", None) == "" # no leading + + +# ── upload_bundle ──────────────────────────────────────────────────── + + +@pytest.mark.asyncio +async def test_upload_bundle_content_addressed_key(): + fake = _FakeS3() + client = _make_client(fake_s3=fake) + data = b"hello phase c" + sha = hashlib.sha256(data).hexdigest() + + ref = await client.upload_bundle( + data, app_tag="radar", phase="phase_c", run_id="42", netuid=1, + ) + + assert isinstance(ref, BundleRef) + assert ref.key == f"radar/1/phase_c/42/{sha[:16]}.tar" + assert ref.size == len(data) + # Tags carry only the queryable subset. + assert ref.tags == {"app_tag": "radar", "phase": "phase_c", "netuid": "1"} + # Metadata carries the full schema view + caller's extra fields. + assert ref.metadata["sha256"] == sha + assert ref.metadata["run_id"] == "42" + # Backwards-compat: BundleRef.cid aliases .key for legacy substrate + # audit entries that read ``.cid``. + assert ref.cid == ref.key + + +@pytest.mark.asyncio +async def test_upload_bundle_passes_extra_metadata_through(): + fake = _FakeS3() + client = _make_client(fake_s3=fake) + ref = await client.upload_bundle( + b"x", phase="phase_c", run_id="1", + extra_metadata={"validator_hotkey": "5G", "record_count": "3"}, + ) + assert ref.metadata["validator_hotkey"] == "5G" + assert ref.metadata["record_count"] == "3" + + +@pytest.mark.asyncio +async def test_upload_bundle_without_creds_raises(): + """No access key / secret → no S3 client wired → upload refuses.""" + client = HippiusClient() # no creds, no env + with pytest.raises(RuntimeError, match="S3 credentials"): + await client.upload_bundle(b"x", phase="phase_c", run_id="1") + + +# ── download_bundle ────────────────────────────────────────────────── + + +@pytest.mark.asyncio +async def test_download_bundle_via_s3_returns_bytes(): + fake = _FakeS3() + client = _make_client(fake_s3=fake) + ref = await client.upload_bundle(b"hello", phase="phase_c", run_id="1") + data = await client.download_bundle(ref.key) + assert data == b"hello" + + +@pytest.mark.asyncio +async def test_download_bundle_nosuchkey_falls_through_to_ipfs_gateway(): + """When S3 misses, we silently try the gateway path. Default behaviour + so legacy IPFS CIDs in old audit entries keep resolving.""" + fake = _FakeS3() + client = _make_client(fake_s3=fake) + + # Stub the IPFS gateway side. + async def _ipfs(self, cid): + return b"ipfs-bytes" + + with patch.object(HippiusClient, "_download_from_ipfs", _ipfs): + data = await client.download_bundle("bafymissing") + assert data == b"ipfs-bytes" + + +@pytest.mark.asyncio +async def test_download_bundle_no_backend_raises(): + """No creds, no gateway URL → loud failure.""" + client = HippiusClient() + client._ipfs_gateway = "" # explicitly disable gateway fallback + with pytest.raises(FileNotFoundError): + await client.download_bundle("bafyany") + + +@pytest.mark.asyncio +async def test_download_bundle_empty_arg_raises(): + client = HippiusClient(access_key="ak", secret_key="sk", bucket="b") + with pytest.raises(ValueError): + await client.download_bundle("") + + +# ── head_bundle ────────────────────────────────────────────────────── + + +@pytest.mark.asyncio +async def test_head_bundle_returns_metadata_and_tags(): + fake = _FakeS3() + client = _make_client(fake_s3=fake) + ref = await client.upload_bundle( + b"x", phase="phase_c", run_id="1", + extra_metadata={"validator_hotkey": "5G"}, + ) + head = await client.head_bundle(ref.key) + assert head.key == ref.key + assert head.size == 1 + assert head.metadata["validator_hotkey"] == "5G" + assert head.tags == ref.tags + + +@pytest.mark.asyncio +async def test_head_bundle_missing_raises_file_not_found(): + fake = _FakeS3() + client = _make_client(fake_s3=fake) + with pytest.raises(FileNotFoundError): + await client.head_bundle("not/a/real/key.tar") + + +# ── list_by_metadata ───────────────────────────────────────────────── + + +@pytest.mark.asyncio +async def test_list_by_metadata_prefix_filters(): + fake = _FakeS3() + client = _make_client(fake_s3=fake) + await client.upload_bundle(b"a", phase="phase_c", run_id="1", netuid=1) + await client.upload_bundle(b"b", phase="phase_c", run_id="2", netuid=1) + await client.upload_bundle(b"c", phase="dispatch", run_id="1", netuid=2) + + # Prefix scope: app_tag=radar netuid=1 → only the two phase_c objects. + refs = await client.list_by_metadata(app_tag="radar", netuid=1) + keys = sorted(r.key for r in refs) + assert len(keys) == 2 + assert all(k.startswith("radar/1/") for k in keys) + + +@pytest.mark.asyncio +async def test_list_by_metadata_tag_filter_keeps_matches(): + fake = _FakeS3() + client = _make_client(fake_s3=fake) + await client.upload_bundle(b"a", phase="phase_c", run_id="1") + await client.upload_bundle(b"b", phase="dispatch", run_id="2") + refs = await client.list_by_metadata( + app_tag="radar", match_tags={"phase": "dispatch"}, + ) + assert len(refs) == 1 + assert refs[0].tags["phase"] == "dispatch" + + +# ── End-to-end smoke (per Phase 2 spec) ────────────────────────────── + + +@pytest.mark.asyncio +async def test_smoke_upload_list_head_download_sha_matches(): + """Per spec: upload a 1 KB bundle, list, head, download, sha256 matches.""" + fake = _FakeS3() + client = _make_client(fake_s3=fake) + + payload = b"x" * 1024 + expected_sha = hashlib.sha256(payload).hexdigest() + + ref = await client.upload_bundle( + payload, app_tag="radar", phase="phase_c", run_id="42", netuid=1, + ) + listed = await client.list_by_metadata(app_tag="radar", netuid=1) + assert any(r.key == ref.key for r in listed) + head = await client.head_bundle(ref.key) + assert head.metadata["sha256"] == expected_sha + fetched = await client.download_bundle(ref.key) + assert hashlib.sha256(fetched).hexdigest() == expected_sha diff --git a/tests/test_substrate.py b/tests/test_substrate.py new file mode 100644 index 0000000..2a40c6c --- /dev/null +++ b/tests/test_substrate.py @@ -0,0 +1,269 @@ +"""Tests for shared/substrate.py — Phase C signed-record contract.""" + +from __future__ import annotations + +import dataclasses +import gzip +import hashlib + +import bittensor as bt +import pytest + +from shared.substrate import ( + SCHEMA_VERSION, + SIGNATURE_PREFIX, + PhaseCRecord, + bundle_sha256, + canonical_bytes, + record_from_json, + record_to_json, + records_from_bundle, + records_to_bundle, + sign_record, + verify_record, +) + + +# ── Test fixtures ──────────────────────────────────────────────────── + + +def _seed_hex(label: str) -> str: + """Deterministic 32-byte seed → 0x-prefixed hex string.""" + return "0x" + hashlib.sha256(label.encode()).hexdigest() + + +def _keypair(label: str) -> bt.Keypair: + """Make a deterministic Keypair for tests.""" + return bt.Keypair.create_from_seed(_seed_hex(label)) + + +class _FakeWallet: + """Just enough surface area for sign_record (wallet.hotkey).""" + + def __init__(self, keypair: bt.Keypair): + self.hotkey = keypair + + +@pytest.fixture +def wallet_a() -> _FakeWallet: + return _FakeWallet(_keypair("validator-a")) + + +@pytest.fixture +def wallet_b() -> _FakeWallet: + return _FakeWallet(_keypair("validator-b")) + + +def _make_record(wallet: _FakeWallet | None = None, **overrides) -> PhaseCRecord: + """Build a populated PhaseCRecord with sensible defaults.""" + base = dict( + schema_version=SCHEMA_VERSION, + round_id=42, + block_hash="0xabc123", + task="ts_forecasting", + miner_uid=7, + miner_hotkey="5MinerHotkeyExample", + code_hash="sha256:deadbeef", + architecture_sha256="sha256:11" * 1, + checkpoint_sha256="sha256:22", + metrics={"crps": 0.123, "mase": 0.456, "size": 1_234_567}, + passed_size_gate=True, + flops_verified=True, + eval_status="ok", + validator_uid=3, + validator_hotkey=( + wallet.hotkey.ss58_address if wallet else "5ValidatorHotkeyExample" + ), + validator_block_height=100_000, + timestamp=1_700_000_000.0, + signature="", + ) + base.update(overrides) + return PhaseCRecord(**base) + + +# ── canonical_bytes ─────────────────────────────────────────────────── + + +def test_canonical_bytes_deterministic(wallet_a): + record = _make_record(wallet_a) + a = canonical_bytes(record) + b = canonical_bytes(record) + assert a == b + # Independent reconstruction with the same inputs produces the same bytes. + record2 = _make_record(wallet_a) + assert canonical_bytes(record2) == a + + +def test_canonical_bytes_excludes_signature(wallet_a): + unsigned = _make_record(wallet_a) + signed = sign_record(unsigned, wallet_a) + # Both pre- and post-signing canonical bytes must match — otherwise the + # signature would change the hash it was computed over. + assert canonical_bytes(unsigned) == canonical_bytes(signed) + assert b"signature" not in canonical_bytes(signed) + + +# ── sign_record / verify_record ────────────────────────────────────── + + +def test_sign_verify_roundtrip(wallet_a): + record = _make_record(wallet_a) + signed = sign_record(record, wallet_a) + assert signed.signature.startswith(SIGNATURE_PREFIX) + assert len(bytes.fromhex(signed.signature[len(SIGNATURE_PREFIX):])) == 64 + ok, err = verify_record(signed) + assert ok, err + assert err == "" + + # Explicit expected_hotkey check + ok, err = verify_record(signed, expected_hotkey=wallet_a.hotkey.ss58_address) + assert ok, err + + +def test_verify_wrong_hotkey(wallet_a, wallet_b): + """Signed with A but the record claims hotkey B → verification fails.""" + # Record claims wallet_b but is signed by wallet_a's keypair. We bypass + # sign_record's hotkey-mismatch guard by signing manually. + record = _make_record(wallet_b) # validator_hotkey == B + raw_sig = wallet_a.hotkey.sign(canonical_bytes(record)) + forged = dataclasses.replace( + record, signature=f"{SIGNATURE_PREFIX}{raw_sig.hex()}" + ) + ok, err = verify_record(forged) + assert not ok + assert "signature" in err.lower() + + # And the explicit expected_hotkey path complains about identity. + signed_a = sign_record(_make_record(wallet_a), wallet_a) + ok, err = verify_record( + signed_a, expected_hotkey=wallet_b.hotkey.ss58_address, + ) + assert not ok + assert "validator_hotkey mismatch" in err + + +def test_verify_tampered_record(wallet_a): + signed = sign_record(_make_record(wallet_a), wallet_a) + tampered = dataclasses.replace(signed, metrics={"crps": 0.001}) + ok, err = verify_record(tampered) + assert not ok + assert "signature" in err.lower() + + +def test_verify_bad_signature(wallet_a): + record = _make_record(wallet_a) + garbage = dataclasses.replace(record, signature=f"{SIGNATURE_PREFIX}deadbeef") + ok, err = verify_record(garbage) + assert not ok + + not_hex = dataclasses.replace(record, signature=f"{SIGNATURE_PREFIX}zzzz") + ok, err = verify_record(not_hex) + assert not ok + assert "hex" in err.lower() + + no_prefix = dataclasses.replace(record, signature="abc123") + ok, err = verify_record(no_prefix) + assert not ok + assert SIGNATURE_PREFIX in err + + +def test_verify_bad_schema_version(wallet_a): + signed = sign_record(_make_record(wallet_a), wallet_a) + bumped = dataclasses.replace(signed, schema_version="radar.substrate.v2") + ok, err = verify_record(bumped) + assert not ok + assert "schema_version" in err + + +def test_sign_record_rejects_hotkey_mismatch(wallet_a, wallet_b): + """Refuse to sign a record whose validator_hotkey is someone else's.""" + record = _make_record(wallet_b) # claims wallet_b + with pytest.raises(ValueError, match="validator_hotkey mismatch"): + sign_record(record, wallet_a) # but we hold wallet_a + + +# ── JSON roundtrip ─────────────────────────────────────────────────── + + +def test_record_to_from_json(wallet_a): + signed = sign_record(_make_record(wallet_a), wallet_a) + s = record_to_json(signed) + parsed = record_from_json(s) + assert parsed == signed + # And it still verifies after the JSON trip. + ok, err = verify_record(parsed) + assert ok, err + + +def test_record_from_json_drops_unknown_fields(wallet_a): + signed = sign_record(_make_record(wallet_a), wallet_a) + s = record_to_json(signed) + # Splice in a fake forward-compat field — older parser must ignore it. + spliced = s[:-1] + ',"future_field":"surprise"}' + parsed = record_from_json(spliced) + assert parsed == signed + + +# ── Bundle ──────────────────────────────────────────────────────────── + + +def test_bundle_roundtrip(wallet_a): + records = [ + sign_record( + _make_record(wallet_a, round_id=i, miner_uid=i), + wallet_a, + ) + for i in range(5) + ] + blob = records_to_bundle(records) + parsed = records_from_bundle(blob) + assert len(parsed) == 5 + assert parsed == records + # Each parsed record still verifies cryptographically. + for r in parsed: + ok, err = verify_record(r) + assert ok, err + + +def test_bundle_is_gzipped(wallet_a): + records = [sign_record(_make_record(wallet_a), wallet_a)] + blob = records_to_bundle(records) + # Gzip magic number + assert blob[:2] == b"\x1f\x8b" + # gzip.decompress() succeeds and yields valid JSON. + decompressed = gzip.decompress(blob) + assert decompressed.startswith(b"{") + + +def test_bundle_accepts_plain_json(wallet_a): + """records_from_bundle should also accept uncompressed JSON for debugging.""" + records = [sign_record(_make_record(wallet_a), wallet_a)] + gz = records_to_bundle(records) + plain = gzip.decompress(gz) + parsed = records_from_bundle(plain) + assert parsed == records + + +def test_bundle_sha256_deterministic(wallet_a): + records = [ + sign_record(_make_record(wallet_a, round_id=i), wallet_a) + for i in range(3) + ] + a = bundle_sha256(records_to_bundle(records)) + b = bundle_sha256(records_to_bundle(records)) + assert a == b + assert a.startswith("sha256:") + assert len(a) == len("sha256:") + 64 + + +def test_bundle_rejects_wrong_schema_version(wallet_a): + records = [sign_record(_make_record(wallet_a), wallet_a)] + blob = records_to_bundle(records) + # Forge a bundle whose top-level schema_version doesn't match. + import json + payload = json.loads(gzip.decompress(blob)) + payload["schema_version"] = "radar.substrate.v999" + forged = gzip.compress(json.dumps(payload).encode()) + with pytest.raises(ValueError, match="schema_version"): + records_from_bundle(forged) diff --git a/tests/test_substrate_db.py b/tests/test_substrate_db.py new file mode 100644 index 0000000..f28f187 --- /dev/null +++ b/tests/test_substrate_db.py @@ -0,0 +1,515 @@ +"""Tests for TEN-240 Phase 5: substrate_cids schema, DB client, and API endpoints. + +Combines: + * `DataElement` field roundtrip + * `pg_schema` column threading (DDL string + INSERT_SQL + serialisation + helpers) without standing up a real Postgres + * `DatabaseClient.add_experiment` payload shape (with and without CID) + * `database/server.py` add + GET /verify endpoints + * `database/verify.verify_experiment` mocked-Hippius unit tests + * `database/neuron.DatabaseNeuron._init_hippius` wiring + +Each test is intentionally tight and uses no real Postgres / Hippius — the +DB-level invariants are covered structurally (DDL inspection + INSERT_SQL +arity match) so the suite stays fast on CI. +""" + +from __future__ import annotations + +import gzip +import hashlib +import json +from unittest.mock import AsyncMock, MagicMock, patch + +import bittensor as bt +import pytest +from fastapi.testclient import TestClient + +from shared.database import DataElement +from shared.db_client import DatabaseClient +from shared.pg_schema import ( + INSERT_SQL, + SCHEMA_INDEX_DDL, + SCHEMA_TABLE_DDL, + element_to_params, +) +from shared.substrate import ( + SCHEMA_VERSION, + PhaseCRecord, + records_to_bundle, + sign_record, +) + + +# ── DataElement ─────────────────────────────────────────────────────── + + +def test_dataelement_substrate_cids_default_empty(): + elem = DataElement() + assert elem.substrate_cids == [] + # Default factory must not be a shared list across instances. + elem.substrate_cids.append({"cid": "x"}) + assert DataElement().substrate_cids == [] + + +def test_dataelement_to_api_dict_includes_substrate_cids(): + entries = [{ + "kind": "phase_c_record", "validator_hotkey": "5G", "cid": "bafy", + "round_id": 7, + }] + elem = DataElement(substrate_cids=entries) + api = elem.to_api_dict() + assert api["substrate_cids"] == entries + + +def test_dataelement_to_api_dict_tolerates_jsonb_string(): + """Defense-in-depth: if a row arrives with substrate_cids as a JSON + string (no asyncpg JSONB codec), to_api_dict must still emit a list.""" + entries = [{"cid": "bafy"}] + elem = DataElement(substrate_cids=json.dumps(entries)) # type: ignore[arg-type] + api = elem.to_api_dict() + assert api["substrate_cids"] == entries + + +def test_dataelement_from_dict_roundtrip_carries_substrate_cids(): + entries = [{"cid": "bafy", "validator_hotkey": "5G"}] + elem = DataElement(substrate_cids=entries) + parsed = DataElement.from_dict(elem.to_dict()) + assert parsed.substrate_cids == entries + + +# ── pg_schema DDL & serialisation ──────────────────────────────────── + + +def test_schema_table_declares_substrate_cids(): + assert "substrate_cids JSONB NOT NULL DEFAULT '[]'" in SCHEMA_TABLE_DDL + + +def test_schema_index_declares_gin_on_substrate_cids(): + assert "idx_substrate_cids" in SCHEMA_INDEX_DDL + assert "GIN(substrate_cids)" in SCHEMA_INDEX_DDL + + +def test_insert_sql_param_count_includes_substrate_cids(): + """The INSERT statement now binds 24 params; element_to_params produces + a 24-tuple. Mismatched arity here would silently misorder columns at + runtime against a real DB.""" + elem = DataElement( + name="t", code="x", success=True, + substrate_cids=[{"cid": "bafy"}], + ) + params = element_to_params(elem, next_id=1) + assert len(params) == 24 + # Last param is the substrate_cids JSONB string, not raw list. + assert isinstance(params[-1], str) + assert json.loads(params[-1]) == [{"cid": "bafy"}] + # INSERT_SQL declares $24 and 24 columns — basic structural check. + assert "$24" in INSERT_SQL + assert "substrate_cids" in INSERT_SQL + + +# ── DatabaseClient.add_experiment payload shape ────────────────────── + + +class _MockWallet: + class hotkey: + ss58_address = "5FakeValidator" + + @staticmethod + def sign(msg): + return b"\x00" * 64 + + +def _patched_post(client, return_index: int = 7): + mock_resp = MagicMock() + mock_resp.status_code = 200 + mock_resp.json.return_value = {"index": return_index} + mock_resp.raise_for_status = MagicMock() + + mock_http = AsyncMock() + mock_http.post = AsyncMock(return_value=mock_resp) + + return mock_resp, mock_http + + +@pytest.mark.asyncio +async def test_add_experiment_without_substrate_cid_unchanged(): + """No substrate_cid → payload shape is exactly the legacy shape.""" + client = DatabaseClient(db_url="http://fake", wallet=_MockWallet()) + _, mock_http = _patched_post(client) + with patch.object(client, "_get_client", return_value=mock_http): + idx = await client.add_experiment({"name": "x"}) + assert idx == 7 + sent = json.loads(mock_http.post.call_args.kwargs["content"]) + assert sent == {"data": {"name": "x"}} + assert "substrate_cid" not in sent + assert "validator_hotkey" not in sent + + +@pytest.mark.asyncio +async def test_add_experiment_with_substrate_cid_includes_fields(): + client = DatabaseClient(db_url="http://fake", wallet=_MockWallet()) + _, mock_http = _patched_post(client) + with patch.object(client, "_get_client", return_value=mock_http): + idx = await client.add_experiment( + {"name": "x", "round_id": 99}, + substrate_cid="bafycid", + validator_hotkey="5Gxxxxx", + ) + assert idx == 7 + sent = json.loads(mock_http.post.call_args.kwargs["content"]) + assert sent["data"] == {"name": "x", "round_id": 99} + assert sent["substrate_cid"] == "bafycid" + assert sent["validator_hotkey"] == "5Gxxxxx" + + +# ── server.py: add_experiment + GET /verify endpoints ──────────────── + + +class _MockStore: + def __init__(self): + self._elements: list[DataElement] = [] + self.provenance = None + + async def get_size(self): + return len(self._elements) + + async def add(self, element: DataElement) -> int: + element.index = len(self._elements) + self._elements.append(element) + return element.index + + async def get(self, index: int): + for e in self._elements: + if e.index == index: + return e + return None + + +def _server_with_store(): + """Reset module-level injection slots and return a TestClient + store.""" + from database import server + store = _MockStore() + server.set_db(store) + server.set_hippius(None) # default state for the verify tests + return TestClient(server.app), store, server + + +def test_add_experiment_without_substrate_cid_legacy_shape(): + client, store, _ = _server_with_store() + r = client.post("/experiments/add", json={"data": {"name": "exp", "round_id": 1}}) + assert r.status_code == 200 + assert store._elements[0].substrate_cids == [] + + +def test_add_experiment_with_substrate_cid_appends_audit_entry(): + client, store, _ = _server_with_store() + r = client.post( + "/experiments/add", + json={ + "data": {"name": "exp", "round_id": 42, "miner_uid": 5}, + "substrate_cid": "bafyfakecid", + "validator_hotkey": "5Gabc", + }, + ) + assert r.status_code == 200 + elem = store._elements[0] + assert elem.substrate_cids == [{ + "kind": "phase_c_record", + "validator_hotkey": "5Gabc", + "cid": "bafyfakecid", + "round_id": 42, + }] + + +def test_add_experiment_with_artifact_cids_appends_each(): + """Phase 7: artifact_cids land in the same substrate_cids list with + their declared kinds (checkpoint/architecture/training_meta). + Entries without a ``cid`` are dropped server-side so a half-built + dual-write doesn't poison the audit list.""" + client, store, _ = _server_with_store() + r = client.post( + "/experiments/add", + json={ + "data": {"name": "exp", "round_id": 42, "miner_uid": 5}, + "substrate_cid": "bafyrec", + "validator_hotkey": "5Gabc", + "artifact_cids": [ + {"kind": "checkpoint", "cid": "bafyckpt"}, + {"kind": "architecture", "cid": "bafyarch"}, + {"kind": "training_meta", "cid": "bafymeta"}, + {"kind": "noop"}, # missing cid — dropped + ], + }, + ) + assert r.status_code == 200 + cids = store._elements[0].substrate_cids + kinds = [c["kind"] for c in cids] + assert kinds == [ + "phase_c_record", "checkpoint", "architecture", "training_meta", + ] + assert {c["cid"] for c in cids} == { + "bafyrec", "bafyckpt", "bafyarch", "bafymeta", + } + + +def test_add_experiment_artifact_cids_pydantic_rejects_non_dicts(): + """Strict input validation: a string in artifact_cids → 422 Unprocessable. + + Catches validator-side bugs early rather than silently filtering them + out and producing partial audit lists.""" + client, _, _ = _server_with_store() + r = client.post( + "/experiments/add", + json={ + "data": {"name": "exp", "round_id": 42}, + "artifact_cids": ["not a dict"], + }, + ) + assert r.status_code == 422 + + +def test_get_experiment_response_includes_substrate_cids(): + client, store, _ = _server_with_store() + store._elements.append(DataElement( + index=0, name="exp", round_id=42, miner_uid=5, + substrate_cids=[{"cid": "bafy", "validator_hotkey": "5G", "round_id": 42}], + )) + r = client.get("/experiments/0") + assert r.status_code == 200 + body = r.json() + assert body["substrate_cids"] == [ + {"cid": "bafy", "validator_hotkey": "5G", "round_id": 42} + ] + + +def test_verify_endpoint_503_when_hippius_unconfigured(): + client, store, _ = _server_with_store() + store._elements.append(DataElement(index=0, round_id=42, miner_uid=5)) + r = client.get("/experiments/0/verify") + assert r.status_code == 503 + assert "Hippius" in r.json()["detail"] + + +def test_verify_endpoint_404_for_missing_experiment(): + client, _, _ = _server_with_store() + r = client.get("/experiments/99/verify") + assert r.status_code == 404 + + +def test_verify_endpoint_empty_when_no_substrate_cids(): + """An experiment with no CIDs returns an empty verifications list.""" + client, store, server = _server_with_store() + store._elements.append(DataElement(index=0, round_id=42, miner_uid=5)) + server.set_hippius(MagicMock()) # any non-None client + r = client.get("/experiments/0/verify") + assert r.status_code == 200 + body = r.json() + assert body["experiment_id"] == 0 + assert body["substrate_cids"] == [] + assert body["verifications"] == [] + + +def test_verify_endpoint_returns_per_cid_results(): + """End-to-end: signed bundle on mocked Hippius matches a stored experiment.""" + client, store, server = _server_with_store() + + kp = bt.Keypair.create_from_seed( + "0x" + hashlib.sha256(b"validator-1").hexdigest(), + ) + record = PhaseCRecord( + schema_version=SCHEMA_VERSION, + round_id=42, block_hash="0xabc", task="ts_forecasting", + miner_uid=5, miner_hotkey="5MinerKey", + code_hash="sha256:c", architecture_sha256="a", checkpoint_sha256="c", + metrics={"crps": 0.123, "mase": 0.4, "passed_size_gate": True}, + passed_size_gate=True, flops_verified=True, eval_status="ok", + validator_uid=0, validator_hotkey=kp.ss58_address, + validator_block_height=100_000, timestamp=1.0, + ) + + class _W: + hotkey = kp + + signed = sign_record(record, _W()) + bundle = records_to_bundle([signed]) + + elem = DataElement( + index=0, name="exp", round_id=42, miner_uid=5, miner_hotkey="5MinerKey", + objectives={"crps": 0.123, "mase": 0.4, "passed_size_gate": True}, + substrate_cids=[{ + "kind": "phase_c_record", "validator_hotkey": kp.ss58_address, + "cid": "bafycid", "round_id": 42, + }], + ) + store._elements.append(elem) + + fake_hippius = MagicMock() + fake_hippius.download_bundle = AsyncMock(return_value=bundle) + server.set_hippius(fake_hippius) + + r = client.get("/experiments/0/verify") + assert r.status_code == 200 + body = r.json() + assert body["experiment_id"] == 0 + assert len(body["verifications"]) == 1 + v = body["verifications"][0] + assert v["fetchable"] is True + assert v["signature_valid"] is True + assert v["matches_db"] is True + assert v["discrepancies"] == [] + fake_hippius.download_bundle.assert_awaited_once_with("bafycid") + + +def test_verify_endpoint_flags_metric_mismatch(): + """When DB and bundle disagree on a metric, matches_db=False with a diff.""" + client, store, server = _server_with_store() + kp = bt.Keypair.create_from_seed( + "0x" + hashlib.sha256(b"validator-2").hexdigest(), + ) + record = PhaseCRecord( + schema_version=SCHEMA_VERSION, + round_id=42, block_hash="0xabc", task="ts_forecasting", + miner_uid=5, miner_hotkey="5MinerKey", + code_hash="sha256:c", architecture_sha256="a", checkpoint_sha256="c", + metrics={"crps": 0.123}, + passed_size_gate=True, flops_verified=True, eval_status="ok", + validator_uid=0, validator_hotkey=kp.ss58_address, + validator_block_height=100_000, timestamp=1.0, + ) + + class _W: + hotkey = kp + + signed = sign_record(record, _W()) + bundle = records_to_bundle([signed]) + + elem = DataElement( + index=0, round_id=42, miner_uid=5, miner_hotkey="5MinerKey", + objectives={"crps": 0.999}, # DB disagrees + substrate_cids=[{ + "kind": "phase_c_record", "validator_hotkey": kp.ss58_address, + "cid": "bafycid", "round_id": 42, + }], + ) + store._elements.append(elem) + + fake_hippius = MagicMock() + fake_hippius.download_bundle = AsyncMock(return_value=bundle) + server.set_hippius(fake_hippius) + + body = client.get("/experiments/0/verify").json() + v = body["verifications"][0] + assert v["signature_valid"] is True + assert v["matches_db"] is False + assert any("crps" in d for d in v["discrepancies"]) + + +def test_verify_endpoint_fetch_failure_surfaces_in_discrepancies(): + client, store, server = _server_with_store() + elem = DataElement( + index=0, round_id=42, miner_uid=5, + substrate_cids=[{ + "kind": "phase_c_record", "validator_hotkey": "5G", + "cid": "bafycid", "round_id": 42, + }], + ) + store._elements.append(elem) + + fake_hippius = MagicMock() + fake_hippius.download_bundle = AsyncMock(side_effect=RuntimeError("ipfs flap")) + server.set_hippius(fake_hippius) + + body = client.get("/experiments/0/verify").json() + v = body["verifications"][0] + assert v["fetchable"] is False + assert v["signature_valid"] is False + assert any("fetch failed" in d for d in v["discrepancies"]) + + +def test_verify_endpoint_corrupt_bundle_flagged(): + client, store, server = _server_with_store() + elem = DataElement( + index=0, round_id=42, miner_uid=5, + substrate_cids=[{ + "kind": "phase_c_record", "validator_hotkey": "5G", + "cid": "bafycid", "round_id": 42, + }], + ) + store._elements.append(elem) + + fake_hippius = MagicMock() + fake_hippius.download_bundle = AsyncMock(return_value=b"not a bundle") + server.set_hippius(fake_hippius) + + body = client.get("/experiments/0/verify").json() + v = body["verifications"][0] + assert v["fetchable"] is True + assert v["signature_valid"] is False + assert any("parse failed" in d for d in v["discrepancies"]) + + +# ── DatabaseNeuron._init_hippius wiring ────────────────────────────── + + +def test_database_neuron_init_hippius_disabled_returns_none(): + from database.neuron import DatabaseNeuron + neuron = DatabaseNeuron.__new__(DatabaseNeuron) + with patch("database.neuron.Config") as cfg: + cfg.HIPPIUS_ENABLED = False + assert neuron._init_hippius() is None + + +def test_database_neuron_init_hippius_missing_sdk_returns_none(caplog): + from database.neuron import DatabaseNeuron + neuron = DatabaseNeuron.__new__(DatabaseNeuron) + import builtins + real_import = builtins.__import__ + + def _missing(name, *args, **kwargs): + if name == "shared.hippius_client": + raise ImportError("not yet") + return real_import(name, *args, **kwargs) + + with patch("database.neuron.Config") as cfg, \ + patch("builtins.__import__", side_effect=_missing), \ + caplog.at_level("WARNING"): + cfg.HIPPIUS_ENABLED = True + cfg.HIPPIUS_IPFS_API_URL = "" + cfg.HIPPIUS_KEY = "" + cfg.HIPPIUS_SUBSTRATE_RPC = "" + result = neuron._init_hippius() + + assert result is None + assert any("not available yet" in m for m in caplog.messages) + + +# ── Sanity: bundle helper round-trip used by /verify ───────────────── + + +def test_records_from_bundle_handles_gzip_and_plain(): + """Reaffirm the publisher↔server contract that /verify relies on.""" + kp = bt.Keypair.create_from_seed( + "0x" + hashlib.sha256(b"sanity").hexdigest(), + ) + + class _W: + hotkey = kp + + rec = PhaseCRecord( + schema_version=SCHEMA_VERSION, + round_id=1, block_hash="b", task="t", + miner_uid=1, miner_hotkey="h", + code_hash="c", architecture_sha256="a", checkpoint_sha256="c", + metrics={"x": 1}, passed_size_gate=True, flops_verified=True, + eval_status="ok", + validator_uid=0, validator_hotkey=kp.ss58_address, + validator_block_height=1, timestamp=1.0, + ) + signed = sign_record(rec, _W()) + gz = records_to_bundle([signed]) + assert gz[:2] == b"\x1f\x8b" + plain = gzip.decompress(gz) + from shared.substrate import records_from_bundle + assert records_from_bundle(gz) == [signed] + assert records_from_bundle(plain) == [signed] diff --git a/tests/test_substrate_publisher.py b/tests/test_substrate_publisher.py new file mode 100644 index 0000000..2614c3e --- /dev/null +++ b/tests/test_substrate_publisher.py @@ -0,0 +1,398 @@ +"""Tests for validator/substrate_publisher.py.""" + +from __future__ import annotations + +import hashlib +from dataclasses import dataclass +from unittest.mock import AsyncMock + +import bittensor as bt +import pytest + +from shared.substrate import ( + SCHEMA_VERSION, + bundle_sha256, + records_from_bundle, + verify_record, +) +from validator.substrate_publisher import ( + UploadResult, + build_phase_c_records, + publish_phase_c_records, +) + + +# ── Fixtures ────────────────────────────────────────────────────────── + + +def _seed_hex(label: str) -> str: + return "0x" + hashlib.sha256(label.encode()).hexdigest() + + +def _keypair(label: str) -> bt.Keypair: + return bt.Keypair.create_from_seed(_seed_hex(label)) + + +class _FakeWallet: + def __init__(self, kp: bt.Keypair): + self.hotkey = kp + + +@dataclass +class _FakeChallenge: + round_id: int = 42 + + +@dataclass +class _FakeMetagraph: + hotkeys: list # list[str] + + +@dataclass +class _FakeCommitment: + code_hash: str + + +@pytest.fixture +def wallet(): + return _FakeWallet(_keypair("validator")) + + +@pytest.fixture +def metagraph(): + # 4 hotkeys for UIDs 0..3 + return _FakeMetagraph(hotkeys=[ + _keypair(f"miner-{i}").ss58_address for i in range(4) + ]) + + +@pytest.fixture +def commitments(): + return { + 1: _FakeCommitment(code_hash="sha256:miner-one-code"), + 2: _FakeCommitment(code_hash="sha256:miner-two-code"), + 3: _FakeCommitment(code_hash="sha256:miner-three-code"), + } + + +@pytest.fixture +def training_metas(): + return { + 1: { + "architecture_sha256": "arch-hash-1", + "checkpoint_sha256": "ckpt-hash-1", + "status": "success", + }, + 2: { + "architecture_sha256": "arch-hash-2", + "checkpoint_sha256": "ckpt-hash-2", + "status": "success", + }, + 3: { + "architecture_sha256": "arch-hash-3", + "checkpoint_sha256": "ckpt-hash-3", + "status": "success", + }, + } + + +def _eval_ok(crps: float = 0.123) -> dict: + return { + "crps": crps, + "mase": 0.456, + "flops_equivalent_size": 1_500_000, + "passed_size_gate": True, + "flops_verified": True, + } + + +# ── build_phase_c_records ──────────────────────────────────────────── + + +@pytest.mark.asyncio +async def test_build_records_produces_one_per_miner( + wallet, metagraph, commitments, training_metas, +): + eval_results = {1: _eval_ok(), 2: _eval_ok(0.1), 3: _eval_ok(0.2)} + records = await build_phase_c_records( + wallet=wallet, challenge=_FakeChallenge(), + eval_results=eval_results, training_metas=training_metas, + commitments=commitments, metagraph=metagraph, + my_uid=0, current_block=100_000, + task_name="ts_forecasting", block_hash="0xdeadbeef", + ) + assert len(records) == 3 + assert {r.miner_uid for r in records} == {1, 2, 3} + + +@pytest.mark.asyncio +async def test_build_records_eval_status_size_gate_failed( + wallet, metagraph, commitments, training_metas, +): + eval_results = {1: {**_eval_ok(), "passed_size_gate": False}} + records = await build_phase_c_records( + wallet=wallet, challenge=_FakeChallenge(), + eval_results=eval_results, training_metas=training_metas, + commitments=commitments, metagraph=metagraph, + my_uid=0, current_block=100_000, + task_name="ts_forecasting", block_hash="0xabc", + ) + assert records[0].eval_status == "size_gate_failed" + assert records[0].passed_size_gate is False + + +@pytest.mark.asyncio +async def test_build_records_eval_status_ok( + wallet, metagraph, commitments, training_metas, +): + records = await build_phase_c_records( + wallet=wallet, challenge=_FakeChallenge(), + eval_results={1: _eval_ok()}, + training_metas=training_metas, commitments=commitments, + metagraph=metagraph, my_uid=0, current_block=100_000, + task_name="ts_forecasting", block_hash="0xabc", + ) + assert records[0].eval_status == "ok" + assert records[0].passed_size_gate is True + assert records[0].flops_verified is True + + +@pytest.mark.asyncio +async def test_build_records_eval_status_eval_failed( + wallet, metagraph, commitments, training_metas, +): + eval_results = {1: { + "passed_size_gate": True, + "flops_verified": True, + "error": "runner crashed", + }} + records = await build_phase_c_records( + wallet=wallet, challenge=_FakeChallenge(), + eval_results=eval_results, training_metas=training_metas, + commitments=commitments, metagraph=metagraph, + my_uid=0, current_block=100_000, + task_name="ts_forecasting", block_hash="0xabc", + ) + assert records[0].eval_status == "eval_failed" + + +@pytest.mark.asyncio +async def test_build_records_eval_status_no_metric( + wallet, metagraph, commitments, training_metas, +): + """Gate passed, no error, but no numeric metric in dict → no_metric.""" + eval_results = {1: {"passed_size_gate": True, "flops_verified": True}} + records = await build_phase_c_records( + wallet=wallet, challenge=_FakeChallenge(), + eval_results=eval_results, training_metas=training_metas, + commitments=commitments, metagraph=metagraph, + my_uid=0, current_block=100_000, + task_name="ts_forecasting", block_hash="0xabc", + ) + assert records[0].eval_status == "no_metric" + + +@pytest.mark.asyncio +async def test_build_records_signatures_valid( + wallet, metagraph, commitments, training_metas, +): + eval_results = {1: _eval_ok(), 2: _eval_ok(0.1), 3: _eval_ok(0.2)} + records = await build_phase_c_records( + wallet=wallet, challenge=_FakeChallenge(), + eval_results=eval_results, training_metas=training_metas, + commitments=commitments, metagraph=metagraph, + my_uid=0, current_block=100_000, + task_name="ts_forecasting", block_hash="0xabc", + ) + for r in records: + ok, err = verify_record(r, expected_hotkey=wallet.hotkey.ss58_address) + assert ok, err + + +@pytest.mark.asyncio +async def test_build_records_includes_miner_hotkey( + wallet, metagraph, commitments, training_metas, +): + records = await build_phase_c_records( + wallet=wallet, challenge=_FakeChallenge(), + eval_results={2: _eval_ok()}, + training_metas=training_metas, commitments=commitments, + metagraph=metagraph, my_uid=0, current_block=100_000, + task_name="ts_forecasting", block_hash="0xabc", + ) + assert records[0].miner_hotkey == metagraph.hotkeys[2] + assert records[0].miner_hotkey # non-empty + + +@pytest.mark.asyncio +async def test_build_records_includes_code_hash( + wallet, metagraph, commitments, training_metas, +): + records = await build_phase_c_records( + wallet=wallet, challenge=_FakeChallenge(), + eval_results={2: _eval_ok()}, + training_metas=training_metas, commitments=commitments, + metagraph=metagraph, my_uid=0, current_block=100_000, + task_name="ts_forecasting", block_hash="0xabc", + ) + assert records[0].code_hash == "sha256:miner-two-code" + + +@pytest.mark.asyncio +async def test_build_records_tolerates_missing_commitment( + wallet, metagraph, training_metas, +): + """A miner that has an eval result but no commitment yields code_hash=''.""" + records = await build_phase_c_records( + wallet=wallet, challenge=_FakeChallenge(), + eval_results={1: _eval_ok()}, + training_metas=training_metas, commitments={}, + metagraph=metagraph, my_uid=0, current_block=100_000, + task_name="ts_forecasting", block_hash="0xabc", + ) + assert records[0].code_hash == "" + + +@pytest.mark.asyncio +async def test_build_records_filters_non_serialisable_metrics( + wallet, metagraph, commitments, training_metas, +): + """List/dict/NaN values must be dropped before signing.""" + eval_results = {1: { + "crps": 0.1, + "passed_size_gate": True, + "flops_verified": True, + "nan_metric": float("nan"), + "inf_metric": float("inf"), + "list_metric": [1, 2, 3], + "dict_metric": {"nested": True}, + "note": "fine string", + }} + records = await build_phase_c_records( + wallet=wallet, challenge=_FakeChallenge(), + eval_results=eval_results, training_metas=training_metas, + commitments=commitments, metagraph=metagraph, + my_uid=0, current_block=100_000, + task_name="ts_forecasting", block_hash="0xabc", + ) + metrics = records[0].metrics + assert metrics["crps"] == 0.1 + assert metrics["note"] == "fine string" + assert "nan_metric" not in metrics + assert "inf_metric" not in metrics + assert "list_metric" not in metrics + assert "dict_metric" not in metrics + + +@pytest.mark.asyncio +async def test_build_records_includes_sha256_fields( + wallet, metagraph, commitments, training_metas, +): + records = await build_phase_c_records( + wallet=wallet, challenge=_FakeChallenge(), + eval_results={2: _eval_ok()}, + training_metas=training_metas, commitments=commitments, + metagraph=metagraph, my_uid=0, current_block=100_000, + task_name="ts_forecasting", block_hash="0xabc", + ) + assert records[0].architecture_sha256 == "arch-hash-2" + assert records[0].checkpoint_sha256 == "ckpt-hash-2" + + +# ── publish_phase_c_records ────────────────────────────────────────── + + +@pytest.mark.asyncio +async def test_publish_success_returns_result( + wallet, metagraph, commitments, training_metas, +): + records = await build_phase_c_records( + wallet=wallet, challenge=_FakeChallenge(), + eval_results={1: _eval_ok(), 2: _eval_ok(0.2)}, + training_metas=training_metas, commitments=commitments, + metagraph=metagraph, my_uid=0, current_block=100_000, + task_name="ts_forecasting", block_hash="0xabc", + ) + expected = UploadResult( + key="radar/0/phase_c/42/abcdef0123456789.tar", + metadata={"app_tag": "radar", "phase": "phase_c", "run_id": "42"}, + tags={"app_tag": "radar", "phase": "phase_c", "netuid": "0"}, + size=512, etag="deadbeef", + ) + client = type("C", (), {})() + client.upload_bundle = AsyncMock(return_value=expected) + + result = await publish_phase_c_records( + client, records, round_id=42, + validator_hotkey=wallet.hotkey.ss58_address, + ) + + assert result is expected + client.upload_bundle.assert_awaited_once() + # New structured API: positional bundle bytes, then keyword args. + call = client.upload_bundle.call_args + bundle_bytes = call.args[0] + kwargs = call.kwargs + assert kwargs["app_tag"] == "radar" + assert kwargs["phase"] == "phase_c" + assert kwargs["run_id"] == "42" + extra = kwargs["extra_metadata"] + assert extra["schema_version"] == SCHEMA_VERSION + assert extra["validator_hotkey"] == wallet.hotkey.ss58_address + assert extra["record_count"] == "2" + assert extra["bundle_sha256"] == bundle_sha256(bundle_bytes) + # Bundle round-trips back to the same records (cryptographically intact). + parsed = records_from_bundle(bundle_bytes) + assert parsed == records + + +@pytest.mark.asyncio +async def test_publish_failure_returns_none( + wallet, metagraph, commitments, training_metas, caplog, +): + records = await build_phase_c_records( + wallet=wallet, challenge=_FakeChallenge(), + eval_results={1: _eval_ok()}, + training_metas=training_metas, commitments=commitments, + metagraph=metagraph, my_uid=0, current_block=100_000, + task_name="ts_forecasting", block_hash="0xabc", + ) + client = type("C", (), {})() + client.upload_bundle = AsyncMock(side_effect=RuntimeError("hippius down")) + with caplog.at_level("WARNING"): + result = await publish_phase_c_records( + client, records, round_id=42, validator_hotkey="5xyz", + ) + assert result is None + assert any("Substrate publish failed" in m for m in caplog.messages) + + +@pytest.mark.asyncio +async def test_publish_empty_records_no_call(): + client = type("C", (), {})() + client.upload_bundle = AsyncMock() + result = await publish_phase_c_records( + client, records=[], round_id=42, validator_hotkey="5xyz", + ) + assert result is None + client.upload_bundle.assert_not_called() + + +@pytest.mark.asyncio +async def test_publish_no_client_returns_none( + wallet, metagraph, commitments, training_metas, caplog, +): + """A None client (operator hasn't opted in to substrate yet) must not raise.""" + records = await build_phase_c_records( + wallet=wallet, challenge=_FakeChallenge(), + eval_results={1: _eval_ok()}, + training_metas=training_metas, commitments=commitments, + metagraph=metagraph, my_uid=0, current_block=100_000, + task_name="ts_forecasting", block_hash="0xabc", + ) + with caplog.at_level("WARNING"): + result = await publish_phase_c_records( + None, records, round_id=42, validator_hotkey="5xyz", + ) + assert result is None + assert any("no Hippius client" in m for m in caplog.messages) diff --git a/tests/test_validator_substrate_integration.py b/tests/test_validator_substrate_integration.py new file mode 100644 index 0000000..c33d270 --- /dev/null +++ b/tests/test_validator_substrate_integration.py @@ -0,0 +1,272 @@ +"""Integration tests for Phase 4: substrate publishing wired into validator/neuron.py. + +The full `Validator.run_round` pulls in bittensor, subtensor, agent pods, and +trainer dispatch — far too much to drive end-to-end in a unit test. We test +the integration in two layers instead: + + * `Validator._init_hippius`: instantiate `Validator` via ``__new__`` (the + pattern other validator tests already use) and exercise the helper that + Phase 4 added to ``__init__``. + * `run_substrate_publish_step`: the helper Phase 4 inserts after + ``score_round``. Driving it with mocked inputs proves the round-loop + contract: disabled → no client call, enabled → exactly one upload per + round, failure → empty dict + no exception. +""" + +from __future__ import annotations + +import hashlib +from dataclasses import dataclass +from unittest.mock import AsyncMock, patch + +import bittensor as bt +import pytest + +from validator.substrate_publisher import ( + UploadResult, + run_substrate_publish_step, +) + + +# ── Fixtures ────────────────────────────────────────────────────────── + + +def _seed_hex(label: str) -> str: + return "0x" + hashlib.sha256(label.encode()).hexdigest() + + +def _keypair(label: str) -> bt.Keypair: + return bt.Keypair.create_from_seed(_seed_hex(label)) + + +class _FakeWallet: + def __init__(self, kp: bt.Keypair): + self.hotkey = kp + + +@dataclass +class _FakeChallenge: + round_id: int = 7 + + +@dataclass +class _FakeMetagraph: + hotkeys: list + + +@dataclass +class _FakeCommitment: + code_hash: str + + +def _round_inputs(num_miners: int = 3) -> dict: + """Build a realistic-shaped set of run_round locals for the helper.""" + wallet = _FakeWallet(_keypair("validator")) + metagraph = _FakeMetagraph(hotkeys=[ + _keypair(f"miner-{i}").ss58_address for i in range(num_miners + 1) + ]) + eval_results = { + uid: { + "crps": 0.1 + 0.01 * uid, + "mase": 0.5, + "flops_equivalent_size": 1_500_000, + "passed_size_gate": True, + "flops_verified": True, + } + for uid in range(1, num_miners + 1) + } + training_metas = { + uid: { + "architecture_sha256": f"arch-{uid}", + "checkpoint_sha256": f"ckpt-{uid}", + "status": "success", + } + for uid in range(1, num_miners + 1) + } + commitments = { + uid: _FakeCommitment(code_hash=f"sha256:code-{uid}") + for uid in range(1, num_miners + 1) + } + return dict( + wallet=wallet, challenge=_FakeChallenge(), + eval_results=eval_results, training_metas=training_metas, + commitments=commitments, metagraph=metagraph, + my_uid=0, current_block=100_000, + task_name="ts_forecasting", block_hash="0xabc", + ) + + +# ── run_substrate_publish_step (the round-loop hook) ───────────────── + + +@pytest.mark.asyncio +async def test_round_with_hippius_disabled_unchanged(): + """hippius=None mirrors HIPPIUS_ENABLED=false — no client work, empty dict.""" + inputs = _round_inputs() + cids = await run_substrate_publish_step(hippius=None, **inputs) + assert cids == {} + + +@pytest.mark.asyncio +async def test_round_with_hippius_enabled_publishes(): + """A configured client gets called exactly once with the structured args, + and every miner's UID maps to the returned bundle key.""" + inputs = _round_inputs(num_miners=3) + upload = UploadResult( + key="radar/0/phase_c/7/abcdef0123456789.tar", + metadata={"app_tag": "radar", "phase": "phase_c", "run_id": "7"}, + tags={"app_tag": "radar", "phase": "phase_c", "netuid": "0"}, + size=1024, etag="deadbeef", + ) + client = type("C", (), {})() + client.upload_bundle = AsyncMock(return_value=upload) + + cids = await run_substrate_publish_step(hippius=client, **inputs) + + client.upload_bundle.assert_awaited_once() + call = client.upload_bundle.call_args + bundle_bytes = call.args[0] + assert isinstance(bundle_bytes, (bytes, bytearray)) + assert call.kwargs["app_tag"] == "radar" + assert call.kwargs["phase"] == "phase_c" + assert call.kwargs["run_id"] == "7" + assert call.kwargs["extra_metadata"]["record_count"] == "3" + # All miners share the same bundle key (one bundle per round). + assert cids == {1: upload.key, 2: upload.key, 3: upload.key} + + +@pytest.mark.asyncio +async def test_round_with_hippius_enabled_publish_failure_doesnt_crash(caplog): + """A raising client must not propagate — round must keep running.""" + inputs = _round_inputs() + client = type("C", (), {})() + client.upload_bundle = AsyncMock(side_effect=RuntimeError("network down")) + + with caplog.at_level("WARNING"): + cids = await run_substrate_publish_step(hippius=client, **inputs) + + assert cids == {} + client.upload_bundle.assert_awaited_once() + # The publish layer logs the failure rather than letting it propagate. + assert any("Substrate publish failed" in m for m in caplog.messages) + + +@pytest.mark.asyncio +async def test_round_with_no_eval_results_skips_upload(): + """A round that produced zero eval results should not burn an upload.""" + inputs = _round_inputs() + inputs["eval_results"] = {} + client = type("C", (), {})() + client.upload_bundle = AsyncMock() + + cids = await run_substrate_publish_step(hippius=client, **inputs) + + assert cids == {} + client.upload_bundle.assert_not_called() + + +@pytest.mark.asyncio +async def test_round_publish_returning_none_yields_empty_cids(): + """When the client signals failure (returns None), no CIDs get propagated.""" + inputs = _round_inputs() + client = type("C", (), {})() + client.upload_bundle = AsyncMock(return_value=None) + + cids = await run_substrate_publish_step(hippius=client, **inputs) + + assert cids == {} + client.upload_bundle.assert_awaited_once() + + +# ── Validator._init_hippius wiring ─────────────────────────────────── + + +def _make_validator_for_init_test(): + """Skip __init__ (which constructs bittensor) and return a bare instance.""" + from validator.neuron import Validator + return Validator.__new__(Validator) + + +def test_init_hippius_returns_none_when_disabled(): + """Default config (HIPPIUS_ENABLED=false) yields self.hippius = None.""" + validator = _make_validator_for_init_test() + with patch("validator.neuron.Config") as cfg: + cfg.HIPPIUS_ENABLED = False + assert validator._init_hippius() is None + + +def test_init_hippius_returns_none_when_sdk_missing(caplog): + """HIPPIUS_ENABLED=true with the wrapper not yet shipped: warn + None.""" + validator = _make_validator_for_init_test() + import builtins + + real_import = builtins.__import__ + + def _missing_hippius_client(name, *args, **kwargs): + if name == "shared.hippius_client": + raise ImportError("no such module") + return real_import(name, *args, **kwargs) + + with patch("validator.neuron.Config") as cfg, \ + patch("builtins.__import__", side_effect=_missing_hippius_client), \ + caplog.at_level("WARNING"): + cfg.HIPPIUS_ENABLED = True + cfg.HIPPIUS_IPFS_API_URL = "https://ipfs" + cfg.HIPPIUS_KEY = "k" + cfg.HIPPIUS_SUBSTRATE_RPC = "rpc" + result = validator._init_hippius() + + assert result is None + assert any("not available yet" in m for m in caplog.messages) + + +def test_init_hippius_constructs_when_enabled(): + """With a stub HippiusClient available, init returns the constructed client.""" + validator = _make_validator_for_init_test() + + class _StubHippiusClient: + def __init__(self, **kwargs): + self.kwargs = kwargs + + # Inject a fake `shared.hippius_client` module so the lazy import succeeds. + import sys, types + stub_mod = types.ModuleType("shared.hippius_client") + stub_mod.HippiusClient = _StubHippiusClient + with patch("validator.neuron.Config") as cfg, \ + patch.dict(sys.modules, {"shared.hippius_client": stub_mod}): + cfg.HIPPIUS_ENABLED = True + cfg.HIPPIUS_IPFS_API_URL = "https://ipfs.example" + cfg.HIPPIUS_KEY = "key123" + cfg.HIPPIUS_SUBSTRATE_RPC = "wss://rpc" + client = validator._init_hippius() + + assert isinstance(client, _StubHippiusClient) + assert client.kwargs == { + "ipfs_api_url": "https://ipfs.example", + "hippius_key": "key123", + "substrate_rpc": "wss://rpc", + } + + +def test_init_hippius_swallows_constructor_errors(caplog): + """Hippius client construction failure must downgrade to None, not crash.""" + validator = _make_validator_for_init_test() + + class _BoomClient: + def __init__(self, **kwargs): + raise RuntimeError("ipfs unreachable") + + import sys, types + stub_mod = types.ModuleType("shared.hippius_client") + stub_mod.HippiusClient = _BoomClient + with patch("validator.neuron.Config") as cfg, \ + patch.dict(sys.modules, {"shared.hippius_client": stub_mod}), \ + caplog.at_level("WARNING"): + cfg.HIPPIUS_ENABLED = True + cfg.HIPPIUS_IPFS_API_URL = "" + cfg.HIPPIUS_KEY = "" + cfg.HIPPIUS_SUBSTRATE_RPC = "" + result = validator._init_hippius() + + assert result is None + assert any("Hippius client init failed" in m for m in caplog.messages) diff --git a/tests/test_verify_radar_tool.py b/tests/test_verify_radar_tool.py new file mode 100644 index 0000000..e10c412 --- /dev/null +++ b/tests/test_verify_radar_tool.py @@ -0,0 +1,251 @@ +"""Tests for tools/verify_radar_experiment.py. + +The CLI talks to two external systems (a RADAR API and a Hippius IPFS +gateway). We mock both ends — the API via httpx.AsyncClient, the Hippius +client by patching ``HippiusClient`` inside the tool module. +""" + +from __future__ import annotations + +import hashlib +import io +from unittest.mock import AsyncMock, MagicMock, patch + +import bittensor as bt +import httpx +import pytest + +from shared.substrate import ( + SCHEMA_VERSION, + PhaseCRecord, + records_to_bundle, + sign_record, +) +from tools.verify_radar_experiment import main + + +# ── Fixtures ────────────────────────────────────────────────────────── + + +def _kp(label: str) -> bt.Keypair: + return bt.Keypair.create_from_seed( + "0x" + hashlib.sha256(label.encode()).hexdigest(), + ) + + +def _make_signed_bundle( + *, + keypair: bt.Keypair, + round_id: int = 42, + miner_uid: int = 5, + miner_hotkey: str = "5MinerHK", + metrics: dict | None = None, +) -> bytes: + """Build a single-record gzip bundle, signed by ``keypair``.""" + rec = PhaseCRecord( + schema_version=SCHEMA_VERSION, + round_id=round_id, block_hash="0xabc", task="ts_forecasting", + miner_uid=miner_uid, miner_hotkey=miner_hotkey, + code_hash="sha256:c", architecture_sha256="a", checkpoint_sha256="c", + metrics=metrics if metrics is not None else {"crps": 0.123, "mase": 0.4}, + passed_size_gate=True, flops_verified=True, eval_status="ok", + validator_uid=0, validator_hotkey=keypair.ss58_address, + validator_block_height=100_000, timestamp=1.0, + ) + + class _W: + hotkey = keypair + + return records_to_bundle([sign_record(rec, _W())]) + + +def _api_response( + *, + cids: list[dict] | None = None, + metrics: dict | None = None, + miner_uid: int = 5, + miner_hotkey: str = "5MinerHK", + round_id: int = 42, +) -> dict: + return { + "index": 7, + "round_id": round_id, + "miner_uid": miner_uid, + "miner_hotkey": miner_hotkey, + "results": metrics or {"crps": 0.123, "mase": 0.4}, + "substrate_cids": cids if cids is not None else [], + } + + +class _FakeAsyncClient: + """Minimal httpx.AsyncClient stand-in that returns a preset response.""" + + def __init__(self, *, status_code: int = 200, json_body=None, raise_exc=None): + self.status_code = status_code + self.json_body = json_body + self.raise_exc = raise_exc + + async def __aenter__(self): + return self + + async def __aexit__(self, *exc): + return False + + async def get(self, url): + if self.raise_exc: + raise self.raise_exc + resp = MagicMock() + resp.status_code = self.status_code + resp.json = MagicMock(return_value=self.json_body) + if self.status_code >= 400: + resp.raise_for_status = MagicMock( + side_effect=httpx.HTTPStatusError( + f"{self.status_code}", request=MagicMock(), response=resp, + ), + ) + else: + resp.raise_for_status = MagicMock() + return resp + + +def _patch_api(json_body=None, raise_exc=None): + """Patch httpx.AsyncClient to return ``json_body`` (or raise ``raise_exc``).""" + fake = _FakeAsyncClient(json_body=json_body, raise_exc=raise_exc) + return patch("tools.verify_radar_experiment.httpx.AsyncClient", + lambda *a, **kw: fake) + + +def _patch_hippius(bundle: bytes | None = None, raise_exc: Exception | None = None): + """Replace the lazily-imported HippiusClient with one that yields ``bundle``.""" + fake = MagicMock() + if raise_exc is not None: + fake.download_bundle = AsyncMock(side_effect=raise_exc) + else: + fake.download_bundle = AsyncMock(return_value=bundle) + fake.close = AsyncMock() + # The CLI does ``from shared.hippius_client import HippiusClient`` inside + # _run, so we patch it on the source module. + return patch("shared.hippius_client.HippiusClient", + lambda **kw: fake), fake + + +def _argv(extra: dict | None = None) -> list[str]: + base = { + "--api-url": "http://radar.example.com", + "--experiment-id": "7", + } + if extra: + base.update(extra) + return [a for k, v in base.items() for a in (k, v)] + + +# ── Tests ───────────────────────────────────────────────────────────── + + +def test_verify_tool_all_match_exit_0(capsys): + """Bundle on Hippius matches the API record → exit 0.""" + kp = _kp("validator-1") + bundle = _make_signed_bundle(keypair=kp) + api_body = _api_response(cids=[{ + "kind": "phase_c_record", + "validator_hotkey": kp.ss58_address, + "cid": "bafyok", + "round_id": 42, + }]) + + api_patch = _patch_api(json_body=api_body) + hip_patch, fake = _patch_hippius(bundle=bundle) + with api_patch, hip_patch: + rc = main(_argv()) + assert rc == 0 + out = capsys.readouterr().out + assert "PASS" in out + assert "All substrate records verified" in out + fake.download_bundle.assert_awaited_once_with("bafyok") + + +def test_verify_tool_metric_mismatch_exit_1(capsys): + """Signed metric disagrees with API → exit 1, mismatch shown.""" + kp = _kp("validator-2") + bundle = _make_signed_bundle(keypair=kp, metrics={"crps": 0.123}) + api_body = _api_response( + cids=[{"validator_hotkey": kp.ss58_address, "cid": "bafymis", "round_id": 42}], + metrics={"crps": 0.999}, # different + ) + + api_patch = _patch_api(json_body=api_body) + hip_patch, _ = _patch_hippius(bundle=bundle) + with api_patch, hip_patch: + rc = main(_argv()) + assert rc == 1 + out = capsys.readouterr().out + assert "FAIL" in out + assert "crps" in out # the mismatched field name is in the report + assert "Verification failed" in out + + +def test_verify_tool_no_cids_exit_2(capsys): + """API returns an experiment with empty substrate_cids → exit 2.""" + api_body = _api_response(cids=[]) + + api_patch = _patch_api(json_body=api_body) + with api_patch: + rc = main(_argv()) + assert rc == 2 + assert "no substrate_cids" in capsys.readouterr().out + + +def test_verify_tool_api_unreachable_exit_3(capsys): + """Transport error from httpx → exit 3.""" + api_patch = _patch_api(raise_exc=httpx.ConnectError("connection refused")) + with api_patch: + rc = main(_argv()) + assert rc == 3 + err = capsys.readouterr().err + assert "API unreachable" in err + + +def test_verify_tool_signature_failure_exit_1(capsys): + """A bundle signed by validator A but the API claims B → exit 1.""" + kp_a = _kp("validator-a") + kp_b = _kp("validator-b") + bundle = _make_signed_bundle(keypair=kp_a) + # API claims the CID came from kp_b → expected_hotkey mismatch in verify. + api_body = _api_response(cids=[{ + "validator_hotkey": kp_b.ss58_address, "cid": "bafyforged", "round_id": 42, + }]) + + api_patch = _patch_api(json_body=api_body) + hip_patch, _ = _patch_hippius(bundle=bundle) + with api_patch, hip_patch: + rc = main(_argv()) + assert rc == 1 + out = capsys.readouterr().out + assert "validator_hotkey mismatch" in out + + +def test_verify_tool_fetch_failure_exit_1(capsys): + """IPFS gateway flap surfaces in the per-CID report and exits 1.""" + api_body = _api_response(cids=[ + {"validator_hotkey": "5G", "cid": "bafyflap", "round_id": 42}, + ]) + + api_patch = _patch_api(json_body=api_body) + hip_patch, _ = _patch_hippius(raise_exc=RuntimeError("ipfs down")) + with api_patch, hip_patch: + rc = main(_argv()) + assert rc == 1 + out = capsys.readouterr().out + assert "fetch failed" in out + + +def test_verify_tool_help_renders_clean(capsys): + """--help should print our argparse output (regression: bittensor used + to hijack argparse via its module-load-time logging config).""" + with pytest.raises(SystemExit) as exc: + main(["--help"]) + assert exc.value.code == 0 + out = capsys.readouterr().out + assert "--api-url" in out + assert "--experiment-id" in out + assert "Exit codes" in out diff --git a/tools/README.md b/tools/README.md new file mode 100644 index 0000000..985bd3b --- /dev/null +++ b/tools/README.md @@ -0,0 +1,71 @@ +# RADAR verification tools + +Standalone scripts you can run to prove that a RADAR experiment was honestly +recorded. They depend only on `shared.substrate` (signing/verification) and +`shared.hippius_client` (IPFS download). No RADAR-internal code beyond that +— fork them freely. + +## `verify_radar_experiment.py` + +Fetches an experiment from a RADAR API, then for each substrate CID it +claims to be backed by, downloads the Hippius bundle, sr25519-verifies the +signature, and compares the bundle's signed metric record with what the +API returned. Discrepancies — wrong signature, missing record, mismatched +metric — are surfaced as a per-CID list in the report. + +### Usage + +```bash +python -m tools.verify_radar_experiment \ + --api-url https://radar.example.com \ + --experiment-id 12345 +``` + +`--hippius-ipfs` overrides the IPFS gateway (default: the public Hippius +gateway). Point it at a local IPFS node or any other IPFS-compatible +gateway you trust: + +```bash +python -m tools.verify_radar_experiment \ + --api-url https://radar.example.com \ + --experiment-id 12345 \ + --hippius-ipfs http://localhost:5001 +``` + +### Exit codes + +| Code | Meaning | +|------|--------------------------------------------------------------------| +| 0 | Every substrate record was fetched, signed correctly, and matched the API | +| 1 | Verification failed — signature invalid, missing record, or metric mismatch | +| 2 | Experiment exists but has no `substrate_cids` (nothing to verify) | +| 3 | API unreachable | + +### What it actually checks + +For each CID listed in the experiment's `substrate_cids` array: + +1. **Fetchable.** The bundle is downloadable from the IPFS gateway. +2. **Parseable.** The bundle deserialises into the schema declared in + `shared/substrate.py`. +3. **Signed correctly.** The `(round_id, miner_uid)`-matching record's + sr25519 signature verifies under the validator hotkey claimed in both + the bundle and the audit-trail entry. +4. **Matches the API.** Every metric in the signed record equals the same + metric in the API's `results` dict; `miner_uid` and `miner_hotkey` + agree across both sources. + +If any step fails the per-CID block prints `FAIL` with a `discrepancies` +list. A clean run prints `PASS` for every CID and "All substrate records +verified." + +### Why this exists + +RADAR's centralised database is convenient but it's a single point of +trust. Substrate bundles on Hippius are a parallel, independently +verifiable record: anyone can pull the raw signed payload, check the +sr25519 signature against the validator's public hotkey, and compare it +to what the API claims. If those two records ever disagree, this tool +makes it loud. + +Use this to prove any RADAR experiment was honestly recorded. diff --git a/tools/__init__.py b/tools/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tools/verify_radar_experiment.py b/tools/verify_radar_experiment.py new file mode 100644 index 0000000..57ccb59 --- /dev/null +++ b/tools/verify_radar_experiment.py @@ -0,0 +1,197 @@ +"""Standalone verifier for RADAR experiments. + +Fetch one experiment from a RADAR API, then for each substrate CID it claims +to be backed by, download the Hippius bundle, sr25519-verify the signature, +and compare the bundle's metric record with what the API returned. Self- +contained on purpose: depends only on ``shared.substrate`` and +``shared.hippius_client`` so anyone can fork it to verify a RADAR experiment +without RADAR repo access beyond the public API and an IPFS gateway. + + python -m tools.verify_radar_experiment \\ + --api-url https://radar.example.com --experiment-id 12345 +""" + +from __future__ import annotations + +import argparse +import asyncio +import sys +from typing import Any, Optional + +import httpx + +# shared.substrate / shared.hippius_client are imported lazily inside the +# functions that use them: both transitively import ``bittensor``, which +# constructs an ArgumentParser at module-load time and consumes sys.argv — +# loading them here would short-circuit our own --help handling. + + +_GREEN, _RED, _YELLOW, _BOLD, _RESET = ( + "\x1b[32m", "\x1b[31m", "\x1b[33m", "\x1b[1m", "\x1b[0m", +) + + +def _c(s: str, c: str) -> str: + return f"{c}{s}{_RESET}" if sys.stdout.isatty() else s + + +async def _fetch_experiment(api_url: str, experiment_id: int) -> dict: + """GET /experiments/{id} and return the parsed JSON object.""" + url = f"{api_url.rstrip('/')}/experiments/{experiment_id}" + async with httpx.AsyncClient(timeout=30.0) as client: + resp = await client.get(url) + resp.raise_for_status() + body = resp.json() + if not isinstance(body, dict): + raise ValueError(f"unexpected response shape: {type(body).__name__}") + return body + + +def _scalars(d: Any) -> dict: + if not isinstance(d, dict): + return {} + return {k: v for k, v in d.items() + if isinstance(v, (bool, int, float, str)) or v is None} + + +def _diff(record, api_results: dict, api_uid: int, api_hk: str) -> list[str]: + """Human-readable strings describing record/API disagreements.""" + out: list[str] = [] + rec, db = _scalars(record.metrics), _scalars(api_results) + for k, v in rec.items(): + if k not in db: + out.append(f"metric {k!r} present in record, missing in API") + elif db[k] != v: + out.append(f"metric {k!r}: record={v!r} api={db[k]!r}") + if record.miner_uid != api_uid: + out.append(f"miner_uid: record={record.miner_uid} api={api_uid}") + if api_hk and record.miner_hotkey and record.miner_hotkey != api_hk: + out.append(f"miner_hotkey: record={record.miner_hotkey!r} api={api_hk!r}") + return out + + +async def _verify_one( + hippius, cid_entry: dict, api_round_id: int, + api_uid: int, api_hk: str, api_results: dict, +) -> dict: + """Verify one (cid, validator_hotkey) entry against the API claim.""" + from shared.substrate import records_from_bundle, verify_record + cid = cid_entry.get("cid", "") + expected_hotkey = cid_entry.get("validator_hotkey", "") or None + out: dict[str, Any] = { + "cid": cid, "validator_hotkey": expected_hotkey or "", + "fetchable": False, "signature_valid": False, + "matches": False, "discrepancies": [], + } + if not cid: + out["discrepancies"].append("cid is empty") + return out + try: + data = await hippius.download_bundle(cid) + out["fetchable"] = True + except Exception as e: + out["discrepancies"].append(f"fetch failed: {e}") + return out + try: + records = records_from_bundle(data) + except Exception as e: + out["discrepancies"].append(f"bundle parse failed: {e}") + return out + record = next( + (r for r in records + if r.round_id == api_round_id and r.miner_uid == api_uid), + None, + ) + if record is None: + out["discrepancies"].append( + f"no record matching round_id={api_round_id} miner_uid={api_uid}" + ) + return out + ok, err = verify_record(record, expected_hotkey=expected_hotkey) + out["signature_valid"] = ok + if not ok: + out["discrepancies"].append(f"signature: {err}") + diffs = _diff(record, api_results, api_uid, api_hk) + out["discrepancies"].extend(diffs) + out["matches"] = ok and not diffs + return out + + +def _print_report(experiment_id: int, results: list[dict]) -> int: + """Render results as a report; return 0 on full pass, 1 on any failure.""" + print(_c(f"\n=== RADAR experiment #{experiment_id} verification ===", _BOLD)) + all_pass = True + for v in results: + tag = _c("PASS", _GREEN) if v["matches"] else _c("FAIL", _RED) + print(f"\n[{tag}] CID: {v['cid'] or '(missing)'}") + print(f" validator_hotkey: {v['validator_hotkey'] or '(unset)'}") + print(f" fetchable: {v['fetchable']}") + print(f" signature_valid: {v['signature_valid']}") + for d in v["discrepancies"]: + print(_c(f" - {d}", _YELLOW)) + if not v["matches"]: + all_pass = False + if all_pass: + print(_c("\nAll substrate records verified.", _GREEN)) + return 0 + print(_c("\nVerification failed. See details above.", _RED)) + return 1 + + +async def _run(args: argparse.Namespace) -> int: + try: + experiment = await _fetch_experiment(args.api_url, args.experiment_id) + except Exception as e: + print(_c(f"API unreachable: {e}", _RED), file=sys.stderr) + return 3 + cids = experiment.get("substrate_cids", []) + if not isinstance(cids, list) or not cids: + print(_c( + f"Experiment #{args.experiment_id} has no substrate_cids " + "— nothing to verify.", _YELLOW, + )) + return 2 + api_round = int(experiment.get("round_id", -1)) + api_uid = int(experiment.get("miner_uid", -1)) + api_hk = str(experiment.get("miner_hotkey", "") or "") + api_res = experiment.get("results") or {} + from shared.hippius_client import HippiusClient + hippius = HippiusClient(ipfs_api_url=args.hippius_ipfs) + try: + results = await asyncio.gather(*[ + _verify_one(hippius, c, api_round, api_uid, api_hk, api_res) + for c in cids if isinstance(c, dict) + ]) + finally: + await hippius.close() + return _print_report(args.experiment_id, results) + + +def parse_args(argv: Optional[list[str]] = None) -> argparse.Namespace: + p = argparse.ArgumentParser( + prog="verify_radar_experiment", + description="Independently verify a RADAR experiment against Hippius.", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=( + "Exit codes:\n" + " 0 every substrate record verified and matched the API\n" + " 1 verification failed (signature invalid or metric mismatch)\n" + " 2 experiment has no substrate CIDs (nothing to verify)\n" + " 3 API unreachable" + ), + ) + p.add_argument("--api-url", required=True, + help="RADAR API base URL, e.g. https://radar.example.com") + p.add_argument("--experiment-id", type=int, required=True, + help="Experiment ID to verify") + p.add_argument("--hippius-ipfs", default="", + help="Hippius IPFS gateway URL. Default: public gateway.") + return p.parse_args(argv) + + +def main(argv: Optional[list[str]] = None) -> int: + return asyncio.run(_run(parse_args(argv))) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/validator/coordinator.py b/validator/coordinator.py index 9fddd4f..7c2e818 100644 --- a/validator/coordinator.py +++ b/validator/coordinator.py @@ -125,11 +125,19 @@ def compute_fallback( class TrainingCoordinator: """Orchestrates Phase B dispatch and R2 artifact monitoring.""" - def __init__(self, wallet, metagraph, r2: "R2AuditLog", my_uid: int): + def __init__( + self, wallet, metagraph, r2: "R2AuditLog", my_uid: int, + artifact_store=None, + ): self.wallet = wallet self.metagraph = metagraph self.r2 = r2 self.my_uid = my_uid + # Optional dual-write store (TEN-240 Phase 7). When present, JSON + # artifacts (dispatch records, frontier snapshots) fan out to both + # R2 and Hippius. Falls back to R2-only when None — historical + # behaviour and the default. + self.artifact_store = artifact_store self._fallback_uids: dict[int, set[int]] = {} # round_id → UIDs using proxy def compute_my_jobs( @@ -551,7 +559,7 @@ async def wait_for_checkpoints( return results async def write_dispatch_record(self, round_id: int, results: list[TrainingResult]): - """Write dispatch record to R2 for auditability.""" + """Write dispatch record to R2 (and Hippius if dual-write enabled).""" hotkey = self.wallet.hotkey.ss58_address key = f"round_{round_id}/dispatch/vali_{hotkey}.json" records = [] @@ -564,17 +572,31 @@ async def write_dispatch_record(self, round_id: int, results: list[TrainingResul "training_time_seconds": r.training_time_seconds, "checkpoint_key": r.checkpoint_key, }) - self.r2.upload_json(key, {"dispatcher": hotkey, "round_id": round_id, "jobs": records}) + body = {"dispatcher": hotkey, "round_id": round_id, "jobs": records} + if self.artifact_store is not None: + await self.artifact_store.put_json(key, body, { + "app": "radar", "kind": "dispatch", + "round_id": str(round_id), "validator_hotkey": hotkey, + }) + else: + self.r2.upload_json(key, body) async def write_frontier( self, frontier_data: list[dict], task_name: str = "", ): - """Write current frontier to R2, scoped by task.""" + """Write current frontier to R2 (and Hippius if dual-write enabled).""" if task_name: key = f"frontier/{task_name}/latest.json" else: key = "frontier/latest.json" - self.r2.upload_json(key, {"frontier": frontier_data}) + body = {"frontier": frontier_data} + if self.artifact_store is not None: + await self.artifact_store.put_json(key, body, { + "app": "radar", "kind": "frontier", + "task": task_name or "default", + }) + else: + self.r2.upload_json(key, body) async def prepare_trainers( self, diff --git a/validator/neuron.py b/validator/neuron.py index 98b70dd..241448f 100644 --- a/validator/neuron.py +++ b/validator/neuron.py @@ -45,6 +45,7 @@ ) from validator.evaluator import evaluate_all_checkpoints from validator.pod_manager import pre_validate_code +from validator.substrate_publisher import run_substrate_publish_step logger = logging.getLogger(__name__) @@ -206,12 +207,41 @@ def __init__(self, config: bt.Config): except Exception as e: logger.warning("Cognition-wiki R2 unavailable: %s", e) + # Substrate publishing (best-effort, opt-in via Config.HIPPIUS_ENABLED). + # The Hippius client wrapper from TEN-242 isn't shipped yet, so the + # import is lazy and tolerant: with HIPPIUS_ENABLED=false (the default) + # nothing is imported and `self.hippius` stays None. Operators who + # opt in before the wrapper lands get a warning and the validator + # continues to run normally — substrate publishing is never a + # precondition for weight setting. + self.hippius = self._init_hippius() + + # Dual-write artifact store (TEN-240 Phase 7). Constructed only + # when an operator has explicitly opted in via + # RADAR_DUAL_WRITE_ARTIFACTS — default deploy keeps the historical + # R2-only path. When None, coordinator/etc. fall back to direct + # R2 calls and behave exactly as before. + self.artifact_store = None + if Config.DUAL_WRITE_ARTIFACTS and (self.r2 or self.hippius): + from shared.artifact_store import ArtifactStore + self.artifact_store = ArtifactStore( + r2=self.r2, hippius=self.hippius, + dual_write=True, + allow_fallback=Config.HIPPIUS_ARTIFACT_FALLBACK, + ) + logger.info( + "Artifact dual-write enabled (r2=%s, hippius=%s, fallback=%s)", + bool(self.r2), bool(self.hippius), + Config.HIPPIUS_ARTIFACT_FALLBACK, + ) + # Training coordinator my_uid = self._my_uid() if self.r2: self.coordinator = TrainingCoordinator( wallet=self.wallet, metagraph=self.metagraph, r2=self.r2, my_uid=my_uid, + artifact_store=self.artifact_store, ) else: self.coordinator = None @@ -244,6 +274,28 @@ def _my_uid(self) -> int: return hotkeys.index(hotkey) return -1 + def _init_hippius(self): + """Construct the Hippius client when opted in; return None otherwise.""" + if not Config.HIPPIUS_ENABLED: + return None + try: + from shared.hippius_client import HippiusClient # type: ignore + except ImportError: + logger.warning( + "HIPPIUS_ENABLED=true but shared.hippius_client is not " + "available yet (TEN-242). Substrate publishing disabled." + ) + return None + try: + return HippiusClient( + ipfs_api_url=Config.HIPPIUS_IPFS_API_URL, + hippius_key=Config.HIPPIUS_KEY, + substrate_rpc=Config.HIPPIUS_SUBSTRATE_RPC, + ) + except Exception as e: # noqa: BLE001 — best-effort + logger.warning("Hippius client init failed: %s", e) + return None + def _get_validator_uids( self, miner_uids: Optional[set[int]] = None, @@ -836,8 +888,14 @@ async def _no_endpoints(): round_id=challenge.round_id, ) - # Write to centralized DB - new_idx = await self.db_client.add_experiment(element.to_dict()) + # Write to centralized DB. When this validator published + # a Phase C bundle this round, attach the CID so the row + # carries an audit-trail entry pointing at the bundle. + new_idx = await self.db_client.add_experiment( + element.to_dict(), + substrate_cid=substrate_cids.get(uid, ""), + validator_hotkey=self.wallet.hotkey.ss58_address, + ) if new_idx is not None: element.index = new_idx pareto.update(element) @@ -865,6 +923,25 @@ async def _no_endpoints(): training_metas=training_metas, ) + # Best-effort: publish signed Phase C records to Hippius/substrate. + # Returns {miner_uid: bundle_cid} so Phase 5 can thread CIDs into + # the per-miner DB writes. Empty dict when disabled, no client, or + # publish failed — never raises. + substrate_cids = await run_substrate_publish_step( + hippius=self.hippius, wallet=self.wallet, + challenge=challenge, eval_results=eval_results, + training_metas=training_metas, commitments=commitments, + metagraph=self.metagraph, my_uid=self._my_uid(), + current_block=current_block, task_name=task_name, + block_hash=block_hash, netuid=int(self.netuid), + ) + if substrate_cids: + logger.info( + "Substrate published: round_id=%d cid=%s miners=%d", + challenge.round_id, next(iter(substrate_cids.values())), + len(substrate_cids), + ) + if round_scores: nonzero = {uid: s for uid, s in round_scores.items() if s > 0} logger.info( diff --git a/validator/pod_manager.py b/validator/pod_manager.py index 0c20e6c..ab3fcc7 100644 --- a/validator/pod_manager.py +++ b/validator/pod_manager.py @@ -111,7 +111,7 @@ def _build_agent_env_vars(allowed_urls: str = "") -> dict[str, str]: if allowed_urls: env_vars["AGENT_ALLOWED_URLS"] = allowed_urls env_vars["RADAR_ALLOWED_URLS"] = allowed_urls - # Note: NO Basilica token, NO R2 credentials, NO RADAR_BASILICA_ENV + # Note: NO Basilica token, NO Hippius/R2 credentials, NO RADAR_BASILICA_ENV return env_vars diff --git a/validator/substrate_publisher.py b/validator/substrate_publisher.py new file mode 100644 index 0000000..1454e71 --- /dev/null +++ b/validator/substrate_publisher.py @@ -0,0 +1,298 @@ +"""Phase C → substrate publisher. + +Glue layer between the validator's Phase C output and the substrate publishing +pipeline. Two functions: + + * `build_phase_c_records` — pure (no IO). Takes the inputs the validator + already has at the end of Phase C and returns a list of signed + `PhaseCRecord`s, one per miner that produced an eval result. Testable + without mocking. + * `publish_phase_c_records` — best-effort async wrapper that bundles + + uploads. *Never raises.* Substrate publishing must not be allowed to take + down weight-setting; on any failure the publisher logs a warning and + returns ``None``. + +The Hippius client is the structured `shared.hippius_client.HippiusClient` +(TEN-242): ``await client.upload_bundle(data, *, app_tag, phase, run_id, +netuid, extra_metadata=None) -> BundleRef``. +""" + +from __future__ import annotations + +import logging +import math +import time +from typing import Any, Optional + +from shared.substrate import ( + SCHEMA_VERSION, + PhaseCRecord, + bundle_sha256, + records_to_bundle, + sign_record, +) + +logger = logging.getLogger(__name__) + + +# Phase 2 (TEN-242) ships ``BundleRef`` as the canonical upload return type; +# this re-export keeps existing callers' imports stable. +from shared.hippius_client import BundleRef, UploadResult # noqa: E402,F401 + + +# ── Pure record construction ───────────────────────────────────────── + + +def _filter_metrics(metrics: dict) -> dict: + """Keep only JSON-serialisable scalars (primitives that survive the bundle). + + NaN / inf are dropped because plain ``json.dumps`` can't round-trip them + losslessly across implementations and we never want a record to verify on + one validator and fail to parse on another. + """ + out: dict[str, Any] = {} + for k, v in metrics.items(): + if isinstance(v, bool): + out[k] = v + elif isinstance(v, (int, float)): + if isinstance(v, float) and (math.isnan(v) or math.isinf(v)): + continue + out[k] = v + elif isinstance(v, str): + out[k] = v + elif v is None: + out[k] = v + # Anything else (lists, dicts, numpy scalars, tensors) is silently + # dropped — record bytes must be deterministic across readers. + return out + + +_BOOKKEEPING_KEYS = frozenset({ + "passed_size_gate", "flops_verified", "error", + "flops_equivalent_size", # diagnostic, not the primary objective +}) + + +def _derive_eval_status(metrics: dict) -> str: + """Map an eval-result dict to one of the four documented status strings. + + Priority order matches the spec: a size-gate fail short-circuits everything + else (the metric, even if reported, doesn't count for scoring); an explicit + error string beats "no_metric" because it tells the operator *why*; finally + "no_metric" catches the silent-failure case where the runner produced no + numeric output at all. + """ + if not metrics.get("passed_size_gate", False): + return "size_gate_failed" + if metrics.get("error"): + return "eval_failed" + has_metric = any( + isinstance(v, (int, float)) and not isinstance(v, bool) + and not (isinstance(v, float) and (math.isnan(v) or math.isinf(v))) + for k, v in metrics.items() + if k not in _BOOKKEEPING_KEYS + ) + if not has_metric: + return "no_metric" + return "ok" + + +def _hotkey_for_uid(metagraph, uid: int) -> str: + """Look up a hotkey on the metagraph, tolerating short / missing arrays.""" + hotkeys = getattr(metagraph, "hotkeys", None) or [] + if 0 <= uid < len(hotkeys): + return hotkeys[uid] or "" + return "" + + +def _code_hash_for_uid(commitments: dict, uid: int) -> str: + """Pull `code_hash` off whatever shape the commitments dict carries.""" + c = commitments.get(uid) if commitments else None + if c is None: + return "" + # ImageCommitment dataclass — attribute access. Some call sites may pass + # a plain dict (older code paths / tests), so fall back to .get. + return getattr(c, "code_hash", None) or (c.get("code_hash", "") if isinstance(c, dict) else "") or "" + + +async def build_phase_c_records( + *, + wallet, + challenge, + eval_results: dict[int, dict], + training_metas: dict[int, dict], + commitments: dict, + metagraph, + my_uid: int, + current_block: int, + task_name: str, + block_hash: str, +) -> list[PhaseCRecord]: + """Build one signed PhaseCRecord per miner in `eval_results`. + + Pure — no network, no disk. Async only so callers can `await` it from the + same site that awaits `publish_phase_c_records`. Iteration order follows + the eval_results dict (which is already insertion-ordered in CPython), + giving deterministic record ordering across runs of the same round. + + Records whose miner has no training meta still get a record (with empty + sha256 fields) — the eval result alone is enough to claim the miner was + evaluated. Empty sha256 strings are valid in the schema. + """ + validator_hotkey = wallet.hotkey.ss58_address + timestamp = time.time() + round_id = int(getattr(challenge, "round_id", 0)) + + records: list[PhaseCRecord] = [] + for uid, metrics in eval_results.items(): + meta = training_metas.get(uid, {}) or {} + record = PhaseCRecord( + schema_version=SCHEMA_VERSION, + round_id=round_id, + block_hash=block_hash, + task=task_name, + miner_uid=int(uid), + miner_hotkey=_hotkey_for_uid(metagraph, int(uid)), + code_hash=_code_hash_for_uid(commitments, int(uid)), + architecture_sha256=str(meta.get("architecture_sha256", "")), + checkpoint_sha256=str(meta.get("checkpoint_sha256", "")), + metrics=_filter_metrics(metrics), + passed_size_gate=bool(metrics.get("passed_size_gate", False)), + flops_verified=bool(metrics.get("flops_verified", False)), + eval_status=_derive_eval_status(metrics), + validator_uid=int(my_uid), + validator_hotkey=validator_hotkey, + validator_block_height=int(current_block), + timestamp=timestamp, + ) + records.append(sign_record(record, wallet)) + return records + + +# ── Async upload (best effort) ─────────────────────────────────────── + + +async def publish_phase_c_records( + hippius, + records: list[PhaseCRecord], + round_id: int, + validator_hotkey: str, + *, + netuid: int = 0, + app_tag: str = "radar", +) -> Optional[UploadResult]: + """Upload a Phase C record bundle to Hippius. Best-effort — never raises. + + Substrate publishing is *advisory*: it gives operators a portable, signed + audit trail but is never a precondition for setting weights. If anything + goes wrong (no client wired up, network flap, SDK regression) we log a + warning and return ``None`` so the validator's main loop continues. + + Metadata tags exposed on the upload mirror the bundle's own header so a + Hippius-side discovery query and an in-bundle parse agree on what + everything is. + """ + if not records: + # Nothing to publish — don't burn a no-op upload. + return None + if hippius is None: + logger.warning( + "Substrate publish skipped: no Hippius client configured " + "(round_id=%d, %d records)", round_id, len(records), + ) + return None + try: + bundle = records_to_bundle(records) + digest = bundle_sha256(bundle) + # The structured args go on the S3 key + tags; the rest of the + # schema rides as user metadata so /verify can sanity-check it. + result = await hippius.upload_bundle( + bundle, + app_tag=app_tag, phase="phase_c", + run_id=str(round_id), netuid=netuid, + extra_metadata={ + "schema_version": SCHEMA_VERSION, + "validator_hotkey": validator_hotkey, + "record_count": str(len(records)), + "bundle_sha256": digest, + }, + ) + except Exception as e: # noqa: BLE001 — best-effort by design + logger.warning( + "Substrate publish failed for round_id=%d (%d records): %s", + round_id, len(records), e, + ) + return None + if result is not None: + cid = getattr(result, "cid", None) or getattr(result, "key", "") or ( + result.get("cid", "") if isinstance(result, dict) else "" + ) + logger.info( + "Substrate publish ok: round_id=%d records=%d cid=%s", + round_id, len(records), cid, + ) + return result + + +# ── Round-loop integration helper ──────────────────────────────────── + + +async def run_substrate_publish_step( + *, + hippius, + wallet, + challenge, + eval_results: dict[int, dict], + training_metas: dict[int, dict], + commitments: dict, + metagraph, + my_uid: int, + current_block: int, + task_name: str, + block_hash: str, + netuid: int = 0, +) -> dict[int, str]: + """Build, sign, and publish a Phase C bundle. Best-effort; never raises. + + Returns ``{miner_uid: bundle_cid}`` (all miners share the round's single + bundle CID) so Phase 5 can thread CIDs into per-miner DB writes. Returns + an empty dict when ``hippius`` is None, ``eval_results`` is empty, or + record building / upload fails. + """ + if hippius is None or not eval_results: + return {} + try: + records = await build_phase_c_records( + wallet=wallet, challenge=challenge, + eval_results=eval_results, training_metas=training_metas, + commitments=commitments, metagraph=metagraph, + my_uid=my_uid, current_block=current_block, + task_name=task_name, block_hash=block_hash, + ) + except Exception as e: # noqa: BLE001 — best-effort by design + logger.warning( + "Substrate publishing failed (non-fatal): record build error: %s", e, + ) + return {} + upload_result = await publish_phase_c_records( + hippius, records, + round_id=int(getattr(challenge, "round_id", 0)), + validator_hotkey=wallet.hotkey.ss58_address, + netuid=netuid, + ) + if upload_result is None: + return {} + cid = getattr(upload_result, "cid", None) or ( + upload_result.get("cid", "") if isinstance(upload_result, dict) else "" + ) + if not cid: + return {} + return {int(uid): cid for uid in eval_results} + + +__all__ = [ + "UploadResult", + "build_phase_c_records", + "publish_phase_c_records", + "run_substrate_publish_step", +]