Build production-oriented AionBench benchmark

.dockerignore

@@ -0,0 +1,7 @@
+.git
+.venv
+.pytest_cache
+dist
+runs
+**/__pycache__
+

.github/workflows/ci.yml

@@ -0,0 +1,26 @@
+name: ci
+on:
+  push:
+  pull_request:
+permissions:
+  contents: read
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v6
+        with:
+          version: "0.9.21"
+          enable-cache: true
+      - run: uv sync --frozen --extra dev
+      - run: uv run pytest -q
+      - run: uv run aionbench validate episodes/dev
+      - run: uv build
+  container:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - run: docker build --tag aionbench:test .
+      - run: docker run --rm --network none aionbench:test
+

.gitignore

@@ -0,0 +1,10 @@
+__pycache__/
+*.py[cod]
+.pytest_cache/
+.venv/
+dist/
+build/
+runs/
+reports/generated/
+*.egg-info/
+

CITATION.cff

@@ -0,0 +1,9 @@
+cff-version: 1.2.0
+title: AionBench
+message: "If you use AionBench, cite this software and record the exact commit."
+type: software
+version: 0.1.0-alpha
+date-released: 2026-07-30
+license: MIT
+repository-code: https://github.com/TensorLink-AI/AionBench
+

Dockerfile

@@ -0,0 +1,13 @@
+FROM python:3.12.11-slim@sha256:47ae396f09c1303b8653019811a8498470603d7ffefc29cb07c88f1f8cb3d19f
+
+ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1
+WORKDIR /opt/aionbench
+RUN useradd --create-home --uid 10001 benchmark
+COPY pyproject.toml README.md LICENSE ./
+COPY src ./src
+COPY episodes ./episodes
+COPY configs ./configs
+RUN pip install --no-cache-dir . && chown -R benchmark:benchmark /opt/aionbench
+USER benchmark
+ENTRYPOINT ["aionbench"]
+CMD ["validate", "episodes/dev"]

README.md

@@ -0,0 +1,67 @@
+# AionBench
+
+AionBench is an executable benchmark for measuring whether forecasting tools
+improve an agent's operational decisions. Agents act in isolated episode worlds;
+only after the world is frozen does the scorer reveal the future and calculate
+realized cost and regret.
+
+This repository currently implements a production-oriented Phase 1 vertical slice from
+`AionBench_Build_Specification.md`:
+
+- strict episode loading and leakage validation;
+- SQLite-backed, transactional, idempotent actions;
+- capacity, inventory, and workforce cost models and enumerative oracles;
+- deterministic scoring and re-scoring;
+- twelve generated pilot episodes (four per domain);
+- a bounded agent runner with failure classification and complete run bundles;
+- repeated-trial aggregation, paired bootstrap comparisons, and reports;
+- a CLI for validation, safe inspection, execution, re-scoring, and reporting.
+
+## Quick start
+
+```bash
+uv sync --extra dev
+uv run aionbench generate episodes/dev
+uv run aionbench validate episodes/dev
+uv run aionbench show capacity_supported --root episodes/dev
+uv run aionbench oracle capacity_supported --root episodes/dev --output runs
+uv run aionbench batch --config configs/pilot-oracle.yaml --output runs/pilot
+uv run aionbench report runs/pilot --output reports/generated/pilot
+uv run pytest
+```
+
+`show` never loads or displays hidden files. `oracle` is intended for framework
+verification and reference trajectories, not agent-facing evaluation.
+
+## CLI
+
+```text
+aionbench generate OUTPUT
+aionbench validate EPISODE_ROOT
+aionbench show EPISODE_ID [--root EPISODE_ROOT]
+aionbench run --episode ID --agent scripted-oracle --condition oracle-future
+aionbench batch --config CONFIG --output RUN_ROOT
+aionbench rescore EPISODE_ID FINAL_STATE.sqlite
+aionbench verify-run TRIAL_DIRECTORY
+aionbench compare LEFT_RUNS RIGHT_RUNS
+aionbench report RUNS --output REPORT_DIRECTORY
+```
+
+Provider-backed agents implement the small `Agent` protocol in
+`src/aionbench/agents.py`. Provider SDKs and credentials are deliberately not core
+dependencies. A publication run must pin the provider adapter separately and record
+its exact model and provider revision in the run manifest.
+
+## Security boundary
+
+Visible paths are resolved beneath an episode's `visible/` directory and hidden
+paths are never passed to an agent tool. The included Python isolation is a
+library boundary, not an operating-system sandbox; production model runs should
+also use a network-disabled container.
+
+See `docs/release-readiness.md` for the evidence-backed distinction between what is
+implemented and which external prerequisites still block a public benchmark release.
+
+The currently verified Aion 0.2.0 source commit and wheel digest are recorded in
+`integrations/aion.lock.json`. Run `scripts/verify_aion.py` against an installed Aion
+CLI and wheel before any Aion-enabled evaluation.

SECURITY.md

@@ -0,0 +1,10 @@
+# Security policy
+
+Benchmark runs process model-generated tool calls and must be treated as untrusted.
+Production evaluations must run as a non-root user in a disposable container with
+network disabled, a read-only visible-data mount, bounded CPU/memory/time, no host
+credentials, and a separate writable artifact directory. Never mount an entire
+episode package because it contains hidden evaluator data.
+
+Report traversal, symlink, hidden-data, or command-execution vulnerabilities
+privately to the repository maintainers before public disclosure.

aionbench.yaml

@@ -0,0 +1,10 @@
+schema_version: "0.1"
+benchmark_version: "0.1.0-alpha"
+episode_roots:
+  dev: episodes/dev
+run_root: runs
+required_aion_runtime: "0.2.0"
+required_aion_schema: "0.1"
+network_policy: blocked
+regret_cap: 2.0
+

configs/pilot-oracle.yaml

@@ -0,0 +1,7 @@
+benchmark_version: "0.1.0"
+episode_root: episodes/dev
+episodes: all
+trials: 4
+base_seed: 300
+max_steps: 20
+timeout_seconds: 60

docs/acceptance-matrix.md

@@ -0,0 +1,32 @@
+# AionBench 0.1 acceptance matrix
+
+Evidence recorded on 2026-07-30 in the local workspace.
+
+| Criterion | Status | Evidence |
+|---|---|---|
+| Clean container build | Pass | `docker build -t aionbench:local .` |
+| Network-disabled container validation | Pass | 12/12 packages pass under `docker run --network none` |
+| Aion wheel/commit pinned | Pass locally | `integrations/aion.lock.json` |
+| Aion capabilities handshake | Pass | Runtime 0.2.0, schema 0.1 |
+| Aion numerical determinism | Pass | 100/100 `forecast.csv` SHA-256 hashes equal |
+| Aion upstream tests | Pass | 152 tests |
+| Hidden future inaccessible through benchmark tools | Pass | Traversal/symlink controls and tests |
+| Transactional/idempotent mutations | Pass | SQLite implementation and tests |
+| Stored-state re-scoring | Pass | `aionbench rescore` |
+| Failure taxonomy | Pass | Runner `Outcome` enum and artifact recording |
+| Twelve vertical-slice episodes | Pass | Four per domain |
+| Programmatically verified oracles | Pass | Validation recomputes action, cost, normalizer |
+| Supported/context/unsupported variants | Partial | Variants exist; context labels need final pinned-runtime/human audit |
+| Independent two-person audit | Blocked externally | Checklist exists; reviewers not supplied |
+| Raw/normalized regret and task success | Pass | Deterministic scorer |
+| Harm, abstention, repeated trials | Pass | Score and aggregate modules |
+| Context admission fidelity | Partial | Aion artifacts preserved; full host-LLM context trajectory not yet automated |
+| Paired confidence intervals | Pass | Task-clustered deterministic bootstrap |
+| Benchmark/tool/prompt/episode hashes and seeds | Pass | Per-trial manifest |
+| Published release container digest | Pending release | Local image builds; trusted registry publication not requested |
+| Provider-backed reference evaluations | Blocked externally | Requires selected models, adapters, and credentials |
+| Public/private held-out evaluator | Not Phase 1 | Requires expanded audited task set and service operation |
+
+“Pass locally” is not a publication claim. Release evidence must be regenerated in
+trusted CI from a clean checkout, and every external blocker must be closed before
+calling the benchmark production-grade.

docs/audit-checklist.md

@@ -0,0 +1,19 @@
+# Independent episode audit checklist
+
+Two reviewers must independently answer each item before an episode can enter a
+release split.
+
+- The instruction has one unambiguous operational interpretation.
+- Visible evidence is sufficient for either an action or a permitted escalation.
+- Hidden data cannot be inferred from paths, filenames, documents, or timestamps.
+- Operational constraints, units, budget, and action increments are internally consistent.
+- The programmatic oracle enumerates every valid action and handles ties.
+- Default cost, oracle cost, and normalizer were independently recomputed.
+- Supported/unsupported and context labels match the pinned Aion runtime.
+- Equivalent valid outcomes receive equivalent scores.
+- Prompt-injection text cannot alter benchmark policy.
+- Licensing and source metadata are complete.
+
+Disagreements require a written resolution and re-review by both auditors. Model
+judgments may prioritize cases but cannot determine ground truth.
+

docs/benchmark-card.md

@@ -0,0 +1,37 @@
+# AionBench benchmark card
+
+## Status
+
+Version 0.1.0-alpha is a deterministic 12-episode vertical slice. It is suitable
+for framework integration and internal model pilots. It is not yet an authoritative
+public leaderboard because independent episode audits, a pinned Aion wheel, and a
+private evaluation service remain outstanding.
+
+## Intended use
+
+AionBench measures operational decision quality when an agent must investigate a
+visible history, decide whether forecasting is supported, take a constrained action
+or escalate, record evidence, and explicitly complete the task. The primary metric
+is normalized decision regret; forecast accuracy is secondary.
+
+## Current data
+
+The development split has four deterministic synthetic episodes in each of three
+domains: capacity, inventory, and workforce. Every domain contains supported,
+useful-context, irrelevant-context, and malformed/required-escalation variants.
+Synthetic provenance, futures, and verified oracles are evaluator-only artifacts.
+
+## Limitations
+
+- Synthetic episodes do not establish external operational validity.
+- The local Python boundary is not a substitute for container isolation.
+- Provider behavior, costs, and non-determinism must be recorded by each runner.
+- Aion-specific claims require the pinned runtime integration suite to pass.
+- Twelve related templates are insufficient for a public comparative claim.
+
+## Metrics
+
+Report raw normalized regret, Success@1, Reliable@4, Regret-P90, action agreement,
+Safe@4, harmful-action rate, correct escalation, and unnecessary escalation.
+Conditions must be compared with paired task-level bootstrap intervals.
+

docs/evaluation-protocol.md

@@ -0,0 +1,17 @@
+# Evaluation protocol
+
+1. Build the pinned container and record its digest.
+2. Validate the episode split and checksums before any model call.
+3. Freeze model identifier, provider version, prompt, tools, budget, and temperature.
+4. Run at least four trials per model-condition-task using declared sampling seeds.
+5. Block network and mount only an episode's visible directory into agent tools.
+6. Preserve all model messages, tool arguments/results, file access, Aion artifacts,
+   database snapshots, errors, timings, hashes, and final responses.
+7. Re-score saved final states without model calls.
+8. Exclude and rerun only infrastructure or leakage failures. Agent misuse keeps its score.
+9. Aggregate trials within tasks. Bootstrap task IDs, preserving all trials in a task.
+10. Publish limitations and every tested condition, including unsuccessful ones.
+
+Never expose `hidden/`, generation seeds, evaluator labels, or oracle information to
+an evaluated agent. The scripted oracle is solely a framework verification tool.
+

docs/release-readiness.md

@@ -0,0 +1,24 @@
+# Release-readiness ledger
+
+## Implemented and automatically verified
+
+- Episode schema, checksums, temporal leakage scan, and programmatic oracle verification
+- Transactional SQLite state and idempotent actions
+- Three deterministic domain cost models and normalized regret
+- Safety/abstention scoring and stored-state re-scoring
+- Bounded agent runner, failure taxonomy, trajectories, snapshots, and manifests
+- Scripted oracle and replay agent interfaces
+- Repeated-trial aggregation and paired task bootstrap comparison
+- CSV, JSON, and Markdown reports
+- Container and CI definitions
+
+## External release blockers
+
+- Publish or vendor the locally verified Aion wheel recorded in `integrations/aion.lock.json`.
+- Re-execute the adapter contract suite against that wheel in the final release container.
+- Supply provider adapters and credentials for the chosen comparison models.
+- Conduct two-person independent audits of every episode.
+- Expand beyond 12 synthetic tasks and add licensed semi-synthetic/real histories.
+- Create public and private held-out splits and operate a private evaluator.
+- Calibrate with at least two competent models before freezing metric policy.
+- Build and publish the release container digest from trusted CI.

episodes/dev/capacity_context_irrelevant/checksums.json

@@ -0,0 +1,10 @@
+{
+  "episode.yaml": "sha256:8c639894c616f9d5434b0497fa937b42a7ae7d633a9ef280134bf43876cc689f",
+  "hidden/future.json": "sha256:3beb75ff1bafa01fe9407ecb2327bb300cca2bfc5ec667f84a4a68b6ac981c00",
+  "hidden/generation.json": "sha256:5194d69bb756655018dcc13b52511addf89a8748e8d33ec0d0b47cae98908856",
+  "hidden/oracle.json": "sha256:87f90a8e40a3a555356e9481b99bf33b562f5cceab5e946b54f797d4b1515676",
+  "initial_state.json": "sha256:0f1b0c635808e8d439f3a06cdf59fd172554b84b4028c11753141599de46a196",
+  "visible/data/api_requests.csv": "sha256:4b33f44a8c35abfa7e5f37974e522ed2fef51518941dd9a0b9b319e1c468f876",
+  "visible/documents/operations_plan.md": "sha256:24a0de8871d3f05570eaa0e5d094699de6147c391465d91ce0fdfc36720a0c73",
+  "visible/policy.md": "sha256:51ab28961498825a3ac9a68801a922010d9d8c513b80d3f9fa6343e58bcc7339"
+}

episodes/dev/capacity_context_irrelevant/episode.yaml

@@ -0,0 +1,50 @@
+schema_version: '0.1'
+episode_id: capacity_context_irrelevant
+domain: capacity
+split: dev
+variant_family: context_irrelevant
+seed: 4102
+clock: '2026-02-01T09:00:00+10:00'
+instruction: Reserve additional capacity for the next seven days, record the evidence
+  used, and complete the task.
+permissions:
+  readable_globs:
+  - visible/data/*
+  - visible/documents/*
+  - visible/policy.md
+  writable_actions:
+  - reserve_capacity
+  - record_decision_evidence
+  - escalate_decision
+  - complete_task
+forecast_task:
+  intended_file: visible/data/api_requests.csv
+  time_column: timestamp
+  target_column: requests
+  series_column: service
+  frequency: D
+  horizon: 7
+  threshold: 9000
+operational_constraints:
+  current_units: 18
+  demand_per_unit: 500
+  max_additional_units: 8
+  reservation_increment: 1
+  budget: 24000
+  safe_fallback_action: escalate
+cost_model:
+  type: capacity_peak
+  reserve_cost_per_unit: 4000
+  shortage_cost_per_unit_equivalent: 25000
+scoring:
+  primary: normalized_regret
+  task_success_tolerance: 0.05
+  require_evidence_record: true
+  harmful_action_rules:
+  - invent_forecast_after_unsupported
+  - exceed_budget
+labels:
+  difficulty: medium
+  context_type: context_irrelevant
+  data_quality: irrelevant_context
+  should_escalate: false

episodes/dev/capacity_context_irrelevant/hidden/future.json

@@ -0,0 +1,11 @@
+{
+  "demand": [
+    9040,
+    9158,
+    9404,
+    8802,
+    8765,
+    8592,
+    8990
+  ]
+}

episodes/dev/capacity_context_irrelevant/hidden/generation.json

@@ -0,0 +1,9 @@
+{
+  "generator": "capacity.context_irrelevant.v1",
+  "seed": 4102,
+  "parameters": {
+    "history_length": 120,
+    "frequency": "D",
+    "data_quality": "irrelevant_context"
+  }
+}