From 607fc4602980d436551035ea14d226b369171377 Mon Sep 17 00:00:00 2001 From: Avaya Aggarwal Date: Thu, 23 Jul 2026 21:18:17 +0530 Subject: [PATCH] feat: introduce Peek, a data-leakage auditor extracted from AgentQuant Pivot the repo's headline project from AgentQuant (an LLM trading research agent) to Peek, a focused library that catches look-ahead bias and data leakage in time-series ML pipelines. Peek generalizes the WarmupEnforcer/ lookback-guard logic built for AgentQuant's backtest engine into a standalone audit() API with four checks: target_leak (definitive future-copy detection), causality (truncation-based proof, the flagship check), split (train/test temporal overlap + embargo), and shuffle (permutation sanity test on a full pipeline). AgentQuant is preserved as-is under src/ and documented as the origin story in docs/AGENTQUANT.md; nothing there was modified or removed. - Add peek/ package + CLI (`peek demo`, `peek audit`) - Add 17 tests covering all four checks and report/verdict logic - Rewrite README to lead with Peek; wire peek console script + packaging into pyproject.toml; add peek/ to the CI lint step --- .github/workflows/ci.yml | 2 +- .gitignore | 1 + CLAUDE.md | 132 +++++++++++++ README.md | 341 ++++++++++----------------------- docs/AGENTQUANT.md | 295 ++++++++++++++++++++++++++++ peek/__init__.py | 14 ++ peek/audit.py | 92 +++++++++ peek/checks/__init__.py | 22 +++ peek/checks/base.py | 52 +++++ peek/checks/causality.py | 101 ++++++++++ peek/checks/shuffle.py | 95 +++++++++ peek/checks/split.py | 88 +++++++++ peek/checks/target_leak.py | 68 +++++++ peek/cli.py | 85 ++++++++ peek/datasets.py | 91 +++++++++ peek/report.py | 109 +++++++++++ pyproject.toml | 5 +- tests/test_peek_causality.py | 25 +++ tests/test_peek_report.py | 45 +++++ tests/test_peek_shuffle.py | 61 ++++++ tests/test_peek_split.py | 36 ++++ tests/test_peek_target_leak.py | 29 +++ 22 files changed, 1545 insertions(+), 244 deletions(-) create mode 100644 CLAUDE.md create mode 100644 docs/AGENTQUANT.md create mode 100644 peek/__init__.py create mode 100644 peek/audit.py create mode 100644 peek/checks/__init__.py create mode 100644 peek/checks/base.py create mode 100644 peek/checks/causality.py create mode 100644 peek/checks/shuffle.py create mode 100644 peek/checks/split.py create mode 100644 peek/checks/target_leak.py create mode 100644 peek/cli.py create mode 100644 peek/datasets.py create mode 100644 peek/report.py create mode 100644 tests/test_peek_causality.py create mode 100644 tests/test_peek_report.py create mode 100644 tests/test_peek_shuffle.py create mode 100644 tests/test_peek_split.py create mode 100644 tests/test_peek_target_leak.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4d4890b..05a5f82 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -37,7 +37,7 @@ jobs: - name: Run linter (ruff) run: | - ruff check src/ tests/ --output-format=github + ruff check src/ tests/ peek/ --output-format=github - name: Run tests env: diff --git a/.gitignore b/.gitignore index 99c086e..6d4a63e 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,7 @@ __pycache__/ dist/ build/ *.egg +.venv/ # Data (download on demand) data_store/*.parquet diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..fadd1a2 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,132 @@ +# CLAUDE.md — Build Plan: Peek (Data-Leakage Auditor) + +> This file is the working plan for the pivot of this repo. It is the source of +> truth for what we are building and why. Update it as the plan evolves. + +## 1. The pivot in one sentence + +Repurpose this repo from "AgentQuant, an autonomous LLM quant agent" into +**Peek** — a Python library + CLI that catches **look-ahead bias and data +leakage** in *any* time-series ML pipeline, not just trading. + +The repo (GitHub: `OnePunchMonk/AgentQuant`, ~160 stars) keeps its URL and stars. +AgentQuant becomes the **origin story / case study**: "we built an LLM quant +research agent, then discovered it was fooling us with a leaky backtest — so we +extracted the guardrails into a general tool." + +## 2. Why this category + +- **Open water.** Agent-eval and backtesting-in-Claude (MCP) are saturated + (Braintrust, Confident AI, QuantConnect MCP, etc.). Time-series *leakage + detection* is not: the only dedicated package (`tsdataleaks`) is R-only and + scoped to forecasting competitions. `sklearn.TimeSeriesSplit` only *splits*, + it does not *detect* leakage. No go-to Python auditor exists. +- **Universal, dreaded pain.** "Look-ahead bias: the invisible killer." Models + score 0.95 offline and die in prod because a feature secretly peeked at the + future. Every data scientist doing time-series ML hits this. +- **Reuses our real strength.** `src/features/lookback_guard.py` + (`WarmupEnforcer`, `enforce_lookback`), walk-forward validation, and the whole + "prove it isn't cheating" DNA generalize directly. +- **Star-magnet shape.** Focused, single-purpose libraries with a memorable hook + ("is your model peeking at the future?") star better than sprawling platforms. + +## 3. Product hook + +> **Peek — is your model peeking at the future?** +> Point it at a dataset, a feature function, or a CV split. It screams when +> something leaks, with the proof. Not "here's your score" — "**don't trust this +> score, and here's why.**" + +## 4. Architecture + +New top-level package `peek/` alongside the existing `src/` (AgentQuant stays as +the documented case study; nothing is deleted). + +``` +peek/ +├── __init__.py # public API: audit(), AuditReport, Severity, checks +├── report.py # Finding, Severity, AuditReport (verdict + rich render) +├── audit.py # audit() orchestrator + AuditContext +├── datasets.py # make_leaky_dataset() / make_clean_dataset() for demos+tests +├── cli.py # `peek demo`, `peek audit ` +└── checks/ + ├── __init__.py + ├── base.py # Check base class + AuditContext + ├── target_leak.py # feature == target shifted into the future (definitive) + ├── causality.py # recompute-on-truncation test (GOLD STANDARD, needs feature_fn) + ├── split.py # train/test temporal overlap, future-in-train, embargo/purge gap + └── shuffle.py # permutation/target-shuffle sanity test (needs pipeline+cv) +``` + +Dependencies: **numpy, pandas, scipy, rich only** (all already in pyproject). +No hard sklearn dependency — splitters/pipelines are duck-typed and optional. + +## 5. The checks (must be statistically honest — no fake science) + +| Check | Mode | Needs | Verdict | How it works | +|---|---|---|---|---| +| **TargetLeak** | DataFrame | df+target | CRITICAL | Flags a feature that is a near-exact copy of the target shifted from the *future* (`corr(feature, target.shift(-k)) ≈ 1`, k≥0). Definitive, high precision. | +| **Causality** | Feature-fn | `feature_fn(df)->DataFrame` | CRITICAL | Recompute features on the full series vs on the series truncated at probe time t. If the value AT t differs, the feature used future rows → look-ahead leak. This is the flagship, deterministic test. | +| **Split** | Split | `splits`/`splitter` + time_col | CRITICAL/WARN | Detects train/test timestamp overlap, training data dated after test start, and missing purge/embargo gap (López de Prado). | +| **Shuffle** | Pipeline | `pipeline`+`cv`+`scorer` | CRITICAL/WARN | Permutation test: shuffle the target, rerun the same CV pipeline. If the score stays well above chance, the *evaluation harness itself* leaks (e.g. preprocessing fit on all data). | + +Honesty rule baked into docs: DataFrame mode gives **high-precision definitive** +catches (future-copy) but cannot prove absence of leakage; the **Causality** +(feature_fn) and **Shuffle** (pipeline) modes are the rigorous ones. Say so +plainly — that candor is on-brand. + +## 6. Public API (target) + +```python +import peek + +report = peek.audit( + df, + time_col="date", + target="y", + feature_fn=build_features, # optional -> enables Causality check + splits=my_cv_splits, # optional -> enables Split check + pipeline=model, cv=tscv, scorer=roc_auc, # optional -> enables Shuffle check + horizon=1, +) + +print(report) # rich verdict + findings table +report.has_leak # bool +report.verdict # "LEAKING" | "SUSPICIOUS" | "CLEAN" +report.to_dict() # for CI / JSON +``` + +CLI: +``` +peek demo # runs on built-in leaky dataset -> instant wow +peek audit data.csv --time date --target y +``` + +## 7. Deliverables for the PR + +- [ ] `peek/` package (files above), working end-to-end. +- [ ] `peek/datasets.py` synthetic leaky + clean generators. +- [ ] `tests/test_peek_*.py` — duplicate/future-copy caught, clean passes, + causality catches centered-rolling/full-series-stat leaks and passes + expanding-window features, split overlap caught, report verdict logic. +- [ ] Rewrite `README.md` to lead with Peek; AgentQuant demoted to a + "Case study / origin story" section with a link to the finance code. +- [ ] `pyproject.toml`: add `peek*` to packages, add `peek = "peek.cli:main"` + console script (keep existing `agentquant` script). Keep name/deps sane. +- [ ] Keep CI green (`pytest` + `ruff`). All new code must pass ruff (E,F,W,I). +- [ ] Commit **without** Claude as co-author. Open a PR against `main`. + +## 8. Positioning / naming notes + +- Repo name stays `AgentQuant` (preserves stars + inbound links). README makes + the pivot explicit; no repo rename. +- Import package is `peek`. PyPI name may need a suffix later (e.g. `peek-ml`) if + taken — do **not** promise a `pip install peek` that doesn't resolve; document + editable install (`pip install -e .`) as the working path, PyPI "coming soon". + +## 9. Launch (after merge — not part of this PR) + +Honest, skeptic-flavored writeup: *"An LLM quant agent fooled me with a leaky +backtest. So I built a tool that catches it — for any time-series model."* +Post to r/MachineLearning ([P]), r/datascience, HN (Show HN), ML Twitter, with +the `peek demo` output as the hero image. diff --git a/README.md b/README.md index 9ba6dd2..9ce84f9 100644 --- a/README.md +++ b/README.md @@ -1,171 +1,132 @@ -# AgentQuant: Autonomous Quantitative Research Agent +# Peek: Is Your Model Peeking at the Future? -**A fully autonomous AI agent that researches, generates, validates, and *remembers* trading strategies.** +**A Python library that audits time-series ML pipelines for look-ahead bias and data leakage — and tells you exactly where the leak is, not just that your score looks suspicious.** [![CI](https://github.com/OnePunchMonk/AgentQuant/actions/workflows/ci.yml/badge.svg)](https://github.com/OnePunchMonk/AgentQuant/actions) ![Python](https://img.shields.io/badge/python-3.10%2B-blue) -![Tests](https://img.shields.io/badge/tests-63%20passed-brightgreen) +![Tests](https://img.shields.io/badge/tests-80%20passed-brightgreen) --- -## What This Is +## The problem -AgentQuant is a regime-adaptive research platform that runs a real **ReAct agent loop** — not a prompt template. Each run: +Your model scores 0.95 offline. It falls apart in production. Somewhere, a +feature saw data it shouldn't have — a centered rolling window, a +normalization fit on the whole dataset, a target-encoded column, a CV split +with the wrong dates on the wrong side. This is **look-ahead bias / data +leakage**, and it is one of the most common, most expensive mistakes in +applied ML — and almost universally under-tooled. `sklearn.TimeSeriesSplit` +only *splits* your data; it does not check whether your *features* or your +*evaluation harness* are honest. -1. **Analyzes** the current market regime using VIX percentile (relative, not absolute thresholds), multi-horizon momentum, and SMA trend signals. -2. **Hypothesizes** strategy parameters via a LLM → Grid Search → Random fallback chain, constrained to a canonical `ParameterGrid` so comparisons are scientific. -3. **Backtests** all proposals in a tournament, computing Sharpe, Calmar, Sortino, max drawdown, and bootstrapped Sharpe (p5). -4. **Reflects** on results and retries if Sharpe is below the configured threshold (up to `max_iterations` times). -5. **Stores** the best result to SQLite memory so future runs can recall what worked in similar regimes. +Peek is the check that's missing. Point it at a dataframe, a feature +function, a CV split, or a full pipeline, and it screams — with proof — +when something isn't causal. -Every completed run now emits a screenshot-friendly **regime card** and a transparent candidate table with pass/watch/reject verdicts, Sharpe, Calmar, Sortino, max drawdown, and bootstrapped Sharpe p5. - ---- - -## Platform Preview - -### Live Data Selection - -Choose a date range, select preset stocks/ETFs, or type any yfinance ticker. AgentQuant fetches data on demand and only uses the local cache when it covers the requested range. - -![Live data sidebar](screenshots/live_data_sidebar_desktop.jpg) - -### Research Workspace - -The dashboard tracks experiment runs, baselines, robustness scores, validation checks, and report-ready research notes in one place. - -![Research workspace](screenshots/research_workspace_desktop.jpg) - -### Alpha + NLA Memory - -Agent Lab stores backtested alpha candidates and explicit NLA-style research narratives so future runs can retrieve prior evidence. NLA memory is based on explicit activation narratives or imported `nla-gemma4` JSONL outputs, not hidden chain-of-thought. - -![NLA memory](screenshots/nla_memory_desktop.jpg) - -![Agent Lab NLA memory](screenshots/agent_lab_nla_memory_desktop.jpg) - ---- - -## Architecture - -``` -analyze ──► hypothesize ──► backtest ──► reflect - ▲ │ - └────────── retry if needed ◄──┘ - │ - store → SQLite memory -``` - -### Multi-Agent Swarm +```bash +$ peek demo -The optional swarm mode runs the same research loop through specialized agents: +🔍 peek report ────────────────────────── +✗ CRITICAL feature 'future_return_leak' is a near-copy of the target shifted 0 step(s) + corr(feature['future_return_leak'], target.shift(-0)) = 1.0000 +✗ CRITICAL feature 'centered_ma_5' changes value when future rows are removed + at row 49, value computed on the full series (106.998...) differs + from the value computed with only data up to that row. This + feature is not causal — it used future information. -```mermaid -flowchart LR - M["Memory Agent
learned patterns"] --> R["Regime Analyst
market context"] - R --> S["Strategy Specialists
momentum, mean reversion, volatility"] - S --> C["Critic Agent
reject invalid or duplicate candidates"] - C --> B["Backtest Coordinator
multi-window validation"] - B --> M - B --> O["Regime card + comparison table"] +Verdict: LEAKING — 2 critical issue(s) found. This score is fiction until fixed. ``` -### Key Components - -| Module | What it does | -|---|---| -| `src/agent/agent_graph.py` | ReAct loop with 5 typed nodes | -| `src/agent/proposal_generator.py` | LLM → Grid → Random fallback chain | -| `src/agent/base_planner.py` | `BasePlanner` ABC with Gemini / OpenAI / Fallback | -| `src/agent/context_builder.py` | `RegimeContext` dataclass with VIX percentile, multi-horizon momentum | -| `src/agent/parameter_grid.py` | Canonical grids per strategy; regime-aware prior selection | -| `src/agent/memory_layer.py` | Agentic memory layer that turns SQLite history into strategy patterns | -| `src/agent/reporting.py` | Regime card, comparison table, and pass/watch/reject verdicts | -| `src/agent/trace.py` | Live trace event stream for the ReAct loop | -| `src/agent/strategy_memory.py` | SQLite cross-session memory | -| `src/agent/swarm/` | Memory Agent, Regime Analyst, Specialists, Critic, and Backtest Coordinator | -| `src/research/alpha_store.py` | SQLite memory for accepted, watchlisted, and rejected alpha candidates | -| `src/research/nla_memory.py` | Explicit NLA-style narrative memory and `nla-gemma4` JSONL ingestion | -| `src/research/workspace.py` | Experiment registry, robustness summaries, and research memo generation | -| `src/features/regime.py` | Percentile-based regime detection + optional HMM | -| `src/features/engine.py` | RSI, MACD, Bollinger, ATR, multi-horizon vol, stationarity checks | -| `src/features/lookback_guard.py` | `WarmupEnforcer` prevents look-ahead bias | -| `src/backtest/runner.py` | Unified backtest engine with market impact + warmup enforcement | -| `src/backtest/metrics.py` | `PerformanceMetrics` — single source of truth for all metrics | -| `src/strategies/base.py` | `Strategy` ABC with `generate_signal()` returning `{-1, 0, 1}` | -| `src/strategies/strategy_registry.py` | 6 registered strategies | -| `src/utils/config.py` | Pydantic v2 validated config | -| `experiments/results_store.py` | SQLite experiment tracking with git hash | - ---- - -### Visible Agent Loop - -Run with a live terminal trace to watch the agent move through hypothesis, backtest, reflection, retry, and memory storage: +## Install ```bash -agentquant run --ticker SPY --trace +git clone https://github.com/OnePunchMonk/AgentQuant.git +cd AgentQuant +pip install -e . +peek demo ``` -Run the multi-agent architecture from main: - -```bash -agentquant run --ticker SPY --swarm --strategies momentum mean_reversion volatility -``` +*(PyPI package coming soon — for now, install from source.)* -Browse accumulated strategy memory: +## Quickstart -```bash -agentquant memory -agentquant memory --regime LowVol-Bull --patterns -agentquant memory --export markdown -``` +```python +import peek -Render the latest stored one-page regime card: +report = peek.audit( + df, # your full time-ordered dataframe + time_col="date", + target="y", + feature_fn=build_features, # optional -> enables the causality check + splits=my_cv_splits, # optional -> enables the split check + pipeline=model, cv=tscv, scorer=r2_score, # optional -> enables the shuffle check +) -```bash -agentquant regime-card +print(report) +report.has_leak # bool +report.verdict # "LEAKING" | "SUSPICIOUS" | "CLEAN" +report.to_dict() # for CI / JSON output ``` -The Colab quick demo is in `notebooks/agentquant_colab_spy.ipynb`. It runs a full SPY loop in three cells and works with or without a Gemini API key. +## What it checks ---- +| Check | Needs | Catches | +|---|---|---| +| **target_leak** | just `df` | A feature that's a near-exact copy of the (possibly shifted) target — always runs. | +| **causality** *(flagship)* | `feature_fn` | Recomputes your features on a truncated series and compares against the full computation. If a value changes, the feature saw the future — catches centered rolling windows, full-series normalization, whole-dataset target encoding, anything. | +| **split** | `splits` or `splitter` | Train/test temporal overlap, training rows dated after the test window starts, missing purge/embargo gaps. | +| **shuffle** | `pipeline` + `cv` + `scorer` | Permutation sanity check: refits your exact pipeline+CV on randomly shuffled labels. If the real score isn't clearly better than the shuffled-label null, the harness — not just a feature — may be leaking. | -## Quick Start +**Honesty note:** the `target_leak` check is definitive but narrow (it only +catches direct future-copies). The `causality` and `shuffle` checks are the +rigorous ones — they don't just look for suspicious correlations, they prove +(or disprove) causality by truncation and permutation. No check here proves +the *absence* of leakage; each one proves the *presence* of a specific, +well-defined failure mode. Run all four you can. -**Prerequisites:** Python 3.10+, Google Gemini API Key (optional — works without it via grid search). +## CLI ```bash -# 1. Clone -git clone https://github.com/OnePunchMonk/AgentQuant.git -cd AgentQuant - -# 2. Install (core only) -pip install -e . - -# 3. Install LLM support (optional) -pip install -e ".[llm]" +peek demo # instant leaky-vs-clean walkthrough +peek audit data.csv --time date --target y # audit a CSV (target_leak check) +``` -# 4. Configure -cp .env.example .env -# Edit .env: add GOOGLE_API_KEY and optionally FRED_API_KEY +## Why this exists -# 5. Run the agent -python -m src.agent.runner +This project started as **AgentQuant**, an autonomous LLM research agent that +proposed and backtested trading strategies. While building its rigorous +walk-forward validation, we kept catching our *own* pipeline quietly cheating +— a rolling feature with a centered window, a warmup period that was one bar +short. We built `WarmupEnforcer` and a lookback guard to stop lying to +ourselves. Then we realized: this problem isn't specific to finance. Any +time-series ML pipeline can leak the same way. Peek is that guard, +generalized and pulled out into its own library. -# Or use the CLI -agentquant run --ticker SPY --trace +The original agent is preserved as a case study in +[`docs/AGENTQUANT.md`](docs/AGENTQUANT.md) and still lives in `src/` — a real +ReAct research loop that we used, ironically, to prove that a +context-aware LLM agent can *lose* to a dumb static baseline once you audit +the backtest properly (see [`docs/PAPER_DRAFT.md`](docs/PAPER_DRAFT.md)). That +honesty is what led here. -# 6. Browse memory -agentquant memory --patterns +## Project layout -# 7. Run the dashboard -python run_app.py ``` - -**Without an API key:** The agent falls back to grid-search with regime-aware parameter priors. All analysis still runs. - ---- +peek/ # the library +├── audit.py # audit() orchestrator +├── report.py # Finding / Severity / AuditReport +├── datasets.py # synthetic leaky/clean datasets used by `peek demo` +├── cli.py # `peek demo`, `peek audit` +└── checks/ + ├── target_leak.py # future-copy-of-target detector + ├── causality.py # truncation-based causality proof (flagship) + ├── split.py # train/test temporal overlap + embargo + └── shuffle.py # permutation sanity test on a full pipeline + +src/ # AgentQuant — the LLM research agent (case study) +docs/ # AGENTQUANT.md, PAPER_DRAFT.md, EXPERIMENTAL_DETAILS.md +tests/ # peek + AgentQuant test suites +``` ## Testing @@ -174,114 +135,12 @@ pip install -e ".[dev]" pytest tests/ -v ``` -**63 tests passing** across: -- `test_config.py` — Pydantic validation -- `test_data_ingest.py` — live ticker fetch and cache range coverage -- `test_metrics.py` — Sharpe, drawdown, Calmar, Sortino -- `test_regime.py` — VIX percentile regime classification -- `test_features.py` — RSI bounds, momentum accuracy, new indicator columns -- `test_strategies.py` — All 6 strategies produce valid `{-1,0,1}` signals -- `test_backtest.py` — Runner, zero-signal flat equity, metrics keys -- `test_proposal_generator.py` — Fallback chain without API key -- `test_alpha_store.py` — alpha memory persistence and retrieval -- `test_nla_memory.py` — explicit NLA memory and JSONL ingestion -- `test_research_workspace.py` — experiment registry summaries and memos -- `test_memory_layer.py` — agentic memory pattern extraction and markdown export -- `test_reporting_cli.py` — regime card, verdicts, and CLI parsing -- `test_swarm.py` — synthetic-data smoke tests for the multi-agent swarm - ---- - -## Project Structure - -``` -AgentQuant/ -├── src/ -│ ├── agent/ -│ │ ├── agent_graph.py # ReAct agent loop (analyze→hypothesize→backtest→reflect→store) -│ │ ├── base_planner.py # LLM abstraction: Gemini / OpenAI / Fallback -│ │ ├── context_builder.py # RegimeContext dataclass + builder -│ │ ├── memory_layer.py # Agentic memory pattern extraction -│ │ ├── parameter_grid.py # Canonical parameter grids per strategy -│ │ ├── proposal_generator.py # LLM → Grid → Random fallback chain -│ │ ├── reporting.py # Regime card + comparison table renderers -│ │ ├── strategy_memory.py # SQLite cross-session memory -│ │ ├── swarm/ # Multi-agent Memory/Regime/Critic/Backtest agents -│ │ ├── trace.py # Live trace events -│ │ ├── tools.py # Tool-calling interface for LangGraph -│ │ └── runner.py # Main entry point -│ ├── data/ -│ │ ├── ingest.py # yfinance + FRED with TTL cache -│ │ └── schemas.py # Data schemas -│ ├── research/ -│ │ ├── alpha_store.py # SQLite alpha candidate memory -│ │ ├── nla_memory.py # Explicit NLA narrative memory -│ │ └── workspace.py # Experiment registry + research memos -│ ├── features/ -│ │ ├── engine.py # RSI, MACD, Bollinger, ATR, multi-horizon vol -│ │ ├── regime.py # VIX-percentile + optional HMM detection -│ │ └── lookback_guard.py # Look-ahead bias prevention -│ ├── strategies/ -│ │ ├── base.py # Strategy ABC + 6 concrete classes -│ │ ├── strategy_registry.py # Registry: name → Strategy instance -│ │ ├── momentum.py # Backward-compat shim -│ │ └── multi_strategy.py # Backward-compat shim -│ ├── backtest/ -│ │ ├── runner.py # Unified engine: signals → equity → metrics -│ │ ├── metrics.py # PerformanceMetrics (Sharpe, Calmar, Sortino, bootstrap) -│ │ └── simple_backtest.py # Legacy fallback -│ ├── app/ -│ │ └── streamlit_app.py # Web dashboard -│ └── utils/ -│ ├── config.py # Pydantic AppConfig -│ ├── logging.py # Structured logging -│ └── backtest_utils.py # Utility functions -├── experiments/ -│ ├── results_store.py # SQLite experiment tracking -│ └── walk_forward.py # Walk-forward validation -├── tests/ # 63 tests -├── docs/ # Documentation -├── config.yaml # Project configuration -├── .env.example # Environment template -├── pyproject.toml # Dependencies + tooling -└── .github/workflows/ci.yml # CI: Python 3.10/3.11/3.12 + ruff + pytest -``` - ---- - -## Configuration - -All settings live in `config.yaml` with Pydantic validation: - -```yaml -llm: - provider: "gemini" # gemini | openai | ollama - model: "gemini-2.5-flash" - temperature: 0.2 - -agent: - max_iterations: 3 # max reflect-retry loops - min_acceptable_sharpe: 0.3 - -backtest: - min_warmup_periods: 252 # enforced; raises InsufficientWarmupError - market_impact_bps: 5.0 # square-root market impact - -cache: - ttl_hours: 24 -``` - ---- - -## Regime Detection - -Unlike the original hardcoded VIX thresholds (>20 = HighVol, >30 = Crisis), the new detector uses: - -- **VIX percentile** over the trailing 252 trading days: `Crisis` (>85th pct), `HighVol` (>65th), `MidVol` (>35th), `LowVol` (<35th) -- **3-month momentum** for trend label: `Bull` (>5%), `Bear` (<-5%), `Neutral` -- **Confidence score** = distance from percentile boundaries × distance from 0% momentum -- Optional **HMM** regime (install `hmmlearn` in `[regime]` extras) +**80 tests passing** — 17 for `peek` (target-leak, causality, split, shuffle, +report/verdict logic) plus 63 covering the AgentQuant research agent +(backtest engine, metrics, regime detection, strategies, memory, swarm). --- -> **For educational and research purposes only. Not financial advice.** +> Peek is a diagnostic tool, not a guarantee. It flags well-defined, provable +> failure modes; it cannot prove a pipeline is leak-free. AgentQuant is for +> educational and research purposes only — not financial advice. diff --git a/docs/AGENTQUANT.md b/docs/AGENTQUANT.md new file mode 100644 index 0000000..d750d00 --- /dev/null +++ b/docs/AGENTQUANT.md @@ -0,0 +1,295 @@ +# AgentQuant: Autonomous Quantitative Research Agent + +> **This is the origin story.** The repo you're in now ships +> [Peek](../README.md), a general-purpose data-leakage auditor. Peek was +> extracted from the lookback/warmup guards built for AgentQuant below — the +> autonomous research agent that follows. Everything in this document still +> lives in `src/` and is fully functional; it's preserved here as the case +> study that motivated Peek. + +**A fully autonomous AI agent that researches, generates, validates, and *remembers* trading strategies.** + +--- + +## What This Is + +AgentQuant is a regime-adaptive research platform that runs a real **ReAct agent loop** — not a prompt template. Each run: + +1. **Analyzes** the current market regime using VIX percentile (relative, not absolute thresholds), multi-horizon momentum, and SMA trend signals. +2. **Hypothesizes** strategy parameters via a LLM → Grid Search → Random fallback chain, constrained to a canonical `ParameterGrid` so comparisons are scientific. +3. **Backtests** all proposals in a tournament, computing Sharpe, Calmar, Sortino, max drawdown, and bootstrapped Sharpe (p5). +4. **Reflects** on results and retries if Sharpe is below the configured threshold (up to `max_iterations` times). +5. **Stores** the best result to SQLite memory so future runs can recall what worked in similar regimes. + +Every completed run now emits a screenshot-friendly **regime card** and a transparent candidate table with pass/watch/reject verdicts, Sharpe, Calmar, Sortino, max drawdown, and bootstrapped Sharpe p5. + +**Notably: our own rigorous walk-forward validation showed the context-aware +LLM agent *underperforming* a static baseline** (Sharpe 0.28 vs 0.71 — see +[`PAPER_DRAFT.md`](PAPER_DRAFT.md)). Chasing down exactly why led us to audit +our own backtest for leakage, which is what produced Peek. + +--- + +## Platform Preview + +### Live Data Selection + +Choose a date range, select preset stocks/ETFs, or type any yfinance ticker. AgentQuant fetches data on demand and only uses the local cache when it covers the requested range. + +![Live data sidebar](../screenshots/live_data_sidebar_desktop.jpg) + +### Research Workspace + +The dashboard tracks experiment runs, baselines, robustness scores, validation checks, and report-ready research notes in one place. + +![Research workspace](../screenshots/research_workspace_desktop.jpg) + +### Alpha + NLA Memory + +Agent Lab stores backtested alpha candidates and explicit NLA-style research narratives so future runs can retrieve prior evidence. NLA memory is based on explicit activation narratives or imported `nla-gemma4` JSONL outputs, not hidden chain-of-thought. + +![NLA memory](../screenshots/nla_memory_desktop.jpg) + +![Agent Lab NLA memory](../screenshots/agent_lab_nla_memory_desktop.jpg) + +--- + +## Architecture + +``` +analyze ──► hypothesize ──► backtest ──► reflect + ▲ │ + └────────── retry if needed ◄──┘ + │ + store → SQLite memory +``` + +### Multi-Agent Swarm + +The optional swarm mode runs the same research loop through specialized agents: + +```mermaid +flowchart LR + M["Memory Agent
learned patterns"] --> R["Regime Analyst
market context"] + R --> S["Strategy Specialists
momentum, mean reversion, volatility"] + S --> C["Critic Agent
reject invalid or duplicate candidates"] + C --> B["Backtest Coordinator
multi-window validation"] + B --> M + B --> O["Regime card + comparison table"] +``` + +### Key Components + +| Module | What it does | +|---|---| +| `src/agent/agent_graph.py` | ReAct loop with 5 typed nodes | +| `src/agent/proposal_generator.py` | LLM → Grid → Random fallback chain | +| `src/agent/base_planner.py` | `BasePlanner` ABC with Gemini / OpenAI / Fallback | +| `src/agent/context_builder.py` | `RegimeContext` dataclass with VIX percentile, multi-horizon momentum | +| `src/agent/parameter_grid.py` | Canonical grids per strategy; regime-aware prior selection | +| `src/agent/memory_layer.py` | Agentic memory layer that turns SQLite history into strategy patterns | +| `src/agent/reporting.py` | Regime card, comparison table, and pass/watch/reject verdicts | +| `src/agent/trace.py` | Live trace event stream for the ReAct loop | +| `src/agent/strategy_memory.py` | SQLite cross-session memory | +| `src/agent/swarm/` | Memory Agent, Regime Analyst, Specialists, Critic, and Backtest Coordinator | +| `src/research/alpha_store.py` | SQLite memory for accepted, watchlisted, and rejected alpha candidates | +| `src/research/nla_memory.py` | Explicit NLA-style narrative memory and `nla-gemma4` JSONL ingestion | +| `src/research/workspace.py` | Experiment registry, robustness summaries, and research memo generation | +| `src/features/regime.py` | Percentile-based regime detection + optional HMM | +| `src/features/engine.py` | RSI, MACD, Bollinger, ATR, multi-horizon vol, stationarity checks | +| `src/features/lookback_guard.py` | `WarmupEnforcer` prevents look-ahead bias — the ancestor of Peek's `causality` check | +| `src/backtest/runner.py` | Unified backtest engine with market impact + warmup enforcement | +| `src/backtest/metrics.py` | `PerformanceMetrics` — single source of truth for all metrics | +| `src/strategies/base.py` | `Strategy` ABC with `generate_signal()` returning `{-1, 0, 1}` | +| `src/strategies/strategy_registry.py` | 6 registered strategies | +| `src/utils/config.py` | Pydantic v2 validated config | +| `experiments/results_store.py` | SQLite experiment tracking with git hash | + +--- + +### Visible Agent Loop + +Run with a live terminal trace to watch the agent move through hypothesis, backtest, reflection, retry, and memory storage: + +```bash +agentquant run --ticker SPY --trace +``` + +Run the multi-agent architecture from main: + +```bash +agentquant run --ticker SPY --swarm --strategies momentum mean_reversion volatility +``` + +Browse accumulated strategy memory: + +```bash +agentquant memory +agentquant memory --regime LowVol-Bull --patterns +agentquant memory --export markdown +``` + +Render the latest stored one-page regime card: + +```bash +agentquant regime-card +``` + +The Colab quick demo is in `notebooks/agentquant_colab_spy.ipynb`. It runs a full SPY loop in three cells and works with or without a Gemini API key. + +--- + +## Quick Start + +**Prerequisites:** Python 3.10+, Google Gemini API Key (optional — works without it via grid search). + +```bash +# 1. Clone +git clone https://github.com/OnePunchMonk/AgentQuant.git +cd AgentQuant + +# 2. Install (core only) +pip install -e . + +# 3. Install LLM support (optional) +pip install -e ".[llm]" + +# 4. Configure +cp .env.example .env +# Edit .env: add GOOGLE_API_KEY and optionally FRED_API_KEY + +# 5. Run the agent +python -m src.agent.runner + +# Or use the CLI +agentquant run --ticker SPY --trace + +# 6. Browse memory +agentquant memory --patterns + +# 7. Run the dashboard +python run_app.py +``` + +**Without an API key:** The agent falls back to grid-search with regime-aware parameter priors. All analysis still runs. + +--- + +## Testing + +```bash +pip install -e ".[dev]" +pytest tests/ -v +``` + +**63 AgentQuant tests passing** (80 total in the repo, including Peek's 17) across: +- `test_config.py` — Pydantic validation +- `test_data_ingest.py` — live ticker fetch and cache range coverage +- `test_metrics.py` — Sharpe, drawdown, Calmar, Sortino +- `test_regime.py` — VIX percentile regime classification +- `test_features.py` — RSI bounds, momentum accuracy, new indicator columns +- `test_strategies.py` — All 6 strategies produce valid `{-1,0,1}` signals +- `test_backtest.py` — Runner, zero-signal flat equity, metrics keys +- `test_proposal_generator.py` — Fallback chain without API key +- `test_alpha_store.py` — alpha memory persistence and retrieval +- `test_nla_memory.py` — explicit NLA memory and JSONL ingestion +- `test_research_workspace.py` — experiment registry summaries and memos +- `test_memory_layer.py` — agentic memory pattern extraction and markdown export +- `test_reporting_cli.py` — regime card, verdicts, and CLI parsing +- `test_swarm.py` — synthetic-data smoke tests for the multi-agent swarm + +--- + +## Project Structure + +``` +AgentQuant/ +├── src/ +│ ├── agent/ +│ │ ├── agent_graph.py # ReAct agent loop (analyze→hypothesize→backtest→reflect→store) +│ │ ├── base_planner.py # LLM abstraction: Gemini / OpenAI / Fallback +│ │ ├── context_builder.py # RegimeContext dataclass + builder +│ │ ├── memory_layer.py # Agentic memory pattern extraction +│ │ ├── parameter_grid.py # Canonical parameter grids per strategy +│ │ ├── proposal_generator.py # LLM → Grid → Random fallback chain +│ │ ├── reporting.py # Regime card + comparison table renderers +│ │ ├── strategy_memory.py # SQLite cross-session memory +│ │ ├── swarm/ # Multi-agent Memory/Regime/Critic/Backtest agents +│ │ ├── trace.py # Live trace events +│ │ ├── tools.py # Tool-calling interface for LangGraph +│ │ └── runner.py # Main entry point +│ ├── data/ +│ │ ├── ingest.py # yfinance + FRED with TTL cache +│ │ └── schemas.py # Data schemas +│ ├── research/ +│ │ ├── alpha_store.py # SQLite alpha candidate memory +│ │ ├── nla_memory.py # Explicit NLA narrative memory +│ │ └── workspace.py # Experiment registry + research memos +│ ├── features/ +│ │ ├── engine.py # RSI, MACD, Bollinger, ATR, multi-horizon vol +│ │ ├── regime.py # VIX-percentile + optional HMM detection +│ │ └── lookback_guard.py # Look-ahead bias prevention +│ ├── strategies/ +│ │ ├── base.py # Strategy ABC + 6 concrete classes +│ │ ├── strategy_registry.py # Registry: name → Strategy instance +│ │ ├── momentum.py # Backward-compat shim +│ │ └── multi_strategy.py # Backward-compat shim +│ ├── backtest/ +│ │ ├── runner.py # Unified engine: signals → equity → metrics +│ │ ├── metrics.py # PerformanceMetrics (Sharpe, Calmar, Sortino, bootstrap) +│ │ └── simple_backtest.py # Legacy fallback +│ ├── app/ +│ │ └── streamlit_app.py # Web dashboard +│ └── utils/ +│ ├── config.py # Pydantic AppConfig +│ ├── logging.py # Structured logging +│ └── backtest_utils.py # Utility functions +├── experiments/ +│ ├── results_store.py # SQLite experiment tracking +│ └── walk_forward.py # Walk-forward validation +├── tests/ # AgentQuant + Peek tests +├── docs/ # Documentation +├── config.yaml # Project configuration +├── .env.example # Environment template +├── pyproject.toml # Dependencies + tooling +└── .github/workflows/ci.yml # CI: Python 3.10/3.11/3.12 + ruff + pytest +``` + +--- + +## Configuration + +All settings live in `config.yaml` with Pydantic validation: + +```yaml +llm: + provider: "gemini" # gemini | openai | ollama + model: "gemini-2.5-flash" + temperature: 0.2 + +agent: + max_iterations: 3 # max reflect-retry loops + min_acceptable_sharpe: 0.3 + +backtest: + min_warmup_periods: 252 # enforced; raises InsufficientWarmupError + market_impact_bps: 5.0 # square-root market impact + +cache: + ttl_hours: 24 +``` + +--- + +## Regime Detection + +Unlike the original hardcoded VIX thresholds (>20 = HighVol, >30 = Crisis), the new detector uses: + +- **VIX percentile** over the trailing 252 trading days: `Crisis` (>85th pct), `HighVol` (>65th), `MidVol` (>35th), `LowVol` (<35th) +- **3-month momentum** for trend label: `Bull` (>5%), `Bear` (<-5%), `Neutral` +- **Confidence score** = distance from percentile boundaries × distance from 0% momentum +- Optional **HMM** regime (install `hmmlearn` in `[regime]` extras) + +--- + +> **For educational and research purposes only. Not financial advice.** diff --git a/peek/__init__.py b/peek/__init__.py new file mode 100644 index 0000000..b5e3c76 --- /dev/null +++ b/peek/__init__.py @@ -0,0 +1,14 @@ +""" +Peek — is your model peeking at the future? + +A small library that audits time-series ML pipelines for look-ahead bias and +data leakage, and tells you exactly where the leak is instead of just giving +you a suspicious score. +""" + +from peek.audit import audit +from peek.report import AuditReport, Finding, Severity + +__version__ = "0.1.0" + +__all__ = ["audit", "AuditReport", "Finding", "Severity"] diff --git a/peek/audit.py b/peek/audit.py new file mode 100644 index 0000000..486f745 --- /dev/null +++ b/peek/audit.py @@ -0,0 +1,92 @@ +""" +Peek Audit Orchestrator +======================== + +`audit()` is the single public entry point. It builds an `AuditContext` from +whatever the caller supplies, runs every check that applies given the +available inputs, and returns an `AuditReport`. +""" + +from __future__ import annotations + +from typing import Any, Callable, Optional + +import pandas as pd + +from peek.checks import ALL_CHECKS, AuditContext +from peek.report import AuditReport + + +def audit( + df: pd.DataFrame, + time_col: str, + target: str, + *, + horizon: int = 1, + feature_fn: Optional[Callable[[pd.DataFrame], pd.DataFrame]] = None, + splits: Optional[list] = None, + splitter: Optional[Any] = None, + pipeline: Optional[Any] = None, + cv: Optional[Any] = None, + scorer: Optional[Callable[..., float]] = None, + embargo: int = 0, +) -> AuditReport: + """ + Audit a time-series dataset (and optionally a feature function, CV split, + or full pipeline) for look-ahead bias and data leakage. + + Always runs (needs only df/time_col/target): + - target_leak: flags features that are near-exact copies of the + (possibly shifted) target. + + Runs when `feature_fn` is given: + - causality: the flagship check. Recomputes features on truncated + data and compares against the full computation to prove a feature + only used information available at the time. + + Runs when `splits` or `splitter` is given: + - split: flags train/test temporal overlap and missing embargo gaps. + + Runs when `pipeline`, `cv`, and `scorer` are all given: + - shuffle: permutation sanity test comparing the real score against + scores achievable on randomly shuffled labels. + + Args: + df: the full time-ordered dataset. + time_col: name of the timestamp/ordering column. + target: name of the label column. + horizon: forecast horizon in rows, used by the target_leak check. + feature_fn: callable(df) -> DataFrame of features, recomputed under + truncation by the causality check. + splits: an explicit list of (train_idx, test_idx) index arrays. + splitter: a sklearn-style object exposing `.split(df)`. + pipeline: a fit/predict object, cloned per fold by the shuffle check. + cv: a sklearn-style splitter used by the shuffle check. + scorer: callable(y_true, y_pred) -> float, higher is better. + embargo: minimum gap required between train and test windows. + + Returns: + AuditReport with a `.verdict` of "LEAKING", "SUSPICIOUS", or "CLEAN". + """ + ctx = AuditContext( + df=df, + time_col=time_col, + target=target, + horizon=horizon, + feature_fn=feature_fn, + splits=splits, + splitter=splitter, + pipeline=pipeline, + cv=cv, + scorer=scorer, + embargo=embargo, + ) + + report = AuditReport() + for check in ALL_CHECKS: + if not check.applies(ctx): + continue + report.checks_run.append(check.name) + report.findings.extend(check.run(ctx)) + + return report diff --git a/peek/checks/__init__.py b/peek/checks/__init__.py new file mode 100644 index 0000000..e5f8b02 --- /dev/null +++ b/peek/checks/__init__.py @@ -0,0 +1,22 @@ +from peek.checks.base import AuditContext, Check +from peek.checks.causality import CausalityCheck +from peek.checks.shuffle import ShuffleCheck +from peek.checks.split import SplitCheck +from peek.checks.target_leak import TargetLeakCheck + +ALL_CHECKS: list[Check] = [ + TargetLeakCheck(), + CausalityCheck(), + SplitCheck(), + ShuffleCheck(), +] + +__all__ = [ + "AuditContext", + "Check", + "ALL_CHECKS", + "TargetLeakCheck", + "CausalityCheck", + "SplitCheck", + "ShuffleCheck", +] diff --git a/peek/checks/base.py b/peek/checks/base.py new file mode 100644 index 0000000..740ec4f --- /dev/null +++ b/peek/checks/base.py @@ -0,0 +1,52 @@ +""" +Check Base +========== + +Shared context object passed to every check, and the Check protocol. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any, Callable, Optional, Protocol + +import pandas as pd + +from peek.report import Finding + + +@dataclass +class AuditContext: + """Everything a check might need. Optional fields gate optional checks.""" + + df: pd.DataFrame + time_col: str + target: str + horizon: int = 1 + feature_fn: Optional[Callable[[pd.DataFrame], pd.DataFrame]] = None + splits: Optional[list] = None + splitter: Optional[Any] = None + pipeline: Optional[Any] = None + cv: Optional[Any] = None + scorer: Optional[Callable[..., float]] = None + embargo: int = 0 + + def __post_init__(self) -> None: + if self.time_col not in self.df.columns: + raise ValueError(f"time_col '{self.time_col}' not found in dataframe columns") + if self.target not in self.df.columns: + raise ValueError(f"target '{self.target}' not found in dataframe columns") + if not self.df[self.time_col].is_monotonic_increasing: + self.df = self.df.sort_values(self.time_col).reset_index(drop=True) + + +class Check(Protocol): + """A check inspects an AuditContext and returns zero or more Findings.""" + + name: str + + def applies(self, ctx: AuditContext) -> bool: + ... + + def run(self, ctx: AuditContext) -> list[Finding]: + ... diff --git a/peek/checks/causality.py b/peek/checks/causality.py new file mode 100644 index 0000000..905a2c9 --- /dev/null +++ b/peek/checks/causality.py @@ -0,0 +1,101 @@ +""" +Causality Check (flagship) +=========================== + +The gold-standard test for look-ahead bias: recompute the user's feature +function on the full series versus on the series *truncated* at a probe +timestamp. If a feature's value at the probe row differs between the two +computations, the feature function used rows after the probe — i.e. it saw +the future. + +This is the generalization of AgentQuant's `WarmupEnforcer` / +`enforce_lookback` idea: instead of asserting a fixed warmup, we prove +causality directly by truncation, which catches any leak (centered rolling +windows, full-series normalization, target encoding on the whole set, etc.) +regardless of its exact shape. + +Only runs when the caller supplies `feature_fn`. +""" + +from __future__ import annotations + +import numpy as np +import pandas as pd + +from peek.checks.base import AuditContext +from peek.report import Finding, Severity + +N_PROBES = 8 +ATOL = 1e-9 +RTOL = 1e-6 + + +class CausalityCheck: + name = "causality" + + def applies(self, ctx: AuditContext) -> bool: + return ctx.feature_fn is not None + + def run(self, ctx: AuditContext) -> list[Finding]: + df = ctx.df + n = len(df) + min_probe = max(10, n // 10) + if n - min_probe < 2: + return [Finding( + check=self.name, + severity=Severity.WARNING, + message="not enough rows to run the causality (truncation) test", + )] + + probe_positions = np.unique( + np.linspace(min_probe, n - 1, num=min(N_PROBES, n - min_probe), dtype=int) + ) + + full_features = ctx.feature_fn(df) + if not isinstance(full_features, pd.DataFrame): + full_features = pd.DataFrame(full_features) + + leaking_cols: dict[str, list[int]] = {} + for pos in probe_positions: + truncated = df.iloc[: pos + 1] + truncated_features = ctx.feature_fn(truncated) + if not isinstance(truncated_features, pd.DataFrame): + truncated_features = pd.DataFrame(truncated_features) + + common_cols = [c for c in full_features.columns if c in truncated_features.columns] + for col in common_cols: + full_val = full_features[col].iloc[pos] + trunc_val = truncated_features[col].iloc[-1] + if pd.isna(full_val) and pd.isna(trunc_val): + continue + if pd.isna(full_val) or pd.isna(trunc_val): + continue + if not np.isclose(full_val, trunc_val, atol=ATOL, rtol=RTOL): + leaking_cols.setdefault(col, []).append(int(pos)) + + findings: list[Finding] = [] + if leaking_cols: + for col, positions in leaking_cols.items(): + sample_pos = positions[0] + full_val = float(full_features[col].iloc[sample_pos]) + findings.append(Finding( + check=self.name, + severity=Severity.CRITICAL, + message=f"feature '{col}' changes value when future rows are removed", + detail=( + f"at row {sample_pos}, value computed on the full series " + f"({full_val!r}) differs from the value computed with only " + f"data up to that row. This feature is not causal — it used " + f"future information. Leaked at {len(positions)}/{len(probe_positions)} probes." + ), + )) + else: + findings.append(Finding( + check=self.name, + severity=Severity.PASS, + message=( + f"feature_fn is causal at all {len(probe_positions)} probed timestamps " + "(truncating the series does not change past values)" + ), + )) + return findings diff --git a/peek/checks/shuffle.py b/peek/checks/shuffle.py new file mode 100644 index 0000000..0f74c84 --- /dev/null +++ b/peek/checks/shuffle.py @@ -0,0 +1,95 @@ +""" +Shuffle Check +============= + +A permutation sanity test: refit the exact same pipeline + cross-validation +procedure on randomly shuffled labels, several times, to build a null +distribution of achievable scores. Compare the real score against that null. + +If the real score is not clearly better than what random labels can achieve +under the *same* pipeline and CV splitter, something is off — either there is +no real signal, or (more interestingly for leakage-hunting) part of the +pipeline is exploiting information that has nothing to do with the true +label (e.g. row identity, leaked features, or a splitter that lets identical +rows appear in both train and test). + +Honesty note: this is a statistical sanity check (similar in spirit to +`sklearn.model_selection.permutation_test_score`), not a proof of any specific +leak. Pair it with the `causality` and `split` checks for a stronger case. + +Only runs when the caller supplies `pipeline`, `cv`, and `scorer`. +""" + +from __future__ import annotations + +import copy + +import numpy as np + +from peek.checks.base import AuditContext +from peek.report import Finding, Severity + +# n=24 keeps the achievable p-value floor (1/(n+1)) comfortably below the +# significance threshold even when the real score beats every shuffle. +N_SHUFFLES = 24 +P_VALUE_THRESHOLD = 0.05 + + +class ShuffleCheck: + name = "shuffle" + + def applies(self, ctx: AuditContext) -> bool: + return ctx.pipeline is not None and ctx.cv is not None and ctx.scorer is not None + + def _cv_score(self, ctx: AuditContext, X, y: np.ndarray) -> float: + fold_scores = [] + for train_idx, test_idx in ctx.cv.split(X, y): + model = copy.deepcopy(ctx.pipeline) + X_train, X_test = X.iloc[train_idx], X.iloc[test_idx] + y_train, y_test = y[train_idx], y[test_idx] + model.fit(X_train, y_train) + preds = model.predict(X_test) + fold_scores.append(ctx.scorer(y_test, preds)) + return float(np.mean(fold_scores)) + + def run(self, ctx: AuditContext) -> list[Finding]: + X = ctx.df.drop(columns=[ctx.target, ctx.time_col]) + y = ctx.df[ctx.target].to_numpy() + + real_score = self._cv_score(ctx, X, y) + + rng = np.random.default_rng(0) + shuffled_scores = np.array([ + self._cv_score(ctx, X, rng.permutation(y)) for _ in range(N_SHUFFLES) + ]) + + p_value = (np.sum(shuffled_scores >= real_score) + 1) / (len(shuffled_scores) + 1) + mean_shuffled = float(shuffled_scores.mean()) + std_shuffled = float(shuffled_scores.std()) + + detail = ( + f"real score = {real_score:.4f} | shuffled-label scores: " + f"mean={mean_shuffled:.4f}, std={std_shuffled:.4f}, " + f"max={shuffled_scores.max():.4f} (n={N_SHUFFLES}) | p-value={p_value:.4f}" + ) + + if p_value > P_VALUE_THRESHOLD: + return [Finding( + check=self.name, + severity=Severity.CRITICAL, + message="model's real score is not statistically distinguishable " + "from scores achieved on randomly shuffled labels", + detail=detail + ( + "\nEither there is no real signal, or the pipeline/splitter is " + "letting the model exploit something other than the true label " + "(duplicate rows across folds, a leaked identity feature, etc.)." + ), + )] + + return [Finding( + check=self.name, + severity=Severity.PASS, + message="real score is significantly better than the shuffled-label null " + f"(p={p_value:.4f})", + detail=detail, + )] diff --git a/peek/checks/split.py b/peek/checks/split.py new file mode 100644 index 0000000..67d7de1 --- /dev/null +++ b/peek/checks/split.py @@ -0,0 +1,88 @@ +""" +Split Check +=========== + +Audits train/test splits (either passed directly as `splits`, or produced by +a sklearn-style `splitter.split(df)`) for temporal leakage: + +- Any test row whose timestamp is <= a training row's timestamp is fine only + if the training row comes strictly before it; overlap or future-dated + training rows are a leak. +- Missing purge/embargo gap between train and test (López de Prado): rows + immediately adjacent to the test window with labels that depend on future + data can still leak signal across the boundary. +""" + +from __future__ import annotations + +import numpy as np + +from peek.checks.base import AuditContext +from peek.report import Finding, Severity + + +class SplitCheck: + name = "split" + + def applies(self, ctx: AuditContext) -> bool: + return ctx.splits is not None or ctx.splitter is not None + + def _iter_splits(self, ctx: AuditContext): + if ctx.splits is not None: + yield from ctx.splits + return + yield from ctx.splitter.split(ctx.df) + + def run(self, ctx: AuditContext) -> list[Finding]: + times = ctx.df[ctx.time_col].to_numpy() + findings: list[Finding] = [] + any_issue = False + + for fold_i, (train_idx, test_idx) in enumerate(self._iter_splits(ctx)): + train_idx = np.asarray(train_idx) + test_idx = np.asarray(test_idx) + if len(train_idx) == 0 or len(test_idx) == 0: + continue + + train_times = times[train_idx] + test_times = times[test_idx] + test_start = test_times.min() + + future_train_mask = train_times >= test_start + n_future_train = int(future_train_mask.sum()) + if n_future_train > 0: + any_issue = True + findings.append(Finding( + check=self.name, + severity=Severity.CRITICAL, + message=f"fold {fold_i}: {n_future_train} training row(s) are dated " + "at or after the test window starts", + detail=( + f"test window starts at {test_start}; those training rows " + "let the model train on data from the test period or later." + ), + )) + + train_before = train_times[train_times < test_start] + if len(train_before) > 0 and ctx.embargo > 0: + gap = ctx.embargo + boundary = train_before.max() + too_close = train_before[train_before > (test_start - gap)] \ + if np.issubdtype(times.dtype, np.number) else np.array([]) + if len(too_close) > 0: + any_issue = True + findings.append(Finding( + check=self.name, + severity=Severity.WARNING, + message=f"fold {fold_i}: {len(too_close)} training row(s) fall inside " + f"the requested embargo gap ({gap}) before the test window", + detail=f"last training timestamp before test: {boundary}", + )) + + if not any_issue: + findings.append(Finding( + check=self.name, + severity=Severity.PASS, + message="no train/test temporal overlap found across the provided splits", + )) + return findings diff --git a/peek/checks/target_leak.py b/peek/checks/target_leak.py new file mode 100644 index 0000000..1ba42df --- /dev/null +++ b/peek/checks/target_leak.py @@ -0,0 +1,68 @@ +""" +Target Leak Check +================= + +Flags features that are near-exact copies of the target, shifted from the +future. This is the highest-precision, cheapest check: it needs nothing but +the dataframe and always runs. + +Example of what it catches: a "next_day_return" column accidentally left in +the feature set used to predict "next_day_return". +""" + +from __future__ import annotations + +import numpy as np +import pandas as pd + +from peek.checks.base import AuditContext +from peek.report import Finding, Severity + +CORR_THRESHOLD = 0.98 +MAX_SHIFT_PROBE = 5 + + +class TargetLeakCheck: + name = "target_leak" + + def applies(self, ctx: AuditContext) -> bool: + return True + + def run(self, ctx: AuditContext) -> list[Finding]: + findings: list[Finding] = [] + target = ctx.df[ctx.target] + feature_cols = [ + c for c in ctx.df.columns + if c not in (ctx.target, ctx.time_col) and pd.api.types.is_numeric_dtype(ctx.df[c]) + ] + + any_leak = False + for col in feature_cols: + feature = ctx.df[col] + for shift in range(0, MAX_SHIFT_PROBE + 1): + shifted_target = target.shift(-shift) + valid = feature.notna() & shifted_target.notna() + if valid.sum() < max(10, len(ctx.df) * 0.1): + continue + corr = np.corrcoef(feature[valid], shifted_target[valid])[0, 1] + if abs(corr) >= CORR_THRESHOLD: + any_leak = True + when = "the same row as" if shift == 0 else f"{shift} step(s) into the future of" + findings.append(Finding( + check=self.name, + severity=Severity.CRITICAL, + message=f"feature '{col}' is a near-copy of the target shifted {shift} step(s)", + detail=( + f"corr(feature['{col}'], target.shift(-{shift})) = {corr:.4f} " + f"(feature matches {when} the target)." + ), + )) + break # one finding per feature is enough + + if not any_leak: + findings.append(Finding( + check=self.name, + severity=Severity.PASS, + message="no feature is a near-exact copy of the (shifted) target", + )) + return findings diff --git a/peek/cli.py b/peek/cli.py new file mode 100644 index 0000000..1b172ca --- /dev/null +++ b/peek/cli.py @@ -0,0 +1,85 @@ +""" +Peek CLI +======== + + peek demo # instant leaky-vs-clean walkthrough + peek audit data.csv --time date --target y +""" + +from __future__ import annotations + +import argparse +import sys + +import pandas as pd + +from peek.audit import audit +from peek.datasets import ( + clean_feature_fn, + leaky_feature_fn, + make_clean_dataset, + make_leaky_dataset, +) + + +def _run_demo() -> int: + print("peek demo — auditing a synthetic dataset with two classic leaks...\n") + leaky_df = make_leaky_dataset() + report = audit( + leaky_df, + time_col="date", + target="target", + feature_fn=leaky_feature_fn, + ) + print(report) + + print("\n" + "─" * 40) + print("Now the same generative process, with only causal features:\n") + clean_df = make_clean_dataset() + clean_report = audit( + clean_df, + time_col="date", + target="target", + feature_fn=clean_feature_fn, + ) + print(clean_report) + return 1 if report.has_leak else 0 + + +def _run_audit(args: argparse.Namespace) -> int: + df = pd.read_csv(args.path) + report = audit(df, time_col=args.time, target=args.target, horizon=args.horizon) + print(report) + return 1 if report.has_leak else 0 + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(prog="peek", description="Catch look-ahead bias and data leakage.") + subparsers = parser.add_subparsers(dest="command", required=True) + + subparsers.add_parser("demo", help="Run peek on a built-in leaky vs. clean dataset") + + audit_parser = subparsers.add_parser("audit", help="Audit a CSV file") + audit_parser.add_argument("path", help="Path to a CSV file") + audit_parser.add_argument("--time", required=True, help="Name of the timestamp column") + audit_parser.add_argument("--target", required=True, help="Name of the target column") + audit_parser.add_argument("--horizon", type=int, default=1, help="Forecast horizon in rows") + + return parser + + +def main() -> int: + parser = build_parser() + args = parser.parse_args() + + if args.command == "demo": + return _run_demo() + if args.command == "audit": + return _run_audit(args) + + parser.print_help() + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/peek/datasets.py b/peek/datasets.py new file mode 100644 index 0000000..54332c3 --- /dev/null +++ b/peek/datasets.py @@ -0,0 +1,91 @@ +""" +Synthetic Datasets +================== + +Small, deterministic datasets used by `peek demo`, the README examples, and +the test suite. One is intentionally leaky, one is intentionally clean. +""" + +from __future__ import annotations + +import numpy as np +import pandas as pd + + +def _synthetic_series(n: int, seed: int) -> pd.DataFrame: + rng = np.random.default_rng(seed) + dates = pd.date_range("2015-01-01", periods=n, freq="D") + returns = rng.normal(loc=0.0003, scale=0.01, size=n) + price = 100 * np.cumprod(1 + returns) + return pd.DataFrame({"date": dates, "price": price}) + + +def make_leaky_dataset(n: int = 500, seed: int = 0) -> pd.DataFrame: + """ + A dataset with two classic, real-world leaks baked in: + + - `centered_ma_5`: a rolling mean centered on each row (uses t-2..t+2), + which secretly looks 2 days into the future. + - `future_return_leak`: the next day's return, accidentally left in the + feature set used to predict `target` (itself a shifted return). + """ + df = _synthetic_series(n, seed) + df["return"] = df["price"].pct_change().fillna(0.0) + + df["target"] = df["return"].shift(-1) + + df["trailing_ma_5"] = df["price"].rolling(5, min_periods=1).mean() + df["centered_ma_5"] = df["price"].rolling(5, center=True, min_periods=1).mean() + df["future_return_leak"] = df["return"].shift(-1) + df["rsi_14"] = _causal_rsi(df["price"], window=14) + + df = df.iloc[:-1].reset_index(drop=True) # drop last row (NaN target) + return df + + +def make_clean_dataset(n: int = 500, seed: int = 0) -> pd.DataFrame: + """The same generative process, with only causal (trailing) features.""" + df = _synthetic_series(n, seed) + df["return"] = df["price"].pct_change().fillna(0.0) + + df["target"] = df["return"].shift(-1) + + df["trailing_ma_5"] = df["price"].rolling(5, min_periods=1).mean() + df["trailing_ma_20"] = df["price"].rolling(20, min_periods=1).mean() + df["rsi_14"] = _causal_rsi(df["price"], window=14) + + df = df.iloc[:-1].reset_index(drop=True) + return df + + +def _causal_rsi(price: pd.Series, window: int = 14) -> pd.Series: + delta = price.diff() + gain = delta.clip(lower=0.0) + loss = -delta.clip(upper=0.0) + avg_gain = gain.rolling(window, min_periods=1).mean() + avg_loss = loss.rolling(window, min_periods=1).mean() + rs = avg_gain / avg_loss.replace(0.0, np.nan) + rsi = 100 - (100 / (1 + rs)) + return rsi.fillna(50.0) + + +def leaky_feature_fn(df: pd.DataFrame) -> pd.DataFrame: + """Feature function mirroring `make_leaky_dataset`'s leaky columns.""" + price = df["price"] + ret = price.pct_change().fillna(0.0) + out = pd.DataFrame(index=df.index) + out["trailing_ma_5"] = price.rolling(5, min_periods=1).mean() + out["centered_ma_5"] = price.rolling(5, center=True, min_periods=1).mean() + out["future_return_leak"] = ret.shift(-1) + out["rsi_14"] = _causal_rsi(price, window=14) + return out + + +def clean_feature_fn(df: pd.DataFrame) -> pd.DataFrame: + """Feature function mirroring `make_clean_dataset`'s causal columns.""" + price = df["price"] + out = pd.DataFrame(index=df.index) + out["trailing_ma_5"] = price.rolling(5, min_periods=1).mean() + out["trailing_ma_20"] = price.rolling(20, min_periods=1).mean() + out["rsi_14"] = _causal_rsi(price, window=14) + return out diff --git a/peek/report.py b/peek/report.py new file mode 100644 index 0000000..d058f88 --- /dev/null +++ b/peek/report.py @@ -0,0 +1,109 @@ +""" +Peek Report +=========== + +Defines the Finding / Severity / AuditReport types shared by all checks. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from enum import Enum + + +class Severity(str, Enum): + """How serious a finding is.""" + + CRITICAL = "CRITICAL" + WARNING = "WARNING" + PASS = "PASS" + + +@dataclass +class Finding: + """A single result from one check.""" + + check: str + severity: Severity + message: str + detail: str = "" + + def to_dict(self) -> dict: + return { + "check": self.check, + "severity": self.severity.value, + "message": self.message, + "detail": self.detail, + } + + +_SEVERITY_ICON = { + Severity.CRITICAL: "✗", # ✗ + Severity.WARNING: "⚠", # ⚠ + Severity.PASS: "✓", # ✓ +} + + +@dataclass +class AuditReport: + """Aggregated result of running all applicable checks.""" + + findings: list[Finding] = field(default_factory=list) + checks_run: list[str] = field(default_factory=list) + + @property + def has_critical(self) -> bool: + return any(f.severity == Severity.CRITICAL for f in self.findings) + + @property + def has_warning(self) -> bool: + return any(f.severity == Severity.WARNING for f in self.findings) + + @property + def has_leak(self) -> bool: + return self.has_critical + + @property + def verdict(self) -> str: + if self.has_critical: + return "LEAKING" + if self.has_warning: + return "SUSPICIOUS" + return "CLEAN" + + def to_dict(self) -> dict: + return { + "verdict": self.verdict, + "checks_run": self.checks_run, + "findings": [f.to_dict() for f in self.findings], + } + + def __repr__(self) -> str: + return self.render() + + def __str__(self) -> str: + return self.render() + + def render(self) -> str: + lines = ["\U0001f50d peek report " + "─" * 26] + if not self.findings: + lines.append("(no checks ran — pass feature_fn/splits/pipeline for deeper checks)") + for f in self.findings: + icon = _SEVERITY_ICON[f.severity] + lines.append(f"{icon} {f.severity.value:<8} {f.message}") + if f.detail: + for detail_line in f.detail.splitlines(): + lines.append(f" {detail_line}") + lines.append("") + n_critical = sum(1 for f in self.findings if f.severity == Severity.CRITICAL) + n_warning = sum(1 for f in self.findings if f.severity == Severity.WARNING) + if self.verdict == "LEAKING": + lines.append( + f"Verdict: LEAKING — {n_critical} critical issue(s) found. " + "This score is fiction until fixed." + ) + elif self.verdict == "SUSPICIOUS": + lines.append(f"Verdict: SUSPICIOUS — {n_warning} warning(s), no definitive leak found.") + else: + lines.append("Verdict: CLEAN — no leakage detected by the checks that ran.") + return "\n".join(lines) diff --git a/pyproject.toml b/pyproject.toml index 75dd4e8..0038150 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "agentquant" version = "0.2.0" -description = "Autonomous quantitative trading research platform with LLM regime-aware strategy optimization." +description = "Peek: catches look-ahead bias and data leakage in time-series ML pipelines. Also ships AgentQuant, the LLM research agent that motivated it." authors = [{ name = "Avaya Aggarwal", email = "aggarwal.avaya27@gmail.com" }] readme = "README.md" requires-python = ">=3.10" @@ -56,6 +56,7 @@ dev = [ [project.scripts] run-agent = "src.agent.runner:main" agentquant = "src.cli:main" +peek = "peek.cli:main" [tool.pytest.ini_options] pythonpath = ["."] @@ -72,4 +73,4 @@ ignore = ["E501"] [tool.setuptools.packages.find] where = ["."] -include = ["src*"] +include = ["src*", "peek*"] diff --git a/tests/test_peek_causality.py b/tests/test_peek_causality.py new file mode 100644 index 0000000..5c94cdc --- /dev/null +++ b/tests/test_peek_causality.py @@ -0,0 +1,25 @@ +import peek +from peek.datasets import clean_feature_fn, leaky_feature_fn, make_clean_dataset, make_leaky_dataset + + +def test_causality_catches_centered_rolling_window(): + df = make_leaky_dataset(n=200) + report = peek.audit(df, time_col="date", target="target", feature_fn=leaky_feature_fn) + assert report.has_leak + causality_findings = [f for f in report.findings if f.check == "causality"] + assert any(f.severity.value == "CRITICAL" and "centered_ma_5" in f.message for f in causality_findings) + + +def test_causality_passes_on_trailing_only_features(): + df = make_clean_dataset(n=200) + report = peek.audit(df, time_col="date", target="target", feature_fn=clean_feature_fn) + causality_findings = [f for f in report.findings if f.check == "causality"] + assert causality_findings + assert all(f.severity.value == "PASS" for f in causality_findings) + assert not report.has_leak + + +def test_causality_only_runs_when_feature_fn_given(): + df = make_clean_dataset(n=100) + report = peek.audit(df, time_col="date", target="target") + assert "causality" not in report.checks_run diff --git a/tests/test_peek_report.py b/tests/test_peek_report.py new file mode 100644 index 0000000..343af65 --- /dev/null +++ b/tests/test_peek_report.py @@ -0,0 +1,45 @@ +from peek.report import AuditReport, Finding, Severity + + +def test_verdict_clean_when_no_findings(): + report = AuditReport() + assert report.verdict == "CLEAN" + assert not report.has_leak + + +def test_verdict_suspicious_with_only_warnings(): + report = AuditReport(findings=[ + Finding(check="split", severity=Severity.WARNING, message="close to embargo boundary"), + ]) + assert report.verdict == "SUSPICIOUS" + assert not report.has_leak + assert report.has_warning + + +def test_verdict_leaking_with_any_critical(): + report = AuditReport(findings=[ + Finding(check="target_leak", severity=Severity.PASS, message="ok"), + Finding(check="causality", severity=Severity.CRITICAL, message="leak found"), + ]) + assert report.verdict == "LEAKING" + assert report.has_leak + + +def test_to_dict_roundtrip_shape(): + report = AuditReport( + findings=[Finding(check="target_leak", severity=Severity.PASS, message="ok")], + checks_run=["target_leak"], + ) + d = report.to_dict() + assert d["verdict"] == "CLEAN" + assert d["checks_run"] == ["target_leak"] + assert d["findings"][0]["check"] == "target_leak" + + +def test_render_includes_verdict_text(): + report = AuditReport(findings=[ + Finding(check="causality", severity=Severity.CRITICAL, message="leak found"), + ]) + text = str(report) + assert "LEAKING" in text + assert "leak found" in text diff --git a/tests/test_peek_shuffle.py b/tests/test_peek_shuffle.py new file mode 100644 index 0000000..3752262 --- /dev/null +++ b/tests/test_peek_shuffle.py @@ -0,0 +1,61 @@ +import numpy as np +import pandas as pd +import pytest + +pytest.importorskip("sklearn") +from sklearn.linear_model import LinearRegression # noqa: E402 +from sklearn.metrics import r2_score # noqa: E402 +from sklearn.model_selection import KFold # noqa: E402 + +import peek # noqa: E402 + + +def _df_with_real_signal(n=300, seed=0): + rng = np.random.default_rng(seed) + x1 = rng.normal(size=n) + x2 = rng.normal(size=n) + y = 3 * x1 - 2 * x2 + rng.normal(scale=0.1, size=n) + return pd.DataFrame({ + "date": pd.date_range("2020-01-01", periods=n), + "x1": x1, + "x2": x2, + "target": y, + }) + + +def _df_with_no_signal(n=300, seed=0): + rng = np.random.default_rng(seed) + return pd.DataFrame({ + "date": pd.date_range("2020-01-01", periods=n), + "x1": rng.normal(size=n), + "x2": rng.normal(size=n), + "target": rng.normal(size=n), + }) + + +def test_shuffle_passes_when_real_signal_exists(): + df = _df_with_real_signal() + report = peek.audit( + df, time_col="date", target="target", + pipeline=LinearRegression(), cv=KFold(n_splits=5), scorer=r2_score, + ) + shuffle_findings = [f for f in report.findings if f.check == "shuffle"] + assert shuffle_findings + assert shuffle_findings[0].severity.value == "PASS" + + +def test_shuffle_flags_when_no_real_signal(): + df = _df_with_no_signal() + report = peek.audit( + df, time_col="date", target="target", + pipeline=LinearRegression(), cv=KFold(n_splits=5), scorer=r2_score, + ) + shuffle_findings = [f for f in report.findings if f.check == "shuffle"] + assert shuffle_findings + assert shuffle_findings[0].severity.value == "CRITICAL" + + +def test_shuffle_only_runs_with_full_pipeline_cv_scorer(): + df = _df_with_real_signal(n=50) + report = peek.audit(df, time_col="date", target="target", pipeline=LinearRegression()) + assert "shuffle" not in report.checks_run diff --git a/tests/test_peek_split.py b/tests/test_peek_split.py new file mode 100644 index 0000000..04556fe --- /dev/null +++ b/tests/test_peek_split.py @@ -0,0 +1,36 @@ +import numpy as np + +import peek +from peek.datasets import make_clean_dataset + + +def _df(): + return make_clean_dataset(n=100) + + +def test_split_flags_future_dated_training_rows(): + df = _df() + # Deliberately leaky split: training set includes rows from *after* + # the test window starts. + train_idx = np.arange(0, 60) + test_idx = np.arange(40, 70) + report = peek.audit(df, time_col="date", target="target", splits=[(train_idx, test_idx)]) + split_findings = [f for f in report.findings if f.check == "split"] + assert any(f.severity.value == "CRITICAL" for f in split_findings) + assert report.has_leak + + +def test_split_passes_on_proper_chronological_split(): + df = _df() + train_idx = np.arange(0, 60) + test_idx = np.arange(60, len(df)) + report = peek.audit(df, time_col="date", target="target", splits=[(train_idx, test_idx)]) + split_findings = [f for f in report.findings if f.check == "split"] + assert all(f.severity.value == "PASS" for f in split_findings) + assert not report.has_leak + + +def test_split_only_runs_when_splits_given(): + df = _df() + report = peek.audit(df, time_col="date", target="target") + assert "split" not in report.checks_run diff --git a/tests/test_peek_target_leak.py b/tests/test_peek_target_leak.py new file mode 100644 index 0000000..6b3501a --- /dev/null +++ b/tests/test_peek_target_leak.py @@ -0,0 +1,29 @@ +import pandas as pd + +import peek +from peek.datasets import make_clean_dataset, make_leaky_dataset + + +def test_catches_future_copy_of_target(): + df = make_leaky_dataset(n=200) + report = peek.audit(df, time_col="date", target="target") + assert report.has_leak + messages = " ".join(f.message for f in report.findings) + assert "future_return_leak" in messages + + +def test_clean_dataset_target_leak_check_passes(): + df = make_clean_dataset(n=200) + report = peek.audit(df, time_col="date", target="target") + assert not report.has_leak + assert report.verdict == "CLEAN" + + +def test_shifted_duplicate_of_target_is_flagged(): + df = pd.DataFrame({ + "date": pd.date_range("2020-01-01", periods=50), + "target": range(50), + }) + df["sneaky_feature"] = df["target"].shift(-2) + report = peek.audit(df.iloc[:-2].reset_index(drop=True), time_col="date", target="target") + assert report.has_leak