Fix backtest rigor and safety gaps (lookback guard, metrics inf, breakout look-ahead)

src/agent/agent_graph.py

@@ -33,6 +33,7 @@ class AgentState(TypedDict, total=False):
     context: Optional[RegimeContext]
     proposals: List[Proposal]
     results: List[Dict[str, Any]]
+    all_results: List[Dict[str, Any]]
     best_result: Optional[Dict[str, Any]]
     iteration: int
     max_iterations: int
@@ -99,6 +100,7 @@ def hypothesize_node(state: AgentState) -> AgentState:
         context=context,
         n_proposals=5,
         strategy_type=strategy_type,
+        prior_results=state.get("all_results"),
     )
 
     state["proposals"] = proposals
@@ -157,6 +159,7 @@ def backtest_node(state: AgentState) -> AgentState:
     # Sort by Sharpe
     results.sort(key=lambda x: x.get("sharpe", 0.0), reverse=True)
     state["results"] = results
+    state["all_results"] = state.get("all_results", []) + results
 
     if results:
         best = results[0]

src/agent/base_planner.py

@@ -9,8 +9,9 @@
 import json
 import logging
 import os
+import time
 from abc import ABC, abstractmethod
-from typing import Any, Dict, List, Optional
+from typing import Any, Callable, Dict, List, Optional, TypeVar
 
 from dotenv import load_dotenv
 
@@ -20,6 +21,35 @@
 
 load_dotenv()
 
+T = TypeVar("T")
+
+_PLACEHOLDER_KEYS = {"your_gemini_api_key_here", "you api key"}
+
+
+def _looks_like_valid_key(key: str) -> bool:
+    """Basic sanity check to reject empty/placeholder/malformed API keys early."""
+    key = key.strip()
+    if not key or key.lower() in _PLACEHOLDER_KEYS:
+        return False
+    return len(key) >= 16 and " " not in key
+
+
+def _call_with_retry(fn: Callable[[], T], max_retries: int, backoff_seconds: float = 1.0) -> T:
+    """Call `fn`, retrying on exception up to `max_retries` times with linear backoff."""
+    last_exc: Optional[Exception] = None
+    for attempt in range(max_retries + 1):
+        try:
+            return fn()
+        except Exception as e:
+            last_exc = e
+            if attempt < max_retries:
+                logger.warning(
+                    "LLM call failed (attempt %d/%d): %s. Retrying...",
+                    attempt + 1, max_retries + 1, e,
+                )
+                time.sleep(backoff_seconds * (attempt + 1))
+    raise last_exc
+
 
 class BasePlanner(ABC):
     """Abstract interface for LLM-based strategy proposal generation."""
@@ -55,7 +85,7 @@ def __init__(self):
         self._model = None
 
     def is_available(self) -> bool:
-        return bool(self._api_key and self._api_key not in ("", "your_gemini_api_key_here", "you api key"))
+        return _looks_like_valid_key(self._api_key)
 
     def _get_model(self):
         if self._model is None:
@@ -69,7 +99,9 @@ def _get_model(self):
 
     def generate_proposals(self, prompt: str, n: int = 5) -> List[Dict[str, Any]]:
         model = self._get_model()
-        response = model.generate_content(prompt)
+        response = _call_with_retry(
+            lambda: model.generate_content(prompt), config.llm.max_retries
+        )
 
         text = response.text.strip()
         return self._parse_json_response(text, n)
@@ -111,7 +143,7 @@ def __init__(self):
         self._temperature = config.llm.temperature
 
     def is_available(self) -> bool:
-        if not self._api_key or self._api_key in ("", "your_gemini_api_key_here", "you api key"):
+        if not _looks_like_valid_key(self._api_key):
             return False
         try:
             from langchain_google_genai import ChatGoogleGenerativeAI  # noqa: F401
@@ -139,7 +171,7 @@ def __init__(self):
         self._api_key = os.getenv("OPENAI_API_KEY", "")
 
     def is_available(self) -> bool:
-        if not self._api_key:
+        if not _looks_like_valid_key(self._api_key):
             return False
         try:
             import openai  # noqa: F401
@@ -151,10 +183,13 @@ def generate_proposals(self, prompt: str, n: int = 5) -> List[Dict[str, Any]]:
         import openai
 
         client = openai.OpenAI(api_key=self._api_key)
-        response = client.chat.completions.create(
-            model="gpt-4o-mini",
-            messages=[{"role": "user", "content": prompt}],
-            temperature=config.llm.temperature,
+        response = _call_with_retry(
+            lambda: client.chat.completions.create(
+                model="gpt-4o-mini",
+                messages=[{"role": "user", "content": prompt}],
+                temperature=config.llm.temperature,
+            ),
+            config.llm.max_retries,
         )
         text = response.choices[0].message.content or ""
         return GeminiPlanner._parse_json_response(None, text, n)

src/agent/proposal_generator.py

@@ -22,12 +22,13 @@
 
 PARAMETER GRID (you MUST select from this list only):
 {param_grid_json}
-
+{prior_results_section}
 TASK:
 1. State which regime characteristic is most relevant to parameter selection.
 2. Explain why longer vs shorter windows are appropriate given current conditions.
 3. Select the {n_proposals} best parameter sets from the grid above, ranked by expected out-of-sample performance.
 4. For each selection, assign a confidence score (0.0-1.0).
+5. If prior attempts from this run are listed above, avoid repeating parameter regions that already underperformed and explain what you are doing differently.
 
 Return a JSON array of objects, each with:
 - All parameter fields from the grid entry you selected
@@ -112,13 +113,21 @@ def generate(
         context: RegimeContext,
         n_proposals: int = 5,
         strategy_type: str = "momentum",
+        prior_results: Optional[List[Dict[str, Any]]] = None,
     ) -> List[Proposal]:
+        """
+        Args:
+            prior_results: Backtest results from earlier iterations *within
+                this run* (each with at least "params" and "sharpe"), so the
+                LLM can reason about which parameter regions already failed
+                instead of resampling blindly each iteration.
+        """
         proposals: List[Proposal] = []
 
         # Try LLM first
         if self.planner.is_available():
             try:
-                llm_proposals = self._llm_generate(context, strategy_type, n_proposals)
+                llm_proposals = self._llm_generate(context, strategy_type, n_proposals, prior_results)
                 proposals.extend(llm_proposals)
                 logger.info("LLM generated %d valid proposals.", len(llm_proposals))
             except Exception as e:
@@ -224,13 +233,18 @@ def _rejected_param_keys(self, context: RegimeContext, strategy_type: str) -> se
         return {tuple(sorted(candidate.params.items())) for candidate in rejected}
 
     def _llm_generate(
-        self, context: RegimeContext, strategy_type: str, n: int
+        self,
+        context: RegimeContext,
+        strategy_type: str,
+        n: int,
+        prior_results: Optional[List[Dict[str, Any]]] = None,
     ) -> List[Proposal]:
         prompt = PROMPT_TEMPLATE.format(
             strategy_type=strategy_type,
             regime_context=context.to_prompt_string(),
             param_grid_json=self.grid.to_json(strategy_type),
             n_proposals=n,
+            prior_results_section=self._format_prior_results(prior_results),
         )
         logger.debug("LLM Prompt:\n%s", prompt)
 
@@ -241,3 +255,14 @@ def _llm_generate(
             if v is not None:
                 validated.append(v)
         return validated
+
+    @staticmethod
+    def _format_prior_results(prior_results: Optional[List[Dict[str, Any]]]) -> str:
+        if not prior_results:
+            return ""
+        lines = ["\nPRIOR ATTEMPTS THIS RUN (avoid repeating underperforming regions):"]
+        for r in prior_results[-10:]:
+            lines.append(
+                f"- params={r.get('params')} -> sharpe={r.get('sharpe', 0.0):.2f}"
+            )
+        return "\n".join(lines) + "\n"

src/backtest/metrics.py

@@ -15,6 +15,11 @@
 
 ANN_FACTOR = 252  # trading days per year
 
+# Sentinel used instead of float("inf") for near-zero drawdown/downside cases
+# so Calmar/Sortino can safely flow into ranking, sorting, and persisted
+# memory without producing inf/NaN downstream.
+MAX_RATIO = 1e6
+
 
 class PerformanceMetrics:
     """Centralised performance metric computations."""
@@ -44,7 +49,9 @@ def calmar(equity: pd.Series, ann_factor: int = ANN_FACTOR) -> float:
             return 0.0
         ann_ret = (eq.iloc[-1] / eq.iloc[0]) ** (ann_factor / len(eq)) - 1
         mdd = PerformanceMetrics.max_drawdown(eq)
-        return float(ann_ret / mdd) if mdd > 1e-9 else float("inf")
+        if mdd <= 1e-9:
+            return MAX_RATIO if ann_ret >= 0 else -MAX_RATIO
+        return float(ann_ret / mdd)
 
     @staticmethod
     def sortino(returns: pd.Series, ann_factor: int = ANN_FACTOR, risk_free: float = 0.0) -> float:
@@ -56,19 +63,33 @@ def sortino(returns: pd.Series, ann_factor: int = ANN_FACTOR, risk_free: float =
         downside = excess[excess < 0]
         downside_std = downside.std() if len(downside) > 1 else 1e-12
         if downside_std < 1e-12:
-            return float("inf")
+            return MAX_RATIO if excess.mean() >= 0 else -MAX_RATIO
         return float((excess.mean() / downside_std) * np.sqrt(ann_factor))
 
     @staticmethod
-    def bootstrap_sharpe(returns: pd.Series, n: int = 200, pct: int = 5) -> float:
-        """5th percentile Sharpe from bootstrapped returns (penalizes lucky results)."""
-        r = returns.dropna()
-        if len(r) < 10:
+    def bootstrap_sharpe(
+        returns: pd.Series, n: int = 200, pct: int = 5, block_size: int = 20
+    ) -> float:
+        """
+        5th percentile Sharpe from a moving-block bootstrap (penalizes lucky
+        results). Uses overlapping blocks of `block_size` consecutive daily
+        returns (rather than an IID resample) so autocorrelation/regime
+        structure in the return series is preserved — an IID resample
+        destroys the serial correlation present in trend/momentum strategies
+        and understates the true uncertainty of the Sharpe estimate.
+        """
+        r = returns.dropna().to_numpy()
+        n_obs = len(r)
+        if n_obs < 10:
             return 0.0
-        sharpes = [
-            PerformanceMetrics.sharpe(r.sample(len(r), replace=True))
-            for _ in range(n)
-        ]
+        block_size = max(1, min(block_size, n_obs))
+        n_blocks = int(np.ceil(n_obs / block_size))
+        rng = np.random.default_rng()
+        sharpes = []
+        for _ in range(n):
+            starts = rng.integers(0, n_obs - block_size + 1, size=n_blocks)
+            sample = np.concatenate([r[s:s + block_size] for s in starts])[:n_obs]
+            sharpes.append(PerformanceMetrics.sharpe(pd.Series(sample)))
         return float(np.percentile(sharpes, pct))
 
     @staticmethod
@@ -98,4 +119,4 @@ def from_equity(equity: pd.Series, bootstrap: bool = False) -> dict:
     def from_returns(returns: pd.Series) -> dict:
         """Compute metrics from a daily returns series."""
         equity = (1 + returns.fillna(0)).cumprod()
-        return PerformanceMetrics.from_equity(equity)
+        return PerformanceMetrics.from_equity(equity)

src/backtest/runner.py

@@ -15,7 +15,6 @@
 from src.backtest.metrics import PerformanceMetrics
 from src.exceptions import (
     BacktestFailedError,
-    InsufficientWarmupError,
     SignalGenerationError,
     StrategyNotFoundError,
 )
@@ -50,7 +49,8 @@ def _apply_transaction_costs(
     Compute per-bar cost series.
     - commission: fraction of trade value
     - slippage: one-way slippage fraction (applied directionally)
-    - market_impact_bps: square-root market impact in basis points
+    - market_impact_bps: flat linear market impact in basis points, applied
+      per unit of trade turnover (not a true square-root impact model)
     """
     trades = signal.diff().abs().fillna(0)
     total_one_way = commission + slippage + (market_impact_bps / 10_000.0)
@@ -76,13 +76,12 @@ def _backtest_single_asset(
     except Exception as e:
         raise SignalGenerationError(f"Signal generation failed for {asset}: {e}") from e
 
-    # Warmup check
+    # Warmup check — insufficient history before eval_start means signals in
+    # the warmup window are unreliable, so this must block the backtest
+    # rather than merely log.
     if eval_start is not None:
         slow_w = params.get("slow_window", params.get("window", config.backtest.min_warmup_periods))
-        try:
-            enforcer.check(df, eval_start, min_window=int(slow_w))
-        except InsufficientWarmupError as e:
-            logger.warning("%s", e)
+        enforcer.check(df, eval_start, min_window=int(slow_w))
 
     # Apply transaction costs
     costs = _apply_transaction_costs(

src/features/engine.py

@@ -7,11 +7,13 @@
 """
 
 import logging
-from typing import Dict, Optional
+from typing import Dict
 
 import numpy as np
 import pandas as pd
 
+from src.features.lookback_guard import enforce_lookback
+
 logger = logging.getLogger(__name__)
 
 
@@ -64,6 +66,7 @@ def _to_series(candidate):
 # Individual indicator functions
 # ---------------------------------------------------------------------------
 
+@enforce_lookback(min_periods=14)
 def _compute_rsi(close: pd.Series, period: int = 14) -> pd.Series:
     """Compute RSI using Wilder's smoothing."""
     delta = close.diff()
@@ -101,6 +104,7 @@ def _compute_bollinger(close: pd.Series, window: int = 20, num_std: float = 2.0)
     return upper, lower, width, pct_b
 
 
+@enforce_lookback(min_periods=14)
 def _compute_atr(high: pd.Series, low: pd.Series, close: pd.Series, period: int = 14) -> pd.Series:
     """Compute Average True Range."""
     prev_close = close.shift(1)

src/features/lookback_guard.py

@@ -21,6 +21,10 @@ def enforce_lookback(min_periods: int):
     Decorator that validates a feature function's output has at least
     `min_periods` non-NaN values.
 
+    Raises InsufficientWarmupError if the computed feature does not have
+    enough valid history — callers must handle this rather than silently
+    trading on an unreliable warmup window.
+
     Usage:
         @enforce_lookback(min_periods=200)
         def compute_sma200(close: pd.Series) -> pd.Series:
@@ -33,10 +37,9 @@ def wrapper(*args, **kwargs):
             if isinstance(result, pd.Series):
                 n_valid = result.notna().sum()
                 if n_valid < min_periods:
-                    logger.warning(
-                        "%s produced only %d valid values but requires %d. "
-                        "Signals in this window may be unreliable.",
-                        func.__name__, n_valid, min_periods,
+                    raise InsufficientWarmupError(
+                        f"{func.__name__} produced only {n_valid} valid values "
+                        f"but requires {min_periods}. Provide more historical data."
                     )
             return result
         return wrapper

src/strategies/base.py

@@ -169,8 +169,11 @@ def generate_signal(self, df: pd.DataFrame, params: Dict[str, Any]) -> pd.Series
         low = _get_low(df).reindex(close.index).ffill()
         window = int(params.get("window", 20))
         threshold = float(params.get("threshold_pct", 0.02))
-        roll_high = high.rolling(window).max()
-        roll_low = low.rolling(window).min()
+        # Use the prior `window` bars only (exclude today) so the breakout
+        # level represents a genuine N-day prior high/low, not one that
+        # already includes today's own high/low.
+        roll_high = high.shift(1).rolling(window).max()
+        roll_low = low.shift(1).rolling(window).min()
         signal = pd.Series(0, index=df.index, dtype=int)
         signal[close > roll_high * (1 + threshold)] = 1
         signal[close < roll_low * (1 - threshold)] = -1