diff --git a/src/agent/agent_graph.py b/src/agent/agent_graph.py index 7590b28..9543e00 100644 --- a/src/agent/agent_graph.py +++ b/src/agent/agent_graph.py @@ -33,6 +33,7 @@ class AgentState(TypedDict, total=False): context: Optional[RegimeContext] proposals: List[Proposal] results: List[Dict[str, Any]] + all_results: List[Dict[str, Any]] best_result: Optional[Dict[str, Any]] iteration: int max_iterations: int @@ -99,6 +100,7 @@ def hypothesize_node(state: AgentState) -> AgentState: context=context, n_proposals=5, strategy_type=strategy_type, + prior_results=state.get("all_results"), ) state["proposals"] = proposals @@ -157,6 +159,7 @@ def backtest_node(state: AgentState) -> AgentState: # Sort by Sharpe results.sort(key=lambda x: x.get("sharpe", 0.0), reverse=True) state["results"] = results + state["all_results"] = state.get("all_results", []) + results if results: best = results[0] diff --git a/src/agent/base_planner.py b/src/agent/base_planner.py index 247448e..5566485 100644 --- a/src/agent/base_planner.py +++ b/src/agent/base_planner.py @@ -9,8 +9,9 @@ import json import logging import os +import time from abc import ABC, abstractmethod -from typing import Any, Dict, List, Optional +from typing import Any, Callable, Dict, List, Optional, TypeVar from dotenv import load_dotenv @@ -20,6 +21,35 @@ load_dotenv() +T = TypeVar("T") + +_PLACEHOLDER_KEYS = {"your_gemini_api_key_here", "you api key"} + + +def _looks_like_valid_key(key: str) -> bool: + """Basic sanity check to reject empty/placeholder/malformed API keys early.""" + key = key.strip() + if not key or key.lower() in _PLACEHOLDER_KEYS: + return False + return len(key) >= 16 and " " not in key + + +def _call_with_retry(fn: Callable[[], T], max_retries: int, backoff_seconds: float = 1.0) -> T: + """Call `fn`, retrying on exception up to `max_retries` times with linear backoff.""" + last_exc: Optional[Exception] = None + for attempt in range(max_retries + 1): + try: + return fn() + except Exception as e: + last_exc = e + if attempt < max_retries: + logger.warning( + "LLM call failed (attempt %d/%d): %s. Retrying...", + attempt + 1, max_retries + 1, e, + ) + time.sleep(backoff_seconds * (attempt + 1)) + raise last_exc + class BasePlanner(ABC): """Abstract interface for LLM-based strategy proposal generation.""" @@ -55,7 +85,7 @@ def __init__(self): self._model = None def is_available(self) -> bool: - return bool(self._api_key and self._api_key not in ("", "your_gemini_api_key_here", "you api key")) + return _looks_like_valid_key(self._api_key) def _get_model(self): if self._model is None: @@ -69,7 +99,9 @@ def _get_model(self): def generate_proposals(self, prompt: str, n: int = 5) -> List[Dict[str, Any]]: model = self._get_model() - response = model.generate_content(prompt) + response = _call_with_retry( + lambda: model.generate_content(prompt), config.llm.max_retries + ) text = response.text.strip() return self._parse_json_response(text, n) @@ -111,7 +143,7 @@ def __init__(self): self._temperature = config.llm.temperature def is_available(self) -> bool: - if not self._api_key or self._api_key in ("", "your_gemini_api_key_here", "you api key"): + if not _looks_like_valid_key(self._api_key): return False try: from langchain_google_genai import ChatGoogleGenerativeAI # noqa: F401 @@ -139,7 +171,7 @@ def __init__(self): self._api_key = os.getenv("OPENAI_API_KEY", "") def is_available(self) -> bool: - if not self._api_key: + if not _looks_like_valid_key(self._api_key): return False try: import openai # noqa: F401 @@ -151,10 +183,13 @@ def generate_proposals(self, prompt: str, n: int = 5) -> List[Dict[str, Any]]: import openai client = openai.OpenAI(api_key=self._api_key) - response = client.chat.completions.create( - model="gpt-4o-mini", - messages=[{"role": "user", "content": prompt}], - temperature=config.llm.temperature, + response = _call_with_retry( + lambda: client.chat.completions.create( + model="gpt-4o-mini", + messages=[{"role": "user", "content": prompt}], + temperature=config.llm.temperature, + ), + config.llm.max_retries, ) text = response.choices[0].message.content or "" return GeminiPlanner._parse_json_response(None, text, n) diff --git a/src/agent/proposal_generator.py b/src/agent/proposal_generator.py index 2abb624..69ac25f 100644 --- a/src/agent/proposal_generator.py +++ b/src/agent/proposal_generator.py @@ -22,12 +22,13 @@ PARAMETER GRID (you MUST select from this list only): {param_grid_json} - +{prior_results_section} TASK: 1. State which regime characteristic is most relevant to parameter selection. 2. Explain why longer vs shorter windows are appropriate given current conditions. 3. Select the {n_proposals} best parameter sets from the grid above, ranked by expected out-of-sample performance. 4. For each selection, assign a confidence score (0.0-1.0). +5. If prior attempts from this run are listed above, avoid repeating parameter regions that already underperformed and explain what you are doing differently. Return a JSON array of objects, each with: - All parameter fields from the grid entry you selected @@ -112,13 +113,21 @@ def generate( context: RegimeContext, n_proposals: int = 5, strategy_type: str = "momentum", + prior_results: Optional[List[Dict[str, Any]]] = None, ) -> List[Proposal]: + """ + Args: + prior_results: Backtest results from earlier iterations *within + this run* (each with at least "params" and "sharpe"), so the + LLM can reason about which parameter regions already failed + instead of resampling blindly each iteration. + """ proposals: List[Proposal] = [] # Try LLM first if self.planner.is_available(): try: - llm_proposals = self._llm_generate(context, strategy_type, n_proposals) + llm_proposals = self._llm_generate(context, strategy_type, n_proposals, prior_results) proposals.extend(llm_proposals) logger.info("LLM generated %d valid proposals.", len(llm_proposals)) except Exception as e: @@ -224,13 +233,18 @@ def _rejected_param_keys(self, context: RegimeContext, strategy_type: str) -> se return {tuple(sorted(candidate.params.items())) for candidate in rejected} def _llm_generate( - self, context: RegimeContext, strategy_type: str, n: int + self, + context: RegimeContext, + strategy_type: str, + n: int, + prior_results: Optional[List[Dict[str, Any]]] = None, ) -> List[Proposal]: prompt = PROMPT_TEMPLATE.format( strategy_type=strategy_type, regime_context=context.to_prompt_string(), param_grid_json=self.grid.to_json(strategy_type), n_proposals=n, + prior_results_section=self._format_prior_results(prior_results), ) logger.debug("LLM Prompt:\n%s", prompt) @@ -241,3 +255,14 @@ def _llm_generate( if v is not None: validated.append(v) return validated + + @staticmethod + def _format_prior_results(prior_results: Optional[List[Dict[str, Any]]]) -> str: + if not prior_results: + return "" + lines = ["\nPRIOR ATTEMPTS THIS RUN (avoid repeating underperforming regions):"] + for r in prior_results[-10:]: + lines.append( + f"- params={r.get('params')} -> sharpe={r.get('sharpe', 0.0):.2f}" + ) + return "\n".join(lines) + "\n" diff --git a/src/backtest/metrics.py b/src/backtest/metrics.py index 2bba397..2bb91b8 100644 --- a/src/backtest/metrics.py +++ b/src/backtest/metrics.py @@ -15,6 +15,11 @@ ANN_FACTOR = 252 # trading days per year +# Sentinel used instead of float("inf") for near-zero drawdown/downside cases +# so Calmar/Sortino can safely flow into ranking, sorting, and persisted +# memory without producing inf/NaN downstream. +MAX_RATIO = 1e6 + class PerformanceMetrics: """Centralised performance metric computations.""" @@ -44,7 +49,9 @@ def calmar(equity: pd.Series, ann_factor: int = ANN_FACTOR) -> float: return 0.0 ann_ret = (eq.iloc[-1] / eq.iloc[0]) ** (ann_factor / len(eq)) - 1 mdd = PerformanceMetrics.max_drawdown(eq) - return float(ann_ret / mdd) if mdd > 1e-9 else float("inf") + if mdd <= 1e-9: + return MAX_RATIO if ann_ret >= 0 else -MAX_RATIO + return float(ann_ret / mdd) @staticmethod def sortino(returns: pd.Series, ann_factor: int = ANN_FACTOR, risk_free: float = 0.0) -> float: @@ -56,19 +63,33 @@ def sortino(returns: pd.Series, ann_factor: int = ANN_FACTOR, risk_free: float = downside = excess[excess < 0] downside_std = downside.std() if len(downside) > 1 else 1e-12 if downside_std < 1e-12: - return float("inf") + return MAX_RATIO if excess.mean() >= 0 else -MAX_RATIO return float((excess.mean() / downside_std) * np.sqrt(ann_factor)) @staticmethod - def bootstrap_sharpe(returns: pd.Series, n: int = 200, pct: int = 5) -> float: - """5th percentile Sharpe from bootstrapped returns (penalizes lucky results).""" - r = returns.dropna() - if len(r) < 10: + def bootstrap_sharpe( + returns: pd.Series, n: int = 200, pct: int = 5, block_size: int = 20 + ) -> float: + """ + 5th percentile Sharpe from a moving-block bootstrap (penalizes lucky + results). Uses overlapping blocks of `block_size` consecutive daily + returns (rather than an IID resample) so autocorrelation/regime + structure in the return series is preserved — an IID resample + destroys the serial correlation present in trend/momentum strategies + and understates the true uncertainty of the Sharpe estimate. + """ + r = returns.dropna().to_numpy() + n_obs = len(r) + if n_obs < 10: return 0.0 - sharpes = [ - PerformanceMetrics.sharpe(r.sample(len(r), replace=True)) - for _ in range(n) - ] + block_size = max(1, min(block_size, n_obs)) + n_blocks = int(np.ceil(n_obs / block_size)) + rng = np.random.default_rng() + sharpes = [] + for _ in range(n): + starts = rng.integers(0, n_obs - block_size + 1, size=n_blocks) + sample = np.concatenate([r[s:s + block_size] for s in starts])[:n_obs] + sharpes.append(PerformanceMetrics.sharpe(pd.Series(sample))) return float(np.percentile(sharpes, pct)) @staticmethod @@ -98,4 +119,4 @@ def from_equity(equity: pd.Series, bootstrap: bool = False) -> dict: def from_returns(returns: pd.Series) -> dict: """Compute metrics from a daily returns series.""" equity = (1 + returns.fillna(0)).cumprod() - return PerformanceMetrics.from_equity(equity) \ No newline at end of file + return PerformanceMetrics.from_equity(equity) diff --git a/src/backtest/runner.py b/src/backtest/runner.py index 6b0bc08..a189181 100644 --- a/src/backtest/runner.py +++ b/src/backtest/runner.py @@ -15,7 +15,6 @@ from src.backtest.metrics import PerformanceMetrics from src.exceptions import ( BacktestFailedError, - InsufficientWarmupError, SignalGenerationError, StrategyNotFoundError, ) @@ -50,7 +49,8 @@ def _apply_transaction_costs( Compute per-bar cost series. - commission: fraction of trade value - slippage: one-way slippage fraction (applied directionally) - - market_impact_bps: square-root market impact in basis points + - market_impact_bps: flat linear market impact in basis points, applied + per unit of trade turnover (not a true square-root impact model) """ trades = signal.diff().abs().fillna(0) total_one_way = commission + slippage + (market_impact_bps / 10_000.0) @@ -76,13 +76,12 @@ def _backtest_single_asset( except Exception as e: raise SignalGenerationError(f"Signal generation failed for {asset}: {e}") from e - # Warmup check + # Warmup check — insufficient history before eval_start means signals in + # the warmup window are unreliable, so this must block the backtest + # rather than merely log. if eval_start is not None: slow_w = params.get("slow_window", params.get("window", config.backtest.min_warmup_periods)) - try: - enforcer.check(df, eval_start, min_window=int(slow_w)) - except InsufficientWarmupError as e: - logger.warning("%s", e) + enforcer.check(df, eval_start, min_window=int(slow_w)) # Apply transaction costs costs = _apply_transaction_costs( diff --git a/src/features/engine.py b/src/features/engine.py index 8d65562..5c3fbdc 100644 --- a/src/features/engine.py +++ b/src/features/engine.py @@ -7,11 +7,13 @@ """ import logging -from typing import Dict, Optional +from typing import Dict import numpy as np import pandas as pd +from src.features.lookback_guard import enforce_lookback + logger = logging.getLogger(__name__) @@ -64,6 +66,7 @@ def _to_series(candidate): # Individual indicator functions # --------------------------------------------------------------------------- +@enforce_lookback(min_periods=14) def _compute_rsi(close: pd.Series, period: int = 14) -> pd.Series: """Compute RSI using Wilder's smoothing.""" delta = close.diff() @@ -101,6 +104,7 @@ def _compute_bollinger(close: pd.Series, window: int = 20, num_std: float = 2.0) return upper, lower, width, pct_b +@enforce_lookback(min_periods=14) def _compute_atr(high: pd.Series, low: pd.Series, close: pd.Series, period: int = 14) -> pd.Series: """Compute Average True Range.""" prev_close = close.shift(1) diff --git a/src/features/lookback_guard.py b/src/features/lookback_guard.py index 8ab30dd..9e123c2 100644 --- a/src/features/lookback_guard.py +++ b/src/features/lookback_guard.py @@ -21,6 +21,10 @@ def enforce_lookback(min_periods: int): Decorator that validates a feature function's output has at least `min_periods` non-NaN values. + Raises InsufficientWarmupError if the computed feature does not have + enough valid history — callers must handle this rather than silently + trading on an unreliable warmup window. + Usage: @enforce_lookback(min_periods=200) def compute_sma200(close: pd.Series) -> pd.Series: @@ -33,10 +37,9 @@ def wrapper(*args, **kwargs): if isinstance(result, pd.Series): n_valid = result.notna().sum() if n_valid < min_periods: - logger.warning( - "%s produced only %d valid values but requires %d. " - "Signals in this window may be unreliable.", - func.__name__, n_valid, min_periods, + raise InsufficientWarmupError( + f"{func.__name__} produced only {n_valid} valid values " + f"but requires {min_periods}. Provide more historical data." ) return result return wrapper diff --git a/src/strategies/base.py b/src/strategies/base.py index 80ae983..66fb6b0 100644 --- a/src/strategies/base.py +++ b/src/strategies/base.py @@ -169,8 +169,11 @@ def generate_signal(self, df: pd.DataFrame, params: Dict[str, Any]) -> pd.Series low = _get_low(df).reindex(close.index).ffill() window = int(params.get("window", 20)) threshold = float(params.get("threshold_pct", 0.02)) - roll_high = high.rolling(window).max() - roll_low = low.rolling(window).min() + # Use the prior `window` bars only (exclude today) so the breakout + # level represents a genuine N-day prior high/low, not one that + # already includes today's own high/low. + roll_high = high.shift(1).rolling(window).max() + roll_low = low.shift(1).rolling(window).min() signal = pd.Series(0, index=df.index, dtype=int) signal[close > roll_high * (1 + threshold)] = 1 signal[close < roll_low * (1 - threshold)] = -1