diff --git a/.jules/bolt.md b/.jules/bolt.md new file mode 100644 index 0000000..09d01d2 --- /dev/null +++ b/.jules/bolt.md @@ -0,0 +1,3 @@ +## 2025-02-09 - O(log N) Time Series Lookups in Backtesting +**Learning:** Backtest engines running repeated interval simulations often re-query historical price arrays. Using `.filter()` to truncate arrays up to an `asOf` date is an O(N) operation per query that allocates intermediate arrays, severely bottlenecking multi-interval backtests. +**Action:** Always prefer O(log N) binary search lookups over O(N) array filtering when interacting with chronologically sorted time-series data, specifically returning index positions to avoid array copying overhead. diff --git a/src/agent/backtestRunner.ts b/src/agent/backtestRunner.ts index e91de2c..1361f12 100644 --- a/src/agent/backtestRunner.ts +++ b/src/agent/backtestRunner.ts @@ -2,7 +2,7 @@ import { randomUUID } from "node:crypto"; import { loadConfig } from "../config/loader.js"; import { getDb } from "../storage/db.js"; import { getBacktestBroker } from "../broker/index.js"; -import { getStockOhlcv, getIndexOhlcv, type Bar } from "../data/sources/dnsePublic.js"; +import { getStockOhlcv, getIndexOhlcv, findLastBarIndex, type Bar } from "../data/sources/dnsePublic.js"; import { DISCOVERY_UNIVERSE, discoverTickers } from "../tools/discover.js"; import { setActiveAsOf } from "./clock.js"; import { runTeamAnalysis } from "./team/index.js"; @@ -298,8 +298,8 @@ export async function runBacktestSession( ); const vnindexAt = (asOf: number): number | null => { - const series = vnindex.filter((b) => b.time <= asOf); - return series.length ? series[series.length - 1]!.close : null; + const idx = findLastBarIndex(vnindex, asOf); + return idx !== -1 ? vnindex[idx]!.close : null; }; const vnindexBaseline = vnindexAt(intervalTurns[0]!); if (vnindexBaseline == null) throw new Error(`no VNINDEX data at first ${interval.label} turn`); @@ -312,8 +312,9 @@ export async function runBacktestSession( throwIfAborted(cb.signal); const dateIso = ictLabel(asOf); const priceOverride = (sym: string): number | null => { - const series = bars[sym]?.filter((b) => b.time <= asOf) ?? []; - return series.length ? series[series.length - 1]!.close : null; + const series = bars[sym] ?? []; + const idx = findLastBarIndex(series, asOf); + return idx !== -1 ? series[idx]!.close : null; }; broker.setPriceOverride(priceOverride); cb.onTurnStart?.({ asOf, dateIso }); diff --git a/src/data/sources/dnsePublic.ts b/src/data/sources/dnsePublic.ts index 151733c..a607cd9 100644 --- a/src/data/sources/dnsePublic.ts +++ b/src/data/sources/dnsePublic.ts @@ -24,6 +24,26 @@ export interface Bar { volume: number; } +/** + * Binary search to find the index of the last bar with time <= asOf. + * This is O(log n) compared to O(n) array filtering. + */ +export function findLastBarIndex(bars: readonly Bar[], asOf: number): number { + let low = 0; + let high = bars.length - 1; + let ans = -1; + while (low <= high) { + const mid = (low + high) >>> 1; + if (bars[mid]!.time <= asOf) { + ans = mid; + low = mid + 1; + } else { + high = mid - 1; + } + } + return ans; +} + async function fetchOhlcs( kind: "stock" | "index", symbol: string, @@ -68,7 +88,9 @@ function clipBars(bars: Bar[]): Bar[] { asOfClock.getStore()?.asOfSec != null || isAsOfOverridden(); if (!hasOverride) return bars; const asOf = nowSec(); - return bars.filter((b) => b.time <= asOf); + const index = findLastBarIndex(bars, asOf); + if (index === -1) return []; + return bars.slice(0, index + 1); } export async function getStockOhlcv(