diff --git a/.jules/bolt.md b/.jules/bolt.md new file mode 100644 index 0000000..6162328 --- /dev/null +++ b/.jules/bolt.md @@ -0,0 +1,3 @@ +## 2024-06-14 - Time-Series Lookup Bottleneck +**Learning:** Found a performance bottleneck specific to this codebase's architecture where backtesting runs `vnindex` and `bars` array lookups per interval using `O(n)` array filtering inside a loop. Since time-series market data is chronologically sorted, `O(n)` filter scans are incredibly inefficient. +**Action:** Implemented a binary search utility (`findLastBarIndex`) in `dnsePublic.ts` and replaced the `filter` calls in the critical loop to enable `O(log n)` lookups, drastically reducing time-complexity for data retrieval during backtests. diff --git a/src/agent/backtestRunner.ts b/src/agent/backtestRunner.ts index e91de2c..7089a7b 100644 --- a/src/agent/backtestRunner.ts +++ b/src/agent/backtestRunner.ts @@ -2,7 +2,7 @@ import { randomUUID } from "node:crypto"; import { loadConfig } from "../config/loader.js"; import { getDb } from "../storage/db.js"; import { getBacktestBroker } from "../broker/index.js"; -import { getStockOhlcv, getIndexOhlcv, type Bar } from "../data/sources/dnsePublic.js"; +import { getStockOhlcv, getIndexOhlcv, findLastBarIndex, type Bar } from "../data/sources/dnsePublic.js"; import { DISCOVERY_UNIVERSE, discoverTickers } from "../tools/discover.js"; import { setActiveAsOf } from "./clock.js"; import { runTeamAnalysis } from "./team/index.js"; @@ -298,8 +298,8 @@ export async function runBacktestSession( ); const vnindexAt = (asOf: number): number | null => { - const series = vnindex.filter((b) => b.time <= asOf); - return series.length ? series[series.length - 1]!.close : null; + const idx = findLastBarIndex(vnindex, asOf); + return idx === -1 ? null : vnindex[idx]!.close; }; const vnindexBaseline = vnindexAt(intervalTurns[0]!); if (vnindexBaseline == null) throw new Error(`no VNINDEX data at first ${interval.label} turn`); @@ -312,8 +312,10 @@ export async function runBacktestSession( throwIfAborted(cb.signal); const dateIso = ictLabel(asOf); const priceOverride = (sym: string): number | null => { - const series = bars[sym]?.filter((b) => b.time <= asOf) ?? []; - return series.length ? series[series.length - 1]!.close : null; + const symBars = bars[sym]; + if (!symBars || symBars.length === 0) return null; + const idx = findLastBarIndex(symBars, asOf); + return idx === -1 ? null : symBars[idx]!.close; }; broker.setPriceOverride(priceOverride); cb.onTurnStart?.({ asOf, dateIso }); diff --git a/src/data/sources/dnsePublic.ts b/src/data/sources/dnsePublic.ts index 151733c..a0f5a3b 100644 --- a/src/data/sources/dnsePublic.ts +++ b/src/data/sources/dnsePublic.ts @@ -45,6 +45,27 @@ async function fetchOhlcs( return (await body.json()) as OhlcvSeries; } +/** + * Uses binary search to find the last index where bar.time <= asOf. + * Time series market data arrays are chronologically sorted. + * @returns The index of the bar, or -1 if no such bar exists. + */ +export function findLastBarIndex(bars: Bar[], asOf: number): number { + let l = 0; + let r = bars.length - 1; + let ans = -1; + while (l <= r) { + const m = Math.floor((l + r) / 2); + if (bars[m]!.time <= asOf) { + ans = m; + l = m + 1; + } else { + r = m - 1; + } + } + return ans; +} + export function seriesToBars(s: OhlcvSeries): Bar[] { const out: Bar[] = []; for (let i = 0; i < s.t.length; i++) { @@ -68,7 +89,11 @@ function clipBars(bars: Bar[]): Bar[] { asOfClock.getStore()?.asOfSec != null || isAsOfOverridden(); if (!hasOverride) return bars; const asOf = nowSec(); - return bars.filter((b) => b.time <= asOf); + + // Use O(log n) binary search instead of O(n) array filter + const idx = findLastBarIndex(bars, asOf); + if (idx === -1) return []; + return bars.slice(0, idx + 1); } export async function getStockOhlcv(