diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 70b8d67..f6d5604 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,7 +30,7 @@ jobs: - name: Setup Node.js uses: actions/setup-node@v4 with: - node-version: 20 + node-version: 22 cache: pnpm - name: Install dependencies diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 75a3ff5..5ba69e6 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -33,7 +33,7 @@ jobs: - name: Setup Node.js uses: actions/setup-node@v4 with: - node-version: 20 + node-version: 22 cache: pnpm registry-url: https://registry.npmjs.org diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml index c169b90..e7c5ba5 100644 --- a/.github/workflows/security.yml +++ b/.github/workflows/security.yml @@ -55,7 +55,7 @@ jobs: - name: Setup Node.js uses: actions/setup-node@v4 with: - node-version: 20 + node-version: 22 cache: pnpm - name: Install dependencies diff --git a/.jules/bolt.md b/.jules/bolt.md new file mode 100644 index 0000000..2946eb6 --- /dev/null +++ b/.jules/bolt.md @@ -0,0 +1,3 @@ +## 2024-06-25 - Time-Series Market Data Filtering Bottleneck +**Learning:** Chronologically sorted market data arrays (`Bar[]`) were being repeatedly traversed with O(n) `.filter()` calls inside the backtest runner. For intervals over long periods, this creates a massive performance bottleneck. +**Action:** Replace O(n) array `.filter()` on sorted time-series arrays with O(log n) binary search (`findLastBarIndex`) and `.slice()` or direct array indexing where only the latest elements are needed. This provides a dramatic performance improvement especially during backtesting. diff --git a/package.json b/package.json index dd2a5c4..119356f 100644 --- a/package.json +++ b/package.json @@ -65,6 +65,14 @@ "yaml": "^2.6.0", "zod": "^3.23.8" }, + "pnpm": { + "overrides": { + "ws": "^8.21.0" + }, + "onlyBuiltDependencies": [ + "better-sqlite3" + ] + }, "devDependencies": { "@changesets/cli": "^2.31.0", "@types/better-sqlite3": "^7.6.11", @@ -75,10 +83,5 @@ "typescript": "^5.6.3", "vitest": "^2.1.3" }, - "pnpm": { - "onlyBuiltDependencies": [ - "better-sqlite3" - ] - }, "packageManager": "pnpm@10.33.2+sha512.a90faf6feeab71ad6c6e57f94e0fe1a12f5dcc22cd754db40ae9593eb6a3e0b6b12e3540218bb37ae083404b1f2ce6db2a4121e979829b4aff94b99f49da1cf8" } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 8e3f135..3d0865b 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -4,6 +4,9 @@ settings: autoInstallPeers: true excludeLinksFromLockfile: false +overrides: + ws: ^8.21.0 + importers: .: @@ -1622,8 +1625,8 @@ packages: wrappy@1.0.2: resolution: {integrity: sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==} - ws@8.20.0: - resolution: {integrity: sha512-sAt8BhgNbzCtgGbt2OxmpuryO63ZoDk/sqaB/znQm94T4fCEsy/yV+7CdC1kJhOU9lboAEU7R3kquuycDoibVA==} + ws@8.21.0: + resolution: {integrity: sha512-Vsp28b7DRcimFQvrqu2Wek3z1iYxDCWqHYB8Qsnk/S4RfaCQzPGPyBNuVjJV3cd6UiKtUtp6sNM77gWvzcCH+g==} engines: {node: '>=10.0.0'} peerDependencies: bufferutil: ^4.0.1 @@ -2590,7 +2593,7 @@ snapshots: type-fest: 4.41.0 widest-line: 5.0.0 wrap-ansi: 9.0.2 - ws: 8.20.0 + ws: 8.21.0 yoga-layout: 3.2.1 optionalDependencies: '@types/react': 18.3.28 @@ -3083,7 +3086,7 @@ snapshots: wrappy@1.0.2: {} - ws@8.20.0: {} + ws@8.21.0: {} yaml@2.8.3: {} diff --git a/src/agent/backtestRunner.ts b/src/agent/backtestRunner.ts index e91de2c..bdba91c 100644 --- a/src/agent/backtestRunner.ts +++ b/src/agent/backtestRunner.ts @@ -2,7 +2,7 @@ import { randomUUID } from "node:crypto"; import { loadConfig } from "../config/loader.js"; import { getDb } from "../storage/db.js"; import { getBacktestBroker } from "../broker/index.js"; -import { getStockOhlcv, getIndexOhlcv, type Bar } from "../data/sources/dnsePublic.js"; +import { getStockOhlcv, getIndexOhlcv, type Bar, findLastBarIndex } from "../data/sources/dnsePublic.js"; import { DISCOVERY_UNIVERSE, discoverTickers } from "../tools/discover.js"; import { setActiveAsOf } from "./clock.js"; import { runTeamAnalysis } from "./team/index.js"; @@ -101,8 +101,10 @@ function parseBacktestInterval(value: string | undefined): { label: string; minu } function intervalCloses(vnindexBars: Bar[], startSec: number, endSec: number, intervalMinutes: number): number[] { + const startIdx = findLastBarIndex(vnindexBars, startSec - 1) + 1; + const endIdx = findLastBarIndex(vnindexBars, endSec); const base = vnindexBars - .filter((b) => b.time >= startSec && b.time <= endSec) + .slice(startIdx, endIdx + 1) .map((b) => b.time) .sort((a, b) => a - b); const step = Math.max(1, Math.round(intervalMinutes / 30)); @@ -298,8 +300,8 @@ export async function runBacktestSession( ); const vnindexAt = (asOf: number): number | null => { - const series = vnindex.filter((b) => b.time <= asOf); - return series.length ? series[series.length - 1]!.close : null; + const lastIdx = findLastBarIndex(vnindex, asOf); + return lastIdx >= 0 ? vnindex[lastIdx]!.close : null; }; const vnindexBaseline = vnindexAt(intervalTurns[0]!); if (vnindexBaseline == null) throw new Error(`no VNINDEX data at first ${interval.label} turn`); @@ -312,8 +314,9 @@ export async function runBacktestSession( throwIfAborted(cb.signal); const dateIso = ictLabel(asOf); const priceOverride = (sym: string): number | null => { - const series = bars[sym]?.filter((b) => b.time <= asOf) ?? []; - return series.length ? series[series.length - 1]!.close : null; + const symBars = bars[sym] ?? []; + const lastIdx = findLastBarIndex(symBars, asOf); + return lastIdx >= 0 ? symBars[lastIdx]!.close : null; }; broker.setPriceOverride(priceOverride); cb.onTurnStart?.({ asOf, dateIso }); diff --git a/src/data/sources/dnsePublic.ts b/src/data/sources/dnsePublic.ts index 151733c..c068971 100644 --- a/src/data/sources/dnsePublic.ts +++ b/src/data/sources/dnsePublic.ts @@ -45,6 +45,22 @@ async function fetchOhlcs( return (await body.json()) as OhlcvSeries; } +export function findLastBarIndex(bars: Bar[], maxTime: number): number { + let low = 0; + let high = bars.length - 1; + let ans = -1; + while (low <= high) { + const mid = Math.floor((low + high) / 2); + if (bars[mid]!.time <= maxTime) { + ans = mid; + low = mid + 1; + } else { + high = mid - 1; + } + } + return ans; +} + export function seriesToBars(s: OhlcvSeries): Bar[] { const out: Bar[] = []; for (let i = 0; i < s.t.length; i++) { @@ -68,7 +84,9 @@ function clipBars(bars: Bar[]): Bar[] { asOfClock.getStore()?.asOfSec != null || isAsOfOverridden(); if (!hasOverride) return bars; const asOf = nowSec(); - return bars.filter((b) => b.time <= asOf); + const idx = findLastBarIndex(bars, asOf); + if (idx === -1) return []; + return bars.slice(0, idx + 1); } export async function getStockOhlcv(