From 660a4a040e3d99bd58765d3b1e4f4995678e5092 Mon Sep 17 00:00:00 2001 From: Maksim Kostromin Date: Mon, 26 Oct 2026 01:22:33 -0400 Subject: [PATCH] feat(updater): closed funds skip fetch and guard, failed new funds keep a catalog-only row, test groups of at most five - Q1: CLOSED_FUNDS (empty, no confirmed closed WisdomTree fund) with the shared closed-fund helpers; the run queue drops closed funds before any request or MAX_FETCHES budget, index rows are marked closed on every run - Q2: a guard rejection was already failed; a new fund whose first refresh failed (guard rejection or any error) now keeps a catalog-only row instead of no row, a new fund no source answered fails instead of publishing an empty meta.json, and a catalog-only row is never passed on as published previous state - Q12: describe groups split to at most five tests (55 tests, 388 expect calls before and after) Refs open-items etfs Q1, Q2, Q12 --- scripts/update-data.test.ts | 80 ++++++++++++++++++++++++++++++++++++- scripts/update-data.ts | 78 ++++++++++++++++++++++++++++++++++-- 2 files changed, 152 insertions(+), 6 deletions(-) diff --git a/scripts/update-data.test.ts b/scripts/update-data.test.ts index 88a3223..f49e63c 100644 --- a/scripts/update-data.test.ts +++ b/scripts/update-data.test.ts @@ -5,8 +5,8 @@ import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { pathToFileURL } from 'node:url'; import { - CONTROL_NAMES, HISTORY_RANGE_PATTERN, RETURNS_BASIS_DERIVED, RETURNS_BASIS_OFFICIAL, - annualizedToTotal, aumSource, cleanHoldingTicker, deriveMetrics, dividendYieldBasisFromKind, formatDate, hasConfiguredFilters, holdingsGuard, inferDistributionFrequency, installSystemCa, + CLOSED_FUNDS, CONTROL_NAMES, HISTORY_RANGE_PATTERN, RETURNS_BASIS_DERIVED, RETURNS_BASIS_OFFICIAL, + annualizedToTotal, aumSource, cleanHoldingTicker, closedFund, deriveMetrics, dividendYieldBasisFromKind, formatDate, hasConfiguredFilters, holdingsGuard, inferDistributionFrequency, installSystemCa, isCertError, isinFromCusip, looksLikeProductPage, mergeDistributionRecords, normalizeHoldingName, nportUrlFor, numberOrNull, parseAumRange, parseCatalogMarkdown, parseChart, parseFundListJson, parseEdgarAtomFilings, parseFundTickerMap, parseNport, parseOfficialDistributions, parseProductPageSummary, parseRange, parseTopHoldingsMarkdown, parseRanges, performanceAsOfDate, priceReturns, readConfig, readKnownRows, resolveControls, @@ -110,7 +110,9 @@ describe('controls', () => { expect(() => resolveControls({}, {}, {}, env)).toThrow(); } }); +}); +describe('controls 2', () => { test('config file: keys equal CONTROL_NAMES and --help, values are strings, the scheduled path equals the defaults', async () => { const file = configFile(); expect(Object.keys(file).sort()).toEqual([...CONTROL_NAMES].sort()); @@ -225,7 +227,9 @@ describe('parsing', () => { const marketOnly = '# GDE WisdomTree Efficient Gold Plus Equity Strategy Fund\n\n### Total Returns\n\nMonth End Performance (8/31/2026)\n\n| Cumulative | 1 Month | 3 Month | YTD | Since Inception* | |\n| --- | --- | --- | --- | --- | --- |\n| Market Price Returns | 1.00% | 2.00% | 3.00% | 4.00% | |'; expect(parseProductPageSummary(marketOnly).officialReturns.monthEnd).toMatchObject({ mo1: 1.00, navReturns: null, indexReturns: null }); }); +}); +describe('parsing 2', () => { test('official distributions: dates and the full tax-character breakdown, blank cells are null, no section is an empty list', () => { const rows = parseOfficialDistributions(distPage('| 9/25/2026 | 9/25/2026 | 9/29/2026 | $0.17000 | $0.00000 | $0.00000 | $0.00000 | $0.17000 |', '| 8/26/2026 | 8/26/2026 | 8/28/2026 | $0.05500 | $0.00000 | $0.00000 | $0.00000 | $0.05500 |')); expect(rows).toHaveLength(2); @@ -276,7 +280,9 @@ describe('parsing', () => { expect(broken.rows.map((row) => row.Weight)).toEqual(['', '', '5']); expect(holdingsGuard(broken.headers, broken.rows, null, ['sum'])).toContain('numeric weight'); }); +}); +describe('parsing 3', () => { test('holdings guard: header-only, renamed weight column, broken weights, a bad weight sum and a halved sheet are rejected', () => { const headers = ['Name', 'Ticker', 'Weight']; const rows = [{ Name: 'A', Ticker: 'A', Weight: '60%' }, { Name: 'B', Ticker: 'B', Weight: '40%' }]; @@ -360,7 +366,9 @@ describe('metrics', () => { expect(dividendYieldBasisFromKind('whatever', 1)).toBe('indicated'); expect(dividendYieldBasisFromKind('official-trailing-12m', null)).toBeNull(); }); +}); +describe('metrics 2', () => { test('dividendYieldBasis is on rebuilt rows too (code follows the stored kind, null with a null yield)', () => { const meta = (yields: any) => ({ ticker: 'AAA', name: 'A', yields, aum: {}, returns: {}, distributions: { rows: [] }, holdings: {}, history: {} }); expect(rowFromMeta(meta({ dividendYield: 2, dividendYieldKind: 'trailing 12-month, published by WisdomTree catalog (a published 0.00% is kept as published)' })).metrics.dividendYieldBasis).toBe('official-trailing-12m'); @@ -569,7 +577,9 @@ describe('pipeline', () => { await writeIndex([...known.values()]); expect(s.index().counts.funds).toBe(3); }); +}); +describe('pipeline 2', () => { test('a second identical run changes no file and no modification time', async () => { const s = new Scenario(); await s.run(); @@ -655,7 +665,9 @@ describe('pipeline', () => { expect(s.index().funds.find((f: any) => f.ticker === 'AAA').metrics.secYield).toBeNull(); expect(s.meta('AAA').nav.value).toBe(50.476); }); +}); +describe('pipeline 3', () => { test('an honest page without a quarter-end table does not get a fabricated quarter-end date', async () => { const s = new Scenario(); const base = s.handler; @@ -698,7 +710,9 @@ describe('pipeline', () => { expect(['AAA', 'BBB', 'CCC'].map((t) => existsSync(new URL(`funds/${t}/meta.json`, s.root)))).toEqual([false, false, false]); await expect(new Scenario().run({ TICKERS: 'AAA ZZZ' })).rejects.toThrow('unknown ticker ZZZ'); }); +}); +describe('pipeline 4', () => { test('a run where every fund failed to get live data exits non-zero', async () => { const s = new Scenario(); await s.run(); @@ -750,6 +764,66 @@ describe('pipeline', () => { expect(s.files().map((path) => readFileSync(path, 'utf8'))).toEqual(before); expect(process.exitCode).toBe(1); }); +}); + +describe('pipeline 5', () => { + test('a closed fund is never fetched or reported failed: its files stay as published, its index row says closed, a rerun writes nothing', async () => { + const entry = { ticker: 'BBB', closed: '2026-10-01', source: 'https://www.sec.gov/Archives/edgar/data/1/notice.htm' }; + expect(() => closedFund([{ ...entry, closed: 'soon' }], 'BBB')).toThrow('CLOSED_FUNDS'); + expect(closedFund([entry], 'bbb')).toEqual(entry); + const s = new Scenario(['AAA', 'BBB', 'CCC']); + const base = s.handler; + const serve = (...rows: string[]) => { s.handler = (url, init) => (url.startsWith('https://r.jina.ai/') && url.endsWith('/bbb') ? new Response(`${productPage('BBB')}\n\n${holdingsSection('| Name | Weight (%) |', ...rows)}`) : base(url, init)); }; + serve('| Alpha Corp | 40.00 |', '| Beta Corp | 30.00 |', '| Gamma Corp | 20.00 |', '| Delta Corp | 10.00 |'); + await s.run({ EDGAR_FALLBACK: 'true' }); + const contents = (): Record => Object.fromEntries(s.files().map((path) => [path.slice(s.dir.length), readFileSync(path, 'utf8')])); + const first = contents(); + const firstRow = s.index().funds.find((row: any) => row.ticker === 'BBB'); + serve('| Alpha Corp | 90.00 |'); // the guard would reject these holdings + s.tickers = ['AAA', 'CCC']; // the live catalog no longer lists the closed fund: TICKERS still accepts it, nothing of it is pruned + CLOSED_FUNDS.push(entry); + try { + s.calls.length = 0; + process.exitCode = 0; + await s.run({ EDGAR_FALLBACK: 'true', TICKERS: 'BBB' }); + expect(s.calls.filter((call) => /bbb/i.test(call.url))).toEqual([]); + expect(process.exitCode).not.toBe(1); + const closed = contents(); + for (const path of Object.keys(first)) if (!path.endsWith('/index.json')) expect([path, closed[path]]).toEqual([path, first[path]]); + const rows = s.index().funds; + expect(rows.map((row: any) => row.ticker)).toEqual(['AAA', 'BBB', 'CCC']); + expect(rows.find((row: any) => row.ticker === 'BBB')).toEqual({ ...firstRow, status: 'closed', closedDate: '2026-10-01', closedSource: entry.source }); + expect(rows.filter((row: any) => row.status === 'closed')).toHaveLength(1); + s.backdate(); + await s.run({ EDGAR_FALLBACK: 'true' }); + expect([s.touched(), contents()]).toEqual([[], closed]); + } finally { + CLOSED_FUNDS.splice(CLOSED_FUNDS.indexOf(entry), 1); + } + }); + + test('a new fund whose first refresh fails (guard rejection, no source answered) is failed with a catalog-only row, never read back as published state', async () => { + const s = new Scenario(['AAA']); + await s.run(); + const base = s.handler; + s.tickers = ['AAA', 'BBB', 'DDD']; + // BBB: fresh top holdings add up to 10, the guard rejects them; DDD: every request about DDD fails + s.handler = (url, init) => (/ddd/i.test(url) ? new Response('down', { status: 500 }) + : url.startsWith('https://r.jina.ai/') && url.endsWith('/bbb') ? new Response(`${productPage('BBB')}\n\n${holdingsSection('| Name | Weight (%) |', '| Alpha Corp | 10.00 |')}`) + : base(url, init)); + await s.run({ EDGAR_FALLBACK: 'true' }); + const rows = s.index().funds; + expect(rows.map((row: any) => [row.ticker, row.dataFile, row.holdings])).toEqual([['AAA', './funds/AAA/meta.json', 1], ['BBB', null, 0], ['DDD', null, 0]]); + for (const row of rows.slice(1)) expect([Object.keys(row.metrics), row.metrics.returnsBasis, row.metrics.tr1y]).toEqual([Object.keys(rows[0].metrics), RETURNS_BASIS_DERIVED, null]); + expect(['BBB', 'DDD'].map((t) => existsSync(new URL(`funds/${t}/`, s.root)))).toEqual([false, false]); + s.backdate(); + await s.run({ EDGAR_FALLBACK: 'true' }); + expect(s.touched()).toEqual([]); + // DDD's product page still fails, Yahoo answers now: the catalog-only row is no published quote to retain + s.handler = (url, init) => (url.startsWith('https://r.jina.ai/') && url.endsWith('/ddd') ? new Response('down', { status: 500 }) : base(url, init)); + await s.run({ TICKERS: 'DDD' }); + expect(s.index().funds.find((row: any) => row.ticker === 'DDD')).toMatchObject({ dataFile: './funds/DDD/meta.json', closePriceValue: 140 }); + }); test('a product page that turns its holdings list into an allocation summary keeps the published list and still refreshes the fund', async () => { const s = new Scenario(['BBB']); @@ -843,7 +917,9 @@ describe('network', () => { await expect(stalled.run({}, { fetchTimeoutMs: 30 })).rejects.toThrow('No catalog rows'); expect([hanging.calls.length > 0, stalled.calls.length > 0, withoutSignal]).toEqual([true, true, 0]); }); +}); +describe('network 2', () => { test('only the SEC receives the SEC contact as User-Agent', async () => { const s = new Scenario(); await s.run({ EDGAR_FALLBACK: 'true' }); diff --git a/scripts/update-data.ts b/scripts/update-data.ts index f2cb74a..31d1d49 100755 --- a/scripts/update-data.ts +++ b/scripts/update-data.ts @@ -236,6 +236,8 @@ const HOLDINGS_HEADERS = ['Name', 'Ticker', 'Identifier', 'Weight', 'Market Valu const HOLDINGS_GUARD_RELAX: Record = { WSPC: ['sum'], // no N-PORT filing yet: the product page lists only its top 10 holdings (about 47% of the fund) }; +/** Funds the issuer closed, each confirmed from an SEC filing or the issuer notice (`source`): never fetched or guarded, last published data kept, index row marked closed. */ +export const CLOSED_FUNDS: ClosedFund[] = []; const MONTHS = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec']; const TRUTHY = new Set(['1', 'true', 'yes', 'y', 'on']); const AUM_BOUNDS = { nano: [0, 10_000_000], micro: [10_000_000, 300_000_000], small: [300_000_000, 2_000_000_000], mid: [2_000_000_000, 10_000_000_000], large: [10_000_000_000, undefined] } as const; @@ -1911,6 +1913,8 @@ async function processFund(fund: CatalogFund, config: UpdaterConfig, previous: J outputNote(`[ ${'chart'.padEnd(9)}] ${fund.ticker}: ${error instanceof Error ? error.message : String(error)}`); } } + // A new fund that no source answered publishes nothing (never an empty meta.json reported updated): it fails and keeps a catalog-only row. + if (!fresh && !previousMeta) throw new Error('no source answered; no data published for this fund yet'); const previousRows = await readPreviousSheet(fund.ticker, 'history'); if (!days.length) { days = previousRows.map((row) => ({ date: toIsoDate(row.Date), close: numberOrNull(row.Close) || 0, adjClose: numberOrNull(row['Adj Close']) || numberOrNull(row.Close) || 0, volume: numberOrNull(row.Volume) || 0 })).filter((row) => row.date && row.close > 0); @@ -2130,6 +2134,25 @@ export function buildIndexRow(input: IndexRowInput): JsonRecord { }; } +/** The index row of a new fund whose first refresh failed: its catalog listing only (TER and inception, no prices, net assets, returns or files; dataFile and metrics are completed by normalizeIndexRow). */ +export function catalogOnlyRow(fund: CatalogFund): JsonRecord { + return buildIndexRow({ + fund, + nav: null, + marketPrice: null, + premiumDiscount: null, + netAssets: null, + asOfLabel: '—', + inceptionDate: fund.inception ? formatDate(fund.inception) : '—', + exchange: fund.exchange || '', + distributions: { frequency: '—', exDate: '—', dividend: '—' }, + returns: {}, + metrics: {}, + holdings: 0, + history: 0, + }); +} + /** Rebuilds an index row offline from a fund's published meta.json; `previous` (an existing index row) only fills what meta does not carry. Unknown metric inputs stay null, never 0. */ export function rowFromMeta(meta: JsonRecord, firstHistoryDate: string | null = null, previous: JsonRecord = {}): JsonRecord { const ticker = String(meta.ticker); @@ -2474,12 +2497,13 @@ export async function runUpdate(config: UpdaterConfig, deadlineMs: number = SOFT let state: JsonRecord = {}; try { state = JSON.parse(await readFile(STATE_FILE, 'utf8')) as JsonRecord; } catch { state = {}; } // The batch is taken from the funds that pass the catalog filters, in ticker order, starting after the cursor (wrapping around). - const eligible = universe.filter((fund) => catalogFilterReasons(fund, config).length === 0); + const eligible = withoutClosedFunds(universe.filter((fund) => catalogFilterReasons(fund, config).length === 0), (fund) => fund.ticker, CLOSED_FUNDS); const cursor = config.maxFetches > 0 && !config.tickers ? String(state.cursor || '') : ''; const ordered = cursor ? eligible.filter((fund) => fund.ticker > cursor).concat(eligible.filter((fund) => fund.ticker <= cursor)) : eligible; const queue = ordered.slice(); const totalAttempts = config.maxFetches > 0 ? Math.min(config.maxFetches, ordered.length) : ordered.length; const results: JsonRecord[] = []; + const catalogOnly: JsonRecord[] = []; let processed = 0; let failures = 0; let fundsWithFreshData = 0; @@ -2497,8 +2521,11 @@ export async function runUpdate(config: UpdaterConfig, deadlineMs: number = SOFT processed += 1; lastDispatched = fund.ticker; const before = await output.before(fund.ticker); + const listed = { ...fund }; + // A catalog-only row (dataFile null, written for a new fund whose first refresh failed) is no published state to fall back on. + const prior = previous.get(fund.ticker); try { - const row = await processFund(fund, config, previous.get(fund.ticker) || {}); + const row = await processFund(fund, config, prior && prior.dataFile ? prior : {}); if (row.__skipped) { await output.result(fund.ticker, before, 'skipped', (row.__skipReasons || ['not eligible']).join(', ')); } else { @@ -2509,6 +2536,8 @@ export async function runUpdate(config: UpdaterConfig, deadlineMs: number = SOFT } } catch (error) { failures += 1; + // A new fund (no meta.json yet) that failed is still listed, from its catalog row only. + if (!(await hasMetaFile(fund.ticker))) catalogOnly.push(catalogOnlyRow(listed)); const message = error instanceof Error ? error.message : String(error); await output.result(fund.ticker, before, 'failed', message); } @@ -2518,11 +2547,13 @@ export async function runUpdate(config: UpdaterConfig, deadlineMs: number = SOFT // Selected funds are refreshed; every other known fund keeps its published row (filtered, bounded, failed or catalog-less runs never shrink the index). const mergedRows = new Map(previous); + for (const row of catalogOnly) mergedRows.set(String(row.ticker), row); for (const row of results) mergedRows.set(String(row.ticker), row); const funds: JsonRecord[] = []; for (const row of [...mergedRows.values()].sort((a, b) => String(a.ticker).localeCompare(String(b.ticker)))) funds.push(normalizeIndexRow(row, await hasMetaFile(String(row.ticker)))); const counts = { funds: funds.length, holdings: funds.reduce((sum, row) => sum + (numberOrNull(row.holdings) || 0), 0), history: funds.reduce((sum, row) => sum + (numberOrNull(row.history) || 0), 0) }; - await writeIndex(funds); + // Closed funds keep their published row as it is, marked closed (written even when this run updated nothing). + await writeIndex(markClosedFunds(funds, CLOSED_FUNDS)); // A TICKERS run never touches the cursor; a full run resets it. if (!config.tickers) await writeIfChanged(STATE_FILE, { cursor: config.maxFetches > 0 ? lastDispatched ?? (cursor || null) : null, savedAt: new Date().toISOString().replace(/\.\d{3}Z$/, 'Z') }); if (stoppedAtDeadline) console.warn(`[ ${'deadline'.padEnd(9)}] soft deadline reached; ${queue.length} funds were not taken this run (the index was still written)`); @@ -2550,7 +2581,7 @@ if ((import.meta as { main?: boolean }).main) { * Weight (%) or Market Weight) are required, >= 80% of the rows carry a numeric weight, the weights add up to * 50..150 and the sheet keeps >= 50% of the previously published rows. `relax` switches off the checks a fund * legitimately fails: 'numeric' and 'sum' (geared, inverse, futures, look-through, top-N lists) and 'count'. - * Rows are arrays in header order or objects keyed by the header text. + * Rows are arrays in header order or objects keyed by the header text. A fund in the brand's CLOSED_FUNDS never reaches the guard. */ export function holdingsGuard(headers: ReadonlyArray, rows: ReadonlyArray, previousRows: number | null | undefined, relax: ReadonlyArray = []): string | null { const previous = Number(previousRows) > 0 ? Number(previousRows) : 0; @@ -2586,4 +2617,43 @@ function holdingsGuardNumber(value: unknown): number | null { if (!/^[-+]?(\d+\.?\d*|\.\d+)(e[-+]?\d+)?$/i.test(text)) return null; return negative ? -Number(text) : Number(text); } + +/** A fund its issuer closed (liquidated, delisted or merged away), confirmed from an official source: an SEC filing or the issuer's notice. */ +export type ClosedFund = { ticker: string; closed: string; source: string }; + +/** + * The brand's CLOSED_FUNDS entry for a ticker, or null. A closed fund is never fetched, never guarded and never reported failed: it keeps + * its last published data and its index row carries `status: 'closed'`, `closedDate` and `closedSource`. A malformed entry throws, so a + * typo (no ISO date, no https source) never ships. + */ +export function closedFund(list: ReadonlyArray, ticker: unknown): ClosedFund | null { + const wanted = String(ticker === null || ticker === undefined ? '' : ticker).trim().toUpperCase(); + let found: ClosedFund | null = null; + for (let i = 0; i < list.length; i++) { + const fund = list[i]; + if (!/^[A-Z0-9][A-Z0-9.-]{0,9}$/.test(fund.ticker) || !/^\d{4}-\d{2}-\d{2}$/.test(fund.closed) || !/^https:\/\/\S+$/.test(fund.source)) throw new Error(`CLOSED_FUNDS: ${fund.ticker} needs an upper-case ticker, an ISO closing date and an https source`); + if (fund.ticker === wanted) found = fund; + } + return found; +} + +/** The index rows with every closed fund marked (`status: 'closed'`, `closedDate`, `closedSource`); other rows are returned as they are. */ +export function markClosedFunds(rows: ReadonlyArray, list: ReadonlyArray): T[] { + return rows.map((row) => { + const fund = closedFund(list, (row as { ticker?: unknown }).ticker); + return fund ? ({ ...row, status: 'closed', closedDate: fund.closed, closedSource: fund.source } as T) : row; + }); +} + +/** A run's queue without the closed funds, which are named once in the log: they are neither fetched nor counted as failed. */ +export function withoutClosedFunds(queue: ReadonlyArray, tickerOf: (item: T) => string, list: ReadonlyArray): T[] { + const closed: string[] = []; + const open: T[] = []; + for (let i = 0; i < queue.length; i++) { + if (closedFund(list, tickerOf(queue[i]))) closed.push(tickerOf(queue[i])); + else open.push(queue[i]); + } + if (closed.length) console.log(`[ closed ] ${closed.join(', ')}: closed by the issuer, published data kept`); + return open; +} //