Skip to content

feat(updater): closed funds skip fetch and guard, failed new funds keep a catalog-only row, test groups of at most five #85

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
daggerok merged 1 commit into from
Oct 26, 2026
Merged

feat(updater): closed funds skip fetch and guard, failed new funds keep a catalog-only row, test groups of at most five #85

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
80 changes: 78 additions & 2 deletions scripts/update-data.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,8 @@ import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { pathToFileURL } from 'node:url';
import {
CONTROL_NAMES, HISTORY_RANGE_PATTERN, RETURNS_BASIS_DERIVED, RETURNS_BASIS_OFFICIAL,
annualizedToTotal, aumSource, cleanHoldingTicker, deriveMetrics, dividendYieldBasisFromKind, formatDate, hasConfiguredFilters, holdingsGuard, inferDistributionFrequency, installSystemCa,
CLOSED_FUNDS, CONTROL_NAMES, HISTORY_RANGE_PATTERN, RETURNS_BASIS_DERIVED, RETURNS_BASIS_OFFICIAL,
annualizedToTotal, aumSource, cleanHoldingTicker, closedFund, deriveMetrics, dividendYieldBasisFromKind, formatDate, hasConfiguredFilters, holdingsGuard, inferDistributionFrequency, installSystemCa,
isCertError, isinFromCusip, looksLikeProductPage, mergeDistributionRecords, normalizeHoldingName, nportUrlFor, numberOrNull, parseAumRange,
parseCatalogMarkdown, parseChart, parseFundListJson, parseEdgarAtomFilings, parseFundTickerMap, parseNport, parseOfficialDistributions,
parseProductPageSummary, parseRange, parseTopHoldingsMarkdown, parseRanges, performanceAsOfDate, priceReturns, readConfig, readKnownRows, resolveControls,
Expand Down Expand Up @@ -110,7 +110,9 @@ describe('controls', () => {
expect(() => resolveControls({}, {}, {}, env)).toThrow();
}
});
});

describe('controls 2', () => {
test('config file: keys equal CONTROL_NAMES and --help, values are strings, the scheduled path equals the defaults', async () => {
const file = configFile();
expect(Object.keys(file).sort()).toEqual([...CONTROL_NAMES].sort());
Expand Down Expand Up @@ -225,7 +227,9 @@ describe('parsing', () => {
const marketOnly = '# GDE WisdomTree Efficient Gold Plus Equity Strategy Fund\n\n### Total Returns\n\nMonth End Performance (8/31/2026)\n\n| Cumulative | 1 Month | 3 Month | YTD | Since Inception* | |\n| --- | --- | --- | --- | --- | --- |\n| Market Price Returns | 1.00% | 2.00% | 3.00% | 4.00% | |';
expect(parseProductPageSummary(marketOnly).officialReturns.monthEnd).toMatchObject({ mo1: 1.00, navReturns: null, indexReturns: null });
});
});

describe('parsing 2', () => {
test('official distributions: dates and the full tax-character breakdown, blank cells are null, no section is an empty list', () => {
const rows = parseOfficialDistributions(distPage('| 9/25/2026 | 9/25/2026 | 9/29/2026 | $0.17000 | $0.00000 | $0.00000 | $0.00000 | $0.17000 |', '| 8/26/2026 | 8/26/2026 | 8/28/2026 | $0.05500 | $0.00000 | $0.00000 | $0.00000 | $0.05500 |'));
expect(rows).toHaveLength(2);
Expand Down Expand Up @@ -276,7 +280,9 @@ describe('parsing', () => {
expect(broken.rows.map((row) => row.Weight)).toEqual(['', '', '5']);
expect(holdingsGuard(broken.headers, broken.rows, null, ['sum'])).toContain('numeric weight');
});
});

describe('parsing 3', () => {
test('holdings guard: header-only, renamed weight column, broken weights, a bad weight sum and a halved sheet are rejected', () => {
const headers = ['Name', 'Ticker', 'Weight'];
const rows = [{ Name: 'A', Ticker: 'A', Weight: '60%' }, { Name: 'B', Ticker: 'B', Weight: '40%' }];
Expand Down Expand Up @@ -360,7 +366,9 @@ describe('metrics', () => {
expect(dividendYieldBasisFromKind('whatever', 1)).toBe('indicated');
expect(dividendYieldBasisFromKind('official-trailing-12m', null)).toBeNull();
});
});

describe('metrics 2', () => {
test('dividendYieldBasis is on rebuilt rows too (code follows the stored kind, null with a null yield)', () => {
const meta = (yields: any) => ({ ticker: 'AAA', name: 'A', yields, aum: {}, returns: {}, distributions: { rows: [] }, holdings: {}, history: {} });
expect(rowFromMeta(meta({ dividendYield: 2, dividendYieldKind: 'trailing 12-month, published by WisdomTree catalog (a published 0.00% is kept as published)' })).metrics.dividendYieldBasis).toBe('official-trailing-12m');
Expand Down Expand Up @@ -569,7 +577,9 @@ describe('pipeline', () => {
await writeIndex([...known.values()]);
expect(s.index().counts.funds).toBe(3);
});
});

describe('pipeline 2', () => {
test('a second identical run changes no file and no modification time', async () => {
const s = new Scenario();
await s.run();
Expand Down Expand Up @@ -655,7 +665,9 @@ describe('pipeline', () => {
expect(s.index().funds.find((f: any) => f.ticker === 'AAA').metrics.secYield).toBeNull();
expect(s.meta('AAA').nav.value).toBe(50.476);
});
});

describe('pipeline 3', () => {
test('an honest page without a quarter-end table does not get a fabricated quarter-end date', async () => {
const s = new Scenario();
const base = s.handler;
Expand Down Expand Up @@ -698,7 +710,9 @@ describe('pipeline', () => {
expect(['AAA', 'BBB', 'CCC'].map((t) => existsSync(new URL(`funds/${t}/meta.json`, s.root)))).toEqual([false, false, false]);
await expect(new Scenario().run({ TICKERS: 'AAA ZZZ' })).rejects.toThrow('unknown ticker ZZZ');
});
});

describe('pipeline 4', () => {
test('a run where every fund failed to get live data exits non-zero', async () => {
const s = new Scenario();
await s.run();
Expand Down Expand Up @@ -750,6 +764,66 @@ describe('pipeline', () => {
expect(s.files().map((path) => readFileSync(path, 'utf8'))).toEqual(before);
expect(process.exitCode).toBe(1);
});
});

describe('pipeline 5', () => {
test('a closed fund is never fetched or reported failed: its files stay as published, its index row says closed, a rerun writes nothing', async () => {
const entry = { ticker: 'BBB', closed: '2026-10-01', source: 'https://www.sec.gov/Archives/edgar/data/1/notice.htm' };
expect(() => closedFund([{ ...entry, closed: 'soon' }], 'BBB')).toThrow('CLOSED_FUNDS');
expect(closedFund([entry], 'bbb')).toEqual(entry);
const s = new Scenario(['AAA', 'BBB', 'CCC']);
const base = s.handler;
const serve = (...rows: string[]) => { s.handler = (url, init) => (url.startsWith('https://r.jina.ai/') && url.endsWith('/bbb') ? new Response(`${productPage('BBB')}\n\n${holdingsSection('| Name | Weight (%) |', ...rows)}`) : base(url, init)); };
serve('| Alpha Corp | 40.00 |', '| Beta Corp | 30.00 |', '| Gamma Corp | 20.00 |', '| Delta Corp | 10.00 |');
await s.run({ EDGAR_FALLBACK: 'true' });
const contents = (): Record<string, string> => Object.fromEntries(s.files().map((path) => [path.slice(s.dir.length), readFileSync(path, 'utf8')]));
const first = contents();
const firstRow = s.index().funds.find((row: any) => row.ticker === 'BBB');
serve('| Alpha Corp | 90.00 |'); // the guard would reject these holdings
s.tickers = ['AAA', 'CCC']; // the live catalog no longer lists the closed fund: TICKERS still accepts it, nothing of it is pruned
CLOSED_FUNDS.push(entry);
try {
s.calls.length = 0;
process.exitCode = 0;
await s.run({ EDGAR_FALLBACK: 'true', TICKERS: 'BBB' });
expect(s.calls.filter((call) => /bbb/i.test(call.url))).toEqual([]);
expect(process.exitCode).not.toBe(1);
const closed = contents();
for (const path of Object.keys(first)) if (!path.endsWith('/index.json')) expect([path, closed[path]]).toEqual([path, first[path]]);
const rows = s.index().funds;
expect(rows.map((row: any) => row.ticker)).toEqual(['AAA', 'BBB', 'CCC']);
expect(rows.find((row: any) => row.ticker === 'BBB')).toEqual({ ...firstRow, status: 'closed', closedDate: '2026-10-01', closedSource: entry.source });
expect(rows.filter((row: any) => row.status === 'closed')).toHaveLength(1);
s.backdate();
await s.run({ EDGAR_FALLBACK: 'true' });
expect([s.touched(), contents()]).toEqual([[], closed]);
} finally {
CLOSED_FUNDS.splice(CLOSED_FUNDS.indexOf(entry), 1);
}
});

test('a new fund whose first refresh fails (guard rejection, no source answered) is failed with a catalog-only row, never read back as published state', async () => {
const s = new Scenario(['AAA']);
await s.run();
const base = s.handler;
s.tickers = ['AAA', 'BBB', 'DDD'];
// BBB: fresh top holdings add up to 10, the guard rejects them; DDD: every request about DDD fails
s.handler = (url, init) => (/ddd/i.test(url) ? new Response('down', { status: 500 })
: url.startsWith('https://r.jina.ai/') && url.endsWith('/bbb') ? new Response(`${productPage('BBB')}\n\n${holdingsSection('| Name | Weight (%) |', '| Alpha Corp | 10.00 |')}`)
: base(url, init));
await s.run({ EDGAR_FALLBACK: 'true' });
const rows = s.index().funds;
expect(rows.map((row: any) => [row.ticker, row.dataFile, row.holdings])).toEqual([['AAA', './funds/AAA/meta.json', 1], ['BBB', null, 0], ['DDD', null, 0]]);
for (const row of rows.slice(1)) expect([Object.keys(row.metrics), row.metrics.returnsBasis, row.metrics.tr1y]).toEqual([Object.keys(rows[0].metrics), RETURNS_BASIS_DERIVED, null]);
expect(['BBB', 'DDD'].map((t) => existsSync(new URL(`funds/${t}/`, s.root)))).toEqual([false, false]);
s.backdate();
await s.run({ EDGAR_FALLBACK: 'true' });
expect(s.touched()).toEqual([]);
// DDD's product page still fails, Yahoo answers now: the catalog-only row is no published quote to retain
s.handler = (url, init) => (url.startsWith('https://r.jina.ai/') && url.endsWith('/ddd') ? new Response('down', { status: 500 }) : base(url, init));
await s.run({ TICKERS: 'DDD' });
expect(s.index().funds.find((row: any) => row.ticker === 'DDD')).toMatchObject({ dataFile: './funds/DDD/meta.json', closePriceValue: 140 });
});

test('a product page that turns its holdings list into an allocation summary keeps the published list and still refreshes the fund', async () => {
const s = new Scenario(['BBB']);
Expand Down Expand Up @@ -843,7 +917,9 @@ describe('network', () => {
await expect(stalled.run({}, { fetchTimeoutMs: 30 })).rejects.toThrow('No catalog rows');
expect([hanging.calls.length > 0, stalled.calls.length > 0, withoutSignal]).toEqual([true, true, 0]);
});
});

describe('network 2', () => {
test('only the SEC receives the SEC contact as User-Agent', async () => {
const s = new Scenario();
await s.run({ EDGAR_FALLBACK: 'true' });
Expand Down
78 changes: 74 additions & 4 deletions scripts/update-data.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -236,6 +236,8 @@ const HOLDINGS_HEADERS = ['Name', 'Ticker', 'Identifier', 'Weight', 'Market Valu
const HOLDINGS_GUARD_RELAX: Record<string, string[]> = {
WSPC: ['sum'], // no N-PORT filing yet: the product page lists only its top 10 holdings (about 47% of the fund)
};
/** Funds the issuer closed, each confirmed from an SEC filing or the issuer notice (`source`): never fetched or guarded, last published data kept, index row marked closed. */
export const CLOSED_FUNDS: ClosedFund[] = [];
const MONTHS = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'];
const TRUTHY = new Set(['1', 'true', 'yes', 'y', 'on']);
const AUM_BOUNDS = { nano: [0, 10_000_000], micro: [10_000_000, 300_000_000], small: [300_000_000, 2_000_000_000], mid: [2_000_000_000, 10_000_000_000], large: [10_000_000_000, undefined] } as const;
Expand Down Expand Up @@ -1911,6 +1913,8 @@ async function processFund(fund: CatalogFund, config: UpdaterConfig, previous: J
outputNote(`[ ${'chart'.padEnd(9)}] ${fund.ticker}: ${error instanceof Error ? error.message : String(error)}`);
}
}
// A new fund that no source answered publishes nothing (never an empty meta.json reported updated): it fails and keeps a catalog-only row.
if (!fresh && !previousMeta) throw new Error('no source answered; no data published for this fund yet');
const previousRows = await readPreviousSheet(fund.ticker, 'history');
if (!days.length) {
days = previousRows.map((row) => ({ date: toIsoDate(row.Date), close: numberOrNull(row.Close) || 0, adjClose: numberOrNull(row['Adj Close']) || numberOrNull(row.Close) || 0, volume: numberOrNull(row.Volume) || 0 })).filter((row) => row.date && row.close > 0);
Expand Down Expand Up @@ -2130,6 +2134,25 @@ export function buildIndexRow(input: IndexRowInput): JsonRecord {
};
}

/** The index row of a new fund whose first refresh failed: its catalog listing only (TER and inception, no prices, net assets, returns or files; dataFile and metrics are completed by normalizeIndexRow). */
export function catalogOnlyRow(fund: CatalogFund): JsonRecord {
return buildIndexRow({
fund,
nav: null,
marketPrice: null,
premiumDiscount: null,
netAssets: null,
asOfLabel: '—',
inceptionDate: fund.inception ? formatDate(fund.inception) : '—',
exchange: fund.exchange || '',
distributions: { frequency: '—', exDate: '—', dividend: '—' },
returns: {},
metrics: {},
holdings: 0,
history: 0,
});
}

/** Rebuilds an index row offline from a fund's published meta.json; `previous` (an existing index row) only fills what meta does not carry. Unknown metric inputs stay null, never 0. */
export function rowFromMeta(meta: JsonRecord, firstHistoryDate: string | null = null, previous: JsonRecord = {}): JsonRecord {
const ticker = String(meta.ticker);
Expand Down Expand Up @@ -2474,12 +2497,13 @@ export async function runUpdate(config: UpdaterConfig, deadlineMs: number = SOFT
let state: JsonRecord = {};
try { state = JSON.parse(await readFile(STATE_FILE, 'utf8')) as JsonRecord; } catch { state = {}; }
// The batch is taken from the funds that pass the catalog filters, in ticker order, starting after the cursor (wrapping around).
const eligible = universe.filter((fund) => catalogFilterReasons(fund, config).length === 0);
const eligible = withoutClosedFunds(universe.filter((fund) => catalogFilterReasons(fund, config).length === 0), (fund) => fund.ticker, CLOSED_FUNDS);
const cursor = config.maxFetches > 0 && !config.tickers ? String(state.cursor || '') : '';
const ordered = cursor ? eligible.filter((fund) => fund.ticker > cursor).concat(eligible.filter((fund) => fund.ticker <= cursor)) : eligible;
const queue = ordered.slice();
const totalAttempts = config.maxFetches > 0 ? Math.min(config.maxFetches, ordered.length) : ordered.length;
const results: JsonRecord[] = [];
const catalogOnly: JsonRecord[] = [];
let processed = 0;
let failures = 0;
let fundsWithFreshData = 0;
Expand All @@ -2497,8 +2521,11 @@ export async function runUpdate(config: UpdaterConfig, deadlineMs: number = SOFT
processed += 1;
lastDispatched = fund.ticker;
const before = await output.before(fund.ticker);
const listed = { ...fund };
// A catalog-only row (dataFile null, written for a new fund whose first refresh failed) is no published state to fall back on.
const prior = previous.get(fund.ticker);
try {
const row = await processFund(fund, config, previous.get(fund.ticker) || {});
const row = await processFund(fund, config, prior && prior.dataFile ? prior : {});
if (row.__skipped) {
await output.result(fund.ticker, before, 'skipped', (row.__skipReasons || ['not eligible']).join(', '));
} else {
Expand All @@ -2509,6 +2536,8 @@ export async function runUpdate(config: UpdaterConfig, deadlineMs: number = SOFT
}
} catch (error) {
failures += 1;
// A new fund (no meta.json yet) that failed is still listed, from its catalog row only.
if (!(await hasMetaFile(fund.ticker))) catalogOnly.push(catalogOnlyRow(listed));
const message = error instanceof Error ? error.message : String(error);
await output.result(fund.ticker, before, 'failed', message);
}
Expand All @@ -2518,11 +2547,13 @@ export async function runUpdate(config: UpdaterConfig, deadlineMs: number = SOFT

// Selected funds are refreshed; every other known fund keeps its published row (filtered, bounded, failed or catalog-less runs never shrink the index).
const mergedRows = new Map<string, JsonRecord>(previous);
for (const row of catalogOnly) mergedRows.set(String(row.ticker), row);
for (const row of results) mergedRows.set(String(row.ticker), row);
const funds: JsonRecord[] = [];
for (const row of [...mergedRows.values()].sort((a, b) => String(a.ticker).localeCompare(String(b.ticker)))) funds.push(normalizeIndexRow(row, await hasMetaFile(String(row.ticker))));
const counts = { funds: funds.length, holdings: funds.reduce((sum, row) => sum + (numberOrNull(row.holdings) || 0), 0), history: funds.reduce((sum, row) => sum + (numberOrNull(row.history) || 0), 0) };
await writeIndex(funds);
// Closed funds keep their published row as it is, marked closed (written even when this run updated nothing).
await writeIndex(markClosedFunds(funds, CLOSED_FUNDS));
// A TICKERS run never touches the cursor; a full run resets it.
if (!config.tickers) await writeIfChanged(STATE_FILE, { cursor: config.maxFetches > 0 ? lastDispatched ?? (cursor || null) : null, savedAt: new Date().toISOString().replace(/\.\d{3}Z$/, 'Z') });
if (stoppedAtDeadline) console.warn(`[ ${'deadline'.padEnd(9)}] soft deadline reached; ${queue.length} funds were not taken this run (the index was still written)`);
Expand Down Expand Up @@ -2550,7 +2581,7 @@ if ((import.meta as { main?: boolean }).main) {
* Weight (%) or Market Weight) are required, >= 80% of the rows carry a numeric weight, the weights add up to
* 50..150 and the sheet keeps >= 50% of the previously published rows. `relax` switches off the checks a fund
* legitimately fails: 'numeric' and 'sum' (geared, inverse, futures, look-through, top-N lists) and 'count'.
* Rows are arrays in header order or objects keyed by the header text.
* Rows are arrays in header order or objects keyed by the header text. A fund in the brand's CLOSED_FUNDS never reaches the guard.
*/
export function holdingsGuard(headers: ReadonlyArray<string>, rows: ReadonlyArray<unknown>, previousRows: number | null | undefined, relax: ReadonlyArray<string> = []): string | null {
const previous = Number(previousRows) > 0 ? Number(previousRows) : 0;
Expand Down Expand Up @@ -2586,4 +2617,43 @@ function holdingsGuardNumber(value: unknown): number | null {
if (!/^[-+]?(\d+\.?\d*|\.\d+)(e[-+]?\d+)?$/i.test(text)) return null;
return negative ? -Number(text) : Number(text);
}

/** A fund its issuer closed (liquidated, delisted or merged away), confirmed from an official source: an SEC filing or the issuer's notice. */
export type ClosedFund = { ticker: string; closed: string; source: string };

/**
* The brand's CLOSED_FUNDS entry for a ticker, or null. A closed fund is never fetched, never guarded and never reported failed: it keeps
* its last published data and its index row carries `status: 'closed'`, `closedDate` and `closedSource`. A malformed entry throws, so a
* typo (no ISO date, no https source) never ships.
*/
export function closedFund(list: ReadonlyArray<ClosedFund>, ticker: unknown): ClosedFund | null {
const wanted = String(ticker === null || ticker === undefined ? '' : ticker).trim().toUpperCase();
let found: ClosedFund | null = null;
for (let i = 0; i < list.length; i++) {
const fund = list[i];
if (!/^[A-Z0-9][A-Z0-9.-]{0,9}$/.test(fund.ticker) || !/^\d{4}-\d{2}-\d{2}$/.test(fund.closed) || !/^https:\/\/\S+$/.test(fund.source)) throw new Error(`CLOSED_FUNDS: ${fund.ticker} needs an upper-case ticker, an ISO closing date and an https source`);
if (fund.ticker === wanted) found = fund;
}
return found;
}

/** The index rows with every closed fund marked (`status: 'closed'`, `closedDate`, `closedSource`); other rows are returned as they are. */
export function markClosedFunds<T extends object>(rows: ReadonlyArray<T>, list: ReadonlyArray<ClosedFund>): T[] {
return rows.map((row) => {
const fund = closedFund(list, (row as { ticker?: unknown }).ticker);
return fund ? ({ ...row, status: 'closed', closedDate: fund.closed, closedSource: fund.source } as T) : row;
});
}

/** A run's queue without the closed funds, which are named once in the log: they are neither fetched nor counted as failed. */
export function withoutClosedFunds<T>(queue: ReadonlyArray<T>, tickerOf: (item: T) => string, list: ReadonlyArray<ClosedFund>): T[] {
const closed: string[] = [];
const open: T[] = [];
for (let i = 0; i < queue.length; i++) {
if (closedFund(list, tickerOf(queue[i]))) closed.push(tickerOf(queue[i]));
else open.push(queue[i]);
}
if (closed.length) console.log(`[ closed ] ${closed.join(', ')}: closed by the issuer, published data kept`);
return open;
}
// </holdings-guard>
Loading