Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 7 additions & 2 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -35,12 +35,17 @@ jobs:
- name: Ingestion-matching differential harness
run: npx vitest run tests/differential

# Pure unit tests (e.g. $currentOp index-build parsing from captured ops).
- name: Unit tests
run: npx vitest run tests/unit

integration:
name: Integration (MongoDB 7 + ClickHouse 26.8)
name: Integration (MongoDB 8.0 + ClickHouse 26.8)
runs-on: ubuntu-latest
services:
mongo:
image: mongo:7
# The MongoDB line deployments run.
image: mongo:8.0
ports: ['27017:27017']
options: >-
--health-cmd "mongosh --quiet --eval 'db.runCommand({ping:1})'"
Expand Down
63 changes: 55 additions & 8 deletions src/http/ledger-viz-route.ts
Original file line number Diff line number Diff line change
Expand Up @@ -281,6 +281,13 @@ const PAGE = `<!doctype html>
<div class="stat"><small>Failed chunks</small><b id="s-failed">–</b></div>
<div class="stat"><small>ETA</small><b id="s-eta">–</b></div>
</div>
<div id="index-progress" style="display:none;margin:-8px 0 12px 2px">
<div id="index-progress-title" style="font-weight:600;margin-bottom:4px"></div>
<div style="height:10px;border-radius:6px;background:var(--line);overflow:hidden">
<div id="index-progress-fill" style="height:100%;width:0%;background:var(--blue);transition:width .6s"></div>
</div>
<div id="index-progress-text" class="hint" style="margin-top:3px"></div>
</div>
<div id="run-progress" style="display:none;margin:-8px 0 6px 2px">
<div style="height:10px;border-radius:6px;background:var(--line);overflow:hidden">
<div id="run-progress-fill" style="height:100%;width:0%;background:var(--green);transition:width .6s"></div>
Expand Down Expand Up @@ -696,25 +703,64 @@ async function buildIndexes(btn) {
btn.disabled = true;
try {
const r = await fetch('/control/build-indexes', { method: 'POST', headers: { 'content-type': 'application/json' }, body: '{}' }).then(x => x.json());
toast(r.started ? '\u2705 Index builds started (' + r.missing + ' collection(s))' : (r.missing === 0 ? '\u2705 All collections already indexed' : '\u26a0\ufe0f Build already running'));
toast(r.started
? '\u2705 Index builds started (' + r.missing + ' collection(s)' + (r.inProgress ? ', ' + r.inProgress + ' already in progress' : '') + ')'
: (r.missing === 0 ? '\u2705 All collections already indexed' : '\u26a0\ufe0f Build already running'));
pollIndexProgress();
} catch (e) { toast('\u274c ' + e.message); }
btn.disabled = false;
}
let idxTimer = null;
function idxPhaseText(op, phases) {
return (op.phase ? 'step ' + op.phase + ' of ' + phases + ': ' : '') + op.phaseLabel +
(op.pct != null ? ' \u00b7 ' + op.pct + '% (' + fmt(op.done) + ' / ' + fmt(op.total) + ')' : '') +
// behind mongos each shard builds its own part; the run waits for the slowest
(op.shards > 1 ? ' \u00b7 slowest of ' + op.shards + ' shards still building: ' + op.shard
: (op.shard ? ' \u00b7 shard ' + op.shard : ''));
}
// Driven by slowTick: covers builds the run starts itself while mapping,
// not only the ones started from the button.
async function pollIndexProgress() {
const p = await fetch('/api/index-progress').then(r => r.json()).catch(() => null);
if (!p) return;
const el = document.getElementById('idx-progress');
const ops = p.serverOps || [];
const rows = [];
if (p.total > 0) rows.push('<div class="check"><span class="ic">' + (p.running ? '\u23f3' : (p.error ? '\u274c' : '\u2705')) + '</span><span class="lbl">' +
p.done.length + '/' + p.total + ' built</span><span class="det">' + esc(p.current ? 'building: ' + p.current : (p.error || 'idle')) + '</span></div>');
for (const op of p.serverOps || []) {
rows.push('<div class="check"><span class="ic">\u23f3</span><span class="lbl">' + esc(op.collection) + '</span><span class="det">' + (op.pct != null ? op.pct + '%' : esc(op.msg || 'in progress')) + '</span></div>');
for (const op of ops) {
rows.push('<div class="check"><span class="ic">\u23f3</span><span class="lbl">' + esc(op.collection) + '</span><span class="det">' + esc(idxPhaseText(op, p.phases)) + '</span></div>');
}
document.getElementById('idx-progress').innerHTML = rows.join('');

// Overview bar: the build the run is blocked on first, then a button
// build, then any other in-flight build on the source db.
const box = document.getElementById('index-progress');
const target = p.auto ? p.auto.collection : (p.running && p.current ? p.current : (ops[0] ? ops[0].collection : null));
const op = target ? ops.find(o => o.collection === target) || null : null;
// ensureIndex on an already-built index returns in milliseconds \u2014 do not
// flash the bar for it
const blip = p.auto && !op && Date.now() - p.auto.startedAtMs < 3000;
document.getElementById('ph2-s').textContent = target && !blip ? 'building\u2026' : 'auto';
if (!target || blip) { box.style.display = 'none'; return; }
box.style.display = '';
document.getElementById('index-progress-title').textContent = 'Building {cd:1,_id:1} index on ' + target +
(p.auto && p.auto.of ? ' \u2014 collection ' + p.auto.position + ' of ' + p.auto.of
: (!p.auto && p.running ? ' \u2014 ' + (p.done.length + 1) + ' of ' + p.total + ' (started from Guide step 2)' : ''));
const fill = document.getElementById('index-progress-fill');
if (op && op.pct != null) {
fill.style.width = op.pct + '%';
fill.style.animation = '';
} else {
fill.style.width = '100%';
fill.style.animation = 'pulse 1.2s ease-in-out infinite';
}
el.innerHTML = rows.join('');
if (p.running && !idxTimer) { idxTimer = setInterval(pollIndexProgress, 3000); }
if (!p.running && idxTimer) { clearInterval(idxTimer); idxTimer = null; }
document.getElementById('index-progress-text').textContent =
(op ? idxPhaseText(op, p.phases)
: 'no server-side progress visible' + (p.serverOpsError ? ': ' + p.serverOpsError : ' (the build may run on another member)')) +
(p.auto ? ' \u00b7 waiting ' + fmtWait(Date.now() - p.auto.startedAtMs) + ' \u00b7 chunk copying starts once the index is ready' : '');
}
function fmtWait(ms) {
const m = Math.max(0, Math.floor(ms / 60000));
return m >= 60 ? Math.floor(m / 60) + 'h ' + (m % 60) + 'm' : (m > 0 ? m + 'm' : Math.floor(ms / 1000) + 's');
}
async function startDry(btn) {
btn.disabled = true;
Expand Down Expand Up @@ -1305,6 +1351,7 @@ async function pollBoundary(btn) {
// digits as a string, and unquoted they looked like a no-op in the UI
function fmtVal(v) { return typeof v === 'string' ? '"' + esc(v) + '"' : esc(String(v)); }
async function slowTick() {
pollIndexProgress().catch(() => {});
try {
const [dlq, report, replay] = await Promise.all([
fetch('/api/dlq?offset=' + dlqOffset).then(r => r.json()),
Expand Down
90 changes: 65 additions & 25 deletions src/runtime/chunk-orchestrator.ts
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ import { SkipCounter, SkipReason } from '../transform/skip-reasons.ts';
import { CoercionCounter } from '../transform/coercions.ts';
import { classifyError } from './error-classifier.ts';
import { discoverCollections } from '../source/discover-collections.ts';
import { parseIndexBuildOps, INDEX_BUILD_PHASES, type IndexBuildOp } from '../source/index-build-ops.ts';
import type { Cursor } from '../types/cursor.ts';
import { createHash } from 'node:crypto';
import { setTimeout as sleep } from 'node:timers/promises';
Expand All @@ -52,6 +53,14 @@ export interface ChunkOrchestratorDeps {
chPressure?: ClickHousePressure;
}

/** Mapping is blocked on this collection's {cd,_id} index (position/of: null outside the map pass). */
export interface AutoIndexState {
collection: string;
position: number | null;
of: number | null;
startedAtMs: number;
}

export interface LedgerEngineStats {
engine: 'ledger';
runId: string;
Expand Down Expand Up @@ -534,9 +543,9 @@ export class ChunkOrchestrator {
return !(d2 && skipEventNames.has(d2.e));
});
}
for (const collection of collections) {
for (const [i, collection] of collections.entries()) {
if (this.stopping) break;
newChunks += await this.mapCollection(collection);
newChunks += await this.mapCollection(collection, { position: i + 1, of: collections.length });
}
if (mapPass === 0) {
this.logger.info({ mapped: this.collectionDefaults.size }, 'All collections mapped — global claiming starts');
Expand Down Expand Up @@ -653,7 +662,7 @@ export class ChunkOrchestrator {
* bounds, cut the chunk grid (+ null-cd sentinel), persist it idempotently.
* No copying happens here — claims are global afterwards.
*/
private async mapCollection(collection: string): Promise<number> {
private async mapCollection(collection: string, at?: { position: number; of: number }): Promise<number> {
const { config, mongoReader, ledger } = this.d;
this.currentCollection = collection; // mapping progress visible in UI
const log = this.logger.child({ collection });
Expand All @@ -662,7 +671,16 @@ export class ChunkOrchestrator {
if (!(await mongoReader.hasRequiredIndex(collection))) {
log.info('Building {cd:1,_id:1} index (this can take a long time on large collections — progress in the UI)');
}
await mongoReader.ensureIndex(collection);
// State is set around ensureIndex itself, not the check above: an
// in-progress build (another pod's, the UI button's, or one a restart
// left behind) is listed as present, yet ensureIndex joins and blocks
// on it all the same.
this.autoIndex = { collection, position: at?.position ?? null, of: at?.of ?? null, startedAtMs: Date.now() };
try {
await mongoReader.ensureIndex(collection);
} finally {
this.autoIndex = null;
}

const defaults = this.d.hashResolver.resolveCollectionName(collection, config.source.collectionPrefix) ?? undefined;
this.collectionDefaults.set(collection, defaults);
Expand Down Expand Up @@ -2219,18 +2237,32 @@ export class ChunkOrchestrator {
private indexBuild: { running: boolean; total: number; done: string[]; current: string | null; error: string | null } =
{ running: false, total: 0, done: [], current: null, error: null };

/** Kick off {cd,_id} index builds on all collections missing it (background). */
async startIndexBuilds(): Promise<{ started: boolean; missing: number }> {
if (this.indexBuild.running) return { started: false, missing: this.indexBuild.total - this.indexBuild.done.length };
/** The index build the run itself is blocked on during mapping (null when not waiting). */
private autoIndex: AutoIndexState | null = null;

autoIndexState(): AutoIndexState | null {
return this.autoIndex;
}

/**
* Kick off {cd,_id} index builds on all collections missing it (background).
* Builds already in flight are listed as present, so they are joined too —
* otherwise the button reports "all indexed" while a build is running.
*/
async startIndexBuilds(): Promise<{ started: boolean; missing: number; inProgress: number }> {
if (this.indexBuild.running) return { started: false, missing: this.indexBuild.total - this.indexBuild.done.length, inProgress: 0 };
const db = this.d.mongoReader.getDatabase();
const collections = await discoverCollections(db, this.d.config.source.collectionPrefix, this.logger);
const inFlight = new Set((await this.currentIndexBuildOps()).ops.map((op) => op.collection));
const missing: string[] = [];
let inProgress = 0;
for (const name of collections) {
const idx = await db.collection(name).indexes().catch(() => []);
const has = idx.some((i) => (i.key as Record<string, unknown>).cd !== undefined && (i.key as Record<string, unknown>)._id !== undefined);
if (!has) missing.push(name);
else if (inFlight.has(name)) { missing.push(name); inProgress++; }
}
if (missing.length === 0) return { started: false, missing: 0 };
if (missing.length === 0) return { started: false, missing: 0, inProgress: 0 };

this.indexBuild = { running: true, total: missing.length, done: [], current: null, error: null };
void (async () => {
Expand All @@ -2247,30 +2279,38 @@ export class ChunkOrchestrator {
this.indexBuild.running = false;
this.indexBuild.current = null;
})();
return { started: true, missing: missing.length };
return { started: true, missing: missing.length, inProgress };
}

/** Index-build progress incl. live server-side build progress via $currentOp. */
async indexBuildProgress(): Promise<Record<string, unknown>> {
let serverOps: Array<{ collection: string; pct: number | null; msg: string }> = [];
/**
* In-flight {cd,_id} builds on the source db, from $currentOp. Progress is
* best-effort (it needs the inprog privilege), so a failure is returned,
* not thrown — but never swallowed: a silent [] hid the readConcern bug.
*/
private async currentIndexBuildOps(): Promise<{ ops: IndexBuildOp[]; error: string | null }> {
try {
const adminDb = this.d.mongoReader.getDatabase().client.db('admin');
const ops = await adminDb
const db = this.d.mongoReader.getDatabase();
const ops = await db.client.db('admin')
.aggregate([
{ $currentOp: { allUsers: true } },
{ $match: { 'command.createIndexes': { $exists: true } } },
{ $project: { command: 1, progress: 1, msg: 1 } },
])
{ $project: { ns: 1, command: 1, progress: 1, msg: 1, shard: 1 } },
], {
// $currentOp refuses any readConcern but 'local', and the source
// client defaults to 'majority' (MONGO_READ_CONCERN)
readConcern: { level: 'local' },
})
.toArray();
serverOps = ops.map((o: Record<string, unknown>) => ({
collection: String((o.command as Record<string, unknown>)?.createIndexes ?? '?'),
pct: o.progress && (o.progress as Record<string, number>).total
? Math.round(((o.progress as Record<string, number>).done / (o.progress as Record<string, number>).total) * 100)
: null,
msg: String(o.msg ?? ''),
}));
} catch { /* $currentOp may need privileges — progress is then best-effort */ }
return { ...this.indexBuild, serverOps };
return { ops: parseIndexBuildOps(ops, db.databaseName), error: null };
} catch (err) {
return { ops: [], error: (err as Error).message };
}
}

/** Index-build progress: UI-button builds, the run's own mapping-time wait, and live server-side progress. */
async indexBuildProgress(): Promise<Record<string, unknown>> {
const server = await this.currentIndexBuildOps();
return { ...this.indexBuild, auto: this.autoIndex, phases: INDEX_BUILD_PHASES, serverOps: server.ops, serverOpsError: server.error };
}

// -------------------------------------------------------------------------
Expand Down
9 changes: 8 additions & 1 deletion src/runtime/ledger-engine.ts
Original file line number Diff line number Diff line change
Expand Up @@ -139,6 +139,8 @@ export async function runLedgerEngine(config: Config, logger: Logger): Promise<v
// not actively copying.
const dryState: { status: string; stats: Record<string, unknown> | null; error: string | null } =
{ status: 'not_run', stats: null, error: null };
// a dry run maps — and so builds missing indexes — on its own orchestrator
let dryOrchestrator: ChunkOrchestrator | null = null;

async function startDryRun(): Promise<{ started: boolean; reason?: string }> {
if (dryState.status === 'running') return { started: false, reason: 'dry run already running' };
Expand Down Expand Up @@ -171,13 +173,15 @@ export async function runLedgerEngine(config: Config, logger: Logger): Promise<v
config: dryConfig, logger, mongoReader: dryReader, ledger, dlq,
staging: dryStaging, retryPolicy, hashResolver,
});
dryOrchestrator = dryOrch;
await dryOrch.run();
dryState.stats = dryOrch.getStats() as unknown as Record<string, unknown>;
dryState.status = 'completed';
} catch (err) {
dryState.status = 'failed';
dryState.error = (err as Error).message;
} finally {
dryOrchestrator = null;
await dryReader.close().catch(() => {});
await dryStaging.close().catch(() => {});
}
Expand Down Expand Up @@ -378,7 +382,10 @@ export async function runLedgerEngine(config: Config, logger: Logger): Promise<v
waived: await dlq.waive(config.ledger.dryRun ? `${config.ledger.runId}-dry` : config.ledger.runId, req.body?.ids),
}));
app.post('/control/build-indexes', async () => orchestrator.startIndexBuilds());
app.get('/api/index-progress', async () => orchestrator.indexBuildProgress());
app.get('/api/index-progress', async () => {
const progress = await orchestrator.indexBuildProgress();
return progress.auto ? progress : { ...progress, auto: dryOrchestrator?.autoIndexState() ?? null };
});
app.post('/control/dry-run', async () => startDryRun());
app.post<{ Body: { force?: boolean } }>('/control/rebuild-ledger', async (req) => startRebuild(req.body?.force === true));
app.get('/api/rebuild', async () => rebuildState);
Expand Down
Loading
Loading