From 52f0ac01c4aa351765ba44203fb1fb36fe7961fe Mon Sep 17 00:00:00 2001 From: unknown Date: Tue, 1 Sep 2026 07:05:16 +0300 Subject: [PATCH] chore(scripts): report what the language backfill would actually write The dry run said how many posts it would label but not into what, which is the one question a dry run exists to answer. Running it against production made that concrete: "1367 labelled" looks fine, and would have hidden either a healthy corpus or a detector quietly filing every Turkish post under English. It now tallies per language and samples per bucket rather than in scan order. Scan order is by id, which in practice is a run of news bots all writing the same language, so a flat sample of the first twenty says nothing about what is further down. Posts the detector declines to call are sampled too. That bucket is where a language it cannot handle would silently pile up, and a bare count would never say so - reading it is what confirmed the 229 unlabelled posts are link-only bot release notes rather than missed Turkish. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01LP54iXPnpBLkTfg2te3hcn --- scripts/backfill-post-lang.ts | 63 +++++++++++++++++++++++++++++------ 1 file changed, 53 insertions(+), 10 deletions(-) diff --git a/scripts/backfill-post-lang.ts b/scripts/backfill-post-lang.ts index 1d6b581..1e3a628 100644 --- a/scripts/backfill-post-lang.ts +++ b/scripts/backfill-post-lang.ts @@ -33,9 +33,12 @@ import { HeuristicLanguageDetectionService } from "../src/infrastructure/externa /** Rows read per round trip. Large enough to be quick, small enough to stream. */ const BATCH_SIZE = 500; -/** How many detected samples a dry run prints before it stops narrating. */ +/** How many samples a dry run prints per bucket before it stops narrating. */ const DRY_RUN_SAMPLES = 20; +/** Bucket label for posts the detector would not call. */ +const UNDETECTED = "(undetected)"; + const dryRun = process.argv.includes("--dry-run"); const relabelAll = process.argv.includes("--all"); @@ -50,7 +53,9 @@ if (!connectionString) { ); } -const prisma = new PrismaClient({ adapter: new PrismaPg({ connectionString }) }); +const prisma = new PrismaClient({ + adapter: new PrismaPg({ connectionString }), +}); const detector = new HeuristicLanguageDetectionService(); /** @@ -65,6 +70,7 @@ async function backfill(): Promise<{ scanned: number; labelled: number; undetected: number; + perLanguage: Map; }> { const where = relabelAll ? {} : { lang: null }; @@ -73,6 +79,11 @@ async function backfill(): Promise<{ let labelled = 0; let undetected = 0; let samplesPrinted = 0; + // A dry run that says how many posts it would label, without saying into + // what, cannot answer the question the dry run exists for: whether the + // detector is quietly filing one language under another. + const perLanguage = new Map(); + const samplesByLanguage = new Map(); for (;;) { const batch = await prisma.post.findMany({ @@ -91,18 +102,37 @@ async function backfill(): Promise<{ if (!lang) { undetected++; + // Sampled too. An undetected bucket you cannot look inside is + // a blind spot: it is where a language the detector cannot + // handle would silently pile up, and the count alone would + // never say so. + if (dryRun) { + const samples = samplesByLanguage.get(UNDETECTED) ?? []; + if (samples.length < DRY_RUN_SAMPLES) { + samples.push( + post.content.replace(/\s+/g, " ").slice(0, 70), + ); + samplesByLanguage.set(UNDETECTED, samples); + } + } continue; } labelled++; + perLanguage.set(lang, (perLanguage.get(lang) ?? 0) + 1); if (dryRun) { - if (samplesPrinted < DRY_RUN_SAMPLES) { + // Sampled per language rather than in scan order. The first + // rows are whichever ids sort lowest - in practice a run of + // news bots all writing the same language - so a flat sample + // says nothing about the languages further down. + const samples = samplesByLanguage.get(lang) ?? []; + if (samples.length < DRY_RUN_SAMPLES) { + samples.push( + post.content.replace(/\s+/g, " ").slice(0, 70), + ); + samplesByLanguage.set(lang, samples); samplesPrinted++; - const preview = post.content - .replace(/\s+/g, " ") - .slice(0, 70); - console.log(` [${lang}] ${preview}`); } continue; } @@ -117,7 +147,16 @@ async function backfill(): Promise<{ console.log(`scanned ${scanned}, labelled ${labelled}`); } - return { scanned, labelled, undetected }; + if (dryRun) { + for (const [lang, samples] of samplesByLanguage) { + const count = + lang === UNDETECTED ? undetected : (perLanguage.get(lang) ?? 0); + console.log(`\n--- ${lang} (${count}) ---`); + for (const sample of samples) console.log(` ${sample}`); + } + } + + return { scanned, labelled, undetected, perLanguage }; } async function main(): Promise { @@ -127,11 +166,15 @@ async function main(): Promise { })`, ); - const { scanned, labelled, undetected } = await backfill(); + const { scanned, labelled, undetected, perLanguage } = await backfill(); - console.log("---"); + console.log("\n---"); console.log(`scanned: ${scanned}`); console.log(`labelled: ${labelled}`); + for (const [lang, count] of [...perLanguage].sort((a, b) => b[1] - a[1])) { + const share = ((count / labelled) * 100).toFixed(1); + console.log(` ${lang}: ${count} (${share}%)`); + } console.log(`undetected: ${undetected} (left null, reconsidered next run)`); }