From 3eadd48a0307d5fdc45eca488654c6a54bc4bac1 Mon Sep 17 00:00:00 2001
From: Alzer Del Zeyger <75466035+Dx-Alz-xD@users.noreply.github.com>
Date: Thu, 12 Nov 2026 20:09:43 +0530
Subject: [PATCH] feat(scripts): find and remove papers whose file is gone

adds scripts/remove-ghost-papers.mjs, run with npm run ghost-papers. it
sends a HEAD request to the file_url of every record in papers and admins,
the same check the viewer does, and lists the ones that answer 404.
anything else (403, 5xx, timeouts, no url) is left alone

it is a dry run unless --delete is passed. with --delete it saves the full
records to ghost-papers-<time>.json first, then deletes them, and only if
their file_url has not changed since the check. it refuses to delete when
more than half the files are missing, since a moved or renamed bucket
also answers 404

refs #545
---
 .gitignore                      |   5 +-
 package.json                    |   1 +
 scripts/remove-ghost-papers.mjs | 148 ++++++++++++++++++++++++++++++++
 3 files changed, 153 insertions(+), 1 deletion(-)
 create mode 100644 scripts/remove-ghost-papers.mjs

diff --git a/.gitignore b/.gitignore
index 1509e892..6bba8fe2 100644
--- a/.gitignore
+++ b/.gitignore
@@ -44,4 +44,7 @@ yarn-error.log*
 *.tsbuildinfo
 
 # vscode
-./vscode
\ No newline at end of file
+./vscode
+
+# backups written by scripts/remove-ghost-papers.mjs
+ghost-papers-*.json
diff --git a/package.json b/package.json
index 1c869576..97d398ab 100644
--- a/package.json
+++ b/package.json
@@ -6,6 +6,7 @@
   "scripts": {
     "build": "next build",
     "dev": "next dev",
+    "ghost-papers": "node --env-file=.env scripts/remove-ghost-papers.mjs",
     "lint": "next lint",
     "start": "next start"
   },
diff --git a/scripts/remove-ghost-papers.mjs b/scripts/remove-ghost-papers.mjs
new file mode 100644
index 00000000..1f030e95
--- /dev/null
+++ b/scripts/remove-ghost-papers.mjs
@@ -0,0 +1,148 @@
+import { writeFile } from "node:fs/promises";
+import mongoose from "mongoose";
+
+const collections = ["papers", "admins"];
+const parallelChecks = 8;
+const checkTimeoutMs = 15_000;
+
+const shouldDelete = process.argv.includes("--delete");
+const mongoUri = process.env.MONGODB_URI;
+
+if (!mongoUri) {
+  console.error(
+    "set MONGODB_URI, e.g. node --env-file=.env scripts/remove-ghost-papers.mjs",
+  );
+  process.exit(1);
+}
+
+async function fileStatus(url) {
+  if (typeof url !== "string" || !/^https?:\/\//.test(url)) {
+    return "unknown";
+  }
+  try {
+    const response = await fetch(url, {
+      method: "HEAD",
+      signal: AbortSignal.timeout(checkTimeoutMs),
+    });
+    if (response.ok) return "found";
+    if (response.status === 404) return "missing";
+    return "unknown";
+  } catch {
+    return "unknown";
+  }
+}
+
+async function auditCollection(db, name) {
+  const papers = await db
+    .collection(name)
+    .find(
+      {},
+      { projection: { file_url: 1, subject: 1, exam: 1, slot: 1, year: 1 } },
+    )
+    .toArray();
+
+  const missing = [];
+  const unknown = [];
+  let next = 0;
+
+  async function worker() {
+    while (next < papers.length) {
+      const paper = papers[next++];
+      const status = await fileStatus(paper.file_url);
+      if (status === "missing") missing.push(paper);
+      if (status === "unknown") unknown.push(paper);
+    }
+  }
+
+  await Promise.all(Array.from({ length: parallelChecks }, worker));
+  return { total: papers.length, missing, unknown };
+}
+
+function describe(paper) {
+  const details = [paper.subject, paper.exam, paper.slot, paper.year]
+    .filter(Boolean)
+    .join(" | ");
+  return `  ${String(paper._id)}  ${details}\n    ${String(paper.file_url)}`;
+}
+
+async function main() {
+  await mongoose.connect(mongoUri);
+  const db = mongoose.connection.db;
+  const report = {};
+  let ghostCount = 0;
+  let checkedCount = 0;
+
+  for (const name of collections) {
+    const result = await auditCollection(db, name);
+    report[name] = result;
+    ghostCount += result.missing.length;
+    checkedCount += result.total;
+
+    console.log(
+      `\n${name}: ${result.total} checked, ${result.missing.length} missing, ${result.unknown.length} could not be checked`,
+    );
+    if (result.missing.length) {
+      console.log("missing file (404):");
+      result.missing.forEach((paper) => console.log(describe(paper)));
+    }
+    if (result.unknown.length) {
+      console.log("could not be checked, left alone:");
+      result.unknown.forEach((paper) => console.log(describe(paper)));
+    }
+  }
+
+  if (!shouldDelete) {
+    console.log(
+      `\n${ghostCount} ghost records found. nothing was deleted, run again with --delete to remove them`,
+    );
+    return;
+  }
+
+  if (ghostCount === 0) {
+    console.log("\nno ghost records to delete");
+    return;
+  }
+
+  if (ghostCount > checkedCount / 2) {
+    console.error(
+      `\n${ghostCount} of ${checkedCount} files are missing, which looks like a moved or renamed bucket rather than ghost records. nothing was deleted`,
+    );
+    process.exitCode = 1;
+    return;
+  }
+
+  const backup = {};
+  for (const name of collections) {
+    const ids = report[name].missing.map((paper) => paper._id);
+    backup[name] = ids.length
+      ? await db
+          .collection(name)
+          .find({ _id: { $in: ids } })
+          .toArray()
+      : [];
+  }
+  const backupFile = `ghost-papers-${new Date().toISOString().replace(/[:.]/g, "-")}.json`;
+  await writeFile(backupFile, JSON.stringify(backup, null, 2));
+  console.log(`\nsaved the full records to ${backupFile}`);
+
+  for (const name of collections) {
+    const missing = report[name].missing;
+    if (!missing.length) continue;
+    const { deletedCount } = await db.collection(name).deleteMany({
+      $or: missing.map((paper) => ({
+        _id: paper._id,
+        file_url: paper.file_url,
+      })),
+    });
+    console.log(`deleted ${deletedCount} from ${name}`);
+  }
+}
+
+try {
+  await main();
+} catch (error) {
+  console.error(error);
+  process.exitCode = 1;
+} finally {
+  await mongoose.disconnect();
+}