diff --git a/.gitignore b/.gitignore index 1509e89..6bba8fe 100644 --- a/.gitignore +++ b/.gitignore @@ -44,4 +44,7 @@ yarn-error.log* *.tsbuildinfo # vscode -./vscode \ No newline at end of file +./vscode + +# backups written by scripts/remove-ghost-papers.mjs +ghost-papers-*.json diff --git a/package.json b/package.json index 1c86957..97d398a 100644 --- a/package.json +++ b/package.json @@ -6,6 +6,7 @@ "scripts": { "build": "next build", "dev": "next dev", + "ghost-papers": "node --env-file=.env scripts/remove-ghost-papers.mjs", "lint": "next lint", "start": "next start" }, diff --git a/scripts/remove-ghost-papers.mjs b/scripts/remove-ghost-papers.mjs new file mode 100644 index 0000000..1f030e9 --- /dev/null +++ b/scripts/remove-ghost-papers.mjs @@ -0,0 +1,148 @@ +import { writeFile } from "node:fs/promises"; +import mongoose from "mongoose"; + +const collections = ["papers", "admins"]; +const parallelChecks = 8; +const checkTimeoutMs = 15_000; + +const shouldDelete = process.argv.includes("--delete"); +const mongoUri = process.env.MONGODB_URI; + +if (!mongoUri) { + console.error( + "set MONGODB_URI, e.g. node --env-file=.env scripts/remove-ghost-papers.mjs", + ); + process.exit(1); +} + +async function fileStatus(url) { + if (typeof url !== "string" || !/^https?:\/\//.test(url)) { + return "unknown"; + } + try { + const response = await fetch(url, { + method: "HEAD", + signal: AbortSignal.timeout(checkTimeoutMs), + }); + if (response.ok) return "found"; + if (response.status === 404) return "missing"; + return "unknown"; + } catch { + return "unknown"; + } +} + +async function auditCollection(db, name) { + const papers = await db + .collection(name) + .find( + {}, + { projection: { file_url: 1, subject: 1, exam: 1, slot: 1, year: 1 } }, + ) + .toArray(); + + const missing = []; + const unknown = []; + let next = 0; + + async function worker() { + while (next < papers.length) { + const paper = papers[next++]; + const status = await fileStatus(paper.file_url); + if (status === "missing") missing.push(paper); + if (status === "unknown") unknown.push(paper); + } + } + + await Promise.all(Array.from({ length: parallelChecks }, worker)); + return { total: papers.length, missing, unknown }; +} + +function describe(paper) { + const details = [paper.subject, paper.exam, paper.slot, paper.year] + .filter(Boolean) + .join(" | "); + return ` ${String(paper._id)} ${details}\n ${String(paper.file_url)}`; +} + +async function main() { + await mongoose.connect(mongoUri); + const db = mongoose.connection.db; + const report = {}; + let ghostCount = 0; + let checkedCount = 0; + + for (const name of collections) { + const result = await auditCollection(db, name); + report[name] = result; + ghostCount += result.missing.length; + checkedCount += result.total; + + console.log( + `\n${name}: ${result.total} checked, ${result.missing.length} missing, ${result.unknown.length} could not be checked`, + ); + if (result.missing.length) { + console.log("missing file (404):"); + result.missing.forEach((paper) => console.log(describe(paper))); + } + if (result.unknown.length) { + console.log("could not be checked, left alone:"); + result.unknown.forEach((paper) => console.log(describe(paper))); + } + } + + if (!shouldDelete) { + console.log( + `\n${ghostCount} ghost records found. nothing was deleted, run again with --delete to remove them`, + ); + return; + } + + if (ghostCount === 0) { + console.log("\nno ghost records to delete"); + return; + } + + if (ghostCount > checkedCount / 2) { + console.error( + `\n${ghostCount} of ${checkedCount} files are missing, which looks like a moved or renamed bucket rather than ghost records. nothing was deleted`, + ); + process.exitCode = 1; + return; + } + + const backup = {}; + for (const name of collections) { + const ids = report[name].missing.map((paper) => paper._id); + backup[name] = ids.length + ? await db + .collection(name) + .find({ _id: { $in: ids } }) + .toArray() + : []; + } + const backupFile = `ghost-papers-${new Date().toISOString().replace(/[:.]/g, "-")}.json`; + await writeFile(backupFile, JSON.stringify(backup, null, 2)); + console.log(`\nsaved the full records to ${backupFile}`); + + for (const name of collections) { + const missing = report[name].missing; + if (!missing.length) continue; + const { deletedCount } = await db.collection(name).deleteMany({ + $or: missing.map((paper) => ({ + _id: paper._id, + file_url: paper.file_url, + })), + }); + console.log(`deleted ${deletedCount} from ${name}`); + } +} + +try { + await main(); +} catch (error) { + console.error(error); + process.exitCode = 1; +} finally { + await mongoose.disconnect(); +}