From a16a9cc86bc65dbfe1f6d55f6af36cc6a7f31bbd Mon Sep 17 00:00:00 2001 From: Ranjan Mohanty Date: Sun, 15 Nov 2026 00:22:02 +0100 Subject: [PATCH] Doc staleness via git history: context.drift A context doc is drifting when it hasn't been committed for 90+ days and 25+ later commits edited the files it names or added/removed files in the directories it describes (manifests, lockfiles and docs excluded). Local git log only; n/a outside git, in shallow clones and for docs without code references. Symlinked docs are judged by their target and docs with uncommitted edits are skipped. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 13 +++ README.md | 19 ++-- src/checks/doc-history.ts | 226 ++++++++++++++++++++++++++++++++++++++ src/checks/index.ts | 2 + test/doc-history.test.ts | 157 ++++++++++++++++++++++++++ 5 files changed, 409 insertions(+), 8 deletions(-) create mode 100644 src/checks/doc-history.ts create mode 100644 test/doc-history.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index dff367e..d564b17 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,19 @@ All notable changes to this project are documented here. The format is based on [Semantic Versioning](https://semver.org/). Until 1.0, scoring weights may still be tuned in minor releases; any change that moves scores is called out here. +## [Unreleased] + +### Added + +- New check `context.drift` (weight 1): doc staleness from git history (local `git log`, no + network). A context doc is drifting when it hasn't been committed for 90+ days (relative to HEAD) + and 25+ later commits edited the files it names or added/removed files in the directories it + describes. Manifests, lockfiles and other docs don't count. Symlinked docs are judged by their + target, docs with uncommitted edits are skipped, and the check is n/a outside git, in shallow + clones and for docs that reference no code. Score: 1 − 0.5 × the share of drifting docs. + Effect on real repos: excalidraw 88 → 87 (a 537-day-old CLAUDE.md); repos with recently edited + docs gain a passing check (openai-agents-python 85 → 86, vitest context 87 → 88). + ## [0.1.1] - 2026-11-14 ### Changed diff --git a/README.md b/README.md index 2f33494..5784040 100644 --- a/README.md +++ b/README.md @@ -123,13 +123,16 @@ missing sections in your context doc). It never rewrites content you wrote unles ## What it checks -| Category | Weight | Checks | -| ------------------ | ------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| **Context docs** | 30% | `AGENTS.md` / `CLAUDE.md` / `.cursor/rules` / Copilot / Gemini instructions exist · they list exact build, test and lint commands · they describe architecture and key directories · **not stale**: every referenced path and `npm run` script actually exists · not too long (or too thin) · both `AGENTS.md` and `CLAUDE.md` so every agent is covered | -| **Feedback loops** | 25% | A discoverable test command (package.json `test`, pytest config, `make test`, `cargo test`, `go test`) · linter configured (eslint, biome, oxlint, ruff, flake8, clippy, golangci-lint) · typecheck command (`tsc`, mypy, pyright) · CI config · `--run`: test suite passes and is fast | -| **Navigability** | 20% | `node_modules`, `.venv`, `dist`, caches ignored (and not committed) · no source files over 1000 lines · no minified/vendored/generated blobs in agent view · shallow directory tree · README | -| **Type safety** | 15% | TypeScript `strict` across root and package tsconfigs (follows `extends`, incl. workspace packages), `any` / `@ts-ignore` / `@ts-nocheck` density · plain JS penalized · Python type-hint coverage of product code + mypy/pyright config · Go/Rust count as compiler-checked | -| **Safety** | 10% | Committed `.env` / keys / credentials files · AWS, GitHub, Slack, Stripe, OpenAI, Anthropic, Google, npm token patterns · dangerous scripts: `rm -rf ~`, `curl \| sh`, force-push, `npm publish` hidden in `build`/`test`, network calls in install hooks | +| Category | Weight | Checks | +| ------------------ | ------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| **Context docs** | 30% | `AGENTS.md` / `CLAUDE.md` / `.cursor/rules` / Copilot / Gemini instructions exist · they list exact build, test and lint commands · they describe architecture and key directories · **not stale**: every referenced path and `npm run` script actually exists · **not drifting** (git history): a doc untouched for 90+ days while 25+ later commits changed the files it names or added/removed files in the dirs it describes · not too long (or too thin) · both `AGENTS.md` and `CLAUDE.md` so every agent is covered | +| **Feedback loops** | 25% | A discoverable test command (package.json `test`, pytest config, `make test`, `cargo test`, `go test`) · linter configured (eslint, biome, oxlint, ruff, flake8, clippy, golangci-lint) · typecheck command (`tsc`, mypy, pyright) · CI config · `--run`: test suite passes and is fast | +| **Navigability** | 20% | `node_modules`, `.venv`, `dist`, caches ignored (and not committed) · no source files over 1000 lines · no minified/vendored/generated blobs in agent view · shallow directory tree · README | +| **Type safety** | 15% | TypeScript `strict` across root and package tsconfigs (follows `extends`, incl. workspace packages), `any` / `@ts-ignore` / `@ts-nocheck` density · plain JS penalized · Python type-hint coverage of product code + mypy/pyright config · Go/Rust count as compiler-checked | +| **Safety** | 10% | Committed `.env` / keys / credentials files · AWS, GitHub, Slack, Stripe, OpenAI, Anthropic, Google, npm token patterns · dangerous scripts: `rm -rf ~`, `curl \| sh`, force-push, `npm publish` hidden in `build`/`test`, network calls in install hooks | + +Doc drift reads local `git log` only and never touches the network. It is n/a in shallow clones, +so set `fetch-depth: 0` on `actions/checkout` if you want it in CI. Checks that don't apply (e.g. Python checks in a TS repo) are marked n/a and excluded from the score. In polyglot repos, per-language parts are weighted by each language's share of the code, so a tooling @@ -223,7 +226,7 @@ whatever `npm run build` does. - [ ] Deeper Go and Rust checks (unsafe/`interface{}` density, nested `Cargo.toml` workspaces) and Java/Kotlin support - [ ] `--run` for lint/typecheck too, and flaky-test detection (run twice, compare) - [ ] Monorepo awareness: score each workspace package, check nested `AGENTS.md` -- [ ] Doc staleness via git history (doc untouched while referenced code churned) +- [x] Doc staleness via git history (doc untouched while referenced code churned) - [ ] MCP/tooling checks (`.mcp.json`, Claude Code hooks, `.claude/settings.json` permissions) - [ ] Smarter `--fix`: `tsconfig` strict migration plan, `.cursorignore` for generated code, `.env.example` - [ ] SARIF output for GitHub code scanning diff --git a/src/checks/doc-history.ts b/src/checks/doc-history.ts new file mode 100644 index 0000000..523c9d3 --- /dev/null +++ b/src/checks/doc-history.ts @@ -0,0 +1,226 @@ +import { execFileSync } from 'node:child_process'; +import { lstatSync, realpathSync } from 'node:fs'; +import { join, posix, relative, sep } from 'node:path'; +import type { RepoContext } from '../context.js'; +import type { Check, CheckResult, Finding } from '../types.js'; +import { extractPathRefs, isPointerDoc } from './context-docs.js'; +import { findContextDocs } from './docs.js'; + +/** + * Doc staleness from git history (local `git log` only, never the network): a context doc that + * nobody has touched while the code it points at kept changing probably describes code that no + * longer exists in that form. Complements `context.fresh`, which catches references to paths that + * are gone outright. + */ + +/** A doc counts as drifting when it is at least this old (relative to HEAD)... */ +export const MIN_AGE_DAYS = 90; +/** ...and at least this many later commits changed what it references. */ +export const MIN_COMMITS = 25; + +const MAX_DOCS = 20; +const MAX_REFS = 200; +const MAX_COMMITS_SCANNED = 5000; + +function git(root: string, args: string[]): string | null { + try { + return execFileSync('git', args, { + cwd: root, + encoding: 'utf8', + maxBuffer: 64 * 1024 * 1024, + stdio: ['ignore', 'pipe', 'ignore'], + }); + } catch { + return null; + } +} + +/** Manifests, lockfiles and docs change for unrelated reasons (version bumps, new deps). */ +const NOISY_REF = + /(^|\/)(?:package(?:-lock)?\.json|pnpm-lock\.yaml|yarn\.lock|Cargo\.lock|go\.sum|poetry\.lock|uv\.lock|CHANGELOG[^/]*)$|\.(?:md|mdx|txt|rst)$/i; + +export interface DocDrift { + doc: string; + /** Days between the doc's last commit and HEAD. */ + ageDays: number; + /** Later commits that modified a referenced file or added/removed files in a referenced dir. */ + commits: number; + /** Referenced paths that changed. */ + changedRefs: string[]; + refs: number; +} + +const setsCache = new WeakMap; dirSet: Set }>(); + +function pathSets(ctx: RepoContext): { fileSet: Set; dirSet: Set } { + const cached = setsCache.get(ctx); + if (cached) return cached; + const dirSet = new Set(); + for (const f of ctx.files) { + for (let i = f.indexOf('/'); i !== -1; i = f.indexOf('/', i + 1)) dirSet.add(f.slice(0, i)); + } + const sets = { fileSet: new Set(ctx.files), dirSet }; + setsCache.set(ctx, sets); + return sets; +} + +/** Existing, non-noisy paths a doc references, resolved like `context.fresh` does. */ +function docRefs( + ctx: RepoContext, + docPath: string, + content: string, +): { files: string[]; dirs: string[] } { + const files = new Set(); + const dirs = new Set(); + const docDir = posix.dirname(docPath); + const { fileSet, dirSet } = pathSets(ctx); + for (const ref of extractPathRefs(content)) { + const clean = ref.replace(/\/$/, ''); + const candidates = docDir === '.' ? [clean] : [posix.join(docDir, clean), clean]; + for (const c of candidates) { + if (c.startsWith('..') || NOISY_REF.test(c)) continue; + if (fileSet.has(c)) { + files.add(c); + break; + } + if (dirSet.has(c)) { + dirs.add(c); + break; + } + } + } + return { files: [...files].slice(0, MAX_REFS), dirs: [...dirs].slice(0, MAX_REFS) }; +} + +/** How much the code a doc references changed after the doc's last commit. Null if not measurable. */ +export function docDrift( + ctx: RepoContext, + doc: string, + content: string, + headTime: number, +): DocDrift | null { + const { files, dirs } = docRefs(ctx, doc, content); + if (files.length + dirs.length === 0) return null; + const last = git(ctx.root, ['log', '-1', '--format=%H %ct', '--', doc])?.trim(); + if (!last) return null; // untracked: brand new + const [hash, time] = last.split(' '); + const ageDays = Math.floor((headTime - Number(time)) / 86_400); + // Recently edited docs can't be drifting; skip the (more expensive) history walk. + if (ageDays < MIN_AGE_DAYS) { + return { doc, ageDays, commits: 0, changedRefs: [], refs: files.length + dirs.length }; + } + const out = git(ctx.root, [ + 'log', + '--relative', + '--no-merges', + '--no-renames', + '--format=%x00', + '--name-status', + '-n', + String(MAX_COMMITS_SCANNED), + `${hash}..HEAD`, + '--', + ...files, + ...dirs, + ]); + if (out === null) return null; + const fileSet = new Set(files); + const changed = new Set(); + let commits = 0; + for (const commit of out.split('\0').slice(1)) { + let counts = false; + for (const line of commit.split('\n')) { + const m = /^([AMDT])\t(.+)$/.exec(line); + if (!m) continue; + const [, status, path] = m as unknown as [string, string, string]; + if (fileSet.has(path)) { + counts = true; + changed.add(path); + continue; + } + // Inside a referenced directory only structural changes count: files added or removed. + if (status === 'A' || status === 'D') { + const dir = dirs.find((d) => path.startsWith(`${d}/`)); + if (dir) { + counts = true; + changed.add(dir); + } + } + } + if (counts) commits++; + } + return { doc, ageDays, commits, changedRefs: [...changed], refs: files.length + dirs.length }; +} + +/** Repo-relative target of a symlinked doc (null when not a symlink or it points outside). */ +function symlinkTarget(ctx: RepoContext, rel: string): string | null { + try { + if (!lstatSync(join(ctx.root, rel)).isSymbolicLink()) return null; + const target = relative(realpathSync(ctx.root), realpathSync(join(ctx.root, rel))); + return target.startsWith('..') ? null : target.split(sep).join('/'); + } catch { + return null; + } +} + +export function isDrifting(d: DocDrift): boolean { + return d.ageDays >= MIN_AGE_DAYS && d.commits >= MIN_COMMITS; +} + +export const docHistoryCheck: Check = { + id: 'context.drift', + category: 'context', + title: 'Docs keep up with the code they describe (git history)', + weight: 1, + run(ctx): CheckResult { + if (!ctx.isGit) return { score: null, summary: 'Not a git repository' }; + if (git(ctx.root, ['rev-parse', '--is-shallow-repository'])?.trim() === 'true') { + return { + score: null, + summary: 'Shallow clone: needs full git history (in CI, `fetch-depth: 0`)', + }; + } + const head = Number(git(ctx.root, ['log', '-1', '--format=%ct'])?.trim()); + if (!Number.isFinite(head) || head === 0) return { score: null, summary: 'No commits yet' }; + const seen = new Set(); + const drifts: DocDrift[] = []; + for (const d of findContextDocs(ctx)) { + if (drifts.length >= MAX_DOCS) break; + // A symlinked CLAUDE.md never changes itself; its target's history is what matters. + const path = symlinkTarget(ctx, d.path) ?? d.path; + if (isPointerDoc(d.content) || seen.has(path)) continue; + seen.add(path); + const drift = docDrift(ctx, path, d.content, head); + if (!drift) continue; + // A drifting doc with uncommitted edits is being updated right now. + if (isDrifting(drift) && git(ctx.root, ['diff', '--name-only', 'HEAD', '--', path])?.trim()) { + continue; + } + drifts.push(drift); + } + if (drifts.length === 0) { + return { score: null, summary: 'No committed context doc references code paths' }; + } + const stale = drifts.filter(isDrifting); + const findings: Finding[] = stale.map((d) => ({ + severity: 'warn', + file: d.doc, + message: `unchanged for ${d.ageDays} days while ${d.commits} later commits edited the files it names or added/removed files in the directories it describes (${d.changedRefs.length} of its ${d.refs} paths, e.g. ${d.changedRefs + .slice(0, 3) + .map((p) => `\`${p}\``) + .join(', ')})`, + })); + const score = 1 - 0.5 * (stale.length / drifts.length); + return { + score, + summary: + stale.length === 0 + ? `${drifts.length} doc(s) kept up with the code they reference` + : `${stale.length} of ${drifts.length} doc(s) untouched while the code they reference churned`, + findings, + suggestion: stale.length + ? 'Re-read those docs against the current code and update them (even a small edit resets the clock). Docs that describe last year’s code mislead agents with confidence.' + : undefined, + }; + }, +}; diff --git a/src/checks/index.ts b/src/checks/index.ts index 47a7f47..15a76e7 100644 --- a/src/checks/index.ts +++ b/src/checks/index.ts @@ -1,5 +1,6 @@ import type { Check } from '../types.js'; import { contextChecks } from './context-docs.js'; +import { docHistoryCheck } from './doc-history.js'; import { feedbackChecks } from './feedback.js'; import { navigabilityChecks } from './navigability.js'; import { safetyChecks } from './safety.js'; @@ -12,4 +13,5 @@ export const builtinChecks: Check[] = [ ...navigabilityChecks, ...typeSafetyChecks, ...safetyChecks, + docHistoryCheck, ]; diff --git a/test/doc-history.test.ts b/test/doc-history.test.ts new file mode 100644 index 0000000..2855753 --- /dev/null +++ b/test/doc-history.test.ts @@ -0,0 +1,157 @@ +import { execFileSync } from 'node:child_process'; +import { appendFileSync, mkdirSync, mkdtempSync, symlinkSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, join } from 'node:path'; +import { describe, expect, it } from 'vitest'; +import { MIN_AGE_DAYS, MIN_COMMITS } from '../src/checks/doc-history.js'; +import { scan } from '../src/scan.js'; +import type { Report } from '../src/types.js'; + +// Synthetic repos with backdated commits; never real history. + +function check(report: Report, id: string) { + const c = report.checks.find((x) => x.id === id); + if (!c) throw new Error(`no check ${id}`); + return c; +} + +const DAY = 86_400; +const T0 = 1_700_000_000; + +class Repo { + readonly dir = mkdtempSync(join(tmpdir(), 'agent-ready-history-')); + private t = T0; + + constructor() { + this.git('init', '-q'); + this.git('config', 'user.email', 't@example.com'); + this.git('config', 'user.name', 'T'); + } + + git(...args: string[]): string { + const date = `${this.t} +0000`; + return execFileSync('git', args, { + cwd: this.dir, + encoding: 'utf8', + env: { ...process.env, GIT_AUTHOR_DATE: date, GIT_COMMITTER_DATE: date }, + }); + } + + write(rel: string, content: string): void { + mkdirSync(dirname(join(this.dir, rel)), { recursive: true }); + writeFileSync(join(this.dir, rel), content); + } + + commit(files: Record, daysLater = 1): void { + this.t += daysLater * DAY; + for (const [rel, content] of Object.entries(files)) this.write(rel, content); + this.git('add', '-A'); + this.git( + '-c', + 'commit.gpgsign=false', + 'commit', + '-q', + '--allow-empty', + '--no-verify', + '-m', + 'c', + ); + } + + /** `n` commits editing `rel`, one day apart. */ + churn(rel: string, n: number): void { + for (let i = 0; i < n; i++) this.commit({ [rel]: `export const v = ${i + 1};\n` }); + } +} + +const DOC = [ + '# AGENTS.md', + '', + '## Architecture', + '', + '- `src/engine.ts` — the core loop', + '- `src/plugins/` — plugin modules', + '', +].join('\n'); + +function base(): Repo { + const r = new Repo(); + r.commit({ + 'package.json': '{"name":"h","scripts":{"test":"vitest"}}', + 'src/engine.ts': 'export const v = 0;\n', + 'src/plugins/a.ts': 'export const a = 1;\n', + 'AGENTS.md': DOC, + }); + return r; +} + +describe('context.drift', () => { + it('flags a doc left untouched while the files it names churned', async () => { + const r = base(); + r.churn('src/engine.ts', MIN_COMMITS); + r.commit({}, MIN_AGE_DAYS); // time passes + r.commit({ 'README.md': 'x\n' }); + const c = check(await scan(r.dir), 'context.drift'); + expect(c.score).toBe(0.5); + expect(c.findings?.[0]?.file).toBe('AGENTS.md'); + expect(c.findings?.[0]?.message).toMatch( + new RegExp(`unchanged for \\d+ days while ${MIN_COMMITS} later commits`), + ); + expect(c.findings?.[0]?.message).toContain('`src/engine.ts`'); + }); + + it('passes when the doc was updated after the churn', async () => { + const r = base(); + r.churn('src/engine.ts', MIN_COMMITS); + r.commit({}, MIN_AGE_DAYS); + r.commit({ 'AGENTS.md': `${DOC}\nUpdated.\n` }); + const c = check(await scan(r.dir), 'context.drift'); + expect(c.score).toBe(1); + expect(c.summary).toBe('1 doc(s) kept up with the code they reference'); + }); + + it('needs both age and churn', async () => { + const young = base(); + young.churn('src/engine.ts', MIN_COMMITS + 5); // lots of churn, but within the window + expect(check(await scan(young.dir), 'context.drift').score).toBe(1); + const quiet = base(); + quiet.churn('src/engine.ts', 3); + quiet.commit({}, MIN_AGE_DAYS * 3); // old, but the code barely changed + expect(check(await scan(quiet.dir), 'context.drift').score).toBe(1); + }); + + it('counts only files added or removed under a referenced directory', async () => { + const edits = base(); + edits.churn('src/plugins/a.ts', MIN_COMMITS); // edits inside a referenced dir + edits.commit({}, MIN_AGE_DAYS); + expect(check(await scan(edits.dir), 'context.drift').score).toBe(1); + const structural = base(); + for (let i = 0; i < MIN_COMMITS; i++) structural.commit({ [`src/plugins/p${i}.ts`]: '1\n' }); + structural.commit({}, MIN_AGE_DAYS); + expect(check(await scan(structural.dir), 'context.drift').score).toBe(0.5); + }); + + it('skips docs with uncommitted edits and follows symlinked docs', async () => { + const r = base(); + r.churn('src/engine.ts', MIN_COMMITS); + r.commit({}, MIN_AGE_DAYS); + symlinkSync('AGENTS.md', join(r.dir, 'CLAUDE.md')); + r.commit({}); + const before = check(await scan(r.dir), 'context.drift'); + // the symlink is the same doc: reported once, under its target + expect(before.findings?.map((f) => f.file)).toEqual(['AGENTS.md']); + appendFileSync(join(r.dir, 'AGENTS.md'), '\nWork in progress.\n'); + expect(check(await scan(r.dir), 'context.drift').score).toBeNull(); + }); + + it('is n/a without git, in shallow clones and for docs without code references', async () => { + const r = base(); + r.churn('src/engine.ts', 2); + const shallow = mkdtempSync(join(tmpdir(), 'agent-ready-shallow-')); + execFileSync('git', ['clone', '-q', '--depth', '1', `file://${r.dir}`, shallow]); + expect(check(await scan(shallow), 'context.drift').summary).toMatch(/Shallow clone/); + const plain = new Repo(); + plain.commit({ 'AGENTS.md': '# Hi\n\nJust prose.\n', 'a.ts': '1\n' }); + expect(check(await scan(plain.dir), 'context.drift').score).toBeNull(); + }); +});