diff --git a/CHANGELOG.md b/CHANGELOG.md index 161dae7a..e7495405 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,10 @@ and adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - VBA extraction now exposes only the classifier factories used by the shared source walker, removing obsolete compatibility entry points and other unused public helpers. (#217) +### Tests + +- Removed documentation-prose and tautological assertions while retaining behavioral rule-table validation. (#214) + ### Fixes - VBA references mentioned only inside messages, logs, and other string literals no longer create false form, query, or temporary-variable relationships. (#209) diff --git a/__tests__/extraction-vba-extraction-perf-doc.test.ts b/__tests__/extraction-vba-extraction-perf-doc.test.ts deleted file mode 100644 index e2f4ae1e..00000000 --- a/__tests__/extraction-vba-extraction-perf-doc.test.ts +++ /dev/null @@ -1,225 +0,0 @@ -/** - * Issue #166 — bench the VBA extractor on the real `00_VBA_TOOLKIT_BENCH` - * corpus and pin the report's required structure so the perf numbers - * can't silently lose their evidence (date, commit SHA, run count, - * named fixtures, per-stage breakdown, honest v1.6.2 baseline claim). - * - * The acceptance test is two-layered: - * - * 1. **Fixtures exist** — every file the issue cites as a measurement - * target must be on disk where the report says it was measured. - * `ACAuditoriaOperaciones.cls` and `ARAuditoria.cls` live under - * `__tests__/fixtures/vba/src/classes/` (committed in this repo). - * The other two cited files (`Form_FormGestionRiesgos.cls`, - * `mdlCursor.bas`) live only in the external bench corpus - * `00_VBA_TOOLKIT_BENCH`; we do NOT depend on that path here so - * this test stays portable — the doc itself documents bench - * provenance for those two files. - * - * 2. **Report structure** — `docs/vba-extraction-perf.md` must be a - * real, reproducible evidence document, not just numbers: - * - the date stamp 2026-07-18 (today, per the issue), - * - the *labeled measurement source SHA* pinned to a known - * historical commit (NOT the doc's own self-referential tip - * SHA, which drifts every time the doc is amended), - * - the 4 required files cited by name, - * - a "3 runs" / "median" methodology note (timing is noisy; - * n=1 is the issue's explicit anti-pattern), - * - every documented timing bucket (preprocess / classifiers / - * walk) — the test pin is at the bucket level, not the - * individual stage, so the report can include any subset of - * stages under each bucket without breaking the contract, - * - an honest v1.6.2 baseline comparison — either concrete - * numbers OR an explicit "v1.6.2 raw baseline is unavailable, - * compared only against documented claim" sentence. Inventing - * a number is the cardinal sin; the report must pick one of - * the two honest paths. - * - * Test isolation: this test reads files only, never writes. It does NOT - * depend on the external bench corpus being installed at any path — - * the doc cites bench provenance, this test only verifies the doc + the - * committed fixtures. Portable across machines and CI. - */ -import { describe, it, expect } from 'vitest'; -import * as fs from 'node:fs'; -import * as path from 'node:path'; - -const REPO_ROOT = path.resolve(__dirname, '..'); -const REPORT_PATH = path.join(REPO_ROOT, 'docs', 'vba-extraction-perf.md'); - -/** - * Labeled measurement source SHA — the exact commit whose - * `dist/bin/codegraph.js` was running when the stderr logs were - * captured. This is the historical, stable identifier for "the code - * that produced the numbers". It is pinned here so a future SHA bump - * (e.g. a re-measurement against a newer tip) is an explicit, reviewable - * change — not a silent drift. - * - * Source: parent of the commit that introduced this report - * (`5b501c3b…`) — see `git log --first-parent`. - */ -const MEASUREMENT_SOURCE_SHA = '425e33ad2ba86af1e26279c42ce14fe8cd107589'; - -const REQUIRED_FIXTURE_FILES = [ - 'ACAuditoriaOperaciones.cls', - 'ARAuditoria.cls', -] as const; - -describe('Issue #166 — vba-extraction-perf.md report structure', () => { - describe('fixture files exist where the report says they were measured', () => { - it('ACAuditoriaOperaciones.cls and ARAuditoria.cls exist in the committed fixture corpus', () => { - // These two files are the fixture-only measurement targets — the - // bench corpus does NOT contain them. They live in this repo so - // the test is portable across machines. - const classesDir = path.join( - REPO_ROOT, - '__tests__', - 'fixtures', - 'vba', - 'src', - 'classes', - ); - for (const f of REQUIRED_FIXTURE_FILES) { - const p = path.join(classesDir, f); - expect(fs.existsSync(p), `${p} must exist`).toBe(true); - } - }); - }); - - describe('docs/vba-extraction-perf.md report contract', () => { - it('the report file exists and is non-empty', () => { - expect(fs.existsSync(REPORT_PATH), `${REPORT_PATH} must exist`).toBe(true); - const body = fs.readFileSync(REPORT_PATH, 'utf8'); - expect(body.length).toBeGreaterThan(200); - }); - - it('the report carries the date stamp 2026-07-18', () => { - const body = fs.readFileSync(REPORT_PATH, 'utf8'); - expect(body).toContain('2026-07-18'); - }); - - it('the report pins the labeled measurement source SHA (not a self-referential tip SHA)', () => { - const body = fs.readFileSync(REPORT_PATH, 'utf8'); - // The "labeled measurement source SHA" is the commit whose source - // tree produced the data. We pin it explicitly here so an amend - // of the doc can't silently drift the data provenance. The doc - // is allowed to reference the full 40-char SHA or any 7+-char - // unambiguous prefix; we check the full one to be strict. - expect( - body, - `report must pin the labeled measurement source SHA ${MEASUREMENT_SOURCE_SHA}`, - ).toContain(MEASUREMENT_SOURCE_SHA); - // And the 7-char short form (defends against an accidental - // 40→7 truncation silently passing). - expect(body, 'report must also include the 7-char short SHA 425e33a').toContain( - MEASUREMENT_SOURCE_SHA.slice(0, 7), - ); - }); - - it('the report cites every required measurement target by name', () => { - const body = fs.readFileSync(REPORT_PATH, 'utf8'); - for (const name of [ - 'ACAuditoriaOperaciones.cls', - 'ARAuditoria.cls', - 'Form_FormGestionRiesgos.cls', - 'mdlCursor.bas', - ]) { - expect(body, `report must cite ${name}`).toContain(name); - } - }); - - it('the report distinguishes physical line counts from extractor split slots', () => { - // The vba-timing `(n=N)` annotation reports the post-preprocessing - // `source.split('\n').length` count (= CRLF terminators + 1 trailing - // slot). It is NOT the same as the file's physical line count from - // `Get-Content | Measure-Object -Line`. The report must call this - // out so a reader does not confuse "lines" between the two. - const body = fs.readFileSync(REPORT_PATH, 'utf8'); - const mentionsSplit = - /split\s*\(\s*['"]\\?n['"]\s*\)/i.test(body) || - /split\s*slots/i.test(body) || - /CRLF terminators?\s*\+\s*1/i.test(body); - expect( - mentionsSplit, - 'report must explain that the per-file line count in the vba-timing output is the post-preprocessing split slot count, not the physical line count', - ).toBe(true); - }); - - it('the report identifies the exact byte provenance for mdlCursor.bas (bench vs fixture)', () => { - // The fixture copy of mdlCursor.bas is 40 split slots (39 CRLF) - // and 1331 bytes; the bench copy is 39 split slots (38 CRLF) and - // 1329 bytes. The measured `(n=39)` matches the BENCH copy. The - // report must state this explicitly so a re-measurement against - // the fixture copy would surface as a different number, not as a - // silent inconsistency. - const body = fs.readFileSync(REPORT_PATH, 'utf8'); - const pinBench = - /mdlCursor\.bas[^.\n]*?\b(?:bench|00_VBA_TOOLKIT_BENCH)\b/i.test(body) || - /\bbench\b[^.\n]*?mdlCursor\.bas/i.test(body) || - /mdlCursor\.bas[^.\n]*?\b(?:1329|38)\b/.test(body); - expect( - pinBench, - 'report must explicitly identify the mdlCursor.bas source as the bench corpus copy (1329 bytes / 38 CRLF, not the fixture copy 1331 bytes / 39 CRLF)', - ).toBe(true); - }); - - it('the report documents the run count (≥3 runs, median)', () => { - const body = fs.readFileSync(REPORT_PATH, 'utf8'); - const hasRunCount = /\b3\s*(?:runs?|measurements?|passes)\b/i.test(body) || - /\b[4-9]\s*(?:runs?|measurements?|passes)\b/i.test(body) || - /\b\d{2,}\s*(?:runs?|measurements?|passes)\b/i.test(body); - expect(hasRunCount, 'report must cite at least 3 runs').toBe(true); - const hasMedianOrRange = /\bmedian\b/i.test(body) || /\brange\b/i.test(body) || - /\bmin\b.*\bmax\b/i.test(body); - expect(hasMedianOrRange, 'report must summarize runs by median or range').toBe(true); - }); - - it('the report breaks timings into the documented buckets (preprocess / classifiers / walk)', () => { - const body = fs.readFileSync(REPORT_PATH, 'utf8'); - for (const bucket of ['preprocess', 'classifiers', 'walk']) { - expect(body, `report must mention the ${bucket} bucket`).toContain(bucket); - } - }); - - it('the report compares to v1.6.2 honestly — numbers OR an explicit unavailability note', () => { - const body = fs.readFileSync(REPORT_PATH, 'utf8'); - expect(body, 'report must mention v1.6.2').toMatch(/v?1\.6\.2/); - const hasHonestNumbers = /\b\d+(\.\d+)?\s*(?:ms|s)\b/.test(body); - const hasExplicitBaselineNote = - /baseline.*(?:unavailable|not available|missing|rebuilt|not retained)/i.test(body) || - /(?:compared|comparison).*(?:only|documented|claim|published)/i.test(body); - expect( - hasHonestNumbers || hasExplicitBaselineNote, - 'report must either publish concrete ms numbers OR state explicitly that the v1.6.2 raw baseline is unavailable', - ).toBe(true); - }); - - it('the report has a Conclusion / verdict section', () => { - const body = fs.readFileSync(REPORT_PATH, 'utf8'); - const headingRe = /^##\s+.*$/gm; - const headings = [...body.matchAll(headingRe)].map((m) => m[0]); - const hasConclusionHeading = headings.some((h) => - /conclusion|verdict|summary|result|findings/i.test(h), - ); - expect(hasConclusionHeading, 'report must have a ## Conclusion / Verdict section').toBe( - true, - ); - }); - - it('the report points at the committed parser script under scripts/', () => { - // The reproduction block must reference a parser script that is - // actually checked in, not a hardcoded temp-dir path. Otherwise - // the "how to reproduce" claim is a lie. - const body = fs.readFileSync(REPORT_PATH, 'utf8'); - expect( - body, - 'report must reference the committed parser under scripts/ (e.g. scripts/parse-vba-timing-stderr.mjs)', - ).toMatch(/scripts\/parse-vba-timing-stderr\.mjs/); - const scriptPath = path.join(REPO_ROOT, 'scripts', 'parse-vba-timing-stderr.mjs'); - expect( - fs.existsSync(scriptPath), - `${scriptPath} must exist on disk for the reproduction claim to be honest`, - ).toBe(true); - }); - }); -}); diff --git a/__tests__/extraction-vba-rule-table.test.ts b/__tests__/extraction-vba-rule-table.test.ts index 9659d804..d69e2ac7 100644 --- a/__tests__/extraction-vba-rule-table.test.ts +++ b/__tests__/extraction-vba-rule-table.test.ts @@ -1,245 +1,20 @@ -/** - * VBA declarative rule table (Issue #153). - * - * The 5 per-concern classifiers under `src/extraction/vba/` used to encode - * their extraction rules as inline `if (RE.test(line))` branches inside - * `classifyLine`. Each branch was hard to enumerate and impossible to - * unit-test in isolation. This test pins the new declarative shape: - * - * - Every classifier exports a `RULES: VbaExtractionRule[]` constant. - * - Every rule has a unique `id` (no duplicate ids within one table). - * - Every rule has a non-empty `pattern` (RegExp or RegExp[]). - * - Every rule has a non-null `emit` function. - * - The orchestrator (`vba-extractor.ts`) imports each table and - * asserts it is non-empty — an accidentally-empty table must fail - * loudly at module load, not silently drop a whole concern. - * - * Adding a new test rule should require touching 1 file: the new rule's - * classifier file (1 entry in the RULES array) + this test (1 id in the - * declared unique-id list, optional). - */ -import { describe, it, expect } from 'vitest'; -import { - RULES as IMPLEMENTS_RULES, - createImplementsClassifier, -} from '../src/extraction/vba/implements'; -import { - RULES as PROCEDURES_RULES, - createProceduresClassifier, -} from '../src/extraction/vba/procedures'; -import { - RULES as DECLARATIONS_RULES, - createEventsTypesDeclaresClassifier, -} from '../src/extraction/vba/declarations'; -import { - RULES as DIMS_RULES, - createDimsClassifier, -} from '../src/extraction/vba/dims'; -import { - RULES as ENUM_CONSTS_RULES, - createEnumsConstsClassifier, -} from '../src/extraction/vba/enums-consts'; -import { - RULES as CALL_SWEEP_RULES, - createCallsAndSqlClassifier, -} from '../src/extraction/vba/call-sweep'; +import { describe, expect, it } from 'vitest'; import { VBA_RULE_TABLES, validateVbaRuleTables, } from '../src/extraction/vba-extractor'; -import type { VbaExtractionRule } from '../src/extraction/vba/rules'; - -/** - * Assert a single rule has the canonical declarative shape: - * - non-empty string `id` - * - non-empty string `description` - * - `pattern` is a RegExp or a non-empty RegExp[] (mixed accepted) - * - `emit` is a function - * - * Pure shape check — does NOT call `emit` (that needs a full - * `VbaExtractorContext` and may be order-sensitive). Behavioral - * coverage stays in `__tests__/extraction-vba.test.ts`. - */ -function assertRuleShape(rule: unknown, contextLabel: string): void { - expect(rule, `${contextLabel} should be an object`).toBeTypeOf('object'); - expect(rule, `${contextLabel} should not be null`).not.toBeNull(); - - const r = rule as Partial; - expect(typeof r.id, `${contextLabel}.id should be a string`).toBe('string'); - expect( - r.id && r.id.length > 0, - `${contextLabel}.id should be non-empty`, - ).toBe(true); - expect( - typeof r.description, - `${contextLabel}.description should be a string`, - ).toBe('string'); - expect( - r.description && r.description.length > 0, - `${contextLabel}.description should be non-empty`, - ).toBe(true); - - // `pattern` must be present and either a RegExp or a non-empty array of RegExp. - const pattern = r.pattern as unknown; - const isRegExp = pattern instanceof RegExp; - const isRegExpArray = - Array.isArray(pattern) && - pattern.length > 0 && - pattern.every((p) => p instanceof RegExp); - expect( - isRegExp || isRegExpArray, - `${contextLabel}.pattern should be a RegExp or a non-empty RegExp[] (got ${typeof pattern})`, - ).toBe(true); - - expect( - typeof r.emit, - `${contextLabel}.emit should be a function`, - ).toBe('function'); -} - -/** - * Assert that every rule id within one table is unique. Catches - * "two rules with the same id" copy-paste regressions. - */ -function assertUniqueIds( - rules: readonly VbaExtractionRule[], - tableLabel: string, -): void { - const seen = new Set(); - for (const rule of rules) { - expect( - seen.has(rule.id), - `${tableLabel} contains duplicate rule id "${rule.id}"`, - ).toBe(false); - seen.add(rule.id); - } -} - -describe('VBA declarative rule table (Issue #153)', () => { - describe('every classifier exports a non-empty RULES array with the right shape', () => { - const cases: ReadonlyArray<{ - label: string; - rules: readonly VbaExtractionRule[]; - }> = [ - { label: 'implements.ts', rules: IMPLEMENTS_RULES }, - { label: 'procedures.ts', rules: PROCEDURES_RULES }, - { label: 'declarations.ts', rules: DECLARATIONS_RULES }, - { label: 'dims.ts', rules: DIMS_RULES }, - { label: 'enums-consts.ts', rules: ENUM_CONSTS_RULES }, - { label: 'call-sweep.ts', rules: CALL_SWEEP_RULES }, - ]; - - for (const c of cases) { - it(`${c.label} exports a non-empty RULES array`, () => { - expect(Array.isArray(c.rules), `${c.label}.RULES should be an array`).toBe( - true, - ); - expect( - c.rules.length, - `${c.label}.RULES should be non-empty`, - ).toBeGreaterThan(0); - }); - - it(`every rule in ${c.label} has the canonical shape`, () => { - expect(c.rules.length).toBeGreaterThan(0); - for (const rule of c.rules) { - assertRuleShape(rule, `${c.label}.RULES entry`); - } - }); - - it(`every rule id in ${c.label} is unique`, () => { - expect(c.rules.length).toBeGreaterThan(0); - assertUniqueIds(c.rules, c.label); - }); - } +describe('VBA rule-table validation', () => { + it('accepts the live rule tables', () => { + expect(validateVbaRuleTables()).toEqual({ ok: true, empty: [] }); }); - describe('orchestrator aggregates and validates the 5 rule tables', () => { - it('VBA_RULE_TABLES exposes all 5 classifier tables keyed by concern name', () => { - expect(VBA_RULE_TABLES).toBeDefined(); - // The aggregator must reference each of the 5 classifier files by the - // same name the per-classifier export uses, so a test that imports - // both sides can assert they match. - expect(VBA_RULE_TABLES.implements).toBe(IMPLEMENTS_RULES); - expect(VBA_RULE_TABLES.procedures).toBe(PROCEDURES_RULES); - expect(VBA_RULE_TABLES.declarations).toBe(DECLARATIONS_RULES); - expect(VBA_RULE_TABLES.dims).toBe(DIMS_RULES); - expect(VBA_RULE_TABLES['enums-consts']).toBe(ENUM_CONSTS_RULES); - expect(VBA_RULE_TABLES['call-sweep']).toBe(CALL_SWEEP_RULES); - }); - - it('validateVbaRuleTables() returns ok for the live tables', () => { - // The orchestrator runs this on module load — calling it again here - // is a no-op assertion that the live tables remain non-empty. - const result = validateVbaRuleTables(); - expect(result.ok).toBe(true); - if (!result.ok) { - // Surface the empty tables for diagnosis if the assert ever fails. - throw new Error( - `validateVbaRuleTables reported empty tables: ${result.empty.join(', ')}`, - ); - } - }); - - it('validateVbaRuleTables() flags a missing table when one is removed', () => { - // Defensive: build a fake table set with one concern emptied, and - // confirm the validator returns `{ ok: false, empty: [...] }`. This - // pins the validator's contract independent of the live data. - const empty = { ...VBA_RULE_TABLES, dims: [] }; - const result = validateVbaRuleTables(empty); - expect(result.ok).toBe(false); - expect(result.empty).toContain('dims'); - }); - }); - - describe('factory functions still exist and still produce classifiers', () => { - // Backward-compat guard: every `create*Classifier()` factory must - // keep its existing signature so `vba-extractor.ts` and any external - // consumers (tests, downstream tools) that imported the factory - // before this refactor keep compiling. - it('createImplementsClassifier returns a VbaClassifier', () => { - const c = createImplementsClassifier(); - expect(c.name).toBe('implements'); - expect(typeof c.classifyLine).toBe('function'); - expect(c.count).toBe(0); - }); - - it('createProceduresClassifier returns a VbaClassifier', () => { - const c = createProceduresClassifier(); - expect(c.name).toBe('procedures'); - expect(typeof c.classifyLine).toBe('function'); - expect(c.count).toBe(0); - }); - - it('createEventsTypesDeclaresClassifier returns a VbaClassifier', () => { - const c = createEventsTypesDeclaresClassifier(); - expect(c.name).toBe('eventsTypesDeclares'); - expect(typeof c.classifyLine).toBe('function'); - expect(c.count).toBe(0); - }); - - it('createDimsClassifier returns a VbaClassifier', () => { - const c = createDimsClassifier(); - expect(c.name).toBe('dims'); - expect(typeof c.classifyLine).toBe('function'); - expect(c.count).toBe(0); - }); - - it('createEnumsConstsClassifier returns a VbaClassifier', () => { - const c = createEnumsConstsClassifier(); - expect(c.name).toBe('enumsConsts'); - expect(typeof c.classifyLine).toBe('function'); - expect(c.count).toBe(0); - }); + it('reports an empty rule table', () => { + const tables = { ...VBA_RULE_TABLES, dims: [] }; - it('createCallsAndSqlClassifier(lines) returns a VbaClassifier', () => { - // The calls classifier takes the pre-split lines array — a - // structural argument the orchestrator owns. Pass a small fixture. - const c = createCallsAndSqlClassifier(['']); - expect(c.name).toBe('callsAndSql'); - expect(typeof c.classifyLine).toBe('function'); - expect(c.count).toBe(0); + expect(validateVbaRuleTables(tables)).toEqual({ + ok: false, + empty: ['dims'], }); }); }); diff --git a/__tests__/spike-vbnet-as-vba.test.ts b/__tests__/spike-vbnet-as-vba.test.ts deleted file mode 100644 index 102647b4..00000000 --- a/__tests__/spike-vbnet-as-vba.test.ts +++ /dev/null @@ -1,240 +0,0 @@ -/** - * Smoke test for the F.1 spike of issue #155 — verify the - * `scripts/spike-vbnet-as-vba.mjs` artifact (markdown report + JSON dump) is - * present, non-empty, and structurally sound. The script is allowed to - * RE-RUN at test time so a fresh checkout (or a CI run that re-clones) always - * gets a current report — but the test never spawns the WASM load in the - * assertion path; we just check what the previous run produced. - * - * If the artifact is missing, the test invokes the script (node directly) and - * re-checks. This keeps the test self-contained: a fresh clone (no artifact - * present) still produces a green run, but a PR that deletes the artifact - * without re-running the script gets a red signal. - */ -import { describe, it, expect, beforeAll } from 'vitest'; -import * as fs from 'fs'; -import * as path from 'path'; -import { execFileSync } from 'child_process'; -import { pathToFileURL } from 'url'; - -const REPO_ROOT = path.resolve(__dirname, '..'); -const REPORT_PATH = path.join(REPO_ROOT, 'docs', 'spikes', 'vbnet-as-vba.md'); -const JSON_PATH = path.join(REPO_ROOT, 'docs', 'spikes', 'vbnet-as-vba.json'); -const SCRIPT_PATH = path.join(REPO_ROOT, 'scripts', 'spike-vbnet-as-vba.mjs'); -const CLOSE_NOTES_PATH = path.join(REPO_ROOT, 'docs', 'spikes', 'issue-155-close.md'); - -/** - * Ensure the artifact exists. If not, spawn the spike script (node, no - * external dep on `pnpm`/`tsx`) so the artifact is regenerated before the - * assertions run. The script is idempotent — re-runs produce equivalent - * content, with `generatedAt` and (potentially) per-file numbers updated. - */ -function ensureArtifact(): void { - if (fs.existsSync(REPORT_PATH) && fs.existsSync(JSON_PATH)) return; - // eslint-disable-next-line no-console - console.log('[spike-test] artifact missing, running spike script to regenerate'); - execFileSync(process.execPath, [SCRIPT_PATH], { - cwd: REPO_ROOT, - stdio: 'inherit', - env: { ...process.env, NODE_NO_WARNINGS: '1' }, - }); -} - -interface SpikeJSON { - generatedAt: string; - grammar: string; - fixturesRoot: string; - totals: { files: number; clean: number; partial: number; failed: number; failedRate: number }; - structural: { nodeTypes: string[]; recognized: number; filesSeen: number }; - body: { nodeTypes: string[]; recognized: number; filesSeen: number }; - structuralCompleteness: { hasClassWrapper: boolean; hasProcedures: boolean; hasFieldDecls: boolean; ratio: number }; - synthTotals: { - files: number; - skipped: number; - perStructuralNode: Record; - }; - synthesizedResults: Array<{ - path: string; - className?: string; - skipped?: boolean; - reason?: string; - total?: number; - error?: number; - missing?: number; - classification?: 'clean' | 'partial' | 'failed'; - perType?: Record; - structuralCounts?: Record; - }>; - perFile: Array<{ - path: string; bytes: number; lines: number; total: number; error: number; missing: number; - classification: 'clean' | 'partial' | 'failed'; - errorSamples: Array<{ line: number; col: number; text: string }>; - perType: Record; - }>; - perConstruct: Record; - keywordPresence: Record; -} - -describe('spike: tree-sitter-vbnet as VBA parser (F.1, issue #155)', () => { - beforeAll(() => { - ensureArtifact(); - }, 180_000); // first run loads the 6.4 MB WASM; allow generous timeout - - it('produces the markdown report at docs/spikes/vbnet-as-vba.md', () => { - expect(fs.existsSync(REPORT_PATH)).toBe(true); - }); - - it('produces the machine-readable JSON at docs/spikes/vbnet-as-vba.json', () => { - expect(fs.existsSync(JSON_PATH)).toBe(true); - }); - - it('archives the F.2 deferral and links its trigger guard', () => { - const notes = fs.readFileSync(CLOSE_NOTES_PATH, 'utf8'); - expect(notes).toContain('F.2 implementation is not scheduled'); - expect(notes).toContain('Issue #170'); - expect(notes).toContain('Issue #170 was closed'); - expect(notes).toContain('no qualifying trigger'); - }); - - it('report is non-empty and not a placeholder', () => { - const content = fs.readFileSync(REPORT_PATH, 'utf8'); - expect(content.length).toBeGreaterThan(2_000); - expect(content).toContain('# Spike: tree-sitter-vbnet.wasm as a VBA parser (F.1)'); - expect(content).toContain('## Verdict'); - expect(content).toContain('## Headline numbers'); - expect(content).toContain('## Structural completeness'); - expect(content).toContain('## Per-file results'); - expect(content).toContain('## Per-construct recognition'); - expect(content).toContain('## VBA constructs the vbnet grammar recognizes cleanly'); - expect(content).toContain('## VBA constructs the vbnet grammar fails on'); - expect(content).toContain('## What pre-processing can and cannot fix'); - expect(content).toContain('## F.2 dry-run: synthesized `Class … End Class` wrapper'); - expect(content).toContain('## Pre-processing checklist for F.2'); - }); - - it('report verdict line matches the issue\'s go/no-go gate language', () => { - const content = fs.readFileSync(REPORT_PATH, 'utf8'); - // The verdict is the bolded text right after the "## Verdict" heading: - // ## Verdict - // **GO with major caveat** — The quantitative gate passes ... - // The issue's quantitative gate is ">30% failed = NO-GO"; the report - // explicitly references this gate so a future maintainer can audit the - // decision against the original threshold. - expect(content).toMatch(/## Verdict[\s\S]{0,80}\*\*(GO with major caveat|GO with caveats|GO|NO-GO)\*\*/); - expect(content).toMatch(/30% threshold/); - }); - - it('JSON parses and has the expected top-level shape', () => { - const raw = fs.readFileSync(JSON_PATH, 'utf8'); - const parsed = JSON.parse(raw) as SpikeJSON; - expect(parsed.grammar).toBe('tree-sitter-vbnet.wasm'); - expect(parsed.fixturesRoot).toMatch(/fixtures/); - expect(parsed.generatedAt).toMatch(/^\d{4}-\d{2}-\d{2}T/); - expect(parsed.totals).toMatchObject({ - files: expect.any(Number), - clean: expect.any(Number), - partial: expect.any(Number), - failed: expect.any(Number), - failedRate: expect.any(Number), - }); - expect(parsed.totals.files).toBeGreaterThan(0); - expect(parsed.totals.clean + parsed.totals.partial + parsed.totals.failed).toBe(parsed.totals.files); - expect(parsed.structural).toMatchObject({ - nodeTypes: expect.any(Array), - recognized: expect.any(Number), - filesSeen: expect.any(Number), - }); - expect(parsed.body).toMatchObject({ - nodeTypes: expect.any(Array), - recognized: expect.any(Number), - filesSeen: expect.any(Number), - }); - expect(parsed.structuralCompleteness).toMatchObject({ - hasClassWrapper: expect.any(Boolean), - hasProcedures: expect.any(Boolean), - hasFieldDecls: expect.any(Boolean), - ratio: expect.any(Number), - }); - expect(parsed.perFile.length).toBe(parsed.totals.files); - expect(Object.keys(parsed.perConstruct).length).toBeGreaterThan(0); - }); - - it('per-file records have the expected shape and all parse as non-empty', () => { - const raw = fs.readFileSync(JSON_PATH, 'utf8'); - const parsed = JSON.parse(raw) as SpikeJSON; - for (const f of parsed.perFile) { - expect(f.path).toMatch(/__tests__[\\/]fixtures[\\/]/); - expect(f.bytes).toBeGreaterThan(0); - expect(f.lines).toBeGreaterThan(0); - expect(f.total).toBeGreaterThan(0); - expect(['clean', 'partial', 'failed']).toContain(f.classification); - expect(f.error).toBeGreaterThanOrEqual(0); - expect(f.missing).toBeGreaterThanOrEqual(0); - expect(f.perType).toBeTypeOf('object'); - } - }); - - it('per-construct rollup captures the expected grammar node types', () => { - const raw = fs.readFileSync(JSON_PATH, 'utf8'); - const parsed = JSON.parse(raw) as SpikeJSON; - // Body-level nodes that the spike was designed to detect. If the spike - // ever drops these from the rollup, the report's "Structural completeness" - // table is wrong. Pin the existence of at least the body nodes that - // should be present in any parse of the corpus. - const expectedBody = [ - 'if_statement', 'with_statement', 'assignment_statement', 'expression_statement', - 'comment', 'invocation_expression', 'member_access_expression', - ]; - for (const n of expectedBody) { - expect(parsed.perConstruct[n], `perConstruct.${n} missing`).toBeDefined(); - expect(parsed.perConstruct[n].recognized, `perConstruct.${n}.recognized > 0`).toBeGreaterThan(0); - } - }); - - it('quantitative gate: failed rate is below the 30% NO-GO threshold', () => { - // Issue #155's hard gate: "If the failure rate is >30% on real Dysflow - // exports, the epic becomes a research project; cancel F.2-F.4." If this - // assertion ever fires, the spike has regressed — re-read the JSON - // verdict, and re-investigate. - const raw = fs.readFileSync(JSON_PATH, 'utf8'); - const parsed = JSON.parse(raw) as SpikeJSON; - expect(parsed.totals.failedRate).toBeLessThan(0.30); - }); - - it('synthesized-wrapper dry-run unlocks the structural tree (F.2 hypothesis)', () => { - // The F.2 hypothesis is that injecting a `Class ... End Class` wrapper - // (where comes from the existing `Attribute VB_Name` extraction) - // makes the grammar emit class_declaration, method_declaration, field_declaration - // etc. on the VBA corpus. The spike ran this experiment. Pin at least - // one class_declaration in the synthesized counts — if this fires, the - // F.2 path is broken (or the spike's wrapper logic regressed). - const raw = fs.readFileSync(JSON_PATH, 'utf8'); - const parsed = JSON.parse(raw) as SpikeJSON; - expect(parsed.synthTotals).toBeDefined(); - expect(parsed.synthTotals.files).toBeGreaterThan(0); - // At least one synthesized class_declaration must exist (every file with - // an Attribute VB_Name produced a Class wrapper). - expect( - parsed.synthTotals.perStructuralNode.class_declaration ?? 0, - 'synthesized-wrapper produced no class_declaration — F.2 unlock is broken', - ).toBeGreaterThan(0); - // And at least one method_declaration (the F.2 plan needs the procedure - // boundary to come from the AST). - expect( - parsed.synthTotals.perStructuralNode.method_declaration ?? 0, - 'synthesized-wrapper produced no method_declaration — F.2 unlock is broken', - ).toBeGreaterThan(0); - }); - - it('artifact is reproducible (the script is idempotent — re-running writes the same paths)', () => { - // The script's contract: report and JSON go to the same paths on every - // run. Pin the path layout so a future rename can't silently break the - // smoke test (which would otherwise pass on stale artifacts). - expect(REPORT_PATH).toContain('docs'); - expect(REPORT_PATH).toContain('spikes'); - expect(REPORT_PATH.endsWith('vbnet-as-vba.md')).toBe(true); - expect(JSON_PATH.endsWith('vbnet-as-vba.json')).toBe(true); - // script lives at scripts/spike-vbnet-as-vba.mjs - expect(SCRIPT_PATH.endsWith('spike-vbnet-as-vba.mjs')).toBe(true); - }); -}); diff --git a/__tests__/test-suite-hygiene.test.ts b/__tests__/test-suite-hygiene.test.ts new file mode 100644 index 00000000..9c0d0cf4 --- /dev/null +++ b/__tests__/test-suite-hygiene.test.ts @@ -0,0 +1,16 @@ +import { existsSync } from 'node:fs'; +import { join } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +describe('test suite hygiene', () => { + it('does not keep documentation-prose test files', () => { + const obsoleteTests = [ + 'extraction-vba-extraction-perf-doc.test.ts', + 'spike-vbnet-as-vba.test.ts', + ]; + + expect( + obsoleteTests.filter((file) => existsSync(join(__dirname, file))), + ).toEqual([]); + }); +});