Lower LaTeX presentation into the MathExpression semantic core

package.json

@@ -90,6 +90,7 @@
     "odf.js": "^3.0.1",
     "ooxml.js": "^2.16.0",
     "pdf-codec": "^2.2.35",
+    "temml": "0.13.4",
     "zod": "^4.4.3"
   },
   "devDependencies": {

src/index.ts

@@ -316,6 +316,17 @@ export { buildOfficeMath, buildOfficeMathParagraph } from './omml/write';
 export type { OmmlReadResult } from './omml/read';
 export { collectOfficeMathElements, readOfficeMath } from './omml/read';
 
+// --- LaTeX presentation -> the MathExpression semantic core (src/latex/): the string-to-tree half of document-schema.js's two-layer math model, over a pinned temml parser. lowerLatex is the direct entry point for a caller holding a LaTeX string; latexToFormula wraps it into a whole ContentFormula (verbatim presentation + presentation-MathML + lowered content + provenance) ready to embed the way every other formula in this package travels. The lowering is mechanical where notation is unambiguous (\frac -> math:divide, radicals -> math:sqrt/n-th root, scripted Sigma/Product -> sum/prod binders, numeric literals -> exact rationals, subscripts -> distinct symbol identities) and degrades everything context-starved to visible `unparsed` nodes carrying the verbatim source plus a named diagnostic -- never a parse failure, never a silent guess. The markdown read path runs this lowering automatically over markdown-codec's preserved $$ display blocks and \( \) inline spans (readMarkdownContent's third parameter surfaces the diagnostics). lintMathCoherence is the model's read-only audit: it re-parses and re-lowers every stored presentation string and reports divergence from the stored content layer as a warning carrying provenance -- a deliberate layer edit, never something this package re-derives. ---
+export type { LatexDiagnostic, LatexDiagnosticCode, LatexDiagnosticSink } from './latex/diagnostics';
+export { LATEX_DIAGNOSTIC_CODES, MATH_LINT_CODES } from './latex/diagnostics';
+export type { MathLintDiagnostic, MathLintCode } from './latex/diagnostics';
+export type { LatexFormulaOptions, LatexFormulaResult, LatexLoweringResult, LowerLatexOptions } from './latex/lower';
+export { latexToFormula, lowerLatex } from './latex/lower';
+export { lintMathCoherence } from './latex/lint';
+export type { MarkdownMathLoweringOptions } from './markdown/math';
+export { lowerMarkdownMath } from './markdown/math';
+
+
 // --- Format <-> ContentDocument readers and layout algorithms, each independently usable rather than only reachable through the ergonomic conversions below. ---
 export type { ReadDocxContentOptions } from './ooxml/docx/read';
 export { readDocxContent } from './ooxml/docx/read';

src/latex/diagnostics.ts

@@ -0,0 +1,58 @@
+// The LaTeX lowering's degrade-with-diagnostic channel, following the same three-tier failure policy pdf-codec established and src/svg/diagnostics.ts restates for the svg read: input temml cannot parse at all degrades the whole expression to one `unparsed` node, notation the grammar covers degrades construct-by-construct, and nothing is ever a silent guess -- every degradation leaves the verbatim source visible inside the expression tree (the schema's `unparsed` variant) and names itself here. Every code below names one deliberate scope limit of the lowering rules in src/latex/lower.ts; a formula whose every construct is mechanical lowers with zero diagnostics.
+
+export const LATEX_DIAGNOSTIC_CODES = [
+  // temml's parser rejected the string outright (an unknown command, an unmatched brace): the whole expression becomes one `unparsed` node carrying the full verbatim source.
+  'latex/parse-error',
+  // A construct the lowering grammar has no rule for (an integral, an accent, an overline): that construct becomes an `unparsed` node carrying its verbatim source span.
+  'latex/construct-unparsed',
+  // Two operands sit adjacent with no operator between them (juxtaposition -- `mc^2`, `f(x)`, `2(x+1)`): multiplication and function application are the two conventional readings and LaTeX notation cannot say which, so the whole run becomes one `unparsed` node rather than guessing.
+  'latex/juxtaposition-unparsed',
+  // A \text{...} node: prose inside mathematics has no MathExpression reading, so its verbatim source becomes an `unparsed` node.
+  'latex/text-unparsed',
+  // A subscript that is not a simple symbol suffix (`a_{i+1}`, `x_{(n)}`): the grammar has no indexed-access operator, so the whole scripted construct becomes an `unparsed` node.
+  'latex/subscript-unparsed',
+  // A \sum/\prod whose bound is written as a bare glyph or a non-relation (`\sum_i`, `\sum_{i \in S}`) rather than `name = expression`: the binder still lowers, with the missing bound itself an `unparsed` node.
+  'latex/binder-bound-implicit',
+  // A \sum/\prod whose subscript could not be read as a bound at all: the whole binder becomes an `unparsed` node.
+  'latex/binder-bound-unreadable',
+  // A binary/relation operator with no mapping in the core registry (\pm, \approx, \cup, \to): the sequence around it becomes one `unparsed` node rather than dropping or guessing the operator.
+  'latex/operator-unmapped',
+  // A subscript or superscript whose base is itself an `unparsed` construct: scripts attach to nothing lowerable, so the whole scripted span degrades with it.
+  'latex/script-base-unparsed',
+  // An array environment other than the plain matrix family (align, cases, aligned): layout-semantic environments have no MathExpression reading, so the whole environment becomes one `unparsed` node.
+  'latex/array-environment-unparsed',
+  // A binomial or other generalised fraction drawn with delimiters or without a bar: only the plain stacked fraction is unambiguously division.
+  'latex/genfrac-unparsed',
+  // A binary/relation operator this lowering cannot place: a leading operator other than the one unary minus reading, or an operator with no operand on one side (`a + + b`, a trailing `+`).
+  'latex/operator-placement-unparsed',
+  // The prose scanner found and seeded a symbol-table definition -- an informational audit channel, not a degradation: one diagnostic per definition found, so a caller can see exactly what the scanner inferred from the document's own sentences.
+  'symbols/prose-definition-found',
+] as const;
+
+export type LatexDiagnosticCode = (typeof LATEX_DIAGNOSTIC_CODES)[number];
+
+export interface LatexDiagnostic {
+  readonly code: LatexDiagnosticCode;
+  // The verbatim source construct the diagnostic is about -- the same string the corresponding `unparsed` node carries, so a diagnostic and the visible gap in the tree always agree on what degraded.
+  readonly detail?: string;
+}
+
+export type LatexDiagnosticSink = (diagnostic: LatexDiagnostic) => void;
+
+// The coherence lint's own vocabulary, separate from the lowering's because it reports a different phenomenon: not "this construct would not lower" but "this formula's two stored layers no longer agree", which means somebody edited one layer deliberately since the content was last derived.
+export const MATH_LINT_CODES = [
+  // Re-lowering the stored presentation string produced a different expression tree than the stored content layer: a warning carrying provenance, never an automatic re-derivation -- the schema's atomic pair-edit rule says the edit was deliberate and the stored layers stay exactly as stored.
+  'math/coherence-divergence',
+  // A stored presentation string no longer parses at all while the stored content layer holds a lowered (non-unparsed-root) tree: a stronger form of the divergence above, since the presentation layer's own text has become unreadable to the pinned parser.
+  'math/coherence-unparseable-presentation',
+] as const;
+
+export type MathLintCode = (typeof MATH_LINT_CODES)[number];
+
+export interface MathLintDiagnostic {
+  readonly code: MathLintCode;
+  readonly severity: 'warning';
+  // The formula's stored provenance (its source and edit trail), carried into the warning so the reader can see who last touched either layer before judging which side is stale.
+  readonly provenance?: string;
+  readonly detail?: string;
+}

src/latex/lint.test.ts

@@ -0,0 +1,97 @@
+import type { ContentBlock, DocumentPackage } from 'document-schema.js';
+import { CONTENT_FORMAT_VERSION, DOCUMENT_PACKAGE_FORMAT_VERSION } from 'document-schema.js';
+import { describe, expect, it } from 'vitest';
+import { latexToFormula } from './lower';
+import { lintMathCoherence } from './lint';
+import { buildFormulaBlock } from '../model/formula';
+
+// The coherence lint's contract: re-parse, re-lower, compare -- and report divergence as a warning carrying the stored provenance, never as an automatic re-derivation (the schema's atomic pair-edit rule: the layers stay exactly as stored). These tests also pin that the lint WRITES nothing: every assertion re-reads the same package object after linting.
+
+function packageOf(blocks: readonly ContentBlock[]): DocumentPackage {
+  return {
+    formatVersion: DOCUMENT_PACKAGE_FORMAT_VERSION,
+    content: {
+      kind: 'wordprocessing',
+      formatVersion: CONTENT_FORMAT_VERSION,
+      metadata: {},
+      sections: [{ pageSize: { widthPt: 595, heightPt: 842 }, margins: { topPt: 20, rightPt: 20, bottomPt: 20, leftPt: 20 }, blocks: [...blocks] }],
+    },
+  };
+}
+
+function mathBlockOf(latex: string): ContentBlock {
+  return buildFormulaBlock(latexToFormula(latex, { source: 'test:lint' }).formula, { xPt: 0, yPt: 0, widthPt: 0, heightPt: 22 }, 'test:lint');
+}
+
+describe('lintMathCoherence', () => {
+  it('a package whose stored content is exactly the mechanical re-lowering of its presentation stays silent', () => {
+    const pkg = packageOf([mathBlockOf('\\sum_{i=1}^{n} \\frac{1}{i^2}')]);
+    expect(lintMathCoherence(pkg)).toEqual([]);
+  });
+
+  it('a deliberately edited content layer diverges: a warning carrying provenance, and the stored layers are untouched', () => {
+    const block = mathBlockOf('E = mc^2');
+    const pkg = packageOf([block]);
+    // Someone resolved the mc^2 juxtaposition by hand into an explicit multiplication -- a better reading, stored deliberately next to the unchanged presentation.
+    if (block.kind !== 'embeddedObject' || block.document.kind !== 'formula') {
+      throw new Error('expected a formula block');
+    }
+    block.document.formula.content = {
+      kind: 'app',
+      operator: 'math:eq',
+      args: [
+        { kind: 'sym', id: 'symbols:E' },
+        { kind: 'app', operator: 'math:multiply', args: [{ kind: 'sym', id: 'symbols:m' }, { kind: 'app', operator: 'math:pow', args: [{ kind: 'sym', id: 'symbols:c' }, { kind: 'num', numerator: '2', denominator: '1' }] }] },
+      ],
+    };
+    block.document.formula.provenance = { source: 'test:lint', editTrail: ['human edit: resolved the mc^2 juxtaposition into an explicit multiply'] };
+    const warnings = lintMathCoherence(pkg);
+    expect(warnings).toHaveLength(1);
+    expect(warnings[0]?.code).toBe('math/coherence-divergence');
+    expect(warnings[0]?.severity).toBe('warning');
+    expect(warnings[0]?.provenance).toBe('test:lint -> human edit: resolved the mc^2 juxtaposition into an explicit multiply');
+    expect(warnings[0]?.detail).toContain('E = mc^2');
+    // The lint re-derived nothing: the stored content is still the hand-edited tree, byte for byte.
+    expect(block.document.formula.content).toEqual({
+      kind: 'app',
+      operator: 'math:eq',
+      args: [
+        { kind: 'sym', id: 'symbols:E' },
+        { kind: 'app', operator: 'math:multiply', args: [{ kind: 'sym', id: 'symbols:m' }, { kind: 'app', operator: 'math:pow', args: [{ kind: 'sym', id: 'symbols:c' }, { kind: 'num', numerator: '2', denominator: '1' }] }] },
+      ],
+    });
+  });
+
+  it('a stored non-reduced rational still agrees with the reduced re-lowering -- value canonicalisation, not string equality', () => {
+    // '\frac{0.5}{2}' re-lowers the decimal to the reduced 1/2; the stored content spells the same value as an unreduced 2/4. Same expression, different spelling -- the lint compares cross-reduced values and stays silent.
+    const block = mathBlockOf('\\frac{0.5}{2}');
+    if (block.kind !== 'embeddedObject' || block.document.kind !== 'formula') {
+      throw new Error('expected a formula block');
+    }
+    block.document.formula.content = { kind: 'app', operator: 'math:divide', args: [{ kind: 'num', numerator: '2', denominator: '4' }, { kind: 'num', numerator: '2', denominator: '1' }] };
+    expect(lintMathCoherence(packageOf([block]))).toEqual([]);
+  });
+
+  it('an unparseable stored presentation warns only when the stored content is a real lowering', () => {
+    const degraded = mathBlockOf('\\notacommand');
+    const pkgDegraded = packageOf([degraded]);
+    // Stored content is itself an unparsed root (the lowering degraded too), so this stays silent.
+    expect(lintMathCoherence(pkgDegraded)).toEqual([]);
+    const edited = mathBlockOf('\\notacommand');
+    if (edited.kind !== 'embeddedObject' || edited.document.kind !== 'formula') {
+      throw new Error('expected a formula block');
+    }
+    edited.document.formula.content = { kind: 'sym', id: 'symbols:x' };
+    const warnings = lintMathCoherence(packageOf([edited]));
+    expect(warnings.map((warning) => warning.code)).toEqual(['math/coherence-unparseable-presentation']);
+  });
+
+  it('walks formula blocks inside table cells and skips formulas carrying only one layer', () => {
+    const presentationOnly = buildFormulaBlock({ mathml: [], presentation: { latex: 'x^2' } }, { xPt: 0, yPt: 0, widthPt: 0, heightPt: 22 }, 'test');
+    const pkg = packageOf([
+      { kind: 'table', columnWidthsPt: [100], rows: [{ cells: [{ blocks: [mathBlockOf('a + b')] }] }] },
+      presentationOnly,
+    ]);
+    expect(lintMathCoherence(pkg)).toEqual([]);
+  });
+});