4.0.0: tree-form DocumentPackage, definitions tables, $schema versioning, LayoutDocument demotion

package.json

@@ -1,7 +1,7 @@
 {
   "name": "document-schema.js",
   "version": "3.3.0",
-  "description": "The canonical, format-agnostic content and layout schemas shared by ooxml.js, odf.js, and documents.js -- pure Zod schemas, no behaviour.",
+  "description": "The canonical, format-agnostic content and document-package schemas shared by ooxml.js, odf.js, documents.js, pdf-codec, and markdown-codec -- pure Zod schemas, no behaviour.",
   "type": "module",
   "repository": {
     "type": "git",

src/codec.test.ts

@@ -1,14 +1,10 @@
 import { describe, expect, it } from 'vitest';
-import { COLOR_BLACK } from './color';
-import type { ContentCodec, LayoutCodec } from './codec';
-import { CONTENT_FORMAT_VERSION, type ContentDocument } from './content';
-import { LAYOUT_FORMAT_VERSION, type LayoutDocument } from './layout';
-import { DEFAULT_LAYOUT_FONT } from './style';
+import type { ContentCodec } from './codec';
+import type { ContentDocument } from './content';
 
 function wordprocessingDocument(): ContentDocument {
   return {
     kind: 'wordprocessing',
-    formatVersion: CONTENT_FORMAT_VERSION,
     metadata: { title: 'Codec round trip', author: 'document-schema.js' },
     sections: [
       {
@@ -26,32 +22,6 @@ function wordprocessingDocument(): ContentDocument {
   };
 }
 
-function layoutDocument(): LayoutDocument {
-  return {
-    formatVersion: LAYOUT_FORMAT_VERSION,
-    metadata: { title: 'Codec round trip', author: 'document-schema.js' },
-    pages: [
-      {
-        widthPt: 612,
-        heightPt: 792,
-        items: [
-          {
-            kind: 'text',
-            text: 'Hello, codec.',
-            xPt: 72,
-            yPt: 720,
-            font: DEFAULT_LAYOUT_FONT,
-            sizePt: 12,
-            color: COLOR_BLACK,
-            sourcePath: 'sections[0].blocks[0].runs[0]',
-          },
-        ],
-      },
-    ],
-    images: {},
-  };
-}
-
 describe('ContentCodec', () => {
   it('accepts a real implementation carrying both read and write', () => {
     const codec: ContentCodec = {
@@ -94,40 +64,3 @@ describe('ContentCodec', () => {
     expect(codec.read(new Uint8Array([0]), {})).toEqual(wordprocessingDocument());
   });
 });
-
-describe('LayoutCodec', () => {
-  it('accepts a real implementation carrying both read and write, since write is not optional', () => {
-    const codec: LayoutCodec = {
-      read: (bytes) => {
-        expect(bytes).toBeInstanceOf(Uint8Array);
-        return layoutDocument();
-      },
-      write: (layout) => {
-        expect(layout.pages).toHaveLength(1);
-        return new Uint8Array([4, 5, 6]);
-      },
-    };
-
-    const layout = codec.read(new Uint8Array([0]));
-    expect(layout).toEqual(layoutDocument());
-    expect(codec.write(layout)).toEqual(new Uint8Array([4, 5, 6]));
-  });
-
-  it('threads a format-specific TOptions type through both read and write', () => {
-    interface PdfWriteOptions {
-      onFontSubstitution?: (family: string) => void;
-    }
-
-    const codec: LayoutCodec<PdfWriteOptions> = {
-      read: () => layoutDocument(),
-      write: (_layout, options) => {
-        options?.onFontSubstitution?.('Carlito');
-        return new Uint8Array([7]);
-      },
-    };
-
-    let substituted: string | undefined;
-    codec.write(layoutDocument(), { onFontSubstitution: (family) => (substituted = family) });
-    expect(substituted).toBe('Carlito');
-  });
-});

src/codec.ts

@@ -1,20 +1,13 @@
 import type { ContentDocument } from './content';
-import type { LayoutDocument } from './layout';
 
-// These two contracts describe what an individual format's own codec naturally provides -- not what DocumentPackage requires. A codec's read() never returns a DocumentPackage directly; it returns one half. DocumentPackage (content required, layout optional -- see its own doc comment in package.ts) is the *assembled* format produced by composing a ContentCodec.read() result with a *separately run* layout-engine pass. Nothing here constructs a DocumentPackage; that composition happens one level up, in whatever code owns both a ContentCodec and a layout engine for the same format.
+// This contract describes what an individual format's own codec naturally provides -- not what DocumentPackage requires. A codec's read() never returns a DocumentPackage directly; the package is the *assembled* tree produced by decomposing the flat ContentDocument a reader returns (src/package.ts's three laws). Nothing here constructs a DocumentPackage; that composition happens one level up, in whatever code owns both a ContentCodec and a layout engine for the same format.
 
-// The two contracts are asymmetric rather than a single unified DocumentCodec operating on DocumentPackage, because the formats they model are asymmetric. Most formats (docx/pptx/odt/odp/ods/odg/markdown) only ever produce content on read -- layout for them is always a separate, later, engine-driven step, never something their own codec produces directly. PDF is the mirror image: it produces layout cheaply on read, and content only via a separate, expensive, lossy, opt-in reconstruction pass that is emphatically not part of "reading" a PDF. Forcing a single contract shaped like DocumentPackage would mean either fabricating empty content for every non-PDF format's codec (impossible, they have none to fabricate) or making every PDF read pay reconstruction cost it usually doesn't want (the wrong default). Keeping ContentCodec and LayoutCodec separate lets each format implement only the half it actually has.
+// There is deliberately no LayoutCodec alongside this any more. Releases 1.x-3.x exported one -- read() to a LayoutDocument, write() back -- modelling the single format that produces layout cheaply on read: PDF. The whole LayoutDocument family moved to pdf-codec in this major (ExaDev/pdf-codec#65), where it is that codec's own private model, so the interface that described it belongs there too, alongside it. Callers that held a LayoutCodec over a PDF codec hold pdf-codec's own read/write signatures directly once they migrate.
 
 // A format's own content codec: read() decodes that format's bytes into a ContentDocument; write(), where the format supports it, encodes a ContentDocument back into that format's bytes. write() is deliberately optional -- this models a real, permanent asymmetry, not a temporary gap. The odf format (a standalone ODF formula document) has a reader but genuinely no builder at all: recovering structured MathML from rendered glyphs is a categorically different, OCR-adjacent problem than generating them, so no ContentDocument-to-odf writer exists or is planned.
 export interface ContentCodec<TOptions = unknown> {
   read(bytes: Uint8Array, options?: TOptions): ContentDocument;
   write?(content: ContentDocument, options?: TOptions): Uint8Array;
 }
 
-// A format's own layout codec: read() decodes that format's bytes into a LayoutDocument; write() encodes a LayoutDocument back into that format's bytes. Unlike ContentCodec.write, LayoutCodec.write is not optional -- PDF is the only format with a LayoutCodec implementation anywhere in this family, and it always supports both directions (a PDF is written from a LayoutDocument exactly as readily as it is read into one), so there is no real asymmetry here to model.
-export interface LayoutCodec<TOptions = unknown> {
-  read(bytes: Uint8Array, options?: TOptions): LayoutDocument;
-  write(layout: LayoutDocument, options?: TOptions): Uint8Array;
-}
-
-// TOptions is generic per codec rather than a single shared options type across every ContentCodec/LayoutCodec implementation, since each real format's own read/write options are format-specific today (an AbortSignal, a font-substitution callback, a diagnostic sink) and this package has no reason to force them into one shared shape that would either be too narrow for some formats or carry fields meaningless to others.
+// TOptions is generic per codec rather than a single shared options type across every ContentCodec implementation, since each real format's own read/write options are format-specific today (an AbortSignal, a font-substitution callback, a diagnostic sink) and this package has no reason to force them into one shared shape that would either be too narrow for some formats or carry fields meaningless to others.

src/content-json-schema-defs.test.ts

@@ -8,11 +8,26 @@ import {
   ContentListMembershipSchema,
   ContentPageBreakSchema,
   ContentParagraphSchema,
+  ContentPathPointSchema,
+  ContentPathSegmentSchema,
   ContentRunSchema,
+  ContentSheetCellCommentSchema,
+  ContentSheetCellSchema,
+  ContentSheetColumnSchema,
+  ContentSheetImageSchema,
+  ContentSheetPrintRangeSchema,
+  ContentSheetPrintSettingsSchema,
+  ContentSheetRepeatRangeSchema,
+  ContentSheetRowSchema,
+  ContentStrokeSchema,
   ContentStrokeStyleSchema,
+  ContentSubpathSchema,
+  ContentVectorSchema,
+  ContentCellValueSchema,
 } from './content';
 import { CONTENT_DEFS } from './content-json-schema-defs';
-import { BoxSchema, LayoutFrameSchema } from './geometry';
+import { DefinitionEntrySchema, StyleEntrySchema, StyleParagraphPropertiesSchema, StyleRunPropertiesSchema } from './definitions';
+import { BoxSchema, LayoutFrameSchema, MarginsSchema, PageSizeSchema } from './geometry';
 import {
   DimensionVectorSchema,
   ExactRationalSchema,
@@ -28,9 +43,18 @@ import {
   MathUnparsedSchema,
   SymbolTableSchema,
 } from './math';
+import {
+  DrawPageDescriptorSchema,
+  HeadingParagraphSchema,
+  ListParagraphSchema,
+  SectionDescriptorSchema,
+  ShapeDescriptorSchema,
+  SheetDescriptorSchema,
+  SlideDescriptorSchema,
+} from './package-node';
 import { AlignmentSchema } from './style';
 
-// This is the regression test scripts/generate-json-schemas.mjs's own top comment calls for: the only structural defence that generator has against silently drifting away from src/content.ts/src/color.ts/src/geometry.ts/src/style.ts/src/math.ts, since CONTENT_DEFS (content-json-schema-defs.ts) is transcribed by hand rather than generated. Not every entry in CONTENT_DEFS can be checked this way -- ContentBlock/ContentTable/ContentTableRow/ContentTableCell/ContentEmbeddedObjectBlock/MathMlNode/MathMlElement/MathMlAttribute all sit downstream of one of the genuinely un-representable z.custom() nodes (ContentBlockSchema, ContentEmbeddedObjectSchema, MathMlNodeSchema), and ContentFormula/MathExpression/MathApp/MathSum/MathProd/MathMatrix sit downstream of the fourth (MathExpressionSchema, reached through ContentFormulaSchema.content for the first and through the grammar's own recursion for the rest) -- see that module's own top comment -- so a bare z.toJSONSchema() call over their real schema counterpart either throws or degrades to `{}` for the recursive/custom part, which is exactly the problem CONTENT_DEFS exists to work around in the first place. What CAN be checked -- because a real, non-recursive, non-custom exported Zod schema exists for it -- is every leaf and near-leaf fragment: Color, Box, LayoutFrame, Alignment, ContentStrokeStyle, ContentBorder, ContentCellBorders, ContentListMembership, ContentRun, ContentParagraph, ContentImageBlock, ContentPageBreak, ExactRational, DimensionVector, MathPresentation, MathProvenance, MathUncertainty, MathNum, MathQty, MathSym, MathUnparsed, MathSymbolEntry, MathUnit, MathNormalisationContext, SymbolTable. None of these reaches ContentBlockSchema, ContentEmbeddedObjectSchema, MathMlNodeSchema, or MathExpressionSchema from anywhere in its own field tree, so each can be generated live and compared directly.
+// This is the regression test scripts/generate-json-schemas.mjs's own top comment calls for: the only structural defence that generator has against silently drifting away from src/content.ts/src/color.ts/src/geometry.ts/src/style.ts/src/math.ts/src/package-node.ts/src/definitions.ts, since CONTENT_DEFS (content-json-schema-defs.ts) is transcribed by hand rather than generated. Not every entry in CONTENT_DEFS can be checked this way -- ContentBlock/ContentTable/ContentTableRow/ContentTableCell/ContentEmbeddedObject(Block)/MathMlNode/MathMlElement/MathMlAttribute all sit downstream of one of the genuinely un-representable z.custom() nodes (ContentBlockSchema, ContentEmbeddedObjectSchema, MathMlNodeSchema), the seven package-tree group wrappers sit downstream of the tree's own per-kind group schemas (src/package-node.ts, z.custom over recursive guards, reached only through the hand fragments' own children pointers), and ContentFormula/MathExpression/MathApp/MathSum/MathProd/MathMatrix sit downstream of the fourth opaque node (MathExpressionSchema, reached through ContentFormulaSchema.content for the first and through the grammar's own recursion for the rest) -- see that module's own top comment -- so a bare z.toJSONSchema() call over their real schema counterpart either throws or degrades to `{}` for the recursive/custom part, which is exactly the problem CONTENT_DEFS exists to work around in the first place. What CAN be checked -- because a real, non-recursive, non-custom exported Zod schema exists for it -- is every leaf and near-leaf fragment: Color, Box, LayoutFrame, Alignment, ContentStrokeStyle, ContentBorder, ContentCellBorders, ContentListMembership, ContentRun, ContentParagraph, ContentImageBlock, ContentPageBreak, PageSize, Margins, SectionDescriptor, SlideDescriptor, SheetDescriptor, DrawPageDescriptor, ShapeDescriptor, HeadingParagraph, ListParagraph, ContentSheetCell, ContentCellValue, ContentSheetCellComment, ContentSheetColumn, ContentSheetRow, ContentSheetPrintSettings, ContentSheetPrintRange, ContentSheetRepeatRange, ContentSheetImage, ContentStroke, ContentPathPoint, ContentPathSegment, ContentSubpath, ContentVector, StyleParagraphProperties, StyleRunProperties, StyleEntry, DefinitionEntry, ExactRational, DimensionVector, MathPresentation, MathProvenance, MathUncertainty, MathNum, MathQty, MathSym, MathUnparsed, MathSymbolEntry, MathUnit, MathNormalisationContext, SymbolTable. None of these reaches ContentBlockSchema, ContentEmbeddedObjectSchema, MathMlNodeSchema, MathExpressionSchema, or a tree group schema from anywhere in its own field tree, so each can be generated live and compared directly.
 //
 // Comparison strategy: a bare `z.toJSONSchema(SomeSchema)` call, run in isolation, would INLINE every nested schema it encounters (ColorSchema inside ContentRunSchema, AlignmentSchema inside ContentParagraphSchema, etc.) rather than emit the `{ $ref: '#/$defs/X' }` pointers CONTENT_DEFS itself uses -- because those nested schemas aren't registered anywhere. To reproduce the exact cross-reference shape CONTENT_DEFS hand-authors, this test registers the identical set of real schemas under the identical id strings CONTENT_DEFS uses as its own $defs keys, with a `uri` callback matching the `#/$defs/<id>` convention CONTENT_DEFS was written against -- confirmed empirically (see this file's own construction) to make Zod's registry-based multi-schema generation emit exactly that $ref shape for every registered schema referenced from within another. Each per-schema result still carries its own top-level `$schema`/`$id` (since z.toJSONSchema(registry, ...) treats every registered schema as its own standalone root), which CONTENT_DEFS's own nested fragments never have -- those two keys are stripped before comparison, since they're an artefact of testing each fragment as a registry root rather than a real structural difference.
 
@@ -47,6 +71,33 @@ const REGISTERED_SCHEMAS = {
   ContentParagraph: ContentParagraphSchema,
   ContentImageBlock: ContentImageBlockSchema,
   ContentPageBreak: ContentPageBreakSchema,
+  PageSize: PageSizeSchema,
+  Margins: MarginsSchema,
+  SectionDescriptor: SectionDescriptorSchema,
+  SlideDescriptor: SlideDescriptorSchema,
+  SheetDescriptor: SheetDescriptorSchema,
+  DrawPageDescriptor: DrawPageDescriptorSchema,
+  ShapeDescriptor: ShapeDescriptorSchema,
+  HeadingParagraph: HeadingParagraphSchema,
+  ListParagraph: ListParagraphSchema,
+  ContentSheetCell: ContentSheetCellSchema,
+  ContentCellValue: ContentCellValueSchema,
+  ContentSheetCellComment: ContentSheetCellCommentSchema,
+  ContentSheetColumn: ContentSheetColumnSchema,
+  ContentSheetRow: ContentSheetRowSchema,
+  ContentSheetPrintSettings: ContentSheetPrintSettingsSchema,
+  ContentSheetPrintRange: ContentSheetPrintRangeSchema,
+  ContentSheetRepeatRange: ContentSheetRepeatRangeSchema,
+  ContentSheetImage: ContentSheetImageSchema,
+  ContentStroke: ContentStrokeSchema,
+  ContentPathPoint: ContentPathPointSchema,
+  ContentPathSegment: ContentPathSegmentSchema,
+  ContentSubpath: ContentSubpathSchema,
+  ContentVector: ContentVectorSchema,
+  StyleParagraphProperties: StyleParagraphPropertiesSchema,
+  StyleRunProperties: StyleRunPropertiesSchema,
+  StyleEntry: StyleEntrySchema,
+  DefinitionEntry: DefinitionEntrySchema,
   ExactRational: ExactRationalSchema,
   DimensionVector: DimensionVectorSchema,
   MathPresentation: MathPresentationSchema,