From 0d962a4f12f7940ac3ac97079ba30f2b49f4e723 Mon Sep 17 00:00:00 2001 From: Tommaso Sotte <3329248+tomsotte@users.noreply.github.com> Date: Thu, 8 Oct 2026 10:44:19 +0200 Subject: [PATCH 1/2] Add an objectStreams option to write object streams and a cross-reference stream Packs the objects that are not streams into compressed PDF 1.5 object streams, and writes a cross-reference stream instead of the xref table, making documents with many such objects, e.g. tagged ones, much smaller. Off by default; requires a pdfVersion of 1.5 or higher, and cannot be used with encryption or PDF/A-1. --- CHANGELOG.md | 1 + docs/getting_started.md | 11 ++ lib/document.js | 14 ++- lib/object_streams.js | 117 ++++++++++++++++++ lib/reference.js | 8 ++ tests/unit/object_streams.spec.js | 198 ++++++++++++++++++++++++++++++ 6 files changed, 348 insertions(+), 1 deletion(-) create mode 100644 lib/object_streams.js create mode 100644 tests/unit/object_streams.spec.js diff --git a/CHANGELOG.md b/CHANGELOG.md index daca229d..b48fb321 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ### Unreleased +- Add an `objectStreams` option, which packs the objects that are not streams into compressed object streams and writes a cross-reference stream instead of the xref table, making documents with many such objects, e.g. tagged ones, much smaller. It requires a `pdfVersion` of `1.5` or higher, and cannot be used with encryption or PDF/A-1 - Fix the underline, strike, link and goTo of wrapped text, and `boundsOfString()` with a `width`, counting `characterSpacing` and `wordSpacing` twice, so they no longer run past the text and match the same text drawn with `lineBreak: false` - Fix the Node build throwing once a bundler inlines it into an application bundle, `Cannot find module '#standard-fonts/Helvetica'` on the first document from an ESM bundle and `Invalid URL` while importing a CommonJS one, because the standard font metrics and the PDF/A ICC profile were resolved relative to pdfkit's own package directory at import time. Both are now resolved on first use, a standard font that cannot be loaded names itself and the fix in its error, and `registerStdFonts` is exported from the Node build as it already was from the browser build, so a bundle can register the fonts it uses from `pdfkit/standard-fonts/`. The Node build now requires the standard fonts by a path relative to the built file, which replaces the `#standard-fonts/*` import mapping and lets an ESM bundle load them from a copy of `js/standard-fonts` next to it. Fixes #1801 - Fix `doc.list()` drawing the label of a `numbered` or `lettered` item with the line wrapper's own options object, which threw `unsupported number: NaN` for `align` `center` and `right`, applied a word spacing derived from the previous line to the label for `align` `justify`, and sized the underline, strike, link and goTo of every label after the first from the previous item's text diff --git a/docs/getting_started.md b/docs/getting_started.md index 06e01eb7..0b9ba61c 100644 --- a/docs/getting_started.md +++ b/docs/getting_started.md @@ -170,6 +170,17 @@ The `pageLayout` option specifies how pages should be displayed in a PDF viewer: const doc = new PDFDocument({ pageLayout: 'twoColumnLeft' }); +### Object streams + +The `objectStreams` option packs the objects that are not streams (pages, annotations, the +structure elements of a tagged document, etc.) into compressed object streams, and writes a +cross-reference stream instead of the xref table. Documents with many such objects, like tagged +ones, get much smaller, while their content stays the same. + + const doc = new PDFDocument({ pdfVersion: '1.5', objectStreams: true }); + +It requires a `pdfVersion` of `1.5` or higher, and cannot be used with encryption or PDF/A-1. + ## Adding pages The first page of a PDFKit document is added for you automatically when you diff --git a/lib/document.js b/lib/document.js index 4e76a0a5..09a12442 100644 --- a/lib/document.js +++ b/lib/document.js @@ -9,6 +9,7 @@ import PDFReference from './reference'; import PDFPage from './page'; import PDFNameTree from './name_tree'; import PDFSecurity from './security'; +import PDFObjectStreams from './object_streams'; import ColorMixin from './mixins/color'; import VectorMixin from './mixins/vector'; import FontsMixin from './mixins/fonts'; @@ -53,6 +54,10 @@ class PDFDocument extends Readable { this.compress = this.options.compress != null ? this.options.compress : true; + if (options.objectStreams) { + this._objectStreams = new PDFObjectStreams(this, options); + } + this._pageBuffer = []; this._pageBufferStart = 0; @@ -284,8 +289,9 @@ class PDFDocument extends Readable { _refEnd(ref) { this._offsets[ref.id - 1] = ref.offset; if (--this._waiting === 0 && this._ended) { - this._finalize(); + // reset first, as finalizing may end more references this._ended = false; + this._finalize(); } } @@ -342,6 +348,12 @@ class PDFDocument extends Readable { } _finalize() { + if (this._objectStreams) { + this._objectStreams.finalize(); + this.push(null); + return; + } + // generate xref const xRefOffset = this._offset; this._write('xref'); diff --git a/lib/object_streams.js b/lib/object_streams.js new file mode 100644 index 00000000..80dd7878 --- /dev/null +++ b/lib/object_streams.js @@ -0,0 +1,117 @@ +/* +PDFObjectStreams - packs objects without a stream into compressed object +streams, and writes a cross-reference stream instead of the xref table (PDF 1.5) +*/ + +import PDFObject from './object'; +import { fromBinaryString } from './binary'; + +// objects per object stream, and so the most held in memory at a time +const OBJECTS_PER_STREAM = 200; + +class PDFObjectStreams { + constructor(document, options) { + if (document.version < 1.5) { + throw new Error( + "The objectStreams option requires a pdfVersion of '1.5' or higher", + ); + } + if (/^PDF\/A-1/.test(options.subset)) { + throw new Error('PDF/A-1 does not allow object streams'); + } + if (options.userPassword || options.ownerPassword) { + throw new Error( + 'The objectStreams option cannot be used with encryption', + ); + } + + this.document = document; + this.pending = []; + this.locations = new Map(); // object id -> [object stream id, index] + } + + add(ref) { + this.pending.push({ id: ref.id, body: PDFObject.convert(ref.data) }); + if (this.pending.length >= OBJECTS_PER_STREAM) { + this.flush(); + } + } + + flush() { + if (!this.pending.length) { + return; + } + + const objects = this.pending; + this.pending = []; + + // "id offset" pairs, then the objects, offsets relative to First + let offset = 0; + const pairs = []; + for (const { id, body } of objects) { + pairs.push(id, offset); + offset += body.length + 1; + } + const header = pairs.join(' ') + '\n'; + + const stream = this.document.ref({ + Type: 'ObjStm', + N: objects.length, + First: header.length, + }); + stream.end( + fromBinaryString(header + objects.map((o) => o.body).join('\n')), + ); + + objects.forEach(({ id }, index) => { + this.locations.set(id, [stream.id, index]); + }); + } + + finalize() { + const doc = this.document; + this.flush(); + + const xref = doc.ref({ + Type: 'XRef', + Root: doc._root, + Info: doc._info, + ID: [doc._id, doc._id], + }); + const xrefOffset = doc._offset; + if (xrefOffset > 0xffffffff) { + // offsets are written in 4 bytes + throw new Error('The objectStreams option cannot write files over 4 GB'); + } + doc._offsets[xref.id - 1] = xrefOffset; + + // entries of [type, offset or object stream id, generation or index] + const size = doc._offsets.length + 1; + const table = new DataView(new ArrayBuffer(size * 7)); + const put = (id, type, field2, field3) => { + table.setUint8(id * 7, type); + table.setUint32(id * 7 + 1, field2); + table.setUint16(id * 7 + 5, field3); + }; + + put(0, 0, 0, 0xffff); + for (let id = 1; id < size; id++) { + const location = this.locations.get(id); + if (location) { + put(id, 2, ...location); + } else { + put(id, 1, doc._offsets[id - 1], 0); + } + } + + xref.data.Size = size; + xref.data.W = [1, 4, 2]; + xref.end(new Uint8Array(table.buffer)); + + doc._write('startxref'); + doc._write(`${xrefOffset}`); + doc._write('%%EOF'); + } +} + +export default PDFObjectStreams; diff --git a/lib/reference.js b/lib/reference.js index 0e27bd53..057e5a03 100644 --- a/lib/reference.js +++ b/lib/reference.js @@ -44,6 +44,14 @@ class PDFReference extends PDFAbstractReference { } finalize() { + const objectStreams = this.document._objectStreams; + if (objectStreams && !this.buffer.length) { + // written later as part of an object stream + objectStreams.add(this); + this.document._refEnd(this); + return; + } + this.offset = this.document._offset; const encryptFn = this.document._security diff --git a/tests/unit/object_streams.spec.js b/tests/unit/object_streams.spec.js new file mode 100644 index 00000000..1c3515fb --- /dev/null +++ b/tests/unit/object_streams.spec.js @@ -0,0 +1,198 @@ +import { vi } from 'vitest'; +import zlib from 'zlib'; +import PDFDocument from '../../lib/document'; + +const options = { + autoFirstPage: false, + pdfVersion: '1.5', + objectStreams: true, +}; + +// Returns the bytes of a document; `after` can end references once it ended +function build(docOptions, fn = (doc) => doc.addPage(), after) { + const chunks = []; + const push = PDFDocument.prototype.push; + const spy = vi + .spyOn(PDFDocument.prototype, 'push') + .mockImplementation(function (chunk) { + if (chunk) chunks.push(Buffer.from(chunk)); + return push.call(this, chunk); + }); + try { + const doc = new PDFDocument(docOptions); + const state = fn(doc); + doc.end(); + if (after) after(state); + } finally { + spy.mockRestore(); + } + return Buffer.concat(chunks); +} + +// Reads the stream object at offset: its id, dictionary and decoded data +function readStream(file, offset) { + const text = file.toString('latin1', offset, offset + 2000); + const [head, id, dict] = /^(\d+) 0 obj\n<<([\s\S]*?)>>\nstream\n/.exec(text); + const start = offset + head.length; + let data = file.subarray( + start, + start + Number(/\/Length (\d+)/.exec(dict)[1]), + ); + if (dict.includes('/FlateDecode')) data = zlib.inflateSync(data); + return { id: Number(id), dict, data }; +} + +// Resolves every object through the cross-reference stream: id -> the +// offset of a plain object, or the body of an object in an object stream +function parse(file) { + const text = file.toString('latin1'); + const xref = readStream( + file, + Number(/startxref\n(\d+)\n%%EOF\n$/.exec(text)[1]), + ); + expect(xref.dict).toContain('/Type /XRef'); + expect(xref.dict).toContain('/W [1 4 2]'); + const size = Number(/\/Size (\d+)/.exec(xref.dict)[1]); + expect(xref.data.length).toBe(size * 7); + + const view = new DataView(xref.data.buffer, xref.data.byteOffset); + const entry = (id) => [ + view.getUint8(id * 7), + view.getUint32(id * 7 + 1), + view.getUint16(id * 7 + 5), + ]; + expect(entry(0)).toEqual([0, 0, 0xffff]); + + const objects = new Map(); + const streams = new Map(); + for (let id = 1; id < size; id++) { + const [type, field2, index] = entry(id); + if (type === 1) { + expect(text.startsWith(`${id} 0 obj\n`, field2)).toBe(true); + objects.set(id, { offset: field2 }); + continue; + } + expect(type).toBe(2); + if (!streams.has(field2)) { + const { dict, data } = readStream(file, entry(field2)[1]); + expect(dict).toContain('/Type /ObjStm'); + const first = Number(/\/First (\d+)/.exec(dict)[1]); + const body = data.toString('latin1'); + const pairs = body.slice(0, first).trim().split(' ').map(Number); + expect(pairs.length).toBe(Number(/\/N (\d+)/.exec(dict)[1]) * 2); + const members = []; + for (let i = 0; i < pairs.length; i += 2) { + const end = pairs[i + 3] ?? body.length - first; + members.push([pairs[i], body.slice(first + pairs[i + 1], first + end)]); + } + streams.set(field2, members); + } + const [memberId, body] = streams.get(field2)[index]; + expect(memberId).toBe(id); + objects.set(id, { body: body.trim() }); + } + return { text, xref, objects, streams }; +} + +const bodies = ({ objects }) => [...objects.values()].map((o) => o.body); + +describe('object streams', () => { + test('are off by default', () => { + const text = build({ autoFirstPage: false }).toString('latin1'); + expect(text).toContain('\nxref\n0 '); + expect(text).toContain('\ntrailer\n'); + expect(text).not.toContain('/ObjStm'); + }); + + test('hold every object without a stream, with a cross-reference stream', () => { + const pdf = parse( + build(options, (doc) => { + doc.addPage(); + doc.text('Hello'); + }), + ); + expect(pdf.text).not.toContain('\nxref\n'); + expect(pdf.text).not.toContain('\ntrailer\n'); + expect(pdf.xref.dict).toMatch(/\/ID \[<[0-9a-f]+> <[0-9a-f]+>\]/); + + const root = Number(/\/Root (\d+) 0 R/.exec(pdf.xref.dict)[1]); + expect(pdf.objects.get(root).body).toContain('/Type /Catalog'); + const info = Number(/\/Info (\d+) 0 R/.exec(pdf.xref.dict)[1]); + expect(pdf.objects.get(info).body).toContain('/Producer'); + + // only streams, e.g. the page content, are left as plain objects + for (const { offset } of pdf.objects.values()) { + if (offset !== undefined) { + expect(pdf.text.slice(offset).split('endobj')[0]).toContain( + '\nstream\n', + ); + } + } + }); + + test('hold up to 200 objects each', () => { + const pdf = parse( + build(options, (doc) => { + doc.addPage(); + for (let i = 0; i < 450; i++) doc.ref({ Index: i }).end(); + }), + ); + const sizes = [...pdf.streams.values()].map((members) => members.length); + expect(sizes.length).toBeGreaterThanOrEqual(3); + expect(Math.max(...sizes)).toBe(200); + const all = bodies(pdf).join(' '); + for (let i = 0; i < 450; i++) expect(all).toContain(`/Index ${i}\n`); + }); + + test('include references ended after the document', () => { + const file = build( + options, + (doc) => { + doc.addPage(); + return doc.ref({ Late: true }); + }, + (late) => late.end(), + ); + expect(bodies(parse(file))).toContainEqual( + expect.stringContaining('/Late true'), + ); + }); + + test('work without compression', () => { + const pdf = parse(build({ ...options, compress: false })); + expect(pdf.xref.dict).not.toContain('/Filter'); + expect(bodies(pdf)).toContainEqual(expect.stringContaining('/Catalog')); + }); + + test('work with tagged PDF/A-2', () => { + const pdf = parse( + build( + { ...options, pdfVersion: '1.7', subset: 'PDF/A-2', tagged: true }, + (doc) => { + doc.addPage(); + doc.text('Hello'); + }, + ), + ); + expect(bodies(pdf)).toContainEqual( + expect.stringContaining('/StructTreeRoot'), + ); + }); + + test('throw when they cannot be used', () => { + expect(() => new PDFDocument({ objectStreams: true })).toThrow(/1\.5/); + expect( + () => + new PDFDocument({ ...options, pdfVersion: '1.7', subset: 'PDF/A-1b' }), + ).toThrow(/PDF\/A-1/); + expect( + () => new PDFDocument({ ...options, userPassword: 'secret' }), + ).toThrow(/encryption/); + expect(() => + build(options, (doc) => { + doc.addPage(); + doc._offset = 2 ** 32; // as if 4 GB were already written + }), + ).toThrow(/4 GB/); + }); +}); From 668a15778de9a41d0f62f88e44356f8a8ec443a7 Mon Sep 17 00:00:00 2001 From: Tommaso Sotte <3329248+tomsotte@users.noreply.github.com> Date: Thu, 8 Oct 2026 11:11:46 +0200 Subject: [PATCH 2/2] Name the cross-reference stream entry constants and reference #1810 --- CHANGELOG.md | 2 +- lib/object_streams.js | 24 ++++++++++++++++-------- tests/unit/object_streams.spec.js | 4 +++- 3 files changed, 20 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b48fb321..5fec6ec5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ### Unreleased -- Add an `objectStreams` option, which packs the objects that are not streams into compressed object streams and writes a cross-reference stream instead of the xref table, making documents with many such objects, e.g. tagged ones, much smaller. It requires a `pdfVersion` of `1.5` or higher, and cannot be used with encryption or PDF/A-1 +- Add an `objectStreams` option, which packs the objects that are not streams into compressed object streams and writes a cross-reference stream instead of the xref table, making documents with many such objects, e.g. tagged ones, much smaller. It requires a `pdfVersion` of `1.5` or higher, and cannot be used with encryption or PDF/A-1. Fixes #1810 - Fix the underline, strike, link and goTo of wrapped text, and `boundsOfString()` with a `width`, counting `characterSpacing` and `wordSpacing` twice, so they no longer run past the text and match the same text drawn with `lineBreak: false` - Fix the Node build throwing once a bundler inlines it into an application bundle, `Cannot find module '#standard-fonts/Helvetica'` on the first document from an ESM bundle and `Invalid URL` while importing a CommonJS one, because the standard font metrics and the PDF/A ICC profile were resolved relative to pdfkit's own package directory at import time. Both are now resolved on first use, a standard font that cannot be loaded names itself and the fix in its error, and `registerStdFonts` is exported from the Node build as it already was from the browser build, so a bundle can register the fonts it uses from `pdfkit/standard-fonts/`. The Node build now requires the standard fonts by a path relative to the built file, which replaces the `#standard-fonts/*` import mapping and lets an ESM bundle load them from a copy of `js/standard-fonts` next to it. Fixes #1801 - Fix `doc.list()` drawing the label of a `numbered` or `lettered` item with the line wrapper's own options object, which threw `unsupported number: NaN` for `align` `center` and `right`, applied a word spacing derived from the previous line to the label for `align` `justify`, and sized the underline, strike, link and goTo of every label after the first from the previous item's text diff --git a/lib/object_streams.js b/lib/object_streams.js index 80dd7878..8c204431 100644 --- a/lib/object_streams.js +++ b/lib/object_streams.js @@ -9,6 +9,13 @@ import { fromBinaryString } from './binary'; // objects per object stream, and so the most held in memory at a time const OBJECTS_PER_STREAM = 200; +// cross-reference stream entries: a 1-byte type, then 4-byte and 2-byte fields +const ENTRY_WIDTHS = [1, 4, 2]; +const ENTRY_SIZE = 7; +const FREE = 0; +const UNCOMPRESSED = 1; +const COMPRESSED = 2; + class PDFObjectStreams { constructor(document, options) { if (document.version < 1.5) { @@ -87,25 +94,26 @@ class PDFObjectStreams { // entries of [type, offset or object stream id, generation or index] const size = doc._offsets.length + 1; - const table = new DataView(new ArrayBuffer(size * 7)); + const table = new DataView(new ArrayBuffer(size * ENTRY_SIZE)); const put = (id, type, field2, field3) => { - table.setUint8(id * 7, type); - table.setUint32(id * 7 + 1, field2); - table.setUint16(id * 7 + 5, field3); + const start = id * ENTRY_SIZE; + table.setUint8(start, type); + table.setUint32(start + 1, field2); + table.setUint16(start + 5, field3); }; - put(0, 0, 0, 0xffff); + put(0, FREE, 0, 0xffff); for (let id = 1; id < size; id++) { const location = this.locations.get(id); if (location) { - put(id, 2, ...location); + put(id, COMPRESSED, ...location); } else { - put(id, 1, doc._offsets[id - 1], 0); + put(id, UNCOMPRESSED, doc._offsets[id - 1], 0); } } xref.data.Size = size; - xref.data.W = [1, 4, 2]; + xref.data.W = ENTRY_WIDTHS; xref.end(new Uint8Array(table.buffer)); doc._write('startxref'); diff --git a/tests/unit/object_streams.spec.js b/tests/unit/object_streams.spec.js index 1c3515fb..546b231f 100644 --- a/tests/unit/object_streams.spec.js +++ b/tests/unit/object_streams.spec.js @@ -8,7 +8,9 @@ const options = { objectStreams: true, }; -// Returns the bytes of a document; `after` can end references once it ended +// Returns the bytes of a document; `after` can end references once it ended. +// Spies on push rather than using logData, which starts after the header is +// written, as the cross-reference offsets count from the start of the file function build(docOptions, fn = (doc) => doc.addPage(), after) { const chunks = []; const push = PDFDocument.prototype.push;