diff --git a/CHANGELOG.md b/CHANGELOG.md index 7bab8503..6c275639 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ### Unreleased +- Add an `objectStreams` option, which packs the objects that are not streams into compressed object streams and writes a cross-reference stream instead of the xref table, making documents with many such objects, e.g. tagged ones, much smaller. It requires a `pdfVersion` of `1.5` or higher, and cannot be used with encryption or PDF/A-1. Fixes #1810 - Fix wrapped text with `align: 'center'` sitting left of the middle of the line when `characterSpacing` or `wordSpacing` is set, and with `align: 'right'` running past the right edge by the `wordSpacing` of each gap between words - Fix `boundsOfString()` with `lineBreak: false` leaving out `wordSpacing`, so its width now matches the rendered text and its underline - Fix the underline, strike, link and goTo of wrapped text, and `boundsOfString()` with a `width`, counting `characterSpacing` and `wordSpacing` twice, so they no longer run past the text and match the same text drawn with `lineBreak: false` diff --git a/docs/getting_started.md b/docs/getting_started.md index 06e01eb7..0b9ba61c 100644 --- a/docs/getting_started.md +++ b/docs/getting_started.md @@ -170,6 +170,17 @@ The `pageLayout` option specifies how pages should be displayed in a PDF viewer: const doc = new PDFDocument({ pageLayout: 'twoColumnLeft' }); +### Object streams + +The `objectStreams` option packs the objects that are not streams (pages, annotations, the +structure elements of a tagged document, etc.) into compressed object streams, and writes a +cross-reference stream instead of the xref table. Documents with many such objects, like tagged +ones, get much smaller, while their content stays the same. + + const doc = new PDFDocument({ pdfVersion: '1.5', objectStreams: true }); + +It requires a `pdfVersion` of `1.5` or higher, and cannot be used with encryption or PDF/A-1. + ## Adding pages The first page of a PDFKit document is added for you automatically when you diff --git a/lib/document.js b/lib/document.js index 4e76a0a5..09a12442 100644 --- a/lib/document.js +++ b/lib/document.js @@ -9,6 +9,7 @@ import PDFReference from './reference'; import PDFPage from './page'; import PDFNameTree from './name_tree'; import PDFSecurity from './security'; +import PDFObjectStreams from './object_streams'; import ColorMixin from './mixins/color'; import VectorMixin from './mixins/vector'; import FontsMixin from './mixins/fonts'; @@ -53,6 +54,10 @@ class PDFDocument extends Readable { this.compress = this.options.compress != null ? this.options.compress : true; + if (options.objectStreams) { + this._objectStreams = new PDFObjectStreams(this, options); + } + this._pageBuffer = []; this._pageBufferStart = 0; @@ -284,8 +289,9 @@ class PDFDocument extends Readable { _refEnd(ref) { this._offsets[ref.id - 1] = ref.offset; if (--this._waiting === 0 && this._ended) { - this._finalize(); + // reset first, as finalizing may end more references this._ended = false; + this._finalize(); } } @@ -342,6 +348,12 @@ class PDFDocument extends Readable { } _finalize() { + if (this._objectStreams) { + this._objectStreams.finalize(); + this.push(null); + return; + } + // generate xref const xRefOffset = this._offset; this._write('xref'); diff --git a/lib/object_streams.js b/lib/object_streams.js new file mode 100644 index 00000000..8c204431 --- /dev/null +++ b/lib/object_streams.js @@ -0,0 +1,125 @@ +/* +PDFObjectStreams - packs objects without a stream into compressed object +streams, and writes a cross-reference stream instead of the xref table (PDF 1.5) +*/ + +import PDFObject from './object'; +import { fromBinaryString } from './binary'; + +// objects per object stream, and so the most held in memory at a time +const OBJECTS_PER_STREAM = 200; + +// cross-reference stream entries: a 1-byte type, then 4-byte and 2-byte fields +const ENTRY_WIDTHS = [1, 4, 2]; +const ENTRY_SIZE = 7; +const FREE = 0; +const UNCOMPRESSED = 1; +const COMPRESSED = 2; + +class PDFObjectStreams { + constructor(document, options) { + if (document.version < 1.5) { + throw new Error( + "The objectStreams option requires a pdfVersion of '1.5' or higher", + ); + } + if (/^PDF\/A-1/.test(options.subset)) { + throw new Error('PDF/A-1 does not allow object streams'); + } + if (options.userPassword || options.ownerPassword) { + throw new Error( + 'The objectStreams option cannot be used with encryption', + ); + } + + this.document = document; + this.pending = []; + this.locations = new Map(); // object id -> [object stream id, index] + } + + add(ref) { + this.pending.push({ id: ref.id, body: PDFObject.convert(ref.data) }); + if (this.pending.length >= OBJECTS_PER_STREAM) { + this.flush(); + } + } + + flush() { + if (!this.pending.length) { + return; + } + + const objects = this.pending; + this.pending = []; + + // "id offset" pairs, then the objects, offsets relative to First + let offset = 0; + const pairs = []; + for (const { id, body } of objects) { + pairs.push(id, offset); + offset += body.length + 1; + } + const header = pairs.join(' ') + '\n'; + + const stream = this.document.ref({ + Type: 'ObjStm', + N: objects.length, + First: header.length, + }); + stream.end( + fromBinaryString(header + objects.map((o) => o.body).join('\n')), + ); + + objects.forEach(({ id }, index) => { + this.locations.set(id, [stream.id, index]); + }); + } + + finalize() { + const doc = this.document; + this.flush(); + + const xref = doc.ref({ + Type: 'XRef', + Root: doc._root, + Info: doc._info, + ID: [doc._id, doc._id], + }); + const xrefOffset = doc._offset; + if (xrefOffset > 0xffffffff) { + // offsets are written in 4 bytes + throw new Error('The objectStreams option cannot write files over 4 GB'); + } + doc._offsets[xref.id - 1] = xrefOffset; + + // entries of [type, offset or object stream id, generation or index] + const size = doc._offsets.length + 1; + const table = new DataView(new ArrayBuffer(size * ENTRY_SIZE)); + const put = (id, type, field2, field3) => { + const start = id * ENTRY_SIZE; + table.setUint8(start, type); + table.setUint32(start + 1, field2); + table.setUint16(start + 5, field3); + }; + + put(0, FREE, 0, 0xffff); + for (let id = 1; id < size; id++) { + const location = this.locations.get(id); + if (location) { + put(id, COMPRESSED, ...location); + } else { + put(id, UNCOMPRESSED, doc._offsets[id - 1], 0); + } + } + + xref.data.Size = size; + xref.data.W = ENTRY_WIDTHS; + xref.end(new Uint8Array(table.buffer)); + + doc._write('startxref'); + doc._write(`${xrefOffset}`); + doc._write('%%EOF'); + } +} + +export default PDFObjectStreams; diff --git a/lib/reference.js b/lib/reference.js index 0e27bd53..057e5a03 100644 --- a/lib/reference.js +++ b/lib/reference.js @@ -44,6 +44,14 @@ class PDFReference extends PDFAbstractReference { } finalize() { + const objectStreams = this.document._objectStreams; + if (objectStreams && !this.buffer.length) { + // written later as part of an object stream + objectStreams.add(this); + this.document._refEnd(this); + return; + } + this.offset = this.document._offset; const encryptFn = this.document._security diff --git a/tests/unit/object_streams.spec.js b/tests/unit/object_streams.spec.js new file mode 100644 index 00000000..546b231f --- /dev/null +++ b/tests/unit/object_streams.spec.js @@ -0,0 +1,200 @@ +import { vi } from 'vitest'; +import zlib from 'zlib'; +import PDFDocument from '../../lib/document'; + +const options = { + autoFirstPage: false, + pdfVersion: '1.5', + objectStreams: true, +}; + +// Returns the bytes of a document; `after` can end references once it ended. +// Spies on push rather than using logData, which starts after the header is +// written, as the cross-reference offsets count from the start of the file +function build(docOptions, fn = (doc) => doc.addPage(), after) { + const chunks = []; + const push = PDFDocument.prototype.push; + const spy = vi + .spyOn(PDFDocument.prototype, 'push') + .mockImplementation(function (chunk) { + if (chunk) chunks.push(Buffer.from(chunk)); + return push.call(this, chunk); + }); + try { + const doc = new PDFDocument(docOptions); + const state = fn(doc); + doc.end(); + if (after) after(state); + } finally { + spy.mockRestore(); + } + return Buffer.concat(chunks); +} + +// Reads the stream object at offset: its id, dictionary and decoded data +function readStream(file, offset) { + const text = file.toString('latin1', offset, offset + 2000); + const [head, id, dict] = /^(\d+) 0 obj\n<<([\s\S]*?)>>\nstream\n/.exec(text); + const start = offset + head.length; + let data = file.subarray( + start, + start + Number(/\/Length (\d+)/.exec(dict)[1]), + ); + if (dict.includes('/FlateDecode')) data = zlib.inflateSync(data); + return { id: Number(id), dict, data }; +} + +// Resolves every object through the cross-reference stream: id -> the +// offset of a plain object, or the body of an object in an object stream +function parse(file) { + const text = file.toString('latin1'); + const xref = readStream( + file, + Number(/startxref\n(\d+)\n%%EOF\n$/.exec(text)[1]), + ); + expect(xref.dict).toContain('/Type /XRef'); + expect(xref.dict).toContain('/W [1 4 2]'); + const size = Number(/\/Size (\d+)/.exec(xref.dict)[1]); + expect(xref.data.length).toBe(size * 7); + + const view = new DataView(xref.data.buffer, xref.data.byteOffset); + const entry = (id) => [ + view.getUint8(id * 7), + view.getUint32(id * 7 + 1), + view.getUint16(id * 7 + 5), + ]; + expect(entry(0)).toEqual([0, 0, 0xffff]); + + const objects = new Map(); + const streams = new Map(); + for (let id = 1; id < size; id++) { + const [type, field2, index] = entry(id); + if (type === 1) { + expect(text.startsWith(`${id} 0 obj\n`, field2)).toBe(true); + objects.set(id, { offset: field2 }); + continue; + } + expect(type).toBe(2); + if (!streams.has(field2)) { + const { dict, data } = readStream(file, entry(field2)[1]); + expect(dict).toContain('/Type /ObjStm'); + const first = Number(/\/First (\d+)/.exec(dict)[1]); + const body = data.toString('latin1'); + const pairs = body.slice(0, first).trim().split(' ').map(Number); + expect(pairs.length).toBe(Number(/\/N (\d+)/.exec(dict)[1]) * 2); + const members = []; + for (let i = 0; i < pairs.length; i += 2) { + const end = pairs[i + 3] ?? body.length - first; + members.push([pairs[i], body.slice(first + pairs[i + 1], first + end)]); + } + streams.set(field2, members); + } + const [memberId, body] = streams.get(field2)[index]; + expect(memberId).toBe(id); + objects.set(id, { body: body.trim() }); + } + return { text, xref, objects, streams }; +} + +const bodies = ({ objects }) => [...objects.values()].map((o) => o.body); + +describe('object streams', () => { + test('are off by default', () => { + const text = build({ autoFirstPage: false }).toString('latin1'); + expect(text).toContain('\nxref\n0 '); + expect(text).toContain('\ntrailer\n'); + expect(text).not.toContain('/ObjStm'); + }); + + test('hold every object without a stream, with a cross-reference stream', () => { + const pdf = parse( + build(options, (doc) => { + doc.addPage(); + doc.text('Hello'); + }), + ); + expect(pdf.text).not.toContain('\nxref\n'); + expect(pdf.text).not.toContain('\ntrailer\n'); + expect(pdf.xref.dict).toMatch(/\/ID \[<[0-9a-f]+> <[0-9a-f]+>\]/); + + const root = Number(/\/Root (\d+) 0 R/.exec(pdf.xref.dict)[1]); + expect(pdf.objects.get(root).body).toContain('/Type /Catalog'); + const info = Number(/\/Info (\d+) 0 R/.exec(pdf.xref.dict)[1]); + expect(pdf.objects.get(info).body).toContain('/Producer'); + + // only streams, e.g. the page content, are left as plain objects + for (const { offset } of pdf.objects.values()) { + if (offset !== undefined) { + expect(pdf.text.slice(offset).split('endobj')[0]).toContain( + '\nstream\n', + ); + } + } + }); + + test('hold up to 200 objects each', () => { + const pdf = parse( + build(options, (doc) => { + doc.addPage(); + for (let i = 0; i < 450; i++) doc.ref({ Index: i }).end(); + }), + ); + const sizes = [...pdf.streams.values()].map((members) => members.length); + expect(sizes.length).toBeGreaterThanOrEqual(3); + expect(Math.max(...sizes)).toBe(200); + const all = bodies(pdf).join(' '); + for (let i = 0; i < 450; i++) expect(all).toContain(`/Index ${i}\n`); + }); + + test('include references ended after the document', () => { + const file = build( + options, + (doc) => { + doc.addPage(); + return doc.ref({ Late: true }); + }, + (late) => late.end(), + ); + expect(bodies(parse(file))).toContainEqual( + expect.stringContaining('/Late true'), + ); + }); + + test('work without compression', () => { + const pdf = parse(build({ ...options, compress: false })); + expect(pdf.xref.dict).not.toContain('/Filter'); + expect(bodies(pdf)).toContainEqual(expect.stringContaining('/Catalog')); + }); + + test('work with tagged PDF/A-2', () => { + const pdf = parse( + build( + { ...options, pdfVersion: '1.7', subset: 'PDF/A-2', tagged: true }, + (doc) => { + doc.addPage(); + doc.text('Hello'); + }, + ), + ); + expect(bodies(pdf)).toContainEqual( + expect.stringContaining('/StructTreeRoot'), + ); + }); + + test('throw when they cannot be used', () => { + expect(() => new PDFDocument({ objectStreams: true })).toThrow(/1\.5/); + expect( + () => + new PDFDocument({ ...options, pdfVersion: '1.7', subset: 'PDF/A-1b' }), + ).toThrow(/PDF\/A-1/); + expect( + () => new PDFDocument({ ...options, userPassword: 'secret' }), + ).toThrow(/encryption/); + expect(() => + build(options, (doc) => { + doc.addPage(); + doc._offset = 2 ** 32; // as if 4 GB were already written + }), + ).toThrow(/4 GB/); + }); +});