Add an objectStreams option to write PDF 1.5 object streams

CHANGELOG.md

@@ -2,6 +2,7 @@
 
 ### Unreleased
 
+- Add an `objectStreams` option, which packs the objects that are not streams into compressed object streams and writes a cross-reference stream instead of the xref table, making documents with many such objects, e.g. tagged ones, much smaller. It requires a `pdfVersion` of `1.5` or higher, and cannot be used with encryption or PDF/A-1. Fixes #1810
 - Fix wrapped text with `align: 'center'` sitting left of the middle of the line when `characterSpacing` or `wordSpacing` is set, and with `align: 'right'` running past the right edge by the `wordSpacing` of each gap between words
 - Fix `boundsOfString()` with `lineBreak: false` leaving out `wordSpacing`, so its width now matches the rendered text and its underline
 - Fix the underline, strike, link and goTo of wrapped text, and `boundsOfString()` with a `width`, counting `characterSpacing` and `wordSpacing` twice, so they no longer run past the text and match the same text drawn with `lineBreak: false`

docs/getting_started.md

@@ -170,6 +170,17 @@ The `pageLayout` option specifies how pages should be displayed in a PDF viewer:
 
     const doc = new PDFDocument({ pageLayout: 'twoColumnLeft' });
 
+### Object streams
+
+The `objectStreams` option packs the objects that are not streams (pages, annotations, the
+structure elements of a tagged document, etc.) into compressed object streams, and writes a
+cross-reference stream instead of the xref table. Documents with many such objects, like tagged
+ones, get much smaller, while their content stays the same.
+
+    const doc = new PDFDocument({ pdfVersion: '1.5', objectStreams: true });
+
+It requires a `pdfVersion` of `1.5` or higher, and cannot be used with encryption or PDF/A-1.
+
 ## Adding pages
 
 The first page of a PDFKit document is added for you automatically when you

lib/document.js

@@ -9,6 +9,7 @@ import PDFReference from './reference';
 import PDFPage from './page';
 import PDFNameTree from './name_tree';
 import PDFSecurity from './security';
+import PDFObjectStreams from './object_streams';
 import ColorMixin from './mixins/color';
 import VectorMixin from './mixins/vector';
 import FontsMixin from './mixins/fonts';
@@ -53,6 +54,10 @@ class PDFDocument extends Readable {
     this.compress =
       this.options.compress != null ? this.options.compress : true;
 
+    if (options.objectStreams) {
+      this._objectStreams = new PDFObjectStreams(this, options);
+    }
+
     this._pageBuffer = [];
     this._pageBufferStart = 0;
 
@@ -284,8 +289,9 @@ class PDFDocument extends Readable {
   _refEnd(ref) {
     this._offsets[ref.id - 1] = ref.offset;
     if (--this._waiting === 0 && this._ended) {
-      this._finalize();
+      // reset first, as finalizing may end more references
       this._ended = false;
+      this._finalize();
     }
   }
 
@@ -342,6 +348,12 @@ class PDFDocument extends Readable {
   }
 
   _finalize() {
+    if (this._objectStreams) {
+      this._objectStreams.finalize();
+      this.push(null);
+      return;
+    }
+
     // generate xref
     const xRefOffset = this._offset;
     this._write('xref');

lib/object_streams.js

@@ -0,0 +1,125 @@
+/*
+PDFObjectStreams - packs objects without a stream into compressed object
+streams, and writes a cross-reference stream instead of the xref table (PDF 1.5)
+*/
+
+import PDFObject from './object';
+import { fromBinaryString } from './binary';
+
+// objects per object stream, and so the most held in memory at a time
+const OBJECTS_PER_STREAM = 200;
+
+// cross-reference stream entries: a 1-byte type, then 4-byte and 2-byte fields
+const ENTRY_WIDTHS = [1, 4, 2];
+const ENTRY_SIZE = 7;
+const FREE = 0;
+const UNCOMPRESSED = 1;
+const COMPRESSED = 2;
+
+class PDFObjectStreams {
+  constructor(document, options) {
+    if (document.version < 1.5) {
+      throw new Error(
+        "The objectStreams option requires a pdfVersion of '1.5' or higher",
+      );
+    }
+    if (/^PDF\/A-1/.test(options.subset)) {
+      throw new Error('PDF/A-1 does not allow object streams');
+    }
+    if (options.userPassword || options.ownerPassword) {
+      throw new Error(
+        'The objectStreams option cannot be used with encryption',
+      );
+    }
+
+    this.document = document;
+    this.pending = [];
+    this.locations = new Map(); // object id -> [object stream id, index]
+  }
+
+  add(ref) {
+    this.pending.push({ id: ref.id, body: PDFObject.convert(ref.data) });
+    if (this.pending.length >= OBJECTS_PER_STREAM) {
+      this.flush();
+    }
+  }
+
+  flush() {
+    if (!this.pending.length) {
+      return;
+    }
+
+    const objects = this.pending;
+    this.pending = [];
+
+    // "id offset" pairs, then the objects, offsets relative to First
+    let offset = 0;
+    const pairs = [];
+    for (const { id, body } of objects) {
+      pairs.push(id, offset);
+      offset += body.length + 1;
+    }
+    const header = pairs.join(' ') + '\n';
+
+    const stream = this.document.ref({
+      Type: 'ObjStm',
+      N: objects.length,
+      First: header.length,
+    });
+    stream.end(
+      fromBinaryString(header + objects.map((o) => o.body).join('\n')),
+    );
+
+    objects.forEach(({ id }, index) => {
+      this.locations.set(id, [stream.id, index]);
+    });
+  }
+
+  finalize() {
+    const doc = this.document;
+    this.flush();
+
+    const xref = doc.ref({
+      Type: 'XRef',
+      Root: doc._root,
+      Info: doc._info,
+      ID: [doc._id, doc._id],
+    });
+    const xrefOffset = doc._offset;
+    if (xrefOffset > 0xffffffff) {
+      // offsets are written in 4 bytes
+      throw new Error('The objectStreams option cannot write files over 4 GB');
+    }
+    doc._offsets[xref.id - 1] = xrefOffset;
+
+    // entries of [type, offset or object stream id, generation or index]
+    const size = doc._offsets.length + 1;
+    const table = new DataView(new ArrayBuffer(size * ENTRY_SIZE));
+    const put = (id, type, field2, field3) => {
+      const start = id * ENTRY_SIZE;
+      table.setUint8(start, type);
+      table.setUint32(start + 1, field2);
+      table.setUint16(start + 5, field3);
+    };
+
+    put(0, FREE, 0, 0xffff);
+    for (let id = 1; id < size; id++) {
+      const location = this.locations.get(id);
+      if (location) {
+        put(id, COMPRESSED, ...location);
+      } else {
+        put(id, UNCOMPRESSED, doc._offsets[id - 1], 0);
+      }
+    }
+
+    xref.data.Size = size;
+    xref.data.W = ENTRY_WIDTHS;
+    xref.end(new Uint8Array(table.buffer));
+
+    doc._write('startxref');
+    doc._write(`${xrefOffset}`);
+    doc._write('%%EOF');
+  }
+}
+
+export default PDFObjectStreams;

lib/reference.js

@@ -44,6 +44,14 @@ class PDFReference extends PDFAbstractReference {
   }
 
   finalize() {
+    const objectStreams = this.document._objectStreams;
+    if (objectStreams && !this.buffer.length) {
+      // written later as part of an object stream
+      objectStreams.add(this);
+      this.document._refEnd(this);
+      return;
+    }
+
     this.offset = this.document._offset;
 
     const encryptFn = this.document._security