Add dictionary support to Zstd and Brotli compressor input streams

.gitattributes

@@ -33,3 +33,4 @@ src/test/resources/test4.xml            eol=lf
 src/test/resources/test?with?spaces.txt eol=lf
 src/test/resources/test.txt             eol=lf
 src/test/resources/COMPRESS-380-input   binary
+src/test/resources/brotli.dictionary.*  binary

src/changes/changes.xml

@@ -210,6 +210,8 @@ The <action> type attribute can be add,update,fix,remove.
       <action type="fix" dev="ggregory" due-to="Gary Gregory">ExactMath now throws ArchiveException instead of IllegalArgumetException/IllegalStateException.</action>
       <!-- ADD -->
       <action type="add" dev="ggregory" due-to="Gary Gregory">Add MemoryLimitException.MemoryLimitException(long, long).</action>
+      <action type="add" dev="ggregory" due-to="Arturo Bernal">Add ZstdCompressorInputStream.Builder.setDict(byte[]) to decompress Zstandard input with a dictionary.</action>
+      <action type="add" dev="ggregory" due-to="Arturo Bernal">Add BrotliCompressorInputStream.Builder.setDict(byte[]) to decompress Brotli input with a raw dictionary.</action>
       <action type="add" dev="ggregory" due-to="Gary Gregory">Add CompressException.CompressException(String, Object...).</action>
       <action type="add" dev="ggregory" due-to="Gary Gregory">Add ArchiveException.ArchiveException(String, Object...).</action>
       <action type="add" dev="ggregory" due-to="Gary Gregory">Add CompressorException.CompressorException(String, Object...).</action>

src/main/java/org/apache/commons/compress/compressors/brotli/BrotliCompressorInputStream.java

@@ -25,6 +25,7 @@
 import org.apache.commons.compress.compressors.CompressorInputStream;
 import org.apache.commons.compress.utils.InputStreamStatistics;
 import org.apache.commons.io.IOUtils;
+import org.apache.commons.io.build.AbstractStreamBuilder;
 import org.apache.commons.io.input.BoundedInputStream;
 import org.brotli.dec.BrotliInputStream;
 
@@ -35,9 +36,76 @@
  */
 public class BrotliCompressorInputStream extends CompressorInputStream implements InputStreamStatistics {
 
+    // @formatter:off
+    /**
+     * Builds a new {@link BrotliCompressorInputStream}.
+     *
+     * <p>
+     * For example:
+     * </p>
+     * <pre>{@code
+     * BrotliCompressorInputStream s = BrotliCompressorInputStream.builder()
+     *   .setPath(path)
+     *   .setDict(dictionary)
+     *   .get();
+     * }
+     * </pre>
+     *
+     * @see #get()
+     * @since 1.29.0
+     */
+    // @formatter:on
+    public static final class Builder extends AbstractStreamBuilder<BrotliCompressorInputStream, Builder> {
+
+        private byte[] dict;
+
+        /**
+         * Constructs a new builder of {@link BrotliCompressorInputStream}.
+         */
+        public Builder() {
+            // empty
+        }
+
+        @Override
+        public BrotliCompressorInputStream get() throws IOException {
+            return new BrotliCompressorInputStream(this);
+        }
+
+        /**
+         * Sets the raw (LZ77) dictionary used to decompress the input.
+         * <p>
+         * The input must have been compressed with the same raw dictionary, which acts as data preceding the stream. The value {@code null} means no
+         * dictionary.
+         * </p>
+         *
+         * @param dict The dictionary buffer.
+         * @return {@code this} instance.
+         */
+        public Builder setDict(final byte[] dict) {
+            this.dict = dict;
+            return this;
+        }
+    }
+
+    /**
+     * Constructs a new builder of {@link BrotliCompressorInputStream}.
+     *
+     * @return a new builder of {@link BrotliCompressorInputStream}.
+     * @since 1.29.0
+     */
+    public static Builder builder() {
+        return new Builder();
+    }
+
     private final BoundedInputStream countingInputStream;
     private final BrotliInputStream brotliInputStream;
 
+    @SuppressWarnings("resource") // Caller closes
+    private BrotliCompressorInputStream(final Builder builder) throws IOException {
+        brotliInputStream = new BrotliInputStream(countingInputStream = BoundedInputStream.builder().setInputStream(builder.getInputStream()).get(),
+                BrotliInputStream.DEFAULT_INTERNAL_BUFFER_SIZE, builder.dict);
+    }
+
     /**
      * Construct a new instance.
      *

src/main/java/org/apache/commons/compress/compressors/zstandard/ZstdCompressorInputStream.java

@@ -25,6 +25,7 @@
 import org.apache.commons.compress.compressors.CompressorInputStream;
 import org.apache.commons.compress.utils.InputStreamStatistics;
 import org.apache.commons.io.IOUtils;
+import org.apache.commons.io.build.AbstractStreamBuilder;
 import org.apache.commons.io.input.BoundedInputStream;
 
 import com.github.luben.zstd.BufferPool;
@@ -44,9 +45,80 @@
  */
 public class ZstdCompressorInputStream extends CompressorInputStream implements InputStreamStatistics {
 
+    // @formatter:off
+    /**
+     * Builds a new {@link ZstdCompressorInputStream}.
+     *
+     * <p>
+     * For example:
+     * </p>
+     * <pre>{@code
+     * ZstdCompressorInputStream s = ZstdCompressorInputStream.builder()
+     *   .setPath(path)
+     *   .setDict(dictionary)
+     *   .get();
+     * }
+     * </pre>
+     *
+     * @see #get()
+     * @since 1.29.0
+     */
+    // @formatter:on
+    public static final class Builder extends AbstractStreamBuilder<ZstdCompressorInputStream, Builder> {
+
+        private byte[] dict;
+
+        /**
+         * Constructs a new builder of {@link ZstdCompressorInputStream}.
+         */
+        public Builder() {
+            // empty
+        }
+
+        @Override
+        public ZstdCompressorInputStream get() throws IOException {
+            return new ZstdCompressorInputStream(this);
+        }
+
+        /**
+         * Sets the dictionary used to decompress the input.
+         * <p>
+         * The input must have been compressed with the same dictionary, for example with {@link ZstdCompressorOutputStream.Builder#setDict(byte[])}. The
+         * dictionary may be a trained Zstandard dictionary or raw content. The value {@code null} means no dictionary.
+         * </p>
+         *
+         * @param dict The dictionary buffer.
+         * @return {@code this} instance.
+         * @see <a href="https://facebook.github.io/zstd/zstd_manual.html#Chapter14">Zstd manual Chapter14</a>
+         */
+        public Builder setDict(final byte[] dict) {
+            this.dict = dict;
+            return this;
+        }
+    }
+
+    /**
+     * Constructs a new builder of {@link ZstdCompressorInputStream}.
+     *
+     * @return a new builder of {@link ZstdCompressorInputStream}.
+     * @since 1.29.0
+     */
+    public static Builder builder() {
+        return new Builder();
+    }
+
     private final BoundedInputStream countingStream;
     private final ZstdInputStream decIS;
 
+    @SuppressWarnings("resource") // Caller closes
+    private ZstdCompressorInputStream(final Builder builder) throws IOException {
+        this.decIS = new ZstdInputStream(
+                countingStream = BoundedInputStream.builder().setInputStream(builder.getInputStream()).setPropagateClose(false).get());
+        if (builder.dict != null) {
+            decIS.setDict(builder.dict);
+        }
+    }
+
     /**
      * Constructs a new input stream that decompresses zstd-compressed data from the specific input stream.
      *

src/test/java/org/apache/commons/compress/compressors/brotli/BrotliCompressorInputStreamTest.java

@@ -21,6 +21,7 @@
 
 import static org.junit.jupiter.api.Assertions.assertArrayEquals;
 import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
 import java.io.ByteArrayOutputStream;
@@ -37,6 +38,33 @@
 
 class BrotliCompressorInputStreamTest extends AbstractTest {
 
+    @Test
+    void testBuilderWithoutDictionary() throws IOException {
+        final byte[] expected = readAllBytes("brotli.testdata.uncompressed");
+        try (BrotliCompressorInputStream in = BrotliCompressorInputStream.builder().setPath(getPath("brotli.testdata.compressed")).get()) {
+            assertArrayEquals(expected, IOUtils.toByteArray(in));
+        }
+    }
+
+    @Test
+    void testDictionary() throws IOException {
+        final byte[] expected = readAllBytes("brotli.dictionary.uncompressed");
+        try (BrotliCompressorInputStream in = BrotliCompressorInputStream.builder()
+                .setPath(getPath("brotli.dictionary.compressed"))
+                .setDict(readAllBytes("brotli.dictionary.dict"))
+                .get()) {
+            assertArrayEquals(expected, IOUtils.toByteArray(in));
+            assertEquals(Files.size(getPath("brotli.dictionary.compressed")), in.getCompressedCount());
+        }
+    }
+
+    @Test
+    void testDictionaryRequired() throws IOException {
+        try (BrotliCompressorInputStream in = BrotliCompressorInputStream.builder().setPath(getPath("brotli.dictionary.compressed")).get()) {
+            assertThrows(IOException.class, () -> IOUtils.toByteArray(in));
+        }
+    }
+
     @Test
     void testAvailableShouldReturnZero() throws IOException {
         try (InputStream is = newInputStream("brotli.testdata.compressed");

src/test/java/org/apache/commons/compress/compressors/zstandard/ZstdCompressorInputStreamTest.java

@@ -21,8 +21,10 @@
 
 import static org.junit.jupiter.api.Assertions.assertArrayEquals;
 import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
+import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.File;
 import java.io.IOException;
@@ -40,6 +42,37 @@
 
 class ZstdCompressorInputStreamTest extends AbstractTest {
 
+    private static byte[] compress(final byte[] data, final byte[] dict) throws IOException {
+        final ByteArrayOutputStream out = new ByteArrayOutputStream();
+        try (ZstdCompressorOutputStream zos = ZstdCompressorOutputStream.builder().setOutputStream(out).setDict(dict).get()) {
+            zos.write(data);
+        }
+        return out.toByteArray();
+    }
+
+    @Test
+    void testDictionary() throws IOException {
+        final byte[] dict = readAllBytes("brotli.dictionary.dict");
+        final byte[] data = readAllBytes("brotli.dictionary.uncompressed");
+        final byte[] compressed = compress(data, dict);
+        assertTrue(compressed.length < compress(data, null).length);
+        try (ZstdCompressorInputStream in = ZstdCompressorInputStream.builder()
+                .setInputStream(new ByteArrayInputStream(compressed))
+                .setDict(dict)
+                .get()) {
+            assertArrayEquals(data, IOUtils.toByteArray(in));
+            assertEquals(compressed.length, in.getCompressedCount());
+        }
+    }
+
+    @Test
+    void testDictionaryRequired() throws IOException {
+        final byte[] compressed = compress(readAllBytes("brotli.dictionary.uncompressed"), readAllBytes("brotli.dictionary.dict"));
+        try (ZstdCompressorInputStream in = ZstdCompressorInputStream.builder().setInputStream(new ByteArrayInputStream(compressed)).get()) {
+            assertThrows(IOException.class, () -> IOUtils.toByteArray(in));
+        }
+    }
+
     @Test
     void testCachingIsEnabledByDefaultAndZstdUtilsPresent() {
         assertEquals(ZstdUtils.CachedAvailability.CACHED_AVAILABLE, ZstdUtils.getCachedZstdAvailability());

src/test/resources/brotli.dictionary.dict

@@ -0,0 +1,40 @@
+<div class="product-card"><span class="price">0.99 EUR</span><a href="/catalog/item/0">Add to cart</a></div>
+<div class="product-card"><span class="price">1.99 EUR</span><a href="/catalog/item/1">Add to cart</a></div>
+<div class="product-card"><span class="price">2.99 EUR</span><a href="/catalog/item/2">Add to cart</a></div>
+<div class="product-card"><span class="price">3.99 EUR</span><a href="/catalog/item/3">Add to cart</a></div>
+<div class="product-card"><span class="price">4.99 EUR</span><a href="/catalog/item/4">Add to cart</a></div>
+<div class="product-card"><span class="price">5.99 EUR</span><a href="/catalog/item/5">Add to cart</a></div>
+<div class="product-card"><span class="price">6.99 EUR</span><a href="/catalog/item/6">Add to cart</a></div>
+<div class="product-card"><span class="price">7.99 EUR</span><a href="/catalog/item/7">Add to cart</a></div>
+<div class="product-card"><span class="price">8.99 EUR</span><a href="/catalog/item/8">Add to cart</a></div>
+<div class="product-card"><span class="price">9.99 EUR</span><a href="/catalog/item/9">Add to cart</a></div>
+<div class="product-card"><span class="price">10.99 EUR</span><a href="/catalog/item/10">Add to cart</a></div>
+<div class="product-card"><span class="price">11.99 EUR</span><a href="/catalog/item/11">Add to cart</a></div>
+<div class="product-card"><span class="price">12.99 EUR</span><a href="/catalog/item/12">Add to cart</a></div>
+<div class="product-card"><span class="price">13.99 EUR</span><a href="/catalog/item/13">Add to cart</a></div>
+<div class="product-card"><span class="price">14.99 EUR</span><a href="/catalog/item/14">Add to cart</a></div>
+<div class="product-card"><span class="price">15.99 EUR</span><a href="/catalog/item/15">Add to cart</a></div>
+<div class="product-card"><span class="price">16.99 EUR</span><a href="/catalog/item/16">Add to cart</a></div>
+<div class="product-card"><span class="price">17.99 EUR</span><a href="/catalog/item/17">Add to cart</a></div>
+<div class="product-card"><span class="price">18.99 EUR</span><a href="/catalog/item/18">Add to cart</a></div>
+<div class="product-card"><span class="price">19.99 EUR</span><a href="/catalog/item/19">Add to cart</a></div>
+<div class="product-card"><span class="price">20.99 EUR</span><a href="/catalog/item/20">Add to cart</a></div>
+<div class="product-card"><span class="price">21.99 EUR</span><a href="/catalog/item/21">Add to cart</a></div>
+<div class="product-card"><span class="price">22.99 EUR</span><a href="/catalog/item/22">Add to cart</a></div>
+<div class="product-card"><span class="price">23.99 EUR</span><a href="/catalog/item/23">Add to cart</a></div>
+<div class="product-card"><span class="price">24.99 EUR</span><a href="/catalog/item/24">Add to cart</a></div>
+<div class="product-card"><span class="price">25.99 EUR</span><a href="/catalog/item/25">Add to cart</a></div>
+<div class="product-card"><span class="price">26.99 EUR</span><a href="/catalog/item/26">Add to cart</a></div>
+<div class="product-card"><span class="price">27.99 EUR</span><a href="/catalog/item/27">Add to cart</a></div>
+<div class="product-card"><span class="price">28.99 EUR</span><a href="/catalog/item/28">Add to cart</a></div>
+<div class="product-card"><span class="price">29.99 EUR</span><a href="/catalog/item/29">Add to cart</a></div>
+<div class="product-card"><span class="price">30.99 EUR</span><a href="/catalog/item/30">Add to cart</a></div>
+<div class="product-card"><span class="price">31.99 EUR</span><a href="/catalog/item/31">Add to cart</a></div>
+<div class="product-card"><span class="price">32.99 EUR</span><a href="/catalog/item/32">Add to cart</a></div>
+<div class="product-card"><span class="price">33.99 EUR</span><a href="/catalog/item/33">Add to cart</a></div>
+<div class="product-card"><span class="price">34.99 EUR</span><a href="/catalog/item/34">Add to cart</a></div>
+<div class="product-card"><span class="price">35.99 EUR</span><a href="/catalog/item/35">Add to cart</a></div>
+<div class="product-card"><span class="price">36.99 EUR</span><a href="/catalog/item/36">Add to cart</a></div>
+<div class="product-card"><span class="price">37.99 EUR</span><a href="/catalog/item/37">Add to cart</a></div>
+<div class="product-card"><span class="price">38.99 EUR</span><a href="/catalog/item/38">Add to cart</a></div>
+<div class="product-card"><span class="price">39.99 EUR</span><a href="/catalog/item/39">Add to cart</a></div>

src/test/resources/brotli.dictionary.uncompressed

@@ -0,0 +1,40 @@
+<div class="product-card"><span class="price">20.99 EUR</span><a href="/catalog/item/20">Add to cart</a></div>
+<div class="product-card"><span class="price">21.99 EUR</span><a href="/catalog/item/21">Add to cart</a></div>
+<div class="product-card"><span class="price">22.99 EUR</span><a href="/catalog/item/22">Add to cart</a></div>
+<div class="product-card"><span class="price">23.99 EUR</span><a href="/catalog/item/23">Add to cart</a></div>
+<div class="product-card"><span class="price">24.99 EUR</span><a href="/catalog/item/24">Add to cart</a></div>
+<div class="product-card"><span class="price">25.99 EUR</span><a href="/catalog/item/25">Add to cart</a></div>
+<div class="product-card"><span class="price">26.99 EUR</span><a href="/catalog/item/26">Add to cart</a></div>
+<div class="product-card"><span class="price">27.99 EUR</span><a href="/catalog/item/27">Add to cart</a></div>
+<div class="product-card"><span class="price">28.99 EUR</span><a href="/catalog/item/28">Add to cart</a></div>
+<div class="product-card"><span class="price">29.99 EUR</span><a href="/catalog/item/29">Add to cart</a></div>
+<div class="product-card"><span class="price">30.99 EUR</span><a href="/catalog/item/30">Add to cart</a></div>
+<div class="product-card"><span class="price">31.99 EUR</span><a href="/catalog/item/31">Add to cart</a></div>
+<div class="product-card"><span class="price">32.99 EUR</span><a href="/catalog/item/32">Add to cart</a></div>
+<div class="product-card"><span class="price">33.99 EUR</span><a href="/catalog/item/33">Add to cart</a></div>
+<div class="product-card"><span class="price">34.99 EUR</span><a href="/catalog/item/34">Add to cart</a></div>
+<div class="product-card"><span class="price">35.99 EUR</span><a href="/catalog/item/35">Add to cart</a></div>
+<div class="product-card"><span class="price">36.99 EUR</span><a href="/catalog/item/36">Add to cart</a></div>
+<div class="product-card"><span class="price">37.99 EUR</span><a href="/catalog/item/37">Add to cart</a></div>
+<div class="product-card"><span class="price">38.99 EUR</span><a href="/catalog/item/38">Add to cart</a></div>
+<div class="product-card"><span class="price">39.99 EUR</span><a href="/catalog/item/39">Add to cart</a></div>
+<div class="product-card"><span class="price">40.99 EUR</span><a href="/catalog/item/40">Add to cart</a></div>
+<div class="product-card"><span class="price">41.99 EUR</span><a href="/catalog/item/41">Add to cart</a></div>
+<div class="product-card"><span class="price">42.99 EUR</span><a href="/catalog/item/42">Add to cart</a></div>
+<div class="product-card"><span class="price">43.99 EUR</span><a href="/catalog/item/43">Add to cart</a></div>
+<div class="product-card"><span class="price">44.99 EUR</span><a href="/catalog/item/44">Add to cart</a></div>
+<div class="product-card"><span class="price">45.99 EUR</span><a href="/catalog/item/45">Add to cart</a></div>
+<div class="product-card"><span class="price">46.99 EUR</span><a href="/catalog/item/46">Add to cart</a></div>
+<div class="product-card"><span class="price">47.99 EUR</span><a href="/catalog/item/47">Add to cart</a></div>
+<div class="product-card"><span class="price">48.99 EUR</span><a href="/catalog/item/48">Add to cart</a></div>
+<div class="product-card"><span class="price">49.99 EUR</span><a href="/catalog/item/49">Add to cart</a></div>
+<div class="product-card"><span class="price">50.99 EUR</span><a href="/catalog/item/50">Add to cart</a></div>
+<div class="product-card"><span class="price">51.99 EUR</span><a href="/catalog/item/51">Add to cart</a></div>
+<div class="product-card"><span class="price">52.99 EUR</span><a href="/catalog/item/52">Add to cart</a></div>
+<div class="product-card"><span class="price">53.99 EUR</span><a href="/catalog/item/53">Add to cart</a></div>
+<div class="product-card"><span class="price">54.99 EUR</span><a href="/catalog/item/54">Add to cart</a></div>
+<div class="product-card"><span class="price">55.99 EUR</span><a href="/catalog/item/55">Add to cart</a></div>
+<div class="product-card"><span class="price">56.99 EUR</span><a href="/catalog/item/56">Add to cart</a></div>
+<div class="product-card"><span class="price">57.99 EUR</span><a href="/catalog/item/57">Add to cart</a></div>
+<div class="product-card"><span class="price">58.99 EUR</span><a href="/catalog/item/58">Add to cart</a></div>
+<div class="product-card"><span class="price">59.99 EUR</span><a href="/catalog/item/59">Add to cart</a></div>