diff --git a/.gitattributes b/.gitattributes index 72348c0062d..7a2fdba42dd 100644 --- a/.gitattributes +++ b/.gitattributes @@ -33,3 +33,4 @@ src/test/resources/test4.xml eol=lf src/test/resources/test?with?spaces.txt eol=lf src/test/resources/test.txt eol=lf src/test/resources/COMPRESS-380-input binary +src/test/resources/brotli.dictionary.* binary diff --git a/src/changes/changes.xml b/src/changes/changes.xml index 4ddedbd37e2..889d7bec613 100644 --- a/src/changes/changes.xml +++ b/src/changes/changes.xml @@ -210,6 +210,8 @@ The type attribute can be add,update,fix,remove. ExactMath now throws ArchiveException instead of IllegalArgumetException/IllegalStateException. Add MemoryLimitException.MemoryLimitException(long, long). + Add ZstdCompressorInputStream.Builder.setDict(byte[]) to decompress Zstandard input with a dictionary. + Add BrotliCompressorInputStream.Builder.setDict(byte[]) to decompress Brotli input with a raw dictionary. Add CompressException.CompressException(String, Object...). Add ArchiveException.ArchiveException(String, Object...). Add CompressorException.CompressorException(String, Object...). diff --git a/src/main/java/org/apache/commons/compress/compressors/brotli/BrotliCompressorInputStream.java b/src/main/java/org/apache/commons/compress/compressors/brotli/BrotliCompressorInputStream.java index c37b29c932a..f0bcbcef218 100644 --- a/src/main/java/org/apache/commons/compress/compressors/brotli/BrotliCompressorInputStream.java +++ b/src/main/java/org/apache/commons/compress/compressors/brotli/BrotliCompressorInputStream.java @@ -25,6 +25,7 @@ import org.apache.commons.compress.compressors.CompressorInputStream; import org.apache.commons.compress.utils.InputStreamStatistics; import org.apache.commons.io.IOUtils; +import org.apache.commons.io.build.AbstractStreamBuilder; import org.apache.commons.io.input.BoundedInputStream; import org.brotli.dec.BrotliInputStream; @@ -35,9 +36,76 @@ */ public class BrotliCompressorInputStream extends CompressorInputStream implements InputStreamStatistics { + // @formatter:off + /** + * Builds a new {@link BrotliCompressorInputStream}. + * + *

+ * For example: + *

+ * 
{@code
+     * BrotliCompressorInputStream s = BrotliCompressorInputStream.builder()
+     *   .setPath(path)
+     *   .setDict(dictionary)
+     *   .get();
+     * }
+     *
+ * + * @see #get() + * @since 1.29.0 + */ + // @formatter:on + public static final class Builder extends AbstractStreamBuilder { + + private byte[] dict; + + /** + * Constructs a new builder of {@link BrotliCompressorInputStream}. + */ + public Builder() { + // empty + } + + @Override + public BrotliCompressorInputStream get() throws IOException { + return new BrotliCompressorInputStream(this); + } + + /** + * Sets the raw (LZ77) dictionary used to decompress the input. + *

+ * The input must have been compressed with the same raw dictionary, which acts as data preceding the stream. The value {@code null} means no + * dictionary. + *

+ * + * @param dict The dictionary buffer. + * @return {@code this} instance. + */ + public Builder setDict(final byte[] dict) { + this.dict = dict; + return this; + } + } + + /** + * Constructs a new builder of {@link BrotliCompressorInputStream}. + * + * @return a new builder of {@link BrotliCompressorInputStream}. + * @since 1.29.0 + */ + public static Builder builder() { + return new Builder(); + } + private final BoundedInputStream countingInputStream; private final BrotliInputStream brotliInputStream; + @SuppressWarnings("resource") // Caller closes + private BrotliCompressorInputStream(final Builder builder) throws IOException { + brotliInputStream = new BrotliInputStream(countingInputStream = BoundedInputStream.builder().setInputStream(builder.getInputStream()).get(), + BrotliInputStream.DEFAULT_INTERNAL_BUFFER_SIZE, builder.dict); + } + /** * Construct a new instance. * diff --git a/src/main/java/org/apache/commons/compress/compressors/zstandard/ZstdCompressorInputStream.java b/src/main/java/org/apache/commons/compress/compressors/zstandard/ZstdCompressorInputStream.java index 1931bb7c0c4..53beea2f3d0 100644 --- a/src/main/java/org/apache/commons/compress/compressors/zstandard/ZstdCompressorInputStream.java +++ b/src/main/java/org/apache/commons/compress/compressors/zstandard/ZstdCompressorInputStream.java @@ -25,6 +25,7 @@ import org.apache.commons.compress.compressors.CompressorInputStream; import org.apache.commons.compress.utils.InputStreamStatistics; import org.apache.commons.io.IOUtils; +import org.apache.commons.io.build.AbstractStreamBuilder; import org.apache.commons.io.input.BoundedInputStream; import com.github.luben.zstd.BufferPool; @@ -44,9 +45,80 @@ */ public class ZstdCompressorInputStream extends CompressorInputStream implements InputStreamStatistics { + // @formatter:off + /** + * Builds a new {@link ZstdCompressorInputStream}. + * + *

+ * For example: + *

+ * 
{@code
+     * ZstdCompressorInputStream s = ZstdCompressorInputStream.builder()
+     *   .setPath(path)
+     *   .setDict(dictionary)
+     *   .get();
+     * }
+     *
+ * + * @see #get() + * @since 1.29.0 + */ + // @formatter:on + public static final class Builder extends AbstractStreamBuilder { + + private byte[] dict; + + /** + * Constructs a new builder of {@link ZstdCompressorInputStream}. + */ + public Builder() { + // empty + } + + @Override + public ZstdCompressorInputStream get() throws IOException { + return new ZstdCompressorInputStream(this); + } + + /** + * Sets the dictionary used to decompress the input. + *

+ * The input must have been compressed with the same dictionary, for example with {@link ZstdCompressorOutputStream.Builder#setDict(byte[])}. The + * dictionary may be a trained Zstandard dictionary or raw content. The value {@code null} means no dictionary. + *

+ * + * @param dict The dictionary buffer. + * @return {@code this} instance. + * @see Zstd manual Chapter14 + */ + public Builder setDict(final byte[] dict) { + this.dict = dict; + return this; + } + } + + /** + * Constructs a new builder of {@link ZstdCompressorInputStream}. + * + * @return a new builder of {@link ZstdCompressorInputStream}. + * @since 1.29.0 + */ + public static Builder builder() { + return new Builder(); + } + private final BoundedInputStream countingStream; private final ZstdInputStream decIS; + @SuppressWarnings("resource") // Caller closes + private ZstdCompressorInputStream(final Builder builder) throws IOException { + this.decIS = new ZstdInputStream( + countingStream = BoundedInputStream.builder().setInputStream(builder.getInputStream()).setPropagateClose(false).get()); + if (builder.dict != null) { + decIS.setDict(builder.dict); + } + } + /** * Constructs a new input stream that decompresses zstd-compressed data from the specific input stream. * diff --git a/src/test/java/org/apache/commons/compress/compressors/brotli/BrotliCompressorInputStreamTest.java b/src/test/java/org/apache/commons/compress/compressors/brotli/BrotliCompressorInputStreamTest.java index b2744c3d940..c7ff811a256 100644 --- a/src/test/java/org/apache/commons/compress/compressors/brotli/BrotliCompressorInputStreamTest.java +++ b/src/test/java/org/apache/commons/compress/compressors/brotli/BrotliCompressorInputStreamTest.java @@ -21,6 +21,7 @@ import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.ByteArrayOutputStream; @@ -37,6 +38,33 @@ class BrotliCompressorInputStreamTest extends AbstractTest { + @Test + void testBuilderWithoutDictionary() throws IOException { + final byte[] expected = readAllBytes("brotli.testdata.uncompressed"); + try (BrotliCompressorInputStream in = BrotliCompressorInputStream.builder().setPath(getPath("brotli.testdata.compressed")).get()) { + assertArrayEquals(expected, IOUtils.toByteArray(in)); + } + } + + @Test + void testDictionary() throws IOException { + final byte[] expected = readAllBytes("brotli.dictionary.uncompressed"); + try (BrotliCompressorInputStream in = BrotliCompressorInputStream.builder() + .setPath(getPath("brotli.dictionary.compressed")) + .setDict(readAllBytes("brotli.dictionary.dict")) + .get()) { + assertArrayEquals(expected, IOUtils.toByteArray(in)); + assertEquals(Files.size(getPath("brotli.dictionary.compressed")), in.getCompressedCount()); + } + } + + @Test + void testDictionaryRequired() throws IOException { + try (BrotliCompressorInputStream in = BrotliCompressorInputStream.builder().setPath(getPath("brotli.dictionary.compressed")).get()) { + assertThrows(IOException.class, () -> IOUtils.toByteArray(in)); + } + } + @Test void testAvailableShouldReturnZero() throws IOException { try (InputStream is = newInputStream("brotli.testdata.compressed"); diff --git a/src/test/java/org/apache/commons/compress/compressors/zstandard/ZstdCompressorInputStreamTest.java b/src/test/java/org/apache/commons/compress/compressors/zstandard/ZstdCompressorInputStreamTest.java index f1d1bb11183..8103f08c88f 100644 --- a/src/test/java/org/apache/commons/compress/compressors/zstandard/ZstdCompressorInputStreamTest.java +++ b/src/test/java/org/apache/commons/compress/compressors/zstandard/ZstdCompressorInputStreamTest.java @@ -21,8 +21,10 @@ import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; @@ -40,6 +42,37 @@ class ZstdCompressorInputStreamTest extends AbstractTest { + private static byte[] compress(final byte[] data, final byte[] dict) throws IOException { + final ByteArrayOutputStream out = new ByteArrayOutputStream(); + try (ZstdCompressorOutputStream zos = ZstdCompressorOutputStream.builder().setOutputStream(out).setDict(dict).get()) { + zos.write(data); + } + return out.toByteArray(); + } + + @Test + void testDictionary() throws IOException { + final byte[] dict = readAllBytes("brotli.dictionary.dict"); + final byte[] data = readAllBytes("brotli.dictionary.uncompressed"); + final byte[] compressed = compress(data, dict); + assertTrue(compressed.length < compress(data, null).length); + try (ZstdCompressorInputStream in = ZstdCompressorInputStream.builder() + .setInputStream(new ByteArrayInputStream(compressed)) + .setDict(dict) + .get()) { + assertArrayEquals(data, IOUtils.toByteArray(in)); + assertEquals(compressed.length, in.getCompressedCount()); + } + } + + @Test + void testDictionaryRequired() throws IOException { + final byte[] compressed = compress(readAllBytes("brotli.dictionary.uncompressed"), readAllBytes("brotli.dictionary.dict")); + try (ZstdCompressorInputStream in = ZstdCompressorInputStream.builder().setInputStream(new ByteArrayInputStream(compressed)).get()) { + assertThrows(IOException.class, () -> IOUtils.toByteArray(in)); + } + } + @Test void testCachingIsEnabledByDefaultAndZstdUtilsPresent() { assertEquals(ZstdUtils.CachedAvailability.CACHED_AVAILABLE, ZstdUtils.getCachedZstdAvailability()); diff --git a/src/test/resources/brotli.dictionary.compressed b/src/test/resources/brotli.dictionary.compressed new file mode 100644 index 00000000000..cdc3a32962f Binary files /dev/null and b/src/test/resources/brotli.dictionary.compressed differ diff --git a/src/test/resources/brotli.dictionary.dict b/src/test/resources/brotli.dictionary.dict new file mode 100644 index 00000000000..7b2d4ccb7b5 --- /dev/null +++ b/src/test/resources/brotli.dictionary.dict @@ -0,0 +1,40 @@ +
0.99 EURAdd to cart
+
1.99 EURAdd to cart
+
2.99 EURAdd to cart
+
3.99 EURAdd to cart
+
4.99 EURAdd to cart
+
5.99 EURAdd to cart
+
6.99 EURAdd to cart
+
7.99 EURAdd to cart
+
8.99 EURAdd to cart
+
9.99 EURAdd to cart
+
10.99 EURAdd to cart
+
11.99 EURAdd to cart
+
12.99 EURAdd to cart
+
13.99 EURAdd to cart
+
14.99 EURAdd to cart
+
15.99 EURAdd to cart
+
16.99 EURAdd to cart
+
17.99 EURAdd to cart
+
18.99 EURAdd to cart
+
19.99 EURAdd to cart
+
20.99 EURAdd to cart
+
21.99 EURAdd to cart
+
22.99 EURAdd to cart
+
23.99 EURAdd to cart
+
24.99 EURAdd to cart
+
25.99 EURAdd to cart
+
26.99 EURAdd to cart
+
27.99 EURAdd to cart
+
28.99 EURAdd to cart
+
29.99 EURAdd to cart
+
30.99 EURAdd to cart
+
31.99 EURAdd to cart
+
32.99 EURAdd to cart
+
33.99 EURAdd to cart
+
34.99 EURAdd to cart
+
35.99 EURAdd to cart
+
36.99 EURAdd to cart
+
37.99 EURAdd to cart
+
38.99 EURAdd to cart
+
39.99 EURAdd to cart
diff --git a/src/test/resources/brotli.dictionary.uncompressed b/src/test/resources/brotli.dictionary.uncompressed new file mode 100644 index 00000000000..8f708b06245 --- /dev/null +++ b/src/test/resources/brotli.dictionary.uncompressed @@ -0,0 +1,40 @@ +
20.99 EURAdd to cart
+
21.99 EURAdd to cart
+
22.99 EURAdd to cart
+
23.99 EURAdd to cart
+
24.99 EURAdd to cart
+
25.99 EURAdd to cart
+
26.99 EURAdd to cart
+
27.99 EURAdd to cart
+
28.99 EURAdd to cart
+
29.99 EURAdd to cart
+
30.99 EURAdd to cart
+
31.99 EURAdd to cart
+
32.99 EURAdd to cart
+
33.99 EURAdd to cart
+
34.99 EURAdd to cart
+
35.99 EURAdd to cart
+
36.99 EURAdd to cart
+
37.99 EURAdd to cart
+
38.99 EURAdd to cart
+
39.99 EURAdd to cart
+
40.99 EURAdd to cart
+
41.99 EURAdd to cart
+
42.99 EURAdd to cart
+
43.99 EURAdd to cart
+
44.99 EURAdd to cart
+
45.99 EURAdd to cart
+
46.99 EURAdd to cart
+
47.99 EURAdd to cart
+
48.99 EURAdd to cart
+
49.99 EURAdd to cart
+
50.99 EURAdd to cart
+
51.99 EURAdd to cart
+
52.99 EURAdd to cart
+
53.99 EURAdd to cart
+
54.99 EURAdd to cart
+
55.99 EURAdd to cart
+
56.99 EURAdd to cart
+
57.99 EURAdd to cart
+
58.99 EURAdd to cart
+
59.99 EURAdd to cart