Compute the TupleHash element length prefix in long arithmetic and refuse a negative length

core/src/main/java/org/bouncycastle/crypto/digests/XofUtils.java

@@ -6,6 +6,11 @@ public class XofUtils
 {
     public static byte[] leftEncode(long strLen)
     {
+        if (strLen < 0)
+        {
+            throw new IllegalArgumentException("'strLen' cannot be negative");
+        }
+
         byte n = 1;
 
         long v = strLen;
@@ -28,6 +33,11 @@ public static byte[] leftEncode(long strLen)
 
     public static byte[] rightEncode(long strLen)
     {
+        if (strLen < 0)
+        {
+            throw new IllegalArgumentException("'strLen' cannot be negative");
+        }
+
         byte n = 1;
 
         long v = strLen;
@@ -57,8 +67,8 @@ static byte[] encode(byte[] in, int inOff, int len)
     {
         if (in.length == len)
         {
-            return Arrays.concatenate(XofUtils.leftEncode(len * 8), in);
+            return Arrays.concatenate(XofUtils.leftEncode(len * 8L), in);
         }
-        return Arrays.concatenate(XofUtils.leftEncode(len * 8), Arrays.copyOfRange(in, inOff, inOff + len));
+        return Arrays.concatenate(XofUtils.leftEncode(len * 8L), Arrays.copyOfRange(in, inOff, inOff + len));
     }
 }

core/src/test/java/org/bouncycastle/crypto/test/RegressionTest.java

@@ -187,6 +187,7 @@ public class RegressionTest
             new SP80038GTest(),
             new TupleHashTest(),
             new ParallelHashTest(),
+            new XofUtilsTest(),
             new CryptoServiceConstraintsTest(),
             new SymmetricConstraintsTest(),
             new DigestConstraintsTest(),
@@ -215,7 +216,8 @@ public class RegressionTest
             new SCryptTest(),
             new CramerShoupTest(),
             new OpenSSHKeyParsingTests(),
-            new AsymmetricConstraintsTest()
+            new AsymmetricConstraintsTest(),
+            new TupleHashLargeInputTest()
         };
 
     public static Test[] openBSDBCryptTests =

core/src/test/java/org/bouncycastle/crypto/test/TupleHashLargeInputTest.java

@@ -0,0 +1,65 @@
+package org.bouncycastle.crypto.test;
+
+import org.bouncycastle.crypto.digests.CSHAKEDigest;
+import org.bouncycastle.crypto.digests.TupleHash;
+import org.bouncycastle.crypto.digests.XofUtils;
+import org.bouncycastle.util.Arrays;
+import org.bouncycastle.util.Strings;
+import org.bouncycastle.util.test.SimpleTest;
+
+/**
+ * TupleHash over a single element of 2^28 bytes - the smallest element whose length in bits does
+ * not fit in an int. The expected value is built by driving cSHAKE with the byte string NIST
+ * Special Publication 800-185 section 5.3 prescribes, whose encode_string prefix (section 2.3.3)
+ * is computed here in long arithmetic.
+ * <p>
+ * Allocates around 512 MiB transiently, so it belongs in RegressionTest.slowTests.
+ * </p>
+ */
+public class TupleHashLargeInputTest
+    extends SimpleTest
+{
+    private static final int ELEMENT_SIZE = 1 << 28;
+
+    public String getName()
+    {
+        return "TupleHashLargeInput";
+    }
+
+    public void performTest()
+        throws Exception
+    {
+        byte[] data = new byte[ELEMENT_SIZE];
+
+        TupleHash tHash = new TupleHash(128, new byte[0]);
+
+        tHash.update(data, 0, data.length);
+
+        byte[] res = new byte[tHash.getDigestSize()];
+
+        tHash.doFinal(res, 0);
+
+        CSHAKEDigest cshake = new CSHAKEDigest(128, Strings.toByteArray("TupleHash"), new byte[0]);
+
+        byte[] pre = XofUtils.leftEncode(data.length * 8L);
+
+        cshake.update(pre, 0, pre.length);
+        cshake.update(data, 0, data.length);
+
+        byte[] post = XofUtils.rightEncode(res.length * 8L);
+
+        cshake.update(post, 0, post.length);
+
+        byte[] expected = new byte[res.length];
+
+        cshake.doFinal(expected, 0, expected.length);
+
+        isTrue("large element encoded at the wrong length", Arrays.areEqual(expected, res));
+    }
+
+    public static void main(
+        String[] args)
+    {
+        runTest(new TupleHashLargeInputTest());
+    }
+}

core/src/test/java/org/bouncycastle/crypto/test/XofUtilsTest.java

@@ -0,0 +1,76 @@
+package org.bouncycastle.crypto.test;
+
+import org.bouncycastle.crypto.digests.XofUtils;
+import org.bouncycastle.util.Arrays;
+import org.bouncycastle.util.encoders.Hex;
+import org.bouncycastle.util.test.SimpleTest;
+
+/**
+ * left_encode and right_encode from NIST Special Publication 800-185 sections 2.3.1 and 2.3.2,
+ * over lengths either side of the point where a bit count stops fitting in an int.
+ */
+public class XofUtilsTest
+    extends SimpleTest
+{
+    public String getName()
+    {
+        return "XofUtils";
+    }
+
+    public void performTest()
+        throws Exception
+    {
+        testLeftEncode();
+        testRightEncode();
+        testNegativeLength();
+    }
+
+    private void testLeftEncode()
+    {
+        isTrue("left_encode 0", Arrays.areEqual(Hex.decode("0100"), XofUtils.leftEncode(0)));
+        isTrue("left_encode 8", Arrays.areEqual(Hex.decode("0108"), XofUtils.leftEncode(8)));
+        isTrue("left_encode 2048", Arrays.areEqual(Hex.decode("020800"), XofUtils.leftEncode(2048)));
+
+        // the bit length of the smallest byte string whose bit length overflows an int, and of
+        // the largest one a byte array can hold
+        isTrue("left_encode 2^31", Arrays.areEqual(Hex.decode("0480000000"), XofUtils.leftEncode((1L << 28) * 8)));
+        isTrue("left_encode 2^32", Arrays.areEqual(Hex.decode("050100000000"), XofUtils.leftEncode(1L << 32)));
+        isTrue("left_encode max", Arrays.areEqual(Hex.decode("0503fffffff8"), XofUtils.leftEncode(Integer.MAX_VALUE * 8L)));
+    }
+
+    private void testRightEncode()
+    {
+        isTrue("right_encode 0", Arrays.areEqual(Hex.decode("0001"), XofUtils.rightEncode(0)));
+        isTrue("right_encode 512", Arrays.areEqual(Hex.decode("020002"), XofUtils.rightEncode(512)));
+        isTrue("right_encode 2^32", Arrays.areEqual(Hex.decode("010000000005"), XofUtils.rightEncode(1L << 32)));
+    }
+
+    private void testNegativeLength()
+    {
+        testException("'strLen' cannot be negative", "IllegalArgumentException", new TestExceptionOperation()
+        {
+            @Override
+            public void operation()
+                throws Exception
+            {
+                XofUtils.leftEncode(-1L);
+            }
+        });
+
+        testException("'strLen' cannot be negative", "IllegalArgumentException", new TestExceptionOperation()
+        {
+            @Override
+            public void operation()
+                throws Exception
+            {
+                XofUtils.rightEncode(Long.MIN_VALUE);
+            }
+        });
+    }
+
+    public static void main(
+        String[] args)
+    {
+        runTest(new XofUtilsTest());
+    }
+}

docs/releasenotes.md

@@ -23,6 +23,8 @@ Date: 2026, TBD
 
 - The name-constraint host canonicalisation removed a single RFC 1034 root-label dot, the only empty label a name may legally carry, but nothing refused the ones that are not legal: a dNSName, rfc822Name host or uniformResourceIdentifier host such as "example.com.." kept a phantom empty label after the strip and so matched no constraint at all, escaping an excluded subtree naming the host it appears to carry. A tested name whose host carries an empty label - a second trailing dot, a doubled dot or a leading dot - is now refused outright wherever a constraint of that type is in force, rather than canonicalised into a name it is not: removing the extra dots would decide on the caller's behalf that "example.com.." names example.com, which is not how a consumer resolving or comparing the name reads it, and refusing fails closed in both directions where canonicalising would newly admit such a name under a permitted subtree. The single trailing dot is canonicalised as before, a bare "." remains the root label rather than an empty one, and the guard is scoped to the host, so the doubled dot a quoted local part may legally carry is unaffected. Constraints are untouched - one may still begin with a dot, which is how this implementation spells "subdomains only" (github PR #2436).
 
+- TupleHash prefixed an element of 2^28 bytes or more with a length computed in int arithmetic: org.bouncycastle.crypto.digests.XofUtils built the encode_string prefix of NIST SP 800-185 sec. 2.3.3 as left_encode(len * 8) with len an int, so the bit length wrapped before it reached the long parameter it was passed to. A single 256 MiB update wrapped it negative, and left_encode sizes its output by shifting its argument right eight bits at a time, which never reaches zero from a negative value, so the call did not return; at 512 MiB the length wrapped to zero and the element carried the prefix of an empty one, letting two different tuples absorb the same byte string - the ambiguity the tuple encoding of sec. 5.3 exists to prevent. The multiply is now widened, as the other left_encode and right_encode call sites in CSHAKEDigest, KMAC, TupleHash and ParallelHash already were, and left_encode and right_encode refuse a negative length rather than spinning on one, so a negative output length handed to the three-argument doFinal of TupleHash, ParallelHash or KMAC reports IllegalArgumentException instead of not returning. An element below 2^28 bytes is unaffected, the two arithmetics agreeing exactly there.
+
 ### 2.1.3 Additional Features and Functionality
 
 ### 2.1.4 Additional Notes