diff --git a/CHANGELOG.md b/CHANGELOG.md index 82a1695..b92e358 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,12 @@ +# _next_ + +- Add an opt-in `ignoreWhitespace` parameter to `fromHex`, `fromBase32`, + `fromBase64`, and their `tryFrom` counterparts. When `true`, ASCII whitespace + (space, tab, line feed, vertical tab, form feed, carriage return) in the input + is skipped instead of rejected, so line-wrapped payloads such as PEM and MIME + decode without pre-processing. It is `false` by default, so existing output is + unchanged. + # 3.6.1 - Update `README.md` for the 3.6.0 API: the `toBytes` byte encoders, the diff --git a/README.md b/README.md index 8b2f900..78f0b66 100644 --- a/README.md +++ b/README.md @@ -223,6 +223,33 @@ void main() { } ``` +### Whitespace-tolerant decoding + +Line-wrapped payloads (PEM, MIME, HTTP headers) carry newlines and spaces that +strict decoders reject. Pass `ignoreWhitespace: true` to `fromHex`, `fromBase32`, +or `fromBase64` to skip ASCII whitespace (space, `\t`, `\n`, `\v`, `\f`, `\r`) +instead. It is off by default, so ordinary decoding is unchanged. + + + +```dart +import 'package:convertlib/convertlib.dart'; + +void main() { + // A Base-64 body wrapped across lines the way PEM/MIME fold it. + const wrapped = 'SGVsbG8sIHdv\n' + 'cmxkISBGcm9t\n' + 'IGNvbnZlcnRs\n' + 'aWIu\n'; + + print(fromUtf8(fromBase64(wrapped, ignoreWhitespace: true))); + + // Strict decoding (the default) still rejects the whitespace. + print(tryFromBase64(wrapped)); // null + print(tryFromBase64(wrapped, ignoreWhitespace: true) != null); // true +} +``` + ### BigInt ↔ bytes Read a byte sequence as an arbitrary-precision integer and back. Combine with diff --git a/example/whitespace_example.dart b/example/whitespace_example.dart new file mode 100644 index 0000000..cde463b --- /dev/null +++ b/example/whitespace_example.dart @@ -0,0 +1,15 @@ +import 'package:convertlib/convertlib.dart'; + +void main() { + // A Base-64 body wrapped across lines the way PEM/MIME fold it. + const wrapped = 'SGVsbG8sIHdv\n' + 'cmxkISBGcm9t\n' + 'IGNvbnZlcnRs\n' + 'aWIu\n'; + + print(fromUtf8(fromBase64(wrapped, ignoreWhitespace: true))); + + // Strict decoding (the default) still rejects the whitespace. + print(tryFromBase64(wrapped)); // null + print(tryFromBase64(wrapped, ignoreWhitespace: true) != null); // true +} diff --git a/lib/src/base16.dart b/lib/src/base16.dart index 4908745..378315b 100644 --- a/lib/src/base16.dart +++ b/lib/src/base16.dart @@ -4,6 +4,7 @@ import 'dart:typed_data'; import 'codecs/base16.dart'; +import 'core/whitespace.dart'; Base16Codec _codecFromParameters({ bool upper = false, @@ -55,6 +56,9 @@ Uint8List toHexBytes( /// /// Parameters: /// - [input] should be a valid Base-16 (hexadecimal) string. +/// - If [ignoreWhitespace] is true, ASCII whitespace characters (space, tab, +/// line feed, vertical tab, form feed, and carriage return) in the [input] +/// are skipped instead of rejected. It is `false` by default. /// - [codec] is the [Base16Codec] to use. It is derived from the other /// parameters if not provided. /// @@ -66,22 +70,28 @@ Uint8List toHexBytes( Uint8List fromHex( String input, { Base16Codec? codec, + bool ignoreWhitespace = false, }) { codec ??= _codecFromParameters(); - return codec.decoder.convert(input.codeUnits); + List data = input.codeUnits; + if (ignoreWhitespace) { + data = stripWhitespace(data); + } + return codec.decoder.convert(data); } /// Converts a Base-16 string to an 8-bit integer sequence, returning `null` /// instead of throwing when the [input] is not valid. /// /// This is the non-throwing counterpart of [fromHex]. See [fromHex] for the -/// meaning of [codec]. +/// meaning of [codec] and [ignoreWhitespace]. Uint8List? tryFromHex( String input, { Base16Codec? codec, + bool ignoreWhitespace = false, }) { try { - return fromHex(input, codec: codec); + return fromHex(input, codec: codec, ignoreWhitespace: ignoreWhitespace); } on FormatException { return null; } diff --git a/lib/src/base32.dart b/lib/src/base32.dart index 8796fe7..7af6cb7 100644 --- a/lib/src/base32.dart +++ b/lib/src/base32.dart @@ -90,6 +90,9 @@ Uint8List toBase32Bytes( /// - [input] should be a valid base-32 encoded string. /// - If [padding] is true, the [input] may contain padding characters, which /// are ignored during decoding. +/// - If [ignoreWhitespace] is true, ASCII whitespace characters (space, tab, +/// line feed, vertical tab, form feed, and carriage return) in the [input] +/// are skipped instead of rejected. It is `false` by default. /// - [codec] is the [Base32Codec] to use. It is derived from the other /// parameters if not provided. /// @@ -104,23 +107,33 @@ Uint8List fromBase32( String input, { Base32Codec? codec, bool padding = true, + bool ignoreWhitespace = false, }) { codec ??= _codecFromParameters(padding: padding); - return codec.decoder.convert(input.codeUnits); + return codec.decoder.convert( + input.codeUnits, + ignoreWhitespace: ignoreWhitespace, + ); } /// Converts a Base-32 string to an 8-bit integer sequence, returning `null` /// instead of throwing when the [input] is not valid. /// /// This is the non-throwing counterpart of [fromBase32]. See [fromBase32] for -/// the meaning of [codec] and [padding]. +/// the meaning of [codec], [padding], and [ignoreWhitespace]. Uint8List? tryFromBase32( String input, { Base32Codec? codec, bool padding = true, + bool ignoreWhitespace = false, }) { try { - return fromBase32(input, codec: codec, padding: padding); + return fromBase32( + input, + codec: codec, + padding: padding, + ignoreWhitespace: ignoreWhitespace, + ); } on FormatException { return null; } diff --git a/lib/src/base64.dart b/lib/src/base64.dart index ac7b8e7..a0889a5 100644 --- a/lib/src/base64.dart +++ b/lib/src/base64.dart @@ -86,6 +86,10 @@ Uint8List toBase64Bytes( /// - [input] should be a valid base-64 encoded string. /// - If [padding] is true, the [input] may contain padding characters, which /// are ignored during decoding. +/// - If [ignoreWhitespace] is true, ASCII whitespace characters (space, tab, +/// line feed, vertical tab, form feed, and carriage return) in the [input] +/// are skipped instead of rejected. This is useful for line-wrapped payloads +/// such as PEM and MIME. It is `false` by default. /// - [codec] is the [Base64Codec] to use. It is derived from the other /// parameters if not provided. /// @@ -101,23 +105,33 @@ Uint8List fromBase64( String input, { Base64Codec? codec, bool padding = true, + bool ignoreWhitespace = false, }) { codec ??= _codecFromParameters(padding: padding); - return codec.decoder.convert(input.codeUnits); + return codec.decoder.convert( + input.codeUnits, + ignoreWhitespace: ignoreWhitespace, + ); } /// Converts a Base-64 string to an 8-bit integer sequence, returning `null` /// instead of throwing when the [input] is not valid. /// /// This is the non-throwing counterpart of [fromBase64]. See [fromBase64] for -/// the meaning of [codec] and [padding]. +/// the meaning of [codec], [padding], and [ignoreWhitespace]. Uint8List? tryFromBase64( String input, { Base64Codec? codec, bool padding = true, + bool ignoreWhitespace = false, }) { try { - return fromBase64(input, codec: codec, padding: padding); + return fromBase64( + input, + codec: codec, + padding: padding, + ignoreWhitespace: ignoreWhitespace, + ); } on FormatException { return null; } diff --git a/lib/src/codecs/base32.dart b/lib/src/codecs/base32.dart index 344b089..581f565 100644 --- a/lib/src/codecs/base32.dart +++ b/lib/src/codecs/base32.dart @@ -262,7 +262,14 @@ class Base32Decoder extends AlphabetDecoder { }) : super(bits: 5); @override - Uint8List convert(List encoded) { + Uint8List convert(List encoded, {bool ignoreWhitespace = false}) { + // The fast path below decodes fixed 8-character groups, so it cannot skip + // interspersed whitespace. Defer that case to the generic bit-accumulator, + // which tolerates whitespace anywhere without an intermediate buffer. + if (ignoreWhitespace) { + return super.convert(encoded, ignoreWhitespace: true); + } + final table = alphabet; final pad = padding; final tlen = table.length; diff --git a/lib/src/codecs/base64.dart b/lib/src/codecs/base64.dart index f98556d..430279f 100644 --- a/lib/src/codecs/base64.dart +++ b/lib/src/codecs/base64.dart @@ -162,7 +162,14 @@ class Base64Decoder extends AlphabetDecoder { }) : super(bits: 6); @override - Uint8List convert(List encoded) { + Uint8List convert(List encoded, {bool ignoreWhitespace = false}) { + // The fast path below decodes fixed 4-character groups, so it cannot skip + // interspersed whitespace. Defer that case to the generic bit-accumulator, + // which tolerates whitespace anywhere without an intermediate buffer. + if (ignoreWhitespace) { + return super.convert(encoded, ignoreWhitespace: true); + } + final table = alphabet; final pad = padding; final tlen = table.length; diff --git a/lib/src/core/alphabet.dart b/lib/src/core/alphabet.dart index da8dac9..afe18a6 100644 --- a/lib/src/core/alphabet.dart +++ b/lib/src/core/alphabet.dart @@ -115,7 +115,7 @@ class AlphabetDecoder extends ByteDecoder { int get source => bits; @override - Uint8List convert(List encoded) { + Uint8List convert(List encoded, {bool ignoreWhitespace = false}) { final table = alphabet; final tlen = table.length; final pad = padding; @@ -131,6 +131,7 @@ class AlphabetDecoder extends ByteDecoder { p = n = l = 0; for (i = 0; i < len; ++i) { y = encoded[i]; + if (ignoreWhitespace && (y == 0x20 || (y >= 0x09 && y <= 0x0D))) continue; if (y == pad) break; if (y < 0 || y >= tlen || (x = table[y]) < 0) { throw FormatException('Invalid character $y at $i'); @@ -146,6 +147,7 @@ class AlphabetDecoder extends ByteDecoder { for (; i < len; ++i) { y = encoded[i]; + if (ignoreWhitespace && (y == 0x20 || (y >= 0x09 && y <= 0x0D))) continue; if (y != pad) { throw FormatException('Invalid character $y at $i'); } diff --git a/lib/src/core/whitespace.dart b/lib/src/core/whitespace.dart new file mode 100644 index 0000000..6859b9a --- /dev/null +++ b/lib/src/core/whitespace.dart @@ -0,0 +1,43 @@ +// Copyright (c) 2026, Sudipto Chandra +// All rights reserved. Check LICENSE file for details. + +/// Returns [input] with every ASCII whitespace code unit removed. +/// +/// The characters treated as whitespace are the space (`0x20`) and the C0 +/// control range `0x09`..`0x0D`: horizontal tab (`\t`), line feed (`\n`), +/// vertical tab (`\v`), form feed (`\f`), and carriage return (`\r`). These are +/// the characters that line-wrapped encodings such as PEM and MIME insert into +/// otherwise valid Base-16/32/64 payloads. +/// +/// When [input] contains no whitespace, the same instance is returned without +/// any allocation; otherwise a single exactly-sized `List` is allocated +/// and filled in one pass. +/// +/// Every other code unit is preserved exactly, including values above `0xFF`, +/// so that a subsequent decoder still rejects genuinely invalid characters. The +/// returned list is a plain `List` (never a `Uint8List`) for the same +/// reason — a `Uint8List` would truncate wide code units and could mask an +/// invalid character as a valid one. +List stripWhitespace(List input) { + final n = input.length; + int i, c, ws, j; + + // First pass: count whitespace. A clean input allocates nothing and is + // returned untouched, so the flag is free when the payload has no folding. + ws = 0; + for (i = 0; i < n; ++i) { + c = input[i]; + if (c == 0x20 || (c >= 0x09 && c <= 0x0D)) ws++; + } + if (ws == 0) return input; + + // Second pass: compact into a single buffer of the exact final length. + final out = List.filled(n - ws, 0); + j = 0; + for (i = 0; i < n; ++i) { + c = input[i]; + if (c == 0x20 || (c >= 0x09 && c <= 0x0D)) continue; + out[j++] = c; + } + return out; +} diff --git a/test/whitespace_test.dart b/test/whitespace_test.dart new file mode 100644 index 0000000..0dccff9 --- /dev/null +++ b/test/whitespace_test.dart @@ -0,0 +1,147 @@ +import 'dart:convert' as cvt; + +import 'package:convertlib/convertlib.dart'; +import 'package:convertlib/src/core/whitespace.dart'; +import 'package:test/test.dart'; + +import './utils.dart'; + +/// The six ASCII whitespace characters the feature is documented to skip. +const _ws = [' ', '\t', '\n', '\v', '\f', '\r']; + +/// Intersperse [s] with a rotating cycle of whitespace characters, so that +/// whitespace appears before, between, and after the significant characters. +String _lace(String s) { + var sb = StringBuffer(); + for (int i = 0; i < s.length; i++) { + sb.write(_ws[i % _ws.length]); + sb.write(s[i]); + } + sb.write(_ws[s.length % _ws.length]); + return sb.toString(); +} + +void main() { + group('stripWhitespace helper', () { + test('removes each of the six ASCII whitespace characters', () { + // 0x41 = 'A' on both sides of every whitespace code unit. + final input = [ + 0x41, 0x09, 0x41, 0x0A, 0x41, 0x0B, // + 0x41, 0x0C, 0x41, 0x0D, 0x41, 0x20, 0x41, + ]; + expect(stripWhitespace(input), + equals([0x41, 0x41, 0x41, 0x41, 0x41, 0x41, 0x41])); + }); + test('preserves code units wider than a byte (no truncation)', () { + // 0x141 & 0xFF == 0x41 ('A'); a Uint8List store would corrupt it into a + // valid character. It must survive unchanged. + expect(stripWhitespace([0x141, 0x20, 0x142]), equals([0x141, 0x142])); + }); + test('returns the same instance when there is no whitespace', () { + // A clean input must not be copied, so the flag costs no allocation. + final input = [1, 2, 3, 0x100]; + expect(identical(stripWhitespace(input), input), isTrue); + }); + test('an all-whitespace input becomes empty', () { + expect(stripWhitespace([0x20, 0x09, 0x0A, 0x0D]), isEmpty); + }); + test('does not strip near-whitespace controls (0x08, 0x0E)', () { + expect(stripWhitespace([0x08, 0x0E]), equals([0x08, 0x0E])); + }); + }); + + group('Base-64 whitespace-tolerant decode', () { + test('known answer: laced "SGVsbG8=" decodes to "Hello"', () { + final expected = toUtf8('Hello'); + expect(fromBase64('S G V\ns\tbG8 =', ignoreWhitespace: true), + equals(expected)); + }); + test('differential vs dart:convert on a PEM-style block', () { + final expected = randomBytes(200); + // Encode with dart:convert (the external oracle), then wrap at 64 columns + // with CRLF the way a real PEM/MIME body is folded. + final pem = cvt.base64.encode(expected); + var wrapped = StringBuffer(); + for (int i = 0; i < pem.length; i += 64) { + final end = i + 64 > pem.length ? pem.length : i + 64; + wrapped.write(pem.substring(i, end)); + wrapped.write('\r\n'); + } + expect(fromBase64(wrapped.toString(), ignoreWhitespace: true), + equals(expected)); + }); + test('roundtrip with injected whitespace at every length 0..99', () { + for (int len = 0; len < 100; ++len) { + final data = randomBytes(len); + final enc = toBase64(data); + final out = fromBase64(_lace(enc), ignoreWhitespace: true); + expect(out, equals(data), reason: 'length $len'); + } + }); + test('whitespace-only input decodes to empty', () { + expect(fromBase64(' \t\n\r', ignoreWhitespace: true), isEmpty); + }); + test('default (strict) still rejects whitespace', () { + expect(() => fromBase64('SGVs bG8='), throwsFormatException); + expect(() => fromBase64('SGVs\nbG8='), throwsFormatException); + }); + test('non-whitespace invalid characters still throw when tolerant', () { + expect(() => fromBase64('SG!s', ignoreWhitespace: true), + throwsFormatException); + }); + test('a wide code unit is rejected, not truncated, when tolerant', () { + // 0x141 & 0xFF == 0x41 ('A'). If a byte-width buffer truncated it the + // decoder would silently accept it (failure mode #2). It must throw. + final wide = 'SG${String.fromCharCode(0x141)}s'; + expect(() => fromBase64(wide, ignoreWhitespace: true), + throwsFormatException); + }); + test('tryFromBase64 honours the flag', () { + expect(tryFromBase64('SGVs\nbG8='), isNull); + expect(tryFromBase64('SGVs\nbG8=', ignoreWhitespace: true), + equals(toUtf8('Hello'))); + }); + }); + + group('Base-32 whitespace-tolerant decode', () { + test('roundtrip with injected whitespace at every length 0..99', () { + for (int len = 0; len < 100; ++len) { + final data = randomBytes(len); + final enc = toBase32(data); + final out = fromBase32(_lace(enc), ignoreWhitespace: true); + expect(out, equals(data), reason: 'length $len'); + } + }); + test('default (strict) still rejects whitespace', () { + final enc = toBase32([0x66, 0x6f, 0x6f]); + expect(() => fromBase32(_lace(enc)), throwsFormatException); + }); + test('tryFromBase32 honours the flag', () { + final enc = _lace(toBase32([1, 2, 3])); + expect(tryFromBase32(enc), isNull); + expect(tryFromBase32(enc, ignoreWhitespace: true), equals([1, 2, 3])); + }); + }); + + group('Base-16 whitespace-tolerant decode', () { + test('known answer: spaced hex pairs decode correctly', () { + expect(fromHex('de ad\tbe\nef', ignoreWhitespace: true), + equals([0xde, 0xad, 0xbe, 0xef])); + }); + test('roundtrip with injected whitespace at every length 0..99', () { + for (int len = 0; len < 100; ++len) { + final data = randomBytes(len); + final enc = toHex(data); + final out = fromHex(_lace(enc), ignoreWhitespace: true); + expect(out, equals(data), reason: 'length $len'); + } + }); + test('default (strict) still rejects whitespace', () { + expect(() => fromHex('de ad'), throwsFormatException); + }); + test('tryFromHex honours the flag', () { + expect(tryFromHex('de ad'), isNull); + expect(tryFromHex('de ad', ignoreWhitespace: true), equals([0xde, 0xad])); + }); + }); +} diff --git a/test_integration/main.dart b/test_integration/main.dart index 12df536..770c90a 100644 --- a/test_integration/main.dart +++ b/test_integration/main.dart @@ -34,6 +34,8 @@ void main() { print("base64 (bytes) => ${toBase64Bytes(inp)}"); print("tryFromHex('zz') => ${tryFromHex('zz')}"); print("tryFromBase64(valid) => ${tryFromBase64(toBase64(inp))}"); + print("fromBase64(whitespace) => " + "${fromBase64('A/\nE=', ignoreWhitespace: true)}"); print("constantTimeEquals => ${constantTimeEquals(inp, List.of(inp))}"); print(''); }