From 223d0d4caa963f5ed44cb85e5a97977db662c163 Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Sun, 1 Nov 2026 22:56:55 +0900 Subject: [PATCH] JVM bulk loads: search for a subnormal exponent, not any "e-3" (#182) #180 sends a bulk JSON chunk through a per-row Python walk when its text contains "e-3", to catch a subnormal double the engine's parser would read as a BigDecimal. "e-3" is common in ordinary text (hex ids, UUIDs, "name-3"), so such loads walked every chunk. The text is now searched for the exponent json.dumps writes for a subnormal (e-308 to e-329 after a digit) with a compiled pattern: the same C-speed scan, no false hits on ordinary text. Test: UUID-like and "name-3" text never reaches the walk; 5e-324 and the largest subnormal still raise ValueError (per-row path). Closes #182 Co-Authored-By: Claude Opus 5.5 --- src/arcadedb/type_conversion.py | 13 ++++++++++--- tests/test_bulk_json_paths.py | 17 +++++++++++++++++ 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/src/arcadedb/type_conversion.py b/src/arcadedb/type_conversion.py index 2157970a..7df69f8d 100644 --- a/src/arcadedb/type_conversion.py +++ b/src/arcadedb/type_conversion.py @@ -7,6 +7,7 @@ import json import math +import re import sys from datetime import date, datetime, time, timedelta, timezone from decimal import Decimal @@ -746,6 +747,10 @@ def _has_subnormal(value: Any) -> bool: return False +# the exponent json.dumps writes for a subnormal double (5e-324 up to 2.2250738585072009e-308): e-308 to e-329 after a digit +_SUBNORMAL_EXPONENT = re.compile(r"[0-9]e-3(?:0[89]|[12][0-9])") + + def _dumps_checked(rows: Any, **kwargs: Any) -> str: """json.dumps for the bulk paths, refusing a subnormal double. @@ -753,11 +758,13 @@ def _dumps_checked(rows: Any, **kwargs: Any) -> str: BigDecimal, so it was stored as a DECIMAL and read back as ``Decimal`` where every other write path stores a double. ValueError sends the row down the per-row path, which stores the double. Every subnormal is written with - an exponent of -308 or below, so the text is searched first (C speed) and - only a hit pays for the walk over the rows. + an exponent of -308 or below, so the text is searched first (C speed) for + such an exponent after a digit, and only a hit pays for the walk over the + rows. A bare "e-3" would also hit ordinary text (hex ids, UUIDs, "name-3") + and send every such load through the walk. """ text = json.dumps(rows, allow_nan=False, **kwargs) - if "e-3" in text and any(_has_subnormal(row) for row in rows): + if _SUBNORMAL_EXPONENT.search(text) and any(_has_subnormal(row) for row in rows): raise ValueError("a subnormal double cannot go through the JSON bulk path") return text diff --git a/tests/test_bulk_json_paths.py b/tests/test_bulk_json_paths.py index 555a605a..8bb7bb07 100644 --- a/tests/test_bulk_json_paths.py +++ b/tests/test_bulk_json_paths.py @@ -143,3 +143,20 @@ def test_ordinary_rows_stay_on_the_bulk_path(temp_db_path): ) def test_json_bulk_scalar_ok(value, ok): assert type_conversion.json_bulk_scalar_ok(value) is ok + + +def test_text_that_only_looks_like_an_exponent_skips_the_subnormal_walk(monkeypatch): + # "e-3" is common in ordinary text (hex ids, UUIDs, "name-3"): only an exponent a subnormal can have may send a chunk + # through the per-row Python walk, so a load of such text keeps the fast check + calls = [] + real = type_conversion._has_subnormal + monkeypatch.setattr( + type_conversion, "_has_subnormal", lambda row: calls.append(row) or real(row) + ) + rows = [{"id": f"4fe-3a{i}", "tag": "name-3", "x": 1e-30} for i in range(3)] + assert type_conversion.json_bulk_dumps(rows) == json.dumps(rows) + assert calls == [] + with pytest.raises(ValueError): + type_conversion.json_bulk_dumps([{"x": 5e-324}]) + with pytest.raises(ValueError): + type_conversion.json_bulk_dumps([{"x": 2.2250738585072009e-308}])