From 3e4fb2772af3944646664d5119c6e79288047ab9 Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Sun, 1 Nov 2026 19:33:41 +0900 Subject: [PATCH 1/2] native: build_graph_now() rebuilds an existing vector graph (#162) The native shim ran one vectorNeighbors search, which builds a graph only when there is none; on a built index the call returned in about 2 ms and the vectors written since stayed in the delta (agent-memory study: 26 calls over 13 runs, graphRebuildCount 0, graphNodeCount unchanged; the JVM wheel's calls took about 11 s each and grew the graph). A `build` op in arcadedb_n_vindex calls LSMVectorIndex.buildVectorGraphNow() on every bucket index, the call the JPype wrapper makes. warm_up() keeps its one search: it only loads a persisted graph. Regression test on both builds: an index built over 200 vectors, 100 more written with the automatic triggers off (high floor, inactivity timer 0), then build_graph_now() must leave graphNodeCount at 300 and raise graphRebuildCount. Closes #162 Co-Authored-By: Claude Opus 5.5 --- .../org/humemai/nativelib/NativeExtras.java | 9 ++++- src/arcadedb/_native/schema_objects.py | 20 +++++++--- tests/test_vector_index_introspection.py | 37 +++++++++++++++++++ 3 files changed, 59 insertions(+), 7 deletions(-) diff --git a/native/src/main/java/org/humemai/nativelib/NativeExtras.java b/native/src/main/java/org/humemai/nativelib/NativeExtras.java index 4b4872fe..7ee05e9d 100644 --- a/native/src/main/java/org/humemai/nativelib/NativeExtras.java +++ b/native/src/main/java/org/humemai/nativelib/NativeExtras.java @@ -315,8 +315,9 @@ private static String q(final String text) { } /** - * ops: metadata {type, prop} | approximate {type, prop, vector, k, allowed}. The index is the one the type has on the property; its bucket - * indexes are searched in turn (the caller merges). metadata describes the first bucket index, as the JPype wrapper does. + * ops: metadata {type, prop} | approximate {type, prop, vector, k, allowed} | build {type, prop}. The index is the one the type has on the + * property; its bucket indexes are searched in turn (the caller merges). metadata describes the first bucket index, as the JPype wrapper does. + * build rebuilds the graph of every bucket index now (buildVectorGraphNow), as the JPype wrapper's build_graph_now() does (#162). */ @CEntryPoint(name = "arcadedb_n_vindex") @SuppressWarnings("unchecked") @@ -383,6 +384,10 @@ public static CCharPointer vindex(final IsolateThread thread, final long dbh, fi } sb.append(']'); } + case "build" -> { + for (final LSMVectorIndex lsm : parts) + lsm.buildVectorGraphNow(); + } default -> throw new IllegalArgumentException("unknown vector index op " + ArcadeNative.str(op)); } return ArcadeNative.out(sb.append('}').toString()); diff --git a/src/arcadedb/_native/schema_objects.py b/src/arcadedb/_native/schema_objects.py index 52a85e79..ceb647a8 100644 --- a/src/arcadedb/_native/schema_objects.py +++ b/src/arcadedb/_native/schema_objects.py @@ -136,17 +136,27 @@ def countEntries(self): return int(row.get("c")) def warmUp(self): - # the engine loads a persisted graph on the first search: run that search now - self.buildVectorGraphNow() - - def buildVectorGraphNow(self): - # the engine builds the graph on the first search: one search now, on a vector of the index's dimension + # the engine loads a persisted graph on the first search: one search now, on a vector of the index's dimension dims = self.engine_info()["dimensions"] if dims: self._db.query( "sql", "SELECT vectorNeighbors(?, ?, 1)", self._name, [1.0] * dims ).close() + def buildVectorGraphNow(self): + # the engine's own rebuild of every bucket index (LSMVectorIndex.buildVectorGraphNow), as the JPype wrapper calls it: + # a search builds a graph only when there is none, so it left the vectors written since in the delta (#162) + from . import lib + from .extras import _call, _enc + + _call( + "Failed to rebuild vector graph", + "arcadedb_n_vindex", + self._db._h, + lib.b("build"), + lib.b(_enc({"type": self._type, "prop": self._prop})), + ) + def _rid_text(value) -> str: get_rid = getattr(value, "get_rid", None) diff --git a/tests/test_vector_index_introspection.py b/tests/test_vector_index_introspection.py index d3083b2a..f9cf75c9 100644 --- a/tests/test_vector_index_introspection.py +++ b/tests/test_vector_index_introspection.py @@ -283,6 +283,43 @@ def test_alter_database_sets_the_ratio_the_threshold_reads(self, temp_db): assert index.get_stats()["effectiveMutationsThreshold"] == 500 +class TestBuildGraphNow: + """``build_graph_now()`` rebuilds a graph that already exists, so the vectors written since it was built join it (#162). + + The native build ran one search, which builds a graph only when there is none: on a built index the call returned at + once and the new vectors stayed in the delta. The automatic triggers are off (a high floor, no inactivity timer), so + only the call can move the counters. + """ + + ROWS = [ + (f"v{i}", [1.0 + i % 10, 1.0 + i // 10 % 10, 1.0 + i // 100]) + for i in range(300) + ] + + def test_rebuilds_an_existing_graph(self, temp_db): + _make_type(temp_db) + temp_db.command( + "sql", + "ALTER DATABASE `arcadedb.vectorIndex.inactivityRebuildTimeoutMs` 0", + ) + _insert(temp_db, self.ROWS[:200]) + index = temp_db.create_vector_index( + "Doc", "embedding", dimensions=3, mutations_before_rebuild=1_000_000 + ) + before = index.get_stats() + assert before["graphNodeCount"] == 200 + + _insert(temp_db, self.ROWS[200:]) + assert index.get_stats()["graphNodeCount"] == 200 + + index.build_graph_now() + + after = index.get_stats() + assert after["totalVectors"] == len(self.ROWS) + assert after["graphNodeCount"] == len(self.ROWS) + assert after["graphRebuildCount"] > before["graphRebuildCount"] + + class TestWrapperAcrossRebuildAndDrop: """A ``VectorIndex`` held across SQL ``REBUILD INDEX`` keeps serving; across a drop it raises (#116). From 6d3ebecb836ffeccc6fc664d060a89d1f25b5b45 Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Sun, 1 Nov 2026 19:40:52 +0900 Subject: [PATCH 2/2] test: start the #162 graph at 1000 vectors The engine rebuilds a graph under 1000 nodes synchronously on the next search (ASYNC_REBUILD_MIN_GRAPH_SIZE), so at 200 vectors the old native search shim rebuilt too and the test passed on the pre-fix wheel. At 1000 + 100 vectors the pre-fix native wheel fails it (graphNodeCount 1000, expected 1100) and the JVM wheel passes it. Co-Authored-By: Claude Opus 5.5 --- tests/test_vector_index_introspection.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/tests/test_vector_index_introspection.py b/tests/test_vector_index_introspection.py index f9cf75c9..4a6631ea 100644 --- a/tests/test_vector_index_introspection.py +++ b/tests/test_vector_index_introspection.py @@ -288,12 +288,13 @@ class TestBuildGraphNow: The native build ran one search, which builds a graph only when there is none: on a built index the call returned at once and the new vectors stayed in the delta. The automatic triggers are off (a high floor, no inactivity timer), so - only the call can move the counters. + only the call can move the counters. The graph starts at 1000 vectors because the engine rebuilds a smaller one + synchronously on the next search (ASYNC_REBUILD_MIN_GRAPH_SIZE), which hid the defect. """ ROWS = [ (f"v{i}", [1.0 + i % 10, 1.0 + i // 10 % 10, 1.0 + i // 100]) - for i in range(300) + for i in range(1100) ] def test_rebuilds_an_existing_graph(self, temp_db): @@ -302,15 +303,15 @@ def test_rebuilds_an_existing_graph(self, temp_db): "sql", "ALTER DATABASE `arcadedb.vectorIndex.inactivityRebuildTimeoutMs` 0", ) - _insert(temp_db, self.ROWS[:200]) + _insert(temp_db, self.ROWS[:1000]) index = temp_db.create_vector_index( "Doc", "embedding", dimensions=3, mutations_before_rebuild=1_000_000 ) before = index.get_stats() - assert before["graphNodeCount"] == 200 + assert before["graphNodeCount"] == 1000 - _insert(temp_db, self.ROWS[200:]) - assert index.get_stats()["graphNodeCount"] == 200 + _insert(temp_db, self.ROWS[1000:]) + assert index.get_stats()["graphNodeCount"] == 1000 index.build_graph_now()