From 87f50c4a8a50c0af8d1d8f0beb844b11c74f6532 Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Thu, 29 Oct 2026 00:04:10 +0900 Subject: [PATCH 1/2] VectorIndex: re-resolve the Java index by name after REBUILD INDEX (#116) REBUILD INDEX replaces the engine's index object. The wrapper kept the old one, whose isValid() is false but whose search path does not check it, so a held wrapper answered find_nearest with 0 rows and totalVectors 0 and no error. Every public method now calls _check_live() first: one isValid() crossing on the cached object; when it is false, the index is looked up by name, and _lsm_indexes and _pq_ready are reset. A name that no longer resolves raises ArcadeDBError naming the index. find_nearest_by_key and find_nearest_approximate check once and call internal helpers, so a call never pays the crossing twice. The native wrapper resolves by name per statement: _check_live is a no-op there, and get_stats reads the bucket index name afresh since a rebuild renames it. Co-Authored-By: Claude Opus 5.5 --- src/arcadedb/_native/schema_objects.py | 7 +++ src/arcadedb/vector.py | 54 ++++++++++++++++++++---- tests/test_vector_index_introspection.py | 53 +++++++++++++++++++++++ 3 files changed, 106 insertions(+), 8 deletions(-) diff --git a/src/arcadedb/_native/schema_objects.py b/src/arcadedb/_native/schema_objects.py index 3ad1a684..c795af0d 100644 --- a/src/arcadedb/_native/schema_objects.py +++ b/src/arcadedb/_native/schema_objects.py @@ -164,6 +164,11 @@ def __init__(self, database, type_name, prop, name=None): def _resolve_lsm_indexes(self): return [self._shim] + def _check_live(self): + # every search and counter names the index in its statement, so a rebuilt index is found by name on each call: + # there is no cached engine object to go stale (the JPype wrapper re-resolves one here) + return + def _run_search( self, *, query_vector, k, allowed_rids, approximate, ef_search=None ): @@ -283,6 +288,8 @@ def _lookup_query_vector_by_key(self, key): def get_stats(self): """The engine's live counters of the primary bucket index, from its ``schema:index:`` row (the row's own columns left out).""" + # read afresh: REBUILD INDEX builds new bucket indexes, so the name a memoized read holds can be gone + self._shim.__dict__.pop("_engine_info", None) bucket = self._shim.engine_info()["bucket"] rows = self._database.query( "sql", diff --git a/src/arcadedb/vector.py b/src/arcadedb/vector.py index 6c6acf36..fea4e629 100644 --- a/src/arcadedb/vector.py +++ b/src/arcadedb/vector.py @@ -286,7 +286,33 @@ def __init__(self, java_index, database): # Resolved once per wrapper: the underlying LSM indexes and the # PRODUCT-quantization readiness check are stable for the lifetime of # this wrapper, but were previously re-derived on every search (several - # JPype crossings per query). + # JPype crossings per query). They are reset when _check_live() swaps in + # a rebuilt Java index. + self._lsm_indexes = None + self._pq_ready = False + + def _check_live(self): + """Re-resolve the Java index by name when SQL ``REBUILD INDEX`` (or a drop) invalidated it. + + ``REBUILD INDEX`` drops the Java index and builds a new object under the same + name. The old object answers searches and counters empty instead of raising, + so a wrapper held across a rebuild would return nothing without an error. + Costs one JPype crossing, ``isValid()`` on the cached object, per public call; + the lookup by name runs only when the object is no longer valid. + + Raises: + ArcadeDBError: the index no longer exists in the schema. + """ + if self._java_index.isValid(): + return + name = str(self._java_index.getName()) + try: + fresh = self._database._java_db.getSchema().getIndexByName(name) + except Exception as e: + raise ArcadeDBError( + f"Vector index '{name}' no longer exists: it was dropped" + ) from e + self._java_index = fresh self._lsm_indexes = None self._pq_ready = False @@ -614,6 +640,12 @@ def find_nearest( - record: The matching ArcadeDB record (Vertex, Document, or Edge) - score: The similarity score/distance """ + self._check_live() + return self._find_nearest_live(query_vector, k, ef_search, allowed_rids) + + def _find_nearest_live(self, query_vector, k, ef_search, allowed_rids): + # find_nearest after _check_live(): find_nearest_by_key checks once and + # comes here, so a call never pays the isValid() crossing twice. effective_ef_search = self._normalize_ef_search(ef_search) try: @@ -638,6 +670,7 @@ def get_size(self): Returns: int: Number of items currently indexed """ + self._check_live() try: # Both TypeIndex and LSMVectorIndex support countEntries() return self._java_index.countEntries() @@ -654,6 +687,7 @@ def get_metadata(self): ``rerank_oversample`` and ``fused_pq`` are present when the engine has them (ArcadeDB 26.11.1 or later). """ + self._check_live() try: index = self._get_primary_lsm_index() meta = self._get_primary_metadata() @@ -715,6 +749,7 @@ def get_stats(self): >>> stats["vectorCacheHits"], stats["vectorCacheMisses"] (10399017, 1840948) """ + self._check_live() try: raw = self._get_primary_lsm_index().getStats() except Exception as e: @@ -760,13 +795,9 @@ def find_nearest_by_key( Returns: List of tuples: [(record, score), ...] """ + self._check_live() query_vector = self._lookup_query_vector_by_key(key) - return self.find_nearest( - query_vector, - k=k, - ef_search=ef_search, - allowed_rids=allowed_rids, - ) + return self._find_nearest_live(query_vector, k, ef_search, allowed_rids) def find_nearest_approximate( self, @@ -790,8 +821,9 @@ def find_nearest_approximate( Returns: List of tuples: [(record, score), ...] """ + self._check_live() try: - quantization = self.get_quantization() + quantization = self._get_quantization_live() if quantization != "PRODUCT": raise ArcadeDBError( "Approximate search requires quantization=PRODUCT (PQ)" @@ -818,6 +850,10 @@ def get_quantization(self): Returns: str: "NONE", "INT8", "BINARY", or "PRODUCT" """ + self._check_live() + return self._get_quantization_live() + + def _get_quantization_live(self): try: idx_to_check = self._get_primary_lsm_index() @@ -848,6 +884,7 @@ def warm_up(self): ArcadeDBError: the index cannot be warmed, including on an engine older than 26.10.1, which has no ``warmUp()``. """ + self._check_live() try: warmed_any = False for index in self._iter_lsm_indexes(): @@ -868,6 +905,7 @@ def build_graph_now(self): Useful after bulk inserts/updates to avoid waiting for a search-triggered lazy build. For TypeIndex wrappers, rebuilds all underlying LSMVectorIndex instances. """ + self._check_live() try: self._ensure_product_quantization_ready() diff --git a/tests/test_vector_index_introspection.py b/tests/test_vector_index_introspection.py index 0efb1823..29488825 100644 --- a/tests/test_vector_index_introspection.py +++ b/tests/test_vector_index_introspection.py @@ -257,3 +257,56 @@ def test_the_threshold_is_the_larger_of_the_floor_and_the_ratio_term( stats = index.get_stats() assert stats["totalVectors"] == len(self.ROWS) assert stats["effectiveMutationsThreshold"] == expected + + +class TestWrapperAcrossRebuildAndDrop: + """A ``VectorIndex`` held across SQL ``REBUILD INDEX`` keeps serving; across a drop it raises (#116). + + ``REBUILD INDEX`` replaces the engine's index object. The JVM wrapper used to keep the old one, which answered + ``find_nearest`` with no rows and ``totalVectors`` 0 without an error. + """ + + ROWS = [(f"v{i}", [1.0 + i % 7, 1.0 + i // 7, 2.0 + i % 3]) for i in range(40)] + + def _held_index(self, db): + _make_type(db) + _insert(db, self.ROWS) + index = db.create_vector_index("Doc", "embedding", dimensions=3) + assert len(index.find_nearest([1.0, 1.0, 2.0], k=5)) == 5 + assert index.get_stats()["totalVectors"] == len(self.ROWS) + return index + + def test_old_wrapper_serves_after_rebuild_index(self, temp_db): + index = self._held_index(temp_db) + + temp_db.command("sql", "REBUILD INDEX `Doc[embedding]`") + + assert len(index.find_nearest([1.0, 1.0, 2.0], k=5)) == 5 + assert index.get_stats()["totalVectors"] == len(self.ROWS) + assert index.get_size() == len(self.ROWS) + assert index.get_metadata()["index_name"] == "Doc[embedding]" + assert index.get_quantization() == "NONE" + index.warm_up() + + def test_old_wrapper_follows_a_second_rebuild(self, temp_db): + index = self._held_index(temp_db) + + for _ in range(2): + temp_db.command("sql", "REBUILD INDEX `Doc[embedding]`") + assert len(index.find_nearest([1.0, 1.0, 2.0], k=5)) == 5 + assert index.get_stats()["totalVectors"] == len(self.ROWS) + + def test_dropped_index_raises_and_never_answers_empty(self, temp_db): + index = self._held_index(temp_db) + + temp_db.command("sql", "DROP INDEX `Doc[embedding]`") + + with pytest.raises(arcadedb.ArcadeDBError) as excinfo: + index.find_nearest([1.0, 1.0, 2.0], k=5) + if arcadedb.__backend__ == "jvm": + # the JVM wrapper names the index itself; the native one passes the engine's own error on + assert "Doc[embedding]" in str(excinfo.value) + with pytest.raises(arcadedb.ArcadeDBError, match=r"Doc\[embedding\]"): + index.get_stats() + with pytest.raises(arcadedb.ArcadeDBError, match=r"Doc\[embedding\]"): + index.get_size() From 2d7daab882107cac8ff0f263439eecdc892c8972 Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Thu, 29 Oct 2026 00:46:56 +0900 Subject: [PATCH 2/2] test: compare the quantization across the rebuild instead of assuming NONE create_vector_index defaults to quantization="INT8", so the hard-coded "NONE" failed on the JVM legs and native parity (assert 'INT8' == 'NONE', the only failure: 1832 passed JVM, 1572 passed native). The test now reads get_quantization() before REBUILD INDEX and asserts the held wrapper reports the same value after it. Co-Authored-By: Claude Opus 5.5 --- tests/test_vector_index_introspection.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_vector_index_introspection.py b/tests/test_vector_index_introspection.py index 324e2652..3777b0b3 100644 --- a/tests/test_vector_index_introspection.py +++ b/tests/test_vector_index_introspection.py @@ -291,6 +291,7 @@ def _held_index(self, db): def test_old_wrapper_serves_after_rebuild_index(self, temp_db): index = self._held_index(temp_db) + quantization = index.get_quantization() # create_vector_index defaults to INT8 temp_db.command("sql", "REBUILD INDEX `Doc[embedding]`") @@ -298,7 +299,7 @@ def test_old_wrapper_serves_after_rebuild_index(self, temp_db): assert index.get_stats()["totalVectors"] == len(self.ROWS) assert index.get_size() == len(self.ROWS) assert index.get_metadata()["index_name"] == "Doc[embedding]" - assert index.get_quantization() == "NONE" + assert index.get_quantization() == quantization index.warm_up() def test_old_wrapper_follows_a_second_rebuild(self, temp_db):