From 7004b84b1d5f0b3ef38e6e71637f3b9c0e6726ae Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Wed, 4 Nov 2026 11:03:27 +0900 Subject: [PATCH 1/2] Native find_nearest: include_vector=False returns hits without the indexed vector, loaded by RID on first read find_nearest keeps its default: every hit carries its vector, and the default search is the code of the query-as-float32 change (#203), untouched. A new keyword-only include_vector=False asks for hits built without the vector, for callers that need only ids or distances. On the native wheel the search then runs SELECT *, !, !record FROM (SELECT expand(vectorNeighbors(...))) and each hit loads its record once by RID the first time the vector or the whole record is needed. The JVM build accepts the keyword and ignores it (it loads whole records either way). Reworks the earlier lazy-by-default version after its timed A/B: hits not read 0.66 and 0.805 (1536 and 256 dims), but vector read 1.011 and 1.336, so lazy-by-default is a cost for callers that read the vector. Tests run on both backends (values equal lookup_by_rid, keyword accepted) and on the native wheel (default ships the vector with no lookup and no dropped column, opt-in ships none and looks up once per record). Co-Authored-By: Claude Opus 5.5 --- docs/modules/ROOT/pages/how-to/vectors.adoc | 18 +- src/arcadedb/_native/records.py | 28 +++ src/arcadedb/_native/schema_objects.py | 73 +++++++- src/arcadedb/vector.py | 23 ++- tests/conftest.py | 2 + tests/test_native_vector_hit_lazy.py | 133 ++++++++++++++ tests/test_vector_hit_records.py | 186 ++++++++++++++++++++ 7 files changed, 459 insertions(+), 4 deletions(-) create mode 100644 tests/test_native_vector_hit_lazy.py create mode 100644 tests/test_vector_hit_records.py diff --git a/docs/modules/ROOT/pages/how-to/vectors.adoc b/docs/modules/ROOT/pages/how-to/vectors.adoc index df066b24..81f9f260 100644 --- a/docs/modules/ROOT/pages/how-to/vectors.adoc +++ b/docs/modules/ROOT/pages/how-to/vectors.adoc @@ -162,7 +162,7 @@ print([row["name"] for row in rows]) [[find-nearest]] === find_nearest -`VectorIndex.find_nearest(query_vector, k=10, ef_search=None, allowed_rids=None)` searches without a statement. It takes a list or a NumPy array and returns a list of `(record, distance)` tuples, nearest first. Use it when you already hold the index object, or when you have a list of record ids to search within. +`VectorIndex.find_nearest(query_vector, k=10, ef_search=None, allowed_rids=None, *, include_vector=True)` searches without a statement. It takes a list or a NumPy array and returns a list of `(record, distance)` tuples, nearest first. Use it when you already hold the index object, or when you have a list of record ids to search within. On the native package the query vector crosses as one `float32` buffer whatever its form (a list, a `float64` array, or a `float32` array), so you do not need to convert it first. A vector index stores `float32`, so the numbers are rounded the same way as before. The table gives the median `find_nearest` latency with this change over the median without it (95% interval, below 1 is faster), measured on the native wheel `26.11.1.dev0` on one Linux x86_64 laptop with 11 to 12 clean pairs per figure. The wheel built before this change was the baseline. @@ -199,6 +199,22 @@ python-ish python ---- +=== Hits without the vector + +Every hit of `find_nearest` carries its indexed vector, so a search at 1536 dimensions ships and decodes 1536 numbers per hit whether you read them or not. If you need only the ids, the distances or other properties of the hits, pass `include_vector=False`. On the native build each hit is then built without the vector. It loads the vector, with the rest of the record, in one lookup by record id the first time you read it (`get` of the vector, `has_property`, `to_dict`, `get_property_names`, `modify` or `save`), so a loop that reads the vector of every hit pays one lookup per hit and does more work than the default. The values are the ones `db.lookup_by_rid` returns, and a hit deleted since the search raises `ArcadeDBError` on that first read. The arcadedb-jvm package accepts the argument and returns whole records either way. + +[source,python] +---- +for record, distance in index.find_nearest(query, k=2, include_vector=False): + print(record.get("name")) +---- + +[source,text] +---- +python-ish +python +---- + === Filtered search `allowed_rids` restricts `find_nearest` to the records whose ids you list, as strings. The search considers only those records, so it still returns `k` of them. diff --git a/src/arcadedb/_native/records.py b/src/arcadedb/_native/records.py index e925303e..4711c193 100644 --- a/src/arcadedb/_native/records.py +++ b/src/arcadedb/_native/records.py @@ -250,9 +250,17 @@ def __init__( # The dict is never changed in place (a save makes a new one), so the objects modify() hands it to can share it self._base = base self._property_names_cache = None + # names of properties a search hit was built without (see ``_wrap_partial``); None for every other record + self._lazy = None # -- reading + def _fill(self) -> None: + """Load the properties a search hit was built without, in one lookup by RID (engine order); nothing to do for any other record.""" + if self._lazy: + self._props = dict(self._db.lookup_by_rid(self._rid)._props) + self._lazy = None + def _check_open(self) -> None: database = self._database if database is not None and database._closed and self._rid: @@ -263,6 +271,8 @@ def _check_open(self) -> None: def get(self, name: str, convert_types: bool = True) -> Any: self._check_open() + if self._lazy and name in self._lazy: + self._fill() return self._props.get(name) def get_raw(self, name: str) -> Any: @@ -270,14 +280,18 @@ def get_raw(self, name: str) -> Any: def has_property(self, name: str) -> bool: self._check_open() + if self._lazy and name in self._lazy: + self._fill() return name in self._props def get_property_names(self) -> List[str]: self._check_open() + self._fill() return list(self._props) def to_dict(self, convert_types: bool = True) -> Dict[str, Any]: self._check_open() + self._fill() return dict(self._props) def get_rid(self) -> str: @@ -318,6 +332,7 @@ def save(self): not seen. Saving an existing record needs an active transaction, like every write; outside one it raises ArcadeDBError ("Transaction not begun"). Creating a new record is not checked. """ + self._fill() db = self._db if self._rid is None: from .extras import veto_count @@ -399,6 +414,7 @@ def delete(self): def modify(self): self._check_open() + self._fill() if self._rid and self._base is None: self._base = _snap( self._props @@ -525,3 +541,15 @@ def _wrap_dict(database, record: dict, kind: str = "d"): type_name = props.pop("@type", None) kind = props.pop("@cat", kind) return _CLASS_OF.get(kind, Document)(database, type_name, rid, props) + + +def _wrap_partial(database, rid, type_name, kind, props, omitted): + """The wrapper for a search hit built without the properties named in ``omitted`` (the indexed vector: not shipped with every hit). + + Reading one of them, or anything that needs the whole record (``to_dict``, ``get_property_names``, ``modify``, ``save``), loads the + record once by RID; ``get`` and ``has_property`` of any other property are answered from what was shipped. + """ + cls = {"v": Vertex, "e": Edge}.get(kind, Document) + record = cls(database, type_name, rid, props) + record._lazy = frozenset(omitted) + return record diff --git a/src/arcadedb/_native/schema_objects.py b/src/arcadedb/_native/schema_objects.py index a7a011d3..5f69e0d1 100644 --- a/src/arcadedb/_native/schema_objects.py +++ b/src/arcadedb/_native/schema_objects.py @@ -193,6 +193,18 @@ def _query_vector_param(query_vector): ) +def _quote_name(name: str) -> str: + return "`" + name.replace("`", "``") + "`" + + +# find_nearest(include_vector=False): the engine's row of a hit is the record's properties plus the columns @rid, @type and distance +# (ScoredRecordRow; before 26.11.1, ArcadeData/arcadedb#10898, also a nested `record` column holding every property a second time). +# `*` copies the row, `!name` drops the indexed vector and the `record` column. A property of the record called distance or record is +# hidden by those columns in the row, so such a hit holds them back too (_LAZY_COLUMNS) and loads them with the vector. +_LAZY_COLUMNS = ("record", "distance") +_HITS_FROM = "FROM (SELECT expand(vectorNeighbors(?, ?, ?)))" + + class NativeVectorIndex(_BaseVectorIndex): """``VectorIndex`` over ``vectorNeighbors``: ``find_nearest`` returns ``[(record, distance)]`` like the JPype wrapper.""" @@ -210,10 +222,21 @@ def _check_live(self): return def _run_search( - self, *, query_vector, k, allowed_rids, approximate, ef_search=None + self, + *, + query_vector, + k, + allowed_rids, + approximate, + ef_search=None, + include_vector=True, ): if approximate: return self._run_approximate(query_vector, k, allowed_rids) + if not include_vector: + return self._run_search_without_vector( + query_vector, k, allowed_rids, ef_search + ) from .records import _wrap_dict vector = _query_vector_param(query_vector) @@ -257,6 +280,54 @@ def _run_search( out.append((record, float(row.get("distance")))) return self._sort_results(out)[:k] + def _run_search_without_vector(self, query_vector, k, allowed_rids, ef_search): + """``find_nearest(include_vector=False)``: the hits come back without the indexed vector, which loads on first use. + + ``expand(vectorNeighbors(..))`` alone ships every hit's vector, and the engine reads each record to build the row whatever + the columns asked for. Here the statement drops the vector (and the nested ``record`` column of an engine before 26.11.1), + and ``_Mixin._fill`` loads the record, once, by RID, the first time the vector or the whole record is needed. + """ + from .records import _wrap_partial + + vector = _query_vector_param(query_vector) + options = {} + if ef_search is not None: + options["efSearch"] = int(ef_search) + if allowed_rids: + options["filter"] = [_rid_text(r) for r in allowed_rids] + prop = self._shim._prop + dropped = ", ".join( + "!" + _quote_name(name) for name in dict.fromkeys((prop, "record")) + ) + sql = f"SELECT *, {dropped} {_HITS_FROM}" # nosec B608 - dropped is backtick-quoted schema names, the values go through ? parameters + if options: + sql = sql.replace("vectorNeighbors(?, ?, ?)", "vectorNeighbors(?, ?, ?, ?)") + rows = self._database.query( + "sql", sql, self._shim._name, vector, int(k), options + ) + else: + rows = self._database.query("sql", sql, self._shim._name, vector, int(k)) + omitted = (prop, *_LAZY_COLUMNS) + out = [] + for row in rows: + props = row.to_dict() + rid = props.pop("@rid", None) + type_name = props.pop("@type", None) + distance = props.pop("distance") + if rid is None: + record = None + else: + record = _wrap_partial( + self._database, + _rid_text(rid), + type_name, + self._database._kind_of(type_name), + props, + omitted, + ) + out.append((record, float(distance))) + return self._sort_results(out)[:k] + def _run_approximate(self, query_vector, k, allowed_rids): """The PQ-scored search of the engine (no statement runs it): the engine returns (RID, distance), the records are loaded here.""" from . import lib diff --git a/src/arcadedb/vector.py b/src/arcadedb/vector.py index ddc8b32b..3c407068 100644 --- a/src/arcadedb/vector.py +++ b/src/arcadedb/vector.py @@ -566,6 +566,7 @@ def _run_search( allowed_rids, approximate, ef_search=None, + include_vector=True, # accepted for API parity, the JVM build loads whole records ): java_vector = to_java_float_array(query_vector) allowed_rids_set = self._build_allowed_rids_set(allowed_rids) @@ -584,6 +585,8 @@ def find_nearest( k=10, ef_search=None, allowed_rids=None, + *, + include_vector=True, ): """ Find k nearest neighbors to the query vector. @@ -605,6 +608,17 @@ def find_nearest( uses the Java index default/adaptive behavior. allowed_rids: Optional list of RID strings (e.g. ["#1:0", "#2:5"]) to restrict search + include_vector: Keyword only, default ``True``: every hit carries its + indexed vector, as always. Pass ``False`` when you need only the + ids or the distances (or other properties) of the hits. On the + native build each hit is then built without the indexed vector, and + loads it, with the rest of the record, in one lookup by RID the + first time you read it (``get`` of the vector, ``has_property``, + ``to_dict``, ``get_property_names``, ``modify``, ``save``); the + values are those of ``lookup_by_rid``, and a hit deleted since the + search raises ``ArcadeDBError`` on that first read. The JVM build + accepts the argument and ignores it: it loads whole records either + way, so the call costs the same. Returns: List of tuples: [(record, score), ...] @@ -612,9 +626,13 @@ def find_nearest( - score: The similarity score/distance """ self._check_live() - return self._find_nearest_live(query_vector, k, ef_search, allowed_rids) + return self._find_nearest_live( + query_vector, k, ef_search, allowed_rids, include_vector + ) - def _find_nearest_live(self, query_vector, k, ef_search, allowed_rids): + def _find_nearest_live( + self, query_vector, k, ef_search, allowed_rids, include_vector=True + ): # find_nearest after _check_live(): find_nearest_by_key checks once and # comes here, so a call never pays the isValid() crossing twice. effective_ef_search = self._normalize_ef_search(ef_search) @@ -626,6 +644,7 @@ def _find_nearest_live(self, query_vector, k, ef_search, allowed_rids): allowed_rids=allowed_rids, approximate=False, ef_search=effective_ef_search, + include_vector=include_vector, ) except ArcadeDBError: diff --git a/tests/conftest.py b/tests/conftest.py index 0df9d9ad..ccfc4c40 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -43,6 +43,8 @@ collect_ignore.append("test_native_udf.py") # how find_nearest hands the query vector to the native library (the codec's side buffer) collect_ignore.append("test_native_find_nearest_params.py") + # find_nearest(include_vector=False) on the native wheel fills a hit's vector on first use (counts lookups of the native wrapper) + collect_ignore.append("test_native_vector_hit_lazy.py") # the columnar insert's stored records are compared between the JSON and the columnar entry points of the native library collect_ignore.append("test_native_columnar_insert.py") # the rows of an insert_many chunk with date-times, Decimals, bytes and numpy scalars are told apart by the entry points they cross diff --git a/tests/test_native_vector_hit_lazy.py b/tests/test_native_vector_hit_lazy.py new file mode 100644 index 00000000..7b672f73 --- /dev/null +++ b/tests/test_native_vector_hit_lazy.py @@ -0,0 +1,133 @@ +"""Native wheel: ``find_nearest`` keeps its default (hits carry the vector, no extra lookup) and, asked for ``include_vector=False``, +builds a hit without the indexed vector and fills it on first use, with one lookup by RID. + +The values themselves are checked on both wheels in ``test_vector_hit_records.py``; this file counts what is shipped and looked up, +and which statement the search runs. +""" + +import numpy as np +import pytest + +from arcadedb.exceptions import ArcadeDBError + +DIM = 8 + + +@pytest.fixture +def lazy_db(temp_db, monkeypatch): + temp_db.command("sql", "CREATE VERTEX TYPE Doc BUCKETS 4") + temp_db.command("sql", "CREATE PROPERTY Doc.id INTEGER") + temp_db.command("sql", "CREATE PROPERTY Doc.embedding ARRAY_OF_FLOATS") + rng = np.random.default_rng(5) + vecs = rng.standard_normal((40, DIM)).astype(np.float32) + with temp_db.transaction(): + for i in range(40): + temp_db.command( + "sql", + "INSERT INTO Doc SET id = ?, name = ?, embedding = ?", + i, + f"n{i}", + vecs[i].tolist(), + ) + index = temp_db.create_vector_index( + "Doc", "embedding", dimensions=DIM, id_property="id" + ) + lookups = [] + original = type(temp_db).lookup_by_rid + + def counting(self, rid): + lookups.append(str(rid)) + return original(self, rid) + + monkeypatch.setattr(type(temp_db), "lookup_by_rid", counting) + statements = [] + original_query = type(temp_db).query + + def recording(self, language, text, *args, **kwargs): + statements.append(text) + return original_query(self, language, text, *args, **kwargs) + + monkeypatch.setattr(type(temp_db), "query", recording) + return temp_db, index, vecs, lookups, statements + + +def _search_statements(statements): + return [s for s in statements if "vectorNeighbors" in s] + + +def test_the_default_search_ships_the_vector_and_looks_up_nothing(lazy_db): + db, index, vecs, lookups, statements = lazy_db + hits = index.find_nearest(vecs[2], k=6) + assert len(hits) == 6 + for record, _ in hits: + assert "embedding" in record._props + assert record._lazy is None + assert len(record.get("embedding")) == DIM + assert record.has_property("embedding") + assert "embedding" in record.to_dict() + assert lookups == [] + (text,) = _search_statements(statements) + assert "!" not in text and "SELECT *" not in text + + +def test_asking_for_the_default_in_words_is_the_default(lazy_db): + db, index, vecs, lookups, statements = lazy_db + hits = index.find_nearest(vecs[2], k=3, include_vector=True) + assert all("embedding" in record._props for record, _ in hits) + assert lookups == [] + (text,) = _search_statements(statements) + assert "!" not in text + + +def test_an_opt_in_search_ships_no_vector_and_looks_up_nothing(lazy_db): + db, index, vecs, lookups, statements = lazy_db + hits = index.find_nearest(vecs[2], k=6, include_vector=False) + assert len(hits) == 6 + for record, _ in hits: + assert "embedding" not in record._props + assert record.get("id") is not None and record.get("name") + assert record.get_rid() and record.get_type_name() == "Doc" + assert lookups == [] + (text,) = _search_statements(statements) + assert "!`embedding`" in text + + +def test_the_vector_is_one_lookup_per_record_however_it_is_reached(lazy_db): + db, index, vecs, lookups, statements = lazy_db + hits = index.find_nearest(vecs[2], k=4, include_vector=False) + first, second, third, fourth = (r for r, _ in hits) + assert len(first.get("embedding")) == DIM + assert len(first.get("embedding")) == DIM + assert first.has_property("embedding") + assert "embedding" in first.to_dict() + assert "embedding" in first.get_property_names() + assert len(lookups) == 1 + assert second.has_property("embedding") + assert len(lookups) == 2 + assert "embedding" in third.to_dict() + assert "embedding" in fourth.get_property_names() + assert len(lookups) == 4 + assert sorted(lookups) == sorted(r.get_rid() for r, _ in hits) + + +def test_the_loaded_vector_is_the_one_the_default_search_ships(lazy_db): + db, index, vecs, lookups, statements = lazy_db + full = { + r.get_rid(): list(r.get("embedding")) + for r, _ in index.find_nearest(vecs[9], k=5) + } + assert lookups == [] + lean = index.find_nearest(vecs[9], k=5, include_vector=False) + assert lookups == [] + for record, _ in lean: + assert list(record.get("embedding")) == full[record.get_rid()] + assert len(lookups) == 5 + + +def test_the_vector_of_a_closed_database_is_not_looked_up(lazy_db): + db, index, vecs, lookups, statements = lazy_db + record, _ = index.find_nearest(vecs[2], k=1, include_vector=False)[0] + db.close() + with pytest.raises(ArcadeDBError, match="closed"): + record.get("embedding") + assert lookups == [] diff --git a/tests/test_vector_hit_records.py b/tests/test_vector_hit_records.py new file mode 100644 index 00000000..f8774fcd --- /dev/null +++ b/tests/test_vector_hit_records.py @@ -0,0 +1,186 @@ +"""The records a ``find_nearest`` hit holds, on both backends: the same values as a lookup by RID, whichever way it is asked. + +``find_nearest(..., include_vector=False)`` builds a hit without the indexed vector on the native wheel and fills it on first +use (``test_native_vector_hit_lazy.py`` counts that); the JVM wheel accepts the keyword and returns whole records. These tests are +the contract both builds keep: type, property names and their order, every value, the vector included, and a record that can be +modified and saved after a search. The default call (``include_vector=True``) is held to the same contract. +""" + +import numpy as np +import pytest + +DIM = 8 +N_DOC = 48 +N_SUB = 16 +BOTH = pytest.mark.parametrize("include_vector", [True, False]) + + +def _make(temp_db, extra_properties, extra_columns, extra_values): + temp_db.command("sql", "CREATE VERTEX TYPE Doc BUCKETS 4") + temp_db.command("sql", "CREATE VERTEX TYPE SubDoc EXTENDS Doc") + temp_db.command("sql", "CREATE PROPERTY Doc.id INTEGER") + temp_db.command("sql", "CREATE PROPERTY Doc.label STRING") + for statement in extra_properties: + temp_db.command("sql", statement) + temp_db.command("sql", "CREATE PROPERTY Doc.embedding ARRAY_OF_FLOATS") + rng = np.random.default_rng(11) + vecs = rng.standard_normal((N_DOC + N_SUB, DIM)).astype(np.float32) + with temp_db.transaction(): + for i in range(N_DOC + N_SUB): + type_name = "Doc" if i < N_DOC else "SubDoc" + temp_db.command( + "sql", + f"INSERT INTO {type_name} SET id = ?, label = ?{extra_columns}, embedding = ?", + i, + f"label-{i}", + *extra_values(i), + vecs[i].tolist(), + ) + index = temp_db.create_vector_index( + "Doc", "embedding", dimensions=DIM, id_property="id" + ) + return temp_db, index, vecs + + +@pytest.fixture +def hit_db(temp_db): + """Four buckets and a subtype.""" + return _make(temp_db, [], "", lambda i: ()) + + +@pytest.fixture +def clash_db(temp_db): + """As ``hit_db``, plus properties called ``distance`` and ``record`` (the names of the search row's own columns).""" + return _make( + temp_db, + [ + "CREATE PROPERTY Doc.distance DOUBLE", + "CREATE PROPERTY Doc.record STRING", + ], + ", distance = ?, record = ?", + lambda i: (1000.5 + i, f"record-{i}"), + ) + + +def _same_record(hit, ref): + assert hit.get_rid() == ref.get_rid() + assert hit.get_type_name() == ref.get_type_name() + assert type(hit).__name__ == type(ref).__name__ + assert list(hit.get_property_names()) == list(ref.get_property_names()) + assert list(hit.to_dict()) == list(ref.to_dict()) + for name in ref.get_property_names(): + assert hit.has_property(name) + assert hit.get(name) == ref.get(name), name + + +@BOTH +def test_hits_equal_a_lookup_by_rid(hit_db, include_vector): + db, index, vecs = hit_db + seen_types = set() + for q in (0, 5, 40, 50, 60): + hits = index.find_nearest(vecs[q], k=7, include_vector=include_vector) + assert len(hits) == 7 + assert [d for _, d in hits] == sorted(d for _, d in hits) + for record, score in hits: + ref = db.lookup_by_rid(record.get_rid()) + seen_types.add(record.get_type_name()) + _same_record(record, ref) + assert seen_types == {"Doc", "SubDoc"} + + +def test_the_keyword_changes_no_answer(hit_db): + db, index, vecs = hit_db + for q in (1, 20, 55): + full = index.find_nearest(vecs[q], k=9) + lean = index.find_nearest(vecs[q], k=9, include_vector=False) + assert [(r.get_rid(), d) for r, d in full] == [ + (r.get_rid(), d) for r, d in lean + ] + assert [r.get("id") for r, _ in full] == [r.get("id") for r, _ in lean] + for (a, _), (b, _) in zip(full, lean): + assert list(a.get("embedding")) == list(b.get("embedding")) + + +def test_the_keyword_is_keyword_only_and_a_boolean_default(hit_db): + db, index, vecs = hit_db + with pytest.raises(TypeError): + index.find_nearest(vecs[0], 3, None, None, False) + assert index.find_nearest(vecs[0], k=3, include_vector=True) + + +def test_hits_without_the_vector_equal_a_lookup_when_properties_are_named_like_columns( + clash_db, +): + db, index, vecs = clash_db + for q in (0, 5, 40, 50): + for record, score in index.find_nearest(vecs[q], k=7, include_vector=False): + ref = db.lookup_by_rid(record.get_rid()) + _same_record(record, ref) + # the record's own distance property is not the search score + assert record.get("distance") == pytest.approx( + 1000.5 + record.get("id"), rel=0, abs=1e-9 + ) + assert record.get("record") == f"record-{record.get('id')}" + + +@pytest.mark.parametrize("first", ["vector", "has", "names", "dict", "other"]) +def test_first_access_does_not_change_what_the_record_holds(hit_db, first): + db, index, vecs = hit_db + record, _ = index.find_nearest(vecs[3], k=1, include_vector=False)[0] + ref = db.lookup_by_rid(record.get_rid()) + if first == "vector": + assert list(record.get("embedding")) == list(ref.get("embedding")) + elif first == "has": + assert record.has_property("embedding") + elif first == "names": + assert list(record.get_property_names()) == list(ref.get_property_names()) + elif first == "dict": + assert list(record.to_dict()) == list(ref.to_dict()) + else: + assert record.get("id") == ref.get("id") + _same_record(record, ref) + + +@BOTH +def test_a_hit_can_be_modified_and_saved(hit_db, include_vector): + db, index, vecs = hit_db + record, _ = index.find_nearest(vecs[7], k=1, include_vector=include_vector)[0] + rid = record.get_rid() + before = list(db.lookup_by_rid(rid).get("embedding")) + with db.transaction(): + record.modify().set("label", "changed").save() + after = db.lookup_by_rid(rid) + assert after.get("label") == "changed" + assert list(after.get("embedding")) == before + # the search still finds it, with the new label + again, _ = index.find_nearest(vecs[7], k=1, include_vector=include_vector)[0] + assert again.get_rid() == rid + assert again.get("label") == "changed" + + +@BOTH +def test_options_and_by_key_hits_are_full_records(hit_db, include_vector): + db, index, vecs = hit_db + wanted = [r.get_rid() for r, _ in index.find_nearest(vecs[1], k=12)][3:9] + many = [ + index.find_nearest(vecs[1], k=4, ef_search=64, include_vector=include_vector), + index.find_nearest( + vecs[1], k=4, allowed_rids=wanted, include_vector=include_vector + ), + index.find_nearest( + vecs[1], + k=4, + ef_search=64, + allowed_rids=wanted, + include_vector=include_vector, + ), + index.find_nearest_by_key(1, k=4), + ] + for hits in many: + assert len(hits) == 4 + for record, _ in hits: + _same_record(record, db.lookup_by_rid(record.get_rid())) + filtered = index.find_nearest( + vecs[1], k=4, allowed_rids=wanted, include_vector=include_vector + ) + assert {r.get_rid() for r, _ in filtered} <= set(wanted) From ae8b1efc5a78bcb9fcc5a393a25e9503c9d66c38 Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Thu, 5 Nov 2026 11:33:57 +0900 Subject: [PATCH 2/2] find_nearest include_vector: document the measured opt-in figures; compare hit distances like the query-form tests The "Hits without the vector" section of vectors.adoc now carries the timed A/B of the opt-in (native wheel 26.11.1.dev0, engine 26.10.2, k = 10, 12 clean pairs per figure): p50 B/A with 95% intervals for include_vector=False (hits not read, ids read, vector of every hit read) and for the unchanged default call, at 1536 and 256 dimensions. The figures are generated from the study's score file by a script, not typed. The text says the keyword is an opt-in for callers that do not read the vector (256 dimensions: reading every vector afterwards costs more than the default). test_the_keyword_changes_no_answer compared distances with ==, which the JVM build does not guarantee between two calls (main's _assert_same rule): ids compare exactly, distances exactly on the native build and to a few float32 ulps on the JVM build. Co-Authored-By: Claude Opus 5.5 --- docs/modules/ROOT/pages/how-to/vectors.adoc | 35 ++++++++++++++++++++- tests/test_vector_hit_records.py | 16 ++++++++-- 2 files changed, 47 insertions(+), 4 deletions(-) diff --git a/docs/modules/ROOT/pages/how-to/vectors.adoc b/docs/modules/ROOT/pages/how-to/vectors.adoc index 81f9f260..2b56a7d3 100644 --- a/docs/modules/ROOT/pages/how-to/vectors.adoc +++ b/docs/modules/ROOT/pages/how-to/vectors.adoc @@ -201,7 +201,40 @@ python === Hits without the vector -Every hit of `find_nearest` carries its indexed vector, so a search at 1536 dimensions ships and decodes 1536 numbers per hit whether you read them or not. If you need only the ids, the distances or other properties of the hits, pass `include_vector=False`. On the native build each hit is then built without the vector. It loads the vector, with the rest of the record, in one lookup by record id the first time you read it (`get` of the vector, `has_property`, `to_dict`, `get_property_names`, `modify` or `save`), so a loop that reads the vector of every hit pays one lookup per hit and does more work than the default. The values are the ones `db.lookup_by_rid` returns, and a hit deleted since the search raises `ArcadeDBError` on that first read. The arcadedb-jvm package accepts the argument and returns whole records either way. +Every hit of `find_nearest` carries its indexed vector, so a search at 1536 dimensions ships and decodes 1536 numbers per hit whether you read them or not. If you need only the ids, the distances or other properties of the hits, pass `include_vector=False`. On the native build each hit is then built without the vector. It loads the vector, with the rest of the record, in one lookup by record id the first time you read it (`get` of the vector, `has_property`, `to_dict`, `get_property_names`, `modify` or `save`). The values are the ones `db.lookup_by_rid` returns, and a hit deleted since the search raises `ArcadeDBError` on that first read. The arcadedb-jvm package accepts the argument and returns whole records either way, so the call costs the same there. + +This is an opt-in for callers that do not read the vector. The default stays `include_vector=True`: every hit carries its vector, and the default call is the one described above, unchanged. A loop that reads the vector of every hit after passing `include_vector=False` pays one lookup per hit and does more work than the default call, so do not pass it for that. + +The table gives the median `find_nearest` latency over the median of the default call before this change (95% interval, below 1 is faster), with k = 10 on 100,000 vectors per dimension count. It was measured on the native wheel `26.11.1.dev0` (the engine inside it is 26.10.2) on one Linux x86_64 laptop with 12 clean pairs per figure. The default call has not changed: its intervals lie between 0.993 and 1.011. With the keyword set, a search whose hits are not read, or whose ids are read, is faster at both sizes. If every vector is read afterwards, the call takes 1.0 [0.999, 1.004] times as long as the default call at 1536 dimensions and 1.348 [1.34, 1.352] times as long at 256 dimensions. + +[cols="3,2,2",options="header"] +|=== +| Call | 1536 dimensions | 256 dimensions + +| `include_vector=False`, hits not read +| 0.637 [0.633, 0.641] +| 0.8 [0.793, 0.818] + +| `include_vector=False`, hit ids read +| 0.637 [0.633, 0.638] +| 0.797 [0.793, 0.803] + +| `include_vector=False`, then the vector of every hit read +| 1.0 [0.999, 1.004] +| 1.348 [1.34, 1.352] + +| the default call, hits not read (not changed) +| 1.0 [0.996, 1.002] +| 1.005 [0.996, 1.011] + +| the default call, hit ids read (not changed) +| 1.001 [0.997, 1.005] +| 1.002 [0.994, 1.005] + +| the default call, the vector of every hit read (not changed) +| 0.997 [0.995, 1.005] +| 0.997 [0.993, 1.007] +|=== [source,python] ---- diff --git a/tests/test_vector_hit_records.py b/tests/test_vector_hit_records.py index f8774fcd..304ac6b2 100644 --- a/tests/test_vector_hit_records.py +++ b/tests/test_vector_hit_records.py @@ -9,6 +9,8 @@ import numpy as np import pytest +import arcadedb + DIM = 8 N_DOC = 48 N_SUB = 16 @@ -62,6 +64,15 @@ def clash_db(temp_db): ) +def _assert_same_distances(got, expected): + """Exactly on the native build; to a few float32 ulps on the JVM build, whose repeated calls can differ in the last float32 bit + (the rule of ``_assert_same`` in ``test_find_nearest_query_forms.py``).""" + if arcadedb.__backend__ == "native": + assert got == expected + else: + assert got == pytest.approx(expected, rel=1e-6, abs=1e-9) + + def _same_record(hit, ref): assert hit.get_rid() == ref.get_rid() assert hit.get_type_name() == ref.get_type_name() @@ -93,9 +104,8 @@ def test_the_keyword_changes_no_answer(hit_db): for q in (1, 20, 55): full = index.find_nearest(vecs[q], k=9) lean = index.find_nearest(vecs[q], k=9, include_vector=False) - assert [(r.get_rid(), d) for r, d in full] == [ - (r.get_rid(), d) for r, d in lean - ] + assert [r.get_rid() for r, _ in full] == [r.get_rid() for r, _ in lean] + _assert_same_distances([d for _, d in full], [d for _, d in lean]) assert [r.get("id") for r, _ in full] == [r.get("id") for r, _ in lean] for (a, _), (b, _) in zip(full, lean): assert list(a.get("embedding")) == list(b.get("embedding"))