Native find_nearest: include_vector=False returns hits without the indexed vector (explicit opt-in)

docs/modules/ROOT/pages/how-to/vectors.adoc

@@ -162,7 +162,7 @@ print([row["name"] for row in rows])
 [[find-nearest]]
 === find_nearest
 
-`VectorIndex.find_nearest(query_vector, k=10, ef_search=None, allowed_rids=None)` searches without a statement. It takes a list or a NumPy array and returns a list of `(record, distance)` tuples, nearest first. Use it when you already hold the index object, or when you have a list of record ids to search within.
+`VectorIndex.find_nearest(query_vector, k=10, ef_search=None, allowed_rids=None, *, include_vector=True)` searches without a statement. It takes a list or a NumPy array and returns a list of `(record, distance)` tuples, nearest first. Use it when you already hold the index object, or when you have a list of record ids to search within.
 
 On the native package the query vector crosses as one `float32` buffer whatever its form (a list, a `float64` array, or a `float32` array), so you do not need to convert it first. A vector index stores `float32`, so the numbers are rounded the same way as before. The table gives the median `find_nearest` latency with this change over the median without it (95% interval, below 1 is faster), measured on the native wheel `26.11.1.dev0` on one Linux x86_64 laptop with 11 to 12 clean pairs per figure. The wheel built before this change was the baseline.
 
@@ -199,6 +199,55 @@ python-ish
 python
 ----
 
+=== Hits without the vector
+
+Every hit of `find_nearest` carries its indexed vector, so a search at 1536 dimensions ships and decodes 1536 numbers per hit whether you read them or not. If you need only the ids, the distances or other properties of the hits, pass `include_vector=False`. On the native build each hit is then built without the vector. It loads the vector, with the rest of the record, in one lookup by record id the first time you read it (`get` of the vector, `has_property`, `to_dict`, `get_property_names`, `modify` or `save`). The values are the ones `db.lookup_by_rid` returns, and a hit deleted since the search raises `ArcadeDBError` on that first read. The arcadedb-jvm package accepts the argument and returns whole records either way, so the call costs the same there.
+
+This is an opt-in for callers that do not read the vector. The default stays `include_vector=True`: every hit carries its vector, and the default call is the one described above, unchanged. A loop that reads the vector of every hit after passing `include_vector=False` pays one lookup per hit and does more work than the default call, so do not pass it for that.
+
+The table gives the median `find_nearest` latency over the median of the default call before this change (95% interval, below 1 is faster), with k = 10 on 100,000 vectors per dimension count. It was measured on the native wheel `26.11.1.dev0` (the engine inside it is 26.10.2) on one Linux x86_64 laptop with 12 clean pairs per figure. The default call has not changed: its intervals lie between 0.993 and 1.011. With the keyword set, a search whose hits are not read, or whose ids are read, is faster at both sizes. If every vector is read afterwards, the call takes 1.0 [0.999, 1.004] times as long as the default call at 1536 dimensions and 1.348 [1.34, 1.352] times as long at 256 dimensions.
+
+[cols="3,2,2",options="header"]
+|===
+| Call | 1536 dimensions | 256 dimensions
+
+| `include_vector=False`, hits not read
+| 0.637 [0.633, 0.641]
+| 0.8 [0.793, 0.818]
+
+| `include_vector=False`, hit ids read
+| 0.637 [0.633, 0.638]
+| 0.797 [0.793, 0.803]
+
+| `include_vector=False`, then the vector of every hit read
+| 1.0 [0.999, 1.004]
+| 1.348 [1.34, 1.352]
+
+| the default call, hits not read (not changed)
+| 1.0 [0.996, 1.002]
+| 1.005 [0.996, 1.011]
+
+| the default call, hit ids read (not changed)
+| 1.001 [0.997, 1.005]
+| 1.002 [0.994, 1.005]
+
+| the default call, the vector of every hit read (not changed)
+| 0.997 [0.995, 1.005]
+| 0.997 [0.993, 1.007]
+|===
+
+[source,python]
+----
+for record, distance in index.find_nearest(query, k=2, include_vector=False):
+    print(record.get("name"))
+----
+
+[source,text]
+----
+python-ish
+python
+----
+
 === Filtered search
 
 `allowed_rids` restricts `find_nearest` to the records whose ids you list, as strings. The search considers only those records, so it still returns `k` of them.

src/arcadedb/_native/records.py

@@ -250,9 +250,17 @@ def __init__(
         # The dict is never changed in place (a save makes a new one), so the objects modify() hands it to can share it
         self._base = base
         self._property_names_cache = None
+        # names of properties a search hit was built without (see ``_wrap_partial``); None for every other record
+        self._lazy = None
 
     # -- reading
 
+    def _fill(self) -> None:
+        """Load the properties a search hit was built without, in one lookup by RID (engine order); nothing to do for any other record."""
+        if self._lazy:
+            self._props = dict(self._db.lookup_by_rid(self._rid)._props)
+            self._lazy = None
+
     def _check_open(self) -> None:
         database = self._database
         if database is not None and database._closed and self._rid:
@@ -263,21 +271,27 @@ def _check_open(self) -> None:
 
     def get(self, name: str, convert_types: bool = True) -> Any:
         self._check_open()
+        if self._lazy and name in self._lazy:
+            self._fill()
         return self._props.get(name)
 
     def get_raw(self, name: str) -> Any:
         return self.get(name)
 
     def has_property(self, name: str) -> bool:
         self._check_open()
+        if self._lazy and name in self._lazy:
+            self._fill()
         return name in self._props
 
     def get_property_names(self) -> List[str]:
         self._check_open()
+        self._fill()
         return list(self._props)
 
     def to_dict(self, convert_types: bool = True) -> Dict[str, Any]:
         self._check_open()
+        self._fill()
         return dict(self._props)
 
     def get_rid(self) -> str:
@@ -318,6 +332,7 @@ def save(self):
         not seen. Saving an existing record needs an active transaction, like every write; outside one it raises ArcadeDBError
         ("Transaction not begun"). Creating a new record is not checked.
         """
+        self._fill()
         db = self._db
         if self._rid is None:
             from .extras import veto_count
@@ -399,6 +414,7 @@ def delete(self):
 
     def modify(self):
         self._check_open()
+        self._fill()
         if self._rid and self._base is None:
             self._base = _snap(
                 self._props
@@ -525,3 +541,15 @@ def _wrap_dict(database, record: dict, kind: str = "d"):
     type_name = props.pop("@type", None)
     kind = props.pop("@cat", kind)
     return _CLASS_OF.get(kind, Document)(database, type_name, rid, props)
+
+
+def _wrap_partial(database, rid, type_name, kind, props, omitted):
+    """The wrapper for a search hit built without the properties named in ``omitted`` (the indexed vector: not shipped with every hit).
+
+    Reading one of them, or anything that needs the whole record (``to_dict``, ``get_property_names``, ``modify``, ``save``), loads the
+    record once by RID; ``get`` and ``has_property`` of any other property are answered from what was shipped.
+    """
+    cls = {"v": Vertex, "e": Edge}.get(kind, Document)
+    record = cls(database, type_name, rid, props)
+    record._lazy = frozenset(omitted)
+    return record

src/arcadedb/_native/schema_objects.py

@@ -193,6 +193,18 @@ def _query_vector_param(query_vector):
     )
 
 
+def _quote_name(name: str) -> str:
+    return "`" + name.replace("`", "``") + "`"
+
+
+# find_nearest(include_vector=False): the engine's row of a hit is the record's properties plus the columns @rid, @type and distance
+# (ScoredRecordRow; before 26.11.1, ArcadeData/arcadedb#10898, also a nested `record` column holding every property a second time).
+# `*` copies the row, `!name` drops the indexed vector and the `record` column. A property of the record called distance or record is
+# hidden by those columns in the row, so such a hit holds them back too (_LAZY_COLUMNS) and loads them with the vector.
+_LAZY_COLUMNS = ("record", "distance")
+_HITS_FROM = "FROM (SELECT expand(vectorNeighbors(?, ?, ?)))"
+
+
 class NativeVectorIndex(_BaseVectorIndex):
     """``VectorIndex`` over ``vectorNeighbors``: ``find_nearest`` returns ``[(record, distance)]`` like the JPype wrapper."""
 
@@ -210,10 +222,21 @@ def _check_live(self):
         return
 
     def _run_search(
-        self, *, query_vector, k, allowed_rids, approximate, ef_search=None
+        self,
+        *,
+        query_vector,
+        k,
+        allowed_rids,
+        approximate,
+        ef_search=None,
+        include_vector=True,
     ):
         if approximate:
             return self._run_approximate(query_vector, k, allowed_rids)
+        if not include_vector:
+            return self._run_search_without_vector(
+                query_vector, k, allowed_rids, ef_search
+            )
         from .records import _wrap_dict
 
         vector = _query_vector_param(query_vector)
@@ -257,6 +280,54 @@ def _run_search(
             out.append((record, float(row.get("distance"))))
         return self._sort_results(out)[:k]
 
+    def _run_search_without_vector(self, query_vector, k, allowed_rids, ef_search):
+        """``find_nearest(include_vector=False)``: the hits come back without the indexed vector, which loads on first use.
+
+        ``expand(vectorNeighbors(..))`` alone ships every hit's vector, and the engine reads each record to build the row whatever
+        the columns asked for. Here the statement drops the vector (and the nested ``record`` column of an engine before 26.11.1),
+        and ``_Mixin._fill`` loads the record, once, by RID, the first time the vector or the whole record is needed.
+        """
+        from .records import _wrap_partial
+
+        vector = _query_vector_param(query_vector)
+        options = {}
+        if ef_search is not None:
+            options["efSearch"] = int(ef_search)
+        if allowed_rids:
+            options["filter"] = [_rid_text(r) for r in allowed_rids]
+        prop = self._shim._prop
+        dropped = ", ".join(
+            "!" + _quote_name(name) for name in dict.fromkeys((prop, "record"))
+        )
+        sql = f"SELECT *, {dropped} {_HITS_FROM}"  # nosec B608 - dropped is backtick-quoted schema names, the values go through ? parameters
+        if options:
+            sql = sql.replace("vectorNeighbors(?, ?, ?)", "vectorNeighbors(?, ?, ?, ?)")
+            rows = self._database.query(
+                "sql", sql, self._shim._name, vector, int(k), options
+            )
+        else:
+            rows = self._database.query("sql", sql, self._shim._name, vector, int(k))
+        omitted = (prop, *_LAZY_COLUMNS)
+        out = []
+        for row in rows:
+            props = row.to_dict()
+            rid = props.pop("@rid", None)
+            type_name = props.pop("@type", None)
+            distance = props.pop("distance")
+            if rid is None:
+                record = None
+            else:
+                record = _wrap_partial(
+                    self._database,
+                    _rid_text(rid),
+                    type_name,
+                    self._database._kind_of(type_name),
+                    props,
+                    omitted,
+                )
+            out.append((record, float(distance)))
+        return self._sort_results(out)[:k]
+
     def _run_approximate(self, query_vector, k, allowed_rids):
         """The PQ-scored search of the engine (no statement runs it): the engine returns (RID, distance), the records are loaded here."""
         from . import lib

src/arcadedb/vector.py

@@ -566,6 +566,7 @@ def _run_search(
         allowed_rids,
         approximate,
         ef_search=None,
+        include_vector=True,  # accepted for API parity, the JVM build loads whole records
     ):
         java_vector = to_java_float_array(query_vector)
         allowed_rids_set = self._build_allowed_rids_set(allowed_rids)
@@ -584,6 +585,8 @@ def find_nearest(
         k=10,
         ef_search=None,
         allowed_rids=None,
+        *,
+        include_vector=True,
     ):
         """
         Find k nearest neighbors to the query vector.
@@ -605,16 +608,31 @@ def find_nearest(
                 uses the Java index default/adaptive behavior.
             allowed_rids: Optional list of RID strings (e.g. ["#1:0", "#2:5"]) to
                 restrict search
+            include_vector: Keyword only, default ``True``: every hit carries its
+                indexed vector, as always. Pass ``False`` when you need only the
+                ids or the distances (or other properties) of the hits. On the
+                native build each hit is then built without the indexed vector, and
+                loads it, with the rest of the record, in one lookup by RID the
+                first time you read it (``get`` of the vector, ``has_property``,
+                ``to_dict``, ``get_property_names``, ``modify``, ``save``); the
+                values are those of ``lookup_by_rid``, and a hit deleted since the
+                search raises ``ArcadeDBError`` on that first read. The JVM build
+                accepts the argument and ignores it: it loads whole records either
+                way, so the call costs the same.
 
         Returns:
             List of tuples: [(record, score), ...]
             - record: The matching ArcadeDB record (Vertex, Document, or Edge)
             - score: The similarity score/distance
         """
         self._check_live()
-        return self._find_nearest_live(query_vector, k, ef_search, allowed_rids)
+        return self._find_nearest_live(
+            query_vector, k, ef_search, allowed_rids, include_vector
+        )
 
-    def _find_nearest_live(self, query_vector, k, ef_search, allowed_rids):
+    def _find_nearest_live(
+        self, query_vector, k, ef_search, allowed_rids, include_vector=True
+    ):
         # find_nearest after _check_live(): find_nearest_by_key checks once and
         # comes here, so a call never pays the isValid() crossing twice.
         effective_ef_search = self._normalize_ef_search(ef_search)
@@ -626,6 +644,7 @@ def _find_nearest_live(self, query_vector, k, ef_search, allowed_rids):
                 allowed_rids=allowed_rids,
                 approximate=False,
                 ef_search=effective_ef_search,
+                include_vector=include_vector,
             )
 
         except ArcadeDBError:

tests/conftest.py

@@ -43,6 +43,8 @@
     collect_ignore.append("test_native_udf.py")
     # how find_nearest hands the query vector to the native library (the codec's side buffer)
     collect_ignore.append("test_native_find_nearest_params.py")
+    # find_nearest(include_vector=False) on the native wheel fills a hit's vector on first use (counts lookups of the native wrapper)
+    collect_ignore.append("test_native_vector_hit_lazy.py")
     # the columnar insert's stored records are compared between the JSON and the columnar entry points of the native library
     collect_ignore.append("test_native_columnar_insert.py")
     # the rows of an insert_many chunk with date-times, Decimals, bytes and numpy scalars are told apart by the entry points they cross