Native point operations: cut the repeated Python work around the one crossing

src/arcadedb/_native/codec.py

@@ -201,6 +201,21 @@ def encode_params(args, binary=False):
         return (text, None, 0) if binary else text
     if len(values) == 1 and isinstance(values[0], Mapping):
         named = values[0]
+        if _fast:
+            # every value a plain scalar or an RID (a named point read, the record id of a step or an edge end): no float side buffer
+            # can arise, so the cached encoder writes the text, with an RID as the tagged object the hook would have returned. The
+            # text is the same bytes; what is skipped is _prep, the hook closure and the per-call encoder object.
+            out = {}
+            for k, v in named.items():
+                t = type(v)
+                if t is RID:
+                    v = {TAG: ["R", v.text]}
+                elif t not in plain:
+                    break
+                out[k] = v
+            else:
+                text = _dumps({"named": out})
+                return (text, None, 0) if binary else text
         if not all(type(v) in plain for v in named.values()):
             named = {k: _prep(v, binary) for k, v in named.items()}
         return _dump({"named": dict(named)}, binary)
@@ -209,6 +224,16 @@ def encode_params(args, binary=False):
     return _dump({"pos": list(values)}, binary)
 
 
+def encode_lookup(keys, values):
+    """The JSON text ``arcadedb_n_lookup`` takes, ``{"keys": [..], "values": [..]}`` in the tagged encoding of query parameters
+    (the text ``json.dumps(_prep(obj), default=_default)`` gives). Plain values are written by the cached encoder: for them
+    ``_prep`` returns its argument and the hook is never called."""
+    obj = {"keys": [str(k) for k in keys], "values": list(values)}
+    if _fast and set(map(type, obj["values"])) <= _PLAIN:
+        return _dumps(obj)
+    return json.dumps(_prep(obj), default=_default)
+
+
 def _dump(obj, binary):
     """JSON text; binary mode returns ``(text, float32 side buffer bytes or None, float count)``."""
     if not binary:

src/arcadedb/_native/database.py

@@ -16,7 +16,14 @@
 from . import interrupts, lib, needs_jvm
 from .bulk import insert_rows_columnar
 from .codec import _prep as _prep_row
-from .codec import encode_params, json_native, loads, loads_tagged, row_default
+from .codec import (
+    encode_lookup,
+    encode_params,
+    json_native,
+    loads,
+    loads_tagged,
+    row_default,
+)
 
 _LAZY_COMMAND_ROWS = os.environ.get("ARCADEDB_NATIVE_LAZY_CMD", "1") != "0"
 _BINARY_REPLY = os.environ.get("ARCADEDB_NATIVE_BIN_REPLY", "1") != "0"
@@ -73,6 +80,29 @@ def _call(prefix: str, name: str, *args) -> dict:
 _TX_IDLE = '{"ok":true,"active":false,'
 
 
+def _first_row(database, reply: dict):
+    """``NativeResultSet(database, reply).first()`` for a reply that holds the whole result: the first row as a ``NativeResult``, with no
+    cursor object built to hand out one row and be closed. A reply with more rows to fetch, or with a pending read error, takes the cursor.
+    """
+    rows = reply["rows"]
+    if rows and reply["drained"] and not reply.get("rerr") and not database._closed:
+        return NativeResult(rows[0], reply["meta"][0], database)
+    return NativeResultSet(database, reply).first()
+
+
+def _element_of(database, row):
+    """``row.get_element()`` for a row that only the caller holds: the record takes the row's properties as they are (no copy)."""
+    global _wrap_record
+    if row is None:
+        return None
+    if _wrap_record is None:
+        from .records import _wrap as _wrap_record
+    row._check_open()
+    if not row._meta or row._meta[2] not in "vde":
+        return None
+    return _wrap_record(database, row, True)
+
+
 class NativeResult:
     """One row. Values are already Python types (datetime, Decimal, lists, dicts, RID text)."""
 
@@ -287,6 +317,19 @@ def __next__(self) -> NativeResult:
             self._finish()
         raise StopIteration
 
+    def _drain_pairs(self) -> list:
+        """The ``(props, meta)`` of every row, read as :meth:`_drain_dicts` reads them, for a caller that builds its own objects."""
+        out: list = []
+        if not self._readable():
+            return out
+        while True:
+            out.extend(self._buf)
+            self._buf.clear()
+            if not self._fill():
+                break
+        self._finish()
+        return out
+
     def _drain_dicts(self) -> List[Dict[str, Any]]:
         out: List[Dict[str, Any]] = []
         if not self._readable():
@@ -799,7 +842,7 @@ def lookup_by_rid(self, rid: str):
         row = self.query("sql", rid_sql).first()
         if row is None:
             raise ArcadeDBError(f"Failed to lookup RID '{rid}': record not found")
-        return row.get_element()
+        return _element_of(self, row)
 
     def lookup_by_key(self, type_name: str, keys: List[str], values: List[Any]):
         """The first record of ``type_name`` whose ``keys`` properties equal ``values``, wrapped, or None.
@@ -809,20 +852,17 @@ def lookup_by_key(self, type_name: str, keys: List[str], values: List[Any]):
         (a scan is ``query()`` with a ``WHERE``). No statement has that rule, hence this op and not a ``SELECT``.
         """
         self._check_not_closed()
-        from .extras import _enc
-
         reply = loads(
             lib.call(
                 "arcadedb_n_lookup",
                 self._h,
                 lib.b(type_name),
-                lib.b(_enc({"keys": [str(k) for k in keys], "values": list(values)})),
+                lib.b(encode_lookup(keys, values)),
             )
         )
         if not reply.get("ok"):
             raise _error(f"Failed to lookup by key in '{type_name}'", reply)
-        row = NativeResultSet(self, reply).first()
-        return row.get_element() if row is not None else None
+        return _element_of(self, _first_row(self, reply))
 
     def insert_many(
         self,

src/arcadedb/_native/records.py

@@ -226,14 +226,24 @@ class _Mixin:
     _DELETE = "DELETE FROM :rid"
 
     def __init__(
-        self, database, type_name, rid=None, props=None, mutable=False, base=None
+        self,
+        database,
+        type_name,
+        rid=None,
+        props=None,
+        mutable=False,
+        own=False,
+        base=None,
     ):
         self._java_document = _java_object()
         self._database = database
         self._db = database
         self._type = type_name
         self._rid = rid or None
-        self._props: Dict[str, Any] = dict(props or {})
+        # own: the caller built props for this record and keeps no other reference (the row of a lookup), so the dict is taken as it is
+        self._props: Dict[str, Any] = (
+            props if own and props is not None else dict(props or {})
+        )
         self._mutable = mutable or not rid
         # a copy of the values this record held as read (None: not taken yet). A read does not take it: modify(), the first set() of a saved
         # record or save() does, from the values then held, which are the read-time values. save() refuses to overwrite a changed record.
@@ -431,10 +441,12 @@ def new_edge(self, label: str, target: "Vertex", **kwargs) -> "Edge":
 
     def _edges(self, fn: str, labels) -> List["Edge"]:
         self._check_open()
-        args = ", ".join("'" + label.replace("'", "\\'") + "'" for label in labels)
-        edges_sql = f"SELECT expand({fn}({args})) FROM :rid"  # nosec B608 - fn is a fixed function name, labels are escaped literals, the record id is a parameter
+        edges_sql = _edges_sql(fn, labels)
         rows = self._db.query("sql", edges_sql, {"rid": RID(self._rid)})
-        return [_wrap(self._db, row) for row in rows]
+        database = self._db
+        return [
+            _wrap_meta(database, props, meta) for props, meta in rows._drain_pairs()
+        ]
 
     def get_out_edges(self, *labels: str) -> List["Edge"]:
         return self._edges("outE", labels)
@@ -450,9 +462,8 @@ class Edge(_Mixin, _BaseEdge):
 
     def _end(self, fn: str) -> "Vertex":
         self._check_open()
-        end_sql = f"SELECT expand({fn}()) FROM :rid"  # nosec B608 - fn is a fixed function name, the record id is a parameter
-        row = self._db.query("sql", end_sql, {"rid": RID(self._rid)}).first()
-        return _wrap(self._db, row)
+        row = self._db.query("sql", _END_SQL[fn], {"rid": RID(self._rid)}).first()
+        return _wrap(self._db, row, True)
 
     def get_in(self) -> Vertex:
         return self._end("inV")
@@ -463,14 +474,48 @@ def get_out(self) -> Vertex:
 
 _CLASS_OF = {"v": Vertex, "e": Edge}
 
+# The statement of a step or an edge end is built once per function and label list, not once per call: the same text object goes to the
+# engine every time (it keys its statement cache by text). The label list is bounded so a program that builds labels cannot grow it.
+_END_SQL: Dict[str, str] = {
+    fn: f"SELECT expand({fn}()) FROM :rid"  # nosec B608 - fn is a fixed function name, the record id is a parameter
+    for fn in ("inV", "outV")
+}
+_EDGES_SQL: Dict[Any, str] = {}
+_EDGES_SQL_MAX = 512
+
+
+def _edges_sql(fn: str, labels) -> str:
+    key = (fn, labels)
+    try:
+        sql = _EDGES_SQL.get(key)
+    except TypeError:
+        sql = None  # a label that cannot be hashed: building the text below fails as it always did
+    if sql is None:
+        args = ", ".join("'" + label.replace("'", "\\'") + "'" for label in labels)
+        sql = f"SELECT expand({fn}({args})) FROM :rid"  # nosec B608 - fn is a fixed function name, labels are escaped literals, the record id is a parameter
+        if len(_EDGES_SQL) < _EDGES_SQL_MAX:
+            _EDGES_SQL[key] = sql
+    return sql
+
+
+def _wrap(database, row, own=False):
+    """The wrapper for a result row that is a record (vertex, edge or document); None for a projection.
 
-def _wrap(database, row):
-    """The wrapper for a result row that is a record (vertex, edge or document); None for a projection."""
+    ``own``: nobody else holds the row, so the record takes its property dict as it is instead of a copy.
+    """
     meta = row._meta
     if not meta:
         return None
     rid, type_name, kind = meta
-    return _CLASS_OF.get(kind, Document)(database, type_name, rid, row._props)
+    return _CLASS_OF.get(kind, Document)(database, type_name, rid, row._props, own=own)
+
+
+def _wrap_meta(database, props, meta):
+    """:func:`_wrap` for the ``(props, meta)`` pair of a row the caller drained itself (nothing else holds the dict)."""
+    if not meta:
+        return None
+    rid, type_name, kind = meta
+    return _CLASS_OF.get(kind, Document)(database, type_name, rid, props, own=True)
 
 
 def _wrap_dict(database, record: dict, kind: str = "d"):

tests/conftest.py

@@ -75,6 +75,8 @@
     collect_ignore.append("test_native_statement_texts.py")
     # iter_json_batches reads the cursor one batch at a time (the result set's buffer and its _fill are observed)
     collect_ignore.append("test_native_streaming_batches.py")
+    # the Python part of the native point operations (lookup_by_key, edge ends, steps): spies on the codec, the cursor and the row object
+    collect_ignore.append("test_native_point_op_glue.py")
 if not os.path.isfile(
     os.path.join(os.path.dirname(os.path.dirname(__file__)), "docs", "index.md")
 ):