From 545eef379c0233b53272ec69ed39635f057b118a Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Mon, 26 Oct 2026 21:03:02 +0900 Subject: [PATCH 1/2] Native lookup_by_key raises without an index, as the JVM build does Database.lookup_by_key on the native build ran a SELECT ... WHERE, which scans the type when no index serves it, so the same call returned a record on the arcadedb wheel and raised on the arcadedb-jvm wheel (the engine's lookupByKey needs an index and throws IllegalArgumentException). The native build now checks schema:indexes first, with the engine's rule: an LSM_TREE or HASH index on exactly the given properties, in the index's order, on the type or a supertype. Otherwise it raises ArcadeDBError with the engine's message (No index has been created on type ... properties [...], or No key index ... for a FULL_TEXT index). An unknown type still raises the not-found error of the SELECT. The docstring of the JVM method now states the contract. tests/test_lookup_by_key_index.py runs on both backends. Co-Authored-By: Claude Sonnet 5.5 --- src/arcadedb/_native/database.py | 58 +++++++++++++- src/arcadedb/core.py | 9 ++- tests/test_lookup_by_key_index.py | 125 ++++++++++++++++++++++++++++++ 3 files changed, 190 insertions(+), 2 deletions(-) create mode 100644 tests/test_lookup_by_key_index.py diff --git a/src/arcadedb/_native/database.py b/src/arcadedb/_native/database.py index 5463d08b..abc50633 100644 --- a/src/arcadedb/_native/database.py +++ b/src/arcadedb/_native/database.py @@ -679,10 +679,66 @@ def lookup_by_rid(self, rid: str): raise ArcadeDBError(f"Failed to lookup RID '{rid}': record not found") return row.get_element() + # The index kinds that answer "the value equals the key" (the engine's Schema.INDEX_TYPE.isExactKeyLookup()); a FULL_TEXT, vector + # or geospatial index on the same properties does not. + _EXACT_KEY_INDEX_TYPES = frozenset({"LSM_TREE", "HASH"}) + + def _require_key_index(self, type_name: str, keys: List[str]) -> None: + """Raise unless ``type_name`` or a supertype has an exact-key index on exactly ``keys``, as the engine's ``lookupByKey`` does. + + Without this check the ``SELECT`` of ``lookup_by_key`` scans the type when no index serves it, so the call returns a + record on this build and raises on the JVM build. A type that does not exist is left to that ``SELECT``, which names it. + """ + from .schema_objects import NativeIndex + + wanted = [str(k) for k in keys] + listed = "[" + ", ".join(wanted) + "]" + other = ( + None # an index on these properties that cannot answer an exact-key lookup + ) + pending, seen, first = [type_name], set(), True + while pending: + name = pending.pop() + if name in seen: + continue + seen.add(name) + for row in self.query( + "sql", "SELECT FROM schema:indexes WHERE typeName = ?", name + ).to_list(): + index = NativeIndex(row) + if index.getPropertyNames() != wanted: + continue + if str(index.getType()) in self._EXACT_KEY_INDEX_TYPES: + return + other = other or index + types = self.query( + "sql", "SELECT parentTypes FROM schema:types WHERE name = ?", name + ).to_list() + if not types: + if first: + return # no such type: the SELECT raises the engine's own error + continue + first = False + pending.extend(types[0].get("parentTypes") or []) + if other is not None: + raise ArcadeDBError( + f"No key index has been created on type '{type_name}' properties {listed}: " + f"index '{other.getName()}' is {other.getType()} and cannot look up a record by exact key" + ) + raise ArcadeDBError( + f"No index has been created on type '{type_name}' properties {listed}" + ) + def lookup_by_key(self, type_name: str, keys: List[str], values: List[Any]): + """The first record of ``type_name`` whose ``keys`` properties equal ``values``, wrapped, or None. + + Index-based, as on the JVM build: ``keys`` must be exactly the properties of a LSM_TREE or HASH index of the type or one of + its supertypes, in the index's order. Anything else raises ``ArcadeDBError`` (a scan is ``query()`` with a ``WHERE``). + """ self._check_not_closed() - where = " AND ".join(f"`{k}` = ?" for k in keys) try: + self._require_key_index(type_name, keys) + where = " AND ".join(f"`{k}` = ?" for k in keys) lookup_sql = f"SELECT FROM `{type_name}` WHERE {where} LIMIT 1" # nosec B608 - backtick-quoted identifiers; values go through ? parameters row = self.query("sql", lookup_sql, list(values)).first() return row.get_element() if row is not None else None diff --git a/src/arcadedb/core.py b/src/arcadedb/core.py index b828ea6d..c00ae235 100644 --- a/src/arcadedb/core.py +++ b/src/arcadedb/core.py @@ -1320,12 +1320,19 @@ def lookup_by_key(self, type_name: str, keys: List[str], values: List[Any]): Args: type_name: Type name - keys: List of property names (must be indexed) + keys: List of property names. They must be exactly the properties of + a LSM_TREE or HASH index of the type or one of its supertypes, in + the index's order; a FULL_TEXT or vector index does not count values: List of property values Returns: The first matching record, wrapped, or None + Raises: + ArcadeDBError: If no such index exists (the call never scans the type; + use ``query()`` with a ``WHERE`` for that), or the type does not exist. + The native and the JVM build raise alike. + Example: >>> user = db.lookup_by_key("User", ["email"], ["alice@example.com"]) >>> if user is not None: diff --git a/tests/test_lookup_by_key_index.py b/tests/test_lookup_by_key_index.py new file mode 100644 index 00000000..abc0f329 --- /dev/null +++ b/tests/test_lookup_by_key_index.py @@ -0,0 +1,125 @@ +"""lookup_by_key() needs an index on exactly those properties, on both backends. + +The engine's lookupByKey answers from an index only. It raises when the type has none on the given property list (in that order), when +the one it has is a FULL_TEXT or vector index, or when the type does not exist. The JVM backend gets that from the engine. The +native backend used to run a ``SELECT ... WHERE`` instead, which scans the type without an index: the same call returned a record on +one wheel and raised on the other. Both raise the same message now, so a program does not depend on which wheel runs it. +""" + +import pytest + +from arcadedb import ArcadeDBError + + +@pytest.fixture +def docs(temp_db): + """Type D with an unindexed integer ``k``, two more properties ``a`` and ``b``, and a text property ``t``; one record.""" + temp_db.command("sql", "CREATE DOCUMENT TYPE D") + for prop, kind in (("k", "INTEGER"), ("a", "INTEGER"), ("b", "INTEGER")): + temp_db.command("sql", f"CREATE PROPERTY D.{prop} {kind}") + temp_db.command("sql", "CREATE PROPERTY D.t STRING") + with temp_db.transaction(): + temp_db.command( + "sql", "INSERT INTO D SET k = 1, a = 10, b = 20, t = 'hello world'" + ) + return temp_db + + +def test_lookup_on_an_unindexed_key_raises(docs): + with pytest.raises( + ArcadeDBError, + match=r"Failed to lookup by key in 'D'.*No index has been created on type 'D' properties \[k\]", + ): + docs.lookup_by_key("D", ["k"], [1]) + + +def test_lookup_on_an_unindexed_key_raises_also_without_a_match(docs): + with pytest.raises(ArcadeDBError, match="No index has been created"): + docs.lookup_by_key("D", ["k"], [99]) + + +@pytest.mark.parametrize( + "index_kind", ["UNIQUE", "NOTUNIQUE", "UNIQUE_HASH", "NOTUNIQUE_HASH"] +) +def test_lookup_works_once_the_key_is_indexed(docs, index_kind): + with pytest.raises(ArcadeDBError, match="No index has been created"): + docs.lookup_by_key("D", ["k"], [1]) + docs.command("sql", f"CREATE INDEX ON D (k) {index_kind}") + assert docs.lookup_by_key("D", ["k"], [1]).get("a") == 10 + assert docs.lookup_by_key("D", ["k"], [99]) is None + + +def test_lookup_raises_again_after_the_index_is_dropped(docs): + docs.command("sql", "CREATE INDEX ON D (k) UNIQUE") + assert docs.lookup_by_key("D", ["k"], [1]) is not None + docs.command("sql", "DROP INDEX `D[k]`") + with pytest.raises(ArcadeDBError, match="No index has been created"): + docs.lookup_by_key("D", ["k"], [1]) + + +def test_an_index_on_another_property_does_not_serve_the_lookup(docs): + docs.command("sql", "CREATE INDEX ON D (a) UNIQUE") + with pytest.raises(ArcadeDBError, match=r"properties \[k\]"): + docs.lookup_by_key("D", ["k"], [1]) + + +def test_a_composite_index_serves_exactly_its_property_list(docs): + docs.command("sql", "CREATE INDEX ON D (a, b) NOTUNIQUE") + assert docs.lookup_by_key("D", ["a", "b"], [10, 20]).get("k") == 1 + assert docs.lookup_by_key("D", ["a", "b"], [10, 99]) is None + # a prefix of the index, and the same properties in the other order, are not that index + with pytest.raises(ArcadeDBError, match=r"properties \[a\]"): + docs.lookup_by_key("D", ["a"], [10]) + with pytest.raises(ArcadeDBError, match=r"properties \[b, a\]"): + docs.lookup_by_key("D", ["b", "a"], [20, 10]) + + +def test_a_full_text_index_does_not_serve_an_exact_key_lookup(docs): + docs.command("sql", "CREATE INDEX ON D (t) FULL_TEXT") + with pytest.raises( + ArcadeDBError, + match=r"No key index has been created on type 'D' properties \[t\].*FULL_TEXT", + ): + docs.lookup_by_key("D", ["t"], ["hello"]) + + +def test_a_subtype_is_served_by_the_index_of_any_supertype(temp_db): + temp_db.command("sql", "CREATE DOCUMENT TYPE A") + temp_db.command("sql", "CREATE PROPERTY A.k INTEGER") + temp_db.command("sql", "CREATE DOCUMENT TYPE B EXTENDS A") + temp_db.command("sql", "CREATE DOCUMENT TYPE C EXTENDS B") + with temp_db.transaction(): + temp_db.command("sql", "INSERT INTO C SET k = 3, name = 'deep'") + with pytest.raises(ArcadeDBError, match="No index has been created"): + temp_db.lookup_by_key("C", ["k"], [3]) + temp_db.command("sql", "CREATE INDEX ON A (k) UNIQUE") + assert temp_db.lookup_by_key("C", ["k"], [3]).get("name") == "deep" + assert temp_db.lookup_by_key("B", ["k"], [3]).get("name") == "deep" + assert temp_db.lookup_by_key("B", ["k"], [4]) is None + + +def test_a_supertype_is_not_served_by_the_index_of_a_subtype(temp_db): + temp_db.command("sql", "CREATE DOCUMENT TYPE A") + temp_db.command("sql", "CREATE PROPERTY A.k INTEGER") + temp_db.command("sql", "CREATE DOCUMENT TYPE B EXTENDS A") + temp_db.command("sql", "CREATE INDEX ON B (k) UNIQUE") + with temp_db.transaction(): + temp_db.command("sql", "INSERT INTO B SET k = 5, name = 'sub'") + assert temp_db.lookup_by_key("B", ["k"], [5]).get("name") == "sub" + with pytest.raises(ArcadeDBError, match="No index has been created"): + temp_db.lookup_by_key("A", ["k"], [5]) + + +@pytest.mark.parametrize("kind", ["VERTEX", "EDGE"]) +def test_vertex_and_edge_types_follow_the_same_rule(temp_db, kind): + temp_db.command("sql", f"CREATE {kind} TYPE G") + temp_db.command("sql", "CREATE PROPERTY G.k INTEGER") + with pytest.raises(ArcadeDBError, match="No index has been created"): + temp_db.lookup_by_key("G", ["k"], [1]) + temp_db.command("sql", "CREATE INDEX ON G (k) UNIQUE") + assert temp_db.lookup_by_key("G", ["k"], [1]) is None + + +def test_lookup_on_an_unknown_type_names_the_type_not_the_index(docs): + with pytest.raises(ArcadeDBError, match="'Nope' was not found"): + docs.lookup_by_key("Nope", ["k"], [1]) From 68ca636be40cd8ad526b3f9c83ad2daa69bee0e8 Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Mon, 26 Oct 2026 21:06:39 +0900 Subject: [PATCH 2/2] Native lookup_by_key calls the engine's lookupByKey The first version checked schema:indexes before the SELECT, one more query on a hot point-lookup path. The native glue now has an entry point, arcadedb_n_lookup, that calls Database.lookupByKey(type, keys, values) in the image and replies with the first hit as the rows of a statement reply, so Python reads it as it reads a SELECT. The engine's IllegalArgumentException (no LSM_TREE or HASH index on exactly those properties of the type or a supertype) and SchemaException (unknown type) are the error reply, and become the ArcadeDBError that the JVM build raises. No statement has that raise-on-unindexed rule, so this is the case where a Java call is wrapped. _require_key_index and its two schema queries are gone. Co-Authored-By: Claude Sonnet 5.5 --- .../org/humemai/nativelib/NativeExtras.java | 24 ++++++ src/arcadedb/_native/database.py | 77 ++++--------------- src/arcadedb/_native/lib.py | 1 + 3 files changed, 42 insertions(+), 60 deletions(-) diff --git a/native/src/main/java/org/humemai/nativelib/NativeExtras.java b/native/src/main/java/org/humemai/nativelib/NativeExtras.java index 0ef8336e..4b4872fe 100644 --- a/native/src/main/java/org/humemai/nativelib/NativeExtras.java +++ b/native/src/main/java/org/humemai/nativelib/NativeExtras.java @@ -40,6 +40,7 @@ import com.arcadedb.function.CypherFunctionRegistry; import com.arcadedb.function.FunctionRegistry; import com.arcadedb.function.sql.DefaultSQLFunctionFactory; +import com.arcadedb.index.IndexCursor; import com.arcadedb.index.IndexInternal; import com.arcadedb.index.TypeIndex; import com.arcadedb.index.vector.LSMVectorIndex; @@ -54,6 +55,8 @@ import com.arcadedb.integration.exporter.Exporter; import com.arcadedb.integration.importer.Importer; import com.arcadedb.query.sql.executor.CommandContext; +import com.arcadedb.query.sql.executor.InternalResultSet; +import com.arcadedb.query.sql.executor.ResultInternal; import com.arcadedb.query.sql.executor.ResultSet; import com.arcadedb.schema.DocumentType; import com.arcadedb.schema.EdgeType; @@ -284,6 +287,27 @@ public static CCharPointer dbSet(final IsolateThread thread, final long h, final } } + // ---- Database.lookupByKey: no statement raises when no index serves the keys, as the engine's call does + + /** + * The first record of db.lookupByKey(type, keys, values), as the reply of a statement (none or one row, drained) so that Python reads it + * with the reader of a SELECT. argsJson: {"keys":[property names],"values":[tagged values]}. The engine's own exceptions are the error + * reply, as on the JVM build: IllegalArgumentException when no LSM_TREE or HASH index of the type or a supertype is on exactly those + * properties, SchemaException when the type does not exist. + */ + @CEntryPoint(name = "arcadedb_n_lookup") + public static CCharPointer lookup(final IsolateThread thread, final long dbh, final CCharPointer type, final CCharPointer argsJson) { + try { + final Database db = ArcadeNative.db(dbh); + final Map a = args(argsJson); + final IndexCursor cursor = db.lookupByKey(ArcadeNative.str(type), strings(a.get("keys")), objects(a.get("values"))); + return ArcadeNative.out( + replyRows(cursor.hasNext() ? new InternalResultSet(new ResultInternal(cursor.next())) : new InternalResultSet())); + } catch (final Throwable t) { + return ArcadeNative.err(t); + } + } + // ---- vector indexes: what no statement reports (the engine's configuration of an index) and the PQ search no statement runs private static String q(final String text) { diff --git a/src/arcadedb/_native/database.py b/src/arcadedb/_native/database.py index abc50633..4b8fec4e 100644 --- a/src/arcadedb/_native/database.py +++ b/src/arcadedb/_native/database.py @@ -679,71 +679,28 @@ def lookup_by_rid(self, rid: str): raise ArcadeDBError(f"Failed to lookup RID '{rid}': record not found") return row.get_element() - # The index kinds that answer "the value equals the key" (the engine's Schema.INDEX_TYPE.isExactKeyLookup()); a FULL_TEXT, vector - # or geospatial index on the same properties does not. - _EXACT_KEY_INDEX_TYPES = frozenset({"LSM_TREE", "HASH"}) - - def _require_key_index(self, type_name: str, keys: List[str]) -> None: - """Raise unless ``type_name`` or a supertype has an exact-key index on exactly ``keys``, as the engine's ``lookupByKey`` does. - - Without this check the ``SELECT`` of ``lookup_by_key`` scans the type when no index serves it, so the call returns a - record on this build and raises on the JVM build. A type that does not exist is left to that ``SELECT``, which names it. - """ - from .schema_objects import NativeIndex - - wanted = [str(k) for k in keys] - listed = "[" + ", ".join(wanted) + "]" - other = ( - None # an index on these properties that cannot answer an exact-key lookup - ) - pending, seen, first = [type_name], set(), True - while pending: - name = pending.pop() - if name in seen: - continue - seen.add(name) - for row in self.query( - "sql", "SELECT FROM schema:indexes WHERE typeName = ?", name - ).to_list(): - index = NativeIndex(row) - if index.getPropertyNames() != wanted: - continue - if str(index.getType()) in self._EXACT_KEY_INDEX_TYPES: - return - other = other or index - types = self.query( - "sql", "SELECT parentTypes FROM schema:types WHERE name = ?", name - ).to_list() - if not types: - if first: - return # no such type: the SELECT raises the engine's own error - continue - first = False - pending.extend(types[0].get("parentTypes") or []) - if other is not None: - raise ArcadeDBError( - f"No key index has been created on type '{type_name}' properties {listed}: " - f"index '{other.getName()}' is {other.getType()} and cannot look up a record by exact key" - ) - raise ArcadeDBError( - f"No index has been created on type '{type_name}' properties {listed}" - ) - def lookup_by_key(self, type_name: str, keys: List[str], values: List[Any]): """The first record of ``type_name`` whose ``keys`` properties equal ``values``, wrapped, or None. - Index-based, as on the JVM build: ``keys`` must be exactly the properties of a LSM_TREE or HASH index of the type or one of - its supertypes, in the index's order. Anything else raises ``ArcadeDBError`` (a scan is ``query()`` with a ``WHERE``). + Index-based, as on the JVM build: the engine's own ``lookupByKey`` runs in the image, so ``keys`` must be exactly the properties of + a LSM_TREE or HASH index of the type or one of its supertypes, in the index's order. Anything else raises ``ArcadeDBError`` + (a scan is ``query()`` with a ``WHERE``). No statement has that rule, hence this op and not a ``SELECT``. """ self._check_not_closed() - try: - self._require_key_index(type_name, keys) - where = " AND ".join(f"`{k}` = ?" for k in keys) - lookup_sql = f"SELECT FROM `{type_name}` WHERE {where} LIMIT 1" # nosec B608 - backtick-quoted identifiers; values go through ? parameters - row = self.query("sql", lookup_sql, list(values)).first() - return row.get_element() if row is not None else None - except ArcadeDBError as e: - raise ArcadeDBError(f"Failed to lookup by key in '{type_name}': {e}") from e + from .extras import _enc + + reply = loads( + lib.call( + "arcadedb_n_lookup", + self._h, + lib.b(type_name), + lib.b(_enc({"keys": [str(k) for k in keys], "values": list(values)})), + ) + ) + if not reply.get("ok"): + raise _error(f"Failed to lookup by key in '{type_name}'", reply) + row = NativeResultSet(self, reply).first() + return row.get_element() if row is not None else None def insert_many( self, diff --git a/src/arcadedb/_native/lib.py b/src/arcadedb/_native/lib.py index b258d1df..695cbb22 100644 --- a/src/arcadedb/_native/lib.py +++ b/src/arcadedb/_native/lib.py @@ -228,6 +228,7 @@ class _Params(ctypes.Structure): "arcadedb_n_diag": [P], "arcadedb_n_sample_dump": [P], "arcadedb_n_dbset": [P, L, S, S], + "arcadedb_n_lookup": [P, L, S, S], "arcadedb_n_gb_create": [P, L, S], "arcadedb_n_gb": [P, L, S, S], "arcadedb_n_async": [P, L, S, S],