From ac25ec585d4914dffd7358ea3a1374eb613fe30f Mon Sep 17 00:00:00 2001 From: contrueCT Date: Sun, 12 Apr 2026 21:39:46 +0800 Subject: [PATCH 01/16] improve(query): clarify condition resolution semantics Add explicit condition resolution APIs to ConditionQuery while preserving the legacy condition() behavior. Introduce containsCondition(Object), conditionValues(Object), and conditionValue(Object) so callers can distinguish missing, empty, unique, and multi-value results without overloading null semantics. Migrate LABEL-specific consumers in graph/index transactions, serializers, traversers, and stores to use the new APIs for unique-label resolution and conservative fallback behavior. Extend QueryTest and VertexCoreTest to cover absent, conflicting, and multi-value label conditions as well as collectMatchedIndexes() behavior for multi-label and conflicting label queries. --- .../backend/query/ConditionQuery.java | 124 +++++++++++++++--- .../backend/serializer/BinarySerializer.java | 5 +- .../backend/serializer/TextSerializer.java | 5 +- .../hugegraph/backend/store/ram/RamTable.java | 12 +- .../backend/tx/GraphIndexTransaction.java | 31 +++-- .../backend/tx/GraphTransaction.java | 16 ++- .../traversal/algorithm/HugeTraverser.java | 2 +- .../backend/store/hstore/HstoreStore.java | 14 +- .../apache/hugegraph/core/VertexCoreTest.java | 44 +++++++ .../apache/hugegraph/unit/core/QueryTest.java | 46 +++++++ 10 files changed, 261 insertions(+), 38 deletions(-) diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/query/ConditionQuery.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/query/ConditionQuery.java index 063d23aa6d..5b37622514 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/query/ConditionQuery.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/query/ConditionQuery.java @@ -256,24 +256,26 @@ public boolean containsLabelOrUserpropRelation() { return false; } + /** + * Returns the legacy condition value of the specified key. + * + * This method keeps the historical behavior for existing callers: + * + * + * Prefer {@link #conditionValues(Object)} or {@link #conditionValue(Object)} + * for new code that needs explicit semantics. + */ @Watched public T condition(Object key) { List valuesEQ = InsertionOrderUtil.newList(); List valuesIN = InsertionOrderUtil.newList(); - for (Condition c : this.conditions) { - if (c.isRelation()) { - Condition.Relation r = (Condition.Relation) c; - if (r.key().equals(key)) { - if (r.relation() == RelationType.EQ) { - valuesEQ.add(r.value()); - } else if (r.relation() == RelationType.IN) { - Object value = r.value(); - assert value instanceof List; - valuesIN.add(value); - } - } - } - } + this.collectConditionValues(key, valuesEQ, valuesIN); if (valuesEQ.isEmpty() && valuesIN.isEmpty()) { return null; } @@ -323,20 +325,110 @@ public T condition(Object key) { return value; } + /** + * Returns whether there is any top-level relation for the specified key. + */ + public boolean containsCondition(Object key) { + for (Condition c : this.conditions) { + if (c.isRelation()) { + Condition.Relation r = (Condition.Relation) c; + if (r.key().equals(key)) { + return true; + } + } + } + return false; + } + + /** + * Returns the resolved candidate values of the specified key from + * top-level EQ/IN relations. + * + * Use {@link #containsCondition(Object)} to distinguish "no condition" + * from "conditions exist but resolve to an empty intersection". + */ + public Set conditionValues(Object key) { + List valuesEQ = InsertionOrderUtil.newList(); + List valuesIN = InsertionOrderUtil.newList(); + this.collectConditionValues(key, valuesEQ, valuesIN); + if (valuesEQ.isEmpty() && valuesIN.isEmpty()) { + return InsertionOrderUtil.newSet(); + } + return this.resolveConditionValues(valuesEQ, valuesIN); + } + + /** + * Returns the unique resolved value of the specified key from top-level + * EQ/IN relations. + * + * Returns {@code null} when the resolved candidate set is empty. Throws + * if multiple values remain after resolution. + */ + public T conditionValue(Object key) { + Set values = this.conditionValues(key); + if (values.isEmpty()) { + return null; + } + E.checkState(values.size() == 1, + "Illegal key '%s' with more than one value: %s", + key, values); + @SuppressWarnings("unchecked") + T value = (T) values.iterator().next(); + return value; + } + public void unsetCondition(Object key) { this.conditions.removeIf(c -> c.isRelation() && ((Relation) c).key().equals(key)); } public boolean containsCondition(HugeKeys key) { + return this.containsCondition((Object) key); + } + + private void collectConditionValues(Object key, List valuesEQ, + List valuesIN) { for (Condition c : this.conditions) { if (c.isRelation()) { Condition.Relation r = (Condition.Relation) c; if (r.key().equals(key)) { - return true; + if (r.relation() == RelationType.EQ) { + valuesEQ.add(r.value()); + } else if (r.relation() == RelationType.IN) { + Object value = r.value(); + assert value instanceof List; + valuesIN.add(value); + } } } } - return false; + } + + private Set resolveConditionValues(List valuesEQ, + List valuesIN) { + boolean initialized = false; + Set intersectValues = InsertionOrderUtil.newSet(); + for (Object value : valuesEQ) { + List valueAsList = ImmutableList.of(value); + if (!initialized) { + intersectValues.addAll(valueAsList); + initialized = true; + } else { + CollectionUtil.intersectWithModify(intersectValues, + valueAsList); + } + } + for (Object value : valuesIN) { + @SuppressWarnings("unchecked") + List valueAsList = (List) value; + if (!initialized) { + intersectValues.addAll(valueAsList); + initialized = true; + } else { + CollectionUtil.intersectWithModify(intersectValues, + valueAsList); + } + } + return intersectValues; } public boolean containsCondition(Condition.RelationType type) { diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/serializer/BinarySerializer.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/serializer/BinarySerializer.java index 0bb07760a5..6f1eec58a6 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/serializer/BinarySerializer.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/serializer/BinarySerializer.java @@ -674,7 +674,7 @@ private Query writeQueryEdgeRangeCondition(ConditionQuery cq) { if (direction == null) { direction = Directions.OUT; } - Id label = cq.condition(HugeKeys.LABEL); + Id label = cq.conditionValue(HugeKeys.LABEL); BytesBuffer start = BytesBuffer.allocate(BytesBuffer.BUF_EDGE_ID); writePartitionedId(HugeType.EDGE, vertex, start); @@ -722,7 +722,8 @@ private Query writeQueryEdgePrefixCondition(ConditionQuery cq) { int count = 0; BytesBuffer buffer = BytesBuffer.allocate(BytesBuffer.BUF_EDGE_ID); for (HugeKeys key : EdgeId.KEYS) { - Object value = cq.condition(key); + Object value = key == HugeKeys.LABEL ? + cq.conditionValue(key) : cq.condition(key); if (value != null) { count++; diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/serializer/TextSerializer.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/serializer/TextSerializer.java index 2d5cb81ec1..61830b3c54 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/serializer/TextSerializer.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/serializer/TextSerializer.java @@ -457,7 +457,7 @@ private Query writeQueryEdgeRangeCondition(ConditionQuery cq) { if (direction == null) { direction = Directions.OUT; } - Object label = cq.condition(HugeKeys.LABEL); + Object label = cq.conditionValue(HugeKeys.LABEL); List start = new ArrayList<>(cq.conditionsSize()); start.add(writeEntryId((Id) vertex)); @@ -491,7 +491,8 @@ private Query writeQueryEdgePrefixCondition(ConditionQuery cq) { List condParts = new ArrayList<>(cq.conditionsSize()); for (HugeKeys key : EdgeId.KEYS) { - Object value = cq.condition(key); + Object value = key == HugeKeys.LABEL ? + cq.conditionValue(key) : cq.condition(key); if (value == null) { break; } diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/store/ram/RamTable.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/store/ram/RamTable.java index 0e2c58bddc..c282fe384b 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/store/ram/RamTable.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/store/ram/RamTable.java @@ -269,7 +269,7 @@ public boolean matched(Query query) { int conditionsSize = cq.conditionsSize(); Object owner = cq.condition(HugeKeys.OWNER_VERTEX); Directions direction = cq.condition(HugeKeys.DIRECTION); - Id label = cq.condition(HugeKeys.LABEL); + Id label = uniqueLabel(cq); if (direction == null && conditionsSize > 1) { for (Condition cond : cq.conditions()) { @@ -316,7 +316,7 @@ private Iterator query(ConditionQuery query) { if (dir == null) { dir = Directions.BOTH; } - Id label = query.condition(HugeKeys.LABEL); + Id label = uniqueLabel(query); if (label == null) { label = IdGenerator.ZERO; } @@ -377,6 +377,14 @@ private static void ensureNumberId(Id id) { } } + private static Id uniqueLabel(ConditionQuery query) { + java.util.Set labels = query.conditionValues(HugeKeys.LABEL); + if (labels.size() != 1) { + return null; + } + return (Id) labels.iterator().next(); + } + private static long encode(long target, Directions direction, int label) { // TODO: support property assert (label & 0x0fffffff) == label; diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/tx/GraphIndexTransaction.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/tx/GraphIndexTransaction.java index 7388425167..6b358b2619 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/tx/GraphIndexTransaction.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/tx/GraphIndexTransaction.java @@ -415,8 +415,9 @@ private IdHolderList queryByLabel(ConditionQuery query) { HugeType queryType = query.resultType(); IndexLabel il = IndexLabel.label(queryType); validateIndexLabel(il); - Id label = query.condition(HugeKeys.LABEL); - assert label != null; + Id label = query.conditionValue(HugeKeys.LABEL); + E.checkState(label != null, "Expect one label value for query: %s", + query); HugeType indexType; SchemaLabel schemaLabel; @@ -482,7 +483,7 @@ private IdHolderList queryByUserprop(ConditionQuery query) { } Set indexes = this.collectMatchedIndexes(query); if (indexes.isEmpty()) { - Id label = query.condition(HugeKeys.LABEL); + Id label = uniqueLabel(query); throw noIndexException(this.graph(), query, label); } @@ -756,11 +757,16 @@ private PageIds doIndexQueryOnce(IndexLabel indexLabel, @Watched(prefix = "index") private Set collectMatchedIndexes(ConditionQuery query) { ISchemaTransaction schema = this.params().schemaTransaction(); - Id label = query.condition(HugeKeys.LABEL); + boolean hasLabel = query.containsCondition(HugeKeys.LABEL); + Set labels = query.conditionValues(HugeKeys.LABEL); List schemaLabels; - if (label != null) { - // Query has LABEL condition + if (hasLabel && labels.isEmpty()) { + return Collections.emptySet(); + } + if (labels.size() == 1) { + Id label = (Id) labels.iterator().next(); + // Query has one resolved LABEL condition SchemaLabel schemaLabel; if (query.resultType().isVertex()) { schemaLabel = schema.getVertexLabel(label); @@ -773,7 +779,8 @@ private Set collectMatchedIndexes(ConditionQuery query) { } schemaLabels = ImmutableList.of(schemaLabel); } else { - // Query doesn't have LABEL condition + // Query doesn't have LABEL condition or it doesn't resolve + // to a single label, so keep the conservative fallback. if (query.resultType().isVertex()) { schemaLabels = schema.getVertexLabels(); } else if (query.resultType().isEdge()) { @@ -1781,7 +1788,7 @@ protected long removeIndexLeft(ConditionQuery query, } // Check label is matched - Id label = query.condition(HugeKeys.LABEL); + Id label = uniqueLabel(query); // NOTE: original condition query may not have label condition, // which means possibly label == null. if (label != null && !element.schemaLabel().id().equals(label)) { @@ -1981,4 +1988,12 @@ public Long reduce(Long t1, Long t2) { return t1 + t2; } } + + private static Id uniqueLabel(ConditionQuery query) { + Set labels = query.conditionValues(HugeKeys.LABEL); + if (labels.size() != 1) { + return null; + } + return (Id) labels.iterator().next(); + } } diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/tx/GraphTransaction.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/tx/GraphTransaction.java index 5e33e0b3fc..6c4f174d7d 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/tx/GraphTransaction.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/tx/GraphTransaction.java @@ -1059,7 +1059,7 @@ protected Iterator queryEdgesFromBackend(Query query) { ConditionQueryFlatten.flatten((ConditionQuery) query, supportIn).stream(); Stream> edgeIterators = flattenedQueries.map(cq -> { - Id label = cq.condition(HugeKeys.LABEL); + Id label = uniqueLabel(cq); if (this.storeFeatures().supportsFatherAndSubEdgeLabel() && label != null && graph().edgeLabel(label).isFather() && @@ -1389,7 +1389,7 @@ private static boolean matchEdgeSortKeys(ConditionQuery query, boolean matchAll, HugeGraph graph) { assert query.resultType().isEdge(); - Id label = query.condition(HugeKeys.LABEL); + Id label = uniqueLabel(query); if (label == null) { return false; } @@ -1522,7 +1522,7 @@ private Query optimizeQuery(ConditionQuery query) { throw new HugeException("Not supported querying by id and conditions: %s", query); } - Id label = query.condition(HugeKeys.LABEL); + Id label = uniqueLabel(query); // Optimize vertex query if (label != null && query.resultType().isVertex()) { @@ -1914,7 +1914,7 @@ private boolean rightResultFromIndexQuery(Query query, HugeElement elem) { } ConditionQuery cq = (ConditionQuery) query; - if (cq.condition(HugeKeys.LABEL) != null && cq.resultType().isEdge()) { + if (uniqueLabel(cq) != null && cq.resultType().isEdge()) { if (cq.conditions().size() == 1) { // g.E().hasLabel(xxx) return true; @@ -1966,6 +1966,14 @@ private boolean rightResultFromIndexQuery(Query query, HugeElement elem) { return false; } + private static Id uniqueLabel(ConditionQuery query) { + Set labels = query.conditionValues(HugeKeys.LABEL); + if (labels.size() != 1) { + return null; + } + return (Id) labels.iterator().next(); + } + private Iterator filterExpiredResultFromBackend( Query query, Iterator results) { if (this.store().features().supportsTtl() || query.showExpired()) { diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/traversal/algorithm/HugeTraverser.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/traversal/algorithm/HugeTraverser.java index 8122c79080..a19c32a04f 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/traversal/algorithm/HugeTraverser.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/traversal/algorithm/HugeTraverser.java @@ -582,7 +582,7 @@ private void fillFilterBySortKeys(Query query, Id[] edgeLabels, ConditionQuery condQuery = (ConditionQuery) query; if (!GraphTransaction.matchFullEdgeSortKeys(condQuery, this.graph())) { - Id label = condQuery.condition(HugeKeys.LABEL); + Id label = condQuery.conditionValue(HugeKeys.LABEL); E.checkArgument(false, "The properties %s does not match " + "sort keys of edge label '%s'", this.graph().mapPkId2Name(properties.keySet()), diff --git a/hugegraph-server/hugegraph-hstore/src/main/java/org/apache/hugegraph/backend/store/hstore/HstoreStore.java b/hugegraph-server/hugegraph-hstore/src/main/java/org/apache/hugegraph/backend/store/hstore/HstoreStore.java index 6439096674..713511490d 100644 --- a/hugegraph-server/hugegraph-hstore/src/main/java/org/apache/hugegraph/backend/store/hstore/HstoreStore.java +++ b/hugegraph-server/hugegraph-hstore/src/main/java/org/apache/hugegraph/backend/store/hstore/HstoreStore.java @@ -402,8 +402,8 @@ public IdPrefixQuery next() { List queryList = Lists.newArrayList(); if (hugeGraph != null) { for (ConditionQuery conditionQuery : - ConditionQueryFlatten.flatten(cq)) { - Id label = conditionQuery.condition(HugeKeys.LABEL); + ConditionQueryFlatten.flatten(cq)) { + Id label = this.uniqueLabel(conditionQuery); /* Parent type + sortKeys: g.V("V.id").outE("parentLabel") .has("sortKey","value") converted to all subtypes + sortKeys */ if ((this.subEls == null || @@ -455,11 +455,19 @@ public IdPrefixQuery next() { buffer.bytes(), ownerId)); } + private Id uniqueLabel(ConditionQuery query) { + Set labels = query.conditionValues(HugeKeys.LABEL); + if (labels.size() != 1) { + return null; + } + return (Id) labels.iterator().next(); + } + private boolean matchEdgeSortKeys(ConditionQuery query, boolean matchAll, HugeGraph graph) { assert query.resultType().isEdge(); - Id label = query.condition(HugeKeys.LABEL); + Id label = this.uniqueLabel(query); if (label == null) { return false; } diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/VertexCoreTest.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/VertexCoreTest.java index d33f9bb07d..24525bcf37 100644 --- a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/VertexCoreTest.java +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/VertexCoreTest.java @@ -54,6 +54,7 @@ import org.apache.hugegraph.exception.NotAllowException; import org.apache.hugegraph.schema.PropertyKey; import org.apache.hugegraph.schema.SchemaManager; +import org.apache.hugegraph.schema.SchemaLabel; import org.apache.hugegraph.schema.Userdata; import org.apache.hugegraph.schema.VertexLabel; import org.apache.hugegraph.structure.HugeElement; @@ -9076,6 +9077,49 @@ public void testQueryByJointLabels() { Assert.assertEquals(0, vertices.size()); } + @Test + public void testCollectMatchedIndexesByJointLabelsWithIndexedProperties() { + HugeGraph graph = graph(); + initPersonIndex(true); + init5Persons(); + init5Computers(); + init10Vertices(); + + VertexLabel person = graph.vertexLabel("person"); + VertexLabel computer = graph.vertexLabel("computer"); + PropertyKey city = graph.propertyKey("city"); + + ConditionQuery query = new ConditionQuery(HugeType.VERTEX); + query.query(Condition.in(HugeKeys.LABEL, + ImmutableList.of(person.id(), computer.id()))); + query.query(Condition.eq(city.id(), "Beijing")); + + Set matchedIndexes = Whitebox.invoke(params().graphTransaction(), + "indexTx", + "collectMatchedIndexes", + query); + Assert.assertEquals(1, matchedIndexes.size()); + Object matchedIndex = matchedIndexes.iterator().next(); + SchemaLabel schemaLabel = Whitebox.getInternalState(matchedIndex, + "schemaLabel"); + Assert.assertEquals("person", schemaLabel.name()); + + ConditionQuery conflicting = new ConditionQuery(HugeType.VERTEX); + conflicting.eq(HugeKeys.LABEL, person.id()); + conflicting.eq(HugeKeys.LABEL, computer.id()); + conflicting.query(Condition.eq(city.id(), "Beijing")); + + Assert.assertTrue(conflicting.containsCondition(HugeKeys.LABEL)); + Assert.assertEquals(ImmutableSet.of(), + conflicting.conditionValues(HugeKeys.LABEL)); + + matchedIndexes = Whitebox.invoke(params().graphTransaction(), + "indexTx", + "collectMatchedIndexes", + conflicting); + Assert.assertEquals(0, matchedIndexes.size()); + } + @Test public void testQueryByHasIdEmptyList() { HugeGraph graph = graph(); diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/core/QueryTest.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/core/QueryTest.java index 7d48084dbf..bb2951c8ea 100644 --- a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/core/QueryTest.java +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/core/QueryTest.java @@ -48,6 +48,17 @@ public void testOrderBy() { query.orders()); } + @Test + public void testConditionWithoutLabel() { + ConditionQuery query = new ConditionQuery(HugeType.EDGE); + + Assert.assertFalse(query.containsCondition(HugeKeys.LABEL)); + Assert.assertEquals(ImmutableSet.of(), + query.conditionValues(HugeKeys.LABEL)); + Assert.assertNull(query.conditionValue(HugeKeys.LABEL)); + Assert.assertNull(query.condition(HugeKeys.LABEL)); + } + @Test public void testConditionWithEqAndIn() { Id label1 = IdGenerator.of(1); @@ -58,9 +69,33 @@ public void testConditionWithEqAndIn() { query.query(Condition.in(HugeKeys.LABEL, ImmutableList.of(label1, label2))); + Assert.assertTrue(query.containsCondition(HugeKeys.LABEL)); + Assert.assertEquals(ImmutableSet.of(label1), + query.conditionValues(HugeKeys.LABEL)); + Assert.assertEquals(label1, query.conditionValue(HugeKeys.LABEL)); Assert.assertEquals(label1, query.condition(HugeKeys.LABEL)); } + @Test + public void testConditionWithSingleInValues() { + Id label1 = IdGenerator.of(1); + Id label2 = IdGenerator.of(2); + + ConditionQuery query = new ConditionQuery(HugeType.EDGE); + query.query(Condition.in(HugeKeys.LABEL, + ImmutableList.of(label1, label2))); + + Assert.assertTrue(query.containsCondition(HugeKeys.LABEL)); + Assert.assertEquals(ImmutableSet.of(label1, label2), + query.conditionValues(HugeKeys.LABEL)); + Assert.assertThrows(IllegalStateException.class, + () -> query.conditionValue(HugeKeys.LABEL), + e -> Assert.assertContains("Illegal key 'LABEL'", + e.getMessage())); + Assert.assertEquals(ImmutableList.of(label1, label2), + query.condition(HugeKeys.LABEL)); + } + @Test public void testConditionWithConflictingEqAndIn() { Id label1 = IdGenerator.of(1); @@ -73,6 +108,10 @@ public void testConditionWithConflictingEqAndIn() { query.query(Condition.in(HugeKeys.LABEL, ImmutableList.of(label1, label3))); + Assert.assertTrue(query.containsCondition(HugeKeys.LABEL)); + Assert.assertEquals(ImmutableSet.of(), + query.conditionValues(HugeKeys.LABEL)); + Assert.assertNull(query.conditionValue(HugeKeys.LABEL)); Assert.assertNull(query.condition(HugeKeys.LABEL)); } @@ -89,6 +128,13 @@ public void testConditionWithMultipleMatchedInValues() { query.query(Condition.in(HugeKeys.LABEL, ImmutableList.of(label1, label2, label4))); + Assert.assertTrue(query.containsCondition(HugeKeys.LABEL)); + Assert.assertEquals(ImmutableSet.of(label1, label2), + query.conditionValues(HugeKeys.LABEL)); + Assert.assertThrows(IllegalStateException.class, + () -> query.conditionValue(HugeKeys.LABEL), + e -> Assert.assertContains("Illegal key 'LABEL'", + e.getMessage())); Assert.assertThrows(IllegalStateException.class, () -> query.condition(HugeKeys.LABEL), e -> Assert.assertContains("Illegal key 'LABEL'", From c4cf3d5e9085aa624dca36239ffa9bc26bf37cfb Mon Sep 17 00:00:00 2001 From: contrueCT Date: Mon, 13 Apr 2026 22:08:29 +0800 Subject: [PATCH 02/16] test(core): avoid unnecessary data setup in label index regression --- .../main/java/org/apache/hugegraph/core/VertexCoreTest.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/VertexCoreTest.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/VertexCoreTest.java index 24525bcf37..adbdddc24c 100644 --- a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/VertexCoreTest.java +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/VertexCoreTest.java @@ -9081,9 +9081,6 @@ public void testQueryByJointLabels() { public void testCollectMatchedIndexesByJointLabelsWithIndexedProperties() { HugeGraph graph = graph(); initPersonIndex(true); - init5Persons(); - init5Computers(); - init10Vertices(); VertexLabel person = graph.vertexLabel("person"); VertexLabel computer = graph.vertexLabel("computer"); From ed3b7886fea5901220850de837ccc8076de1519a Mon Sep 17 00:00:00 2001 From: contrueCT Date: Thu, 23 Apr 2026 11:59:26 +0800 Subject: [PATCH 03/16] improve(query): consolidate unique label resolution --- .../backend/query/ConditionQuery.java | 84 +++++++++++++------ .../backend/serializer/BinarySerializer.java | 16 +++- .../backend/serializer/TextSerializer.java | 16 +++- .../hugegraph/backend/store/ram/RamTable.java | 12 +-- .../backend/tx/GraphIndexTransaction.java | 23 ++--- .../backend/tx/GraphTransaction.java | 17 ++-- .../traversal/algorithm/HugeTraverser.java | 2 + .../backend/store/hstore/HstoreStore.java | 12 +-- .../apache/hugegraph/core/VertexCoreTest.java | 16 ++++ .../apache/hugegraph/unit/core/QueryTest.java | 39 +++++++++ 10 files changed, 158 insertions(+), 79 deletions(-) diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/query/ConditionQuery.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/query/ConditionQuery.java index 5b37622514..226091ed36 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/query/ConditionQuery.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/query/ConditionQuery.java @@ -268,8 +268,9 @@ public boolean containsLabelOrUserpropRelation() { *
  • throws if multiple values remain after resolving several relations
    • * * - * Prefer {@link #conditionValues(Object)} or {@link #conditionValue(Object)} - * for new code that needs explicit semantics. + * Prefer {@link #conditionValues(Object)}, {@link #uniqueConditionValue(Object)} + * or {@link #conditionValue(Object)} for new code that needs explicit + * semantics. */ @Watched public T condition(Object key) { @@ -290,29 +291,8 @@ public T condition(Object key) { return value; } - boolean initialized = false; - Set intersectValues = InsertionOrderUtil.newSet(); - for (Object value : valuesEQ) { - List valueAsList = ImmutableList.of(value); - if (!initialized) { - intersectValues.addAll(valueAsList); - initialized = true; - } else { - CollectionUtil.intersectWithModify(intersectValues, - valueAsList); - } - } - for (Object value : valuesIN) { - @SuppressWarnings("unchecked") - List valueAsList = (List) value; - if (!initialized) { - intersectValues.addAll(valueAsList); - initialized = true; - } else { - CollectionUtil.intersectWithModify(intersectValues, - valueAsList); - } - } + Set intersectValues = this.resolveConditionValues(valuesEQ, + valuesIN); if (intersectValues.isEmpty()) { return null; @@ -344,8 +324,9 @@ public boolean containsCondition(Object key) { * Returns the resolved candidate values of the specified key from * top-level EQ/IN relations. * - * Use {@link #containsCondition(Object)} to distinguish "no condition" - * from "conditions exist but resolve to an empty intersection". + * Use {@link #containsConditionValues(Object)} to distinguish "no EQ/IN + * condition" from "EQ/IN conditions exist but resolve to an empty + * intersection". */ public Set conditionValues(Object key) { List valuesEQ = InsertionOrderUtil.newList(); @@ -357,6 +338,24 @@ public Set conditionValues(Object key) { return this.resolveConditionValues(valuesEQ, valuesIN); } + /** + * Returns whether there is any top-level EQ/IN relation for the specified + * key. + */ + public boolean containsConditionValues(Object key) { + for (Condition c : this.conditions) { + if (c.isRelation()) { + Condition.Relation r = (Condition.Relation) c; + if (r.key().equals(key) && + (r.relation() == RelationType.EQ || + r.relation() == RelationType.IN)) { + return true; + } + } + } + return false; + } + /** * Returns the unique resolved value of the specified key from top-level * EQ/IN relations. @@ -377,6 +376,24 @@ public T conditionValue(Object key) { return value; } + /** + * Returns the unique resolved value of the specified key from top-level + * EQ/IN relations, or {@code null} if the resolved candidate set doesn't + * contain exactly one value. + * + * Use this method when callers want "single-or-null" semantics instead of + * treating multiple remaining values as an error. + */ + public T uniqueConditionValue(Object key) { + Set values = this.conditionValues(key); + if (values.size() != 1) { + return null; + } + @SuppressWarnings("unchecked") + T value = (T) values.iterator().next(); + return value; + } + public void unsetCondition(Object key) { this.conditions.removeIf(c -> c.isRelation() && ((Relation) c).key().equals(key)); } @@ -385,6 +402,10 @@ public boolean containsCondition(HugeKeys key) { return this.containsCondition((Object) key); } + public boolean containsConditionValues(HugeKeys key) { + return this.containsConditionValues((Object) key); + } + private void collectConditionValues(Object key, List valuesEQ, List valuesIN) { for (Condition c : this.conditions) { @@ -658,6 +679,15 @@ public boolean hasNeqCondition() { return false; } + public boolean hasUserpropNeqCondition() { + for (Condition.Relation r : this.userpropRelations()) { + if (r.relation() == RelationType.NEQ) { + return true; + } + } + return false; + } + public boolean matchUserpropKeys(List keys) { Set conditionKeys = this.userpropKeys(); return !keys.isEmpty() && conditionKeys.containsAll(keys); diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/serializer/BinarySerializer.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/serializer/BinarySerializer.java index 6f1eec58a6..7871c7fdca 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/serializer/BinarySerializer.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/serializer/BinarySerializer.java @@ -674,7 +674,7 @@ private Query writeQueryEdgeRangeCondition(ConditionQuery cq) { if (direction == null) { direction = Directions.OUT; } - Id label = cq.conditionValue(HugeKeys.LABEL); + Id label = (Id) this.edgeIdConditionValue(cq, HugeKeys.LABEL); BytesBuffer start = BytesBuffer.allocate(BytesBuffer.BUF_EDGE_ID); writePartitionedId(HugeType.EDGE, vertex, start); @@ -722,8 +722,7 @@ private Query writeQueryEdgePrefixCondition(ConditionQuery cq) { int count = 0; BytesBuffer buffer = BytesBuffer.allocate(BytesBuffer.BUF_EDGE_ID); for (HugeKeys key : EdgeId.KEYS) { - Object value = key == HugeKeys.LABEL ? - cq.conditionValue(key) : cq.condition(key); + Object value = this.edgeIdConditionValue(cq, key); if (value != null) { count++; @@ -764,6 +763,17 @@ private Query writeQueryEdgePrefixCondition(ConditionQuery cq) { return null; } + private Object edgeIdConditionValue(ConditionQuery cq, HugeKeys key) { + if (key == HugeKeys.LABEL) { + /* + * LABEL may still be represented by multiple top-level EQ/IN + * relations before strict edge-id serialization. + */ + return cq.conditionValue(key); + } + return cq.condition(key); + } + @Override protected Query writeQueryCondition(Query query) { HugeType type = query.resultType(); diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/serializer/TextSerializer.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/serializer/TextSerializer.java index 61830b3c54..cf357d2132 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/serializer/TextSerializer.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/serializer/TextSerializer.java @@ -457,7 +457,7 @@ private Query writeQueryEdgeRangeCondition(ConditionQuery cq) { if (direction == null) { direction = Directions.OUT; } - Object label = cq.conditionValue(HugeKeys.LABEL); + Object label = this.edgeIdConditionValue(cq, HugeKeys.LABEL); List start = new ArrayList<>(cq.conditionsSize()); start.add(writeEntryId((Id) vertex)); @@ -491,8 +491,7 @@ private Query writeQueryEdgePrefixCondition(ConditionQuery cq) { List condParts = new ArrayList<>(cq.conditionsSize()); for (HugeKeys key : EdgeId.KEYS) { - Object value = key == HugeKeys.LABEL ? - cq.conditionValue(key) : cq.condition(key); + Object value = this.edgeIdConditionValue(cq, key); if (value == null) { break; } @@ -517,6 +516,17 @@ private Query writeQueryEdgePrefixCondition(ConditionQuery cq) { return null; } + private Object edgeIdConditionValue(ConditionQuery cq, HugeKeys key) { + if (key == HugeKeys.LABEL) { + /* + * LABEL may still be represented by multiple top-level EQ/IN + * relations before strict edge-id serialization. + */ + return cq.conditionValue(key); + } + return cq.condition(key); + } + @Override protected Query writeQueryCondition(Query query) { ConditionQuery result = (ConditionQuery) query; diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/store/ram/RamTable.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/store/ram/RamTable.java index c282fe384b..850f37ee5f 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/store/ram/RamTable.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/store/ram/RamTable.java @@ -269,7 +269,7 @@ public boolean matched(Query query) { int conditionsSize = cq.conditionsSize(); Object owner = cq.condition(HugeKeys.OWNER_VERTEX); Directions direction = cq.condition(HugeKeys.DIRECTION); - Id label = uniqueLabel(cq); + Id label = cq.uniqueConditionValue(HugeKeys.LABEL); if (direction == null && conditionsSize > 1) { for (Condition cond : cq.conditions()) { @@ -316,7 +316,7 @@ private Iterator query(ConditionQuery query) { if (dir == null) { dir = Directions.BOTH; } - Id label = uniqueLabel(query); + Id label = query.uniqueConditionValue(HugeKeys.LABEL); if (label == null) { label = IdGenerator.ZERO; } @@ -377,14 +377,6 @@ private static void ensureNumberId(Id id) { } } - private static Id uniqueLabel(ConditionQuery query) { - java.util.Set labels = query.conditionValues(HugeKeys.LABEL); - if (labels.size() != 1) { - return null; - } - return (Id) labels.iterator().next(); - } - private static long encode(long target, Directions direction, int label) { // TODO: support property assert (label & 0x0fffffff) == label; diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/tx/GraphIndexTransaction.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/tx/GraphIndexTransaction.java index 6b358b2619..c6f105a0d9 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/tx/GraphIndexTransaction.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/tx/GraphIndexTransaction.java @@ -415,6 +415,8 @@ private IdHolderList queryByLabel(ConditionQuery query) { HugeType queryType = query.resultType(); IndexLabel il = IndexLabel.label(queryType); validateIndexLabel(il); + // Query-by-label builds a label index entry and requires one + // deterministically resolved label instead of best-effort fallback. Id label = query.conditionValue(HugeKeys.LABEL); E.checkState(label != null, "Expect one label value for query: %s", query); @@ -483,7 +485,7 @@ private IdHolderList queryByUserprop(ConditionQuery query) { } Set indexes = this.collectMatchedIndexes(query); if (indexes.isEmpty()) { - Id label = uniqueLabel(query); + Id label = query.uniqueConditionValue(HugeKeys.LABEL); throw noIndexException(this.graph(), query, label); } @@ -757,11 +759,12 @@ private PageIds doIndexQueryOnce(IndexLabel indexLabel, @Watched(prefix = "index") private Set collectMatchedIndexes(ConditionQuery query) { ISchemaTransaction schema = this.params().schemaTransaction(); - boolean hasLabel = query.containsCondition(HugeKeys.LABEL); + boolean hasLabelValues = query.containsConditionValues(HugeKeys.LABEL); Set labels = query.conditionValues(HugeKeys.LABEL); List schemaLabels; - if (hasLabel && labels.isEmpty()) { + if (hasLabelValues && labels.isEmpty()) { + // LABEL EQ/IN conditions resolve to an empty intersection. return Collections.emptySet(); } if (labels.size() == 1) { @@ -952,7 +955,7 @@ private void removeExpiredIndexIfNeeded(HugeIndex index, private static Set matchSingleOrCompositeIndex( ConditionQuery query, Set indexLabels) { - if (query.hasNeqCondition()) { + if (query.hasUserpropNeqCondition()) { return ImmutableSet.of(); } boolean requireRange = query.hasRangeCondition(); @@ -993,7 +996,7 @@ private static Set matchSingleOrCompositeIndex( private static Set matchJointIndexes( ConditionQuery query, Set indexLabels) { - if (query.hasNeqCondition()) { + if (query.hasUserpropNeqCondition()) { return ImmutableSet.of(); } Set queryPropKeys = query.userpropKeys(); @@ -1788,7 +1791,7 @@ protected long removeIndexLeft(ConditionQuery query, } // Check label is matched - Id label = uniqueLabel(query); + Id label = query.uniqueConditionValue(HugeKeys.LABEL); // NOTE: original condition query may not have label condition, // which means possibly label == null. if (label != null && !element.schemaLabel().id().equals(label)) { @@ -1988,12 +1991,4 @@ public Long reduce(Long t1, Long t2) { return t1 + t2; } } - - private static Id uniqueLabel(ConditionQuery query) { - Set labels = query.conditionValues(HugeKeys.LABEL); - if (labels.size() != 1) { - return null; - } - return (Id) labels.iterator().next(); - } } diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/tx/GraphTransaction.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/tx/GraphTransaction.java index 6c4f174d7d..79e1211908 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/tx/GraphTransaction.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/tx/GraphTransaction.java @@ -1059,7 +1059,7 @@ protected Iterator queryEdgesFromBackend(Query query) { ConditionQueryFlatten.flatten((ConditionQuery) query, supportIn).stream(); Stream> edgeIterators = flattenedQueries.map(cq -> { - Id label = uniqueLabel(cq); + Id label = cq.uniqueConditionValue(HugeKeys.LABEL); if (this.storeFeatures().supportsFatherAndSubEdgeLabel() && label != null && graph().edgeLabel(label).isFather() && @@ -1389,7 +1389,7 @@ private static boolean matchEdgeSortKeys(ConditionQuery query, boolean matchAll, HugeGraph graph) { assert query.resultType().isEdge(); - Id label = uniqueLabel(query); + Id label = query.uniqueConditionValue(HugeKeys.LABEL); if (label == null) { return false; } @@ -1522,7 +1522,7 @@ private Query optimizeQuery(ConditionQuery query) { throw new HugeException("Not supported querying by id and conditions: %s", query); } - Id label = uniqueLabel(query); + Id label = query.uniqueConditionValue(HugeKeys.LABEL); // Optimize vertex query if (label != null && query.resultType().isVertex()) { @@ -1914,7 +1914,8 @@ private boolean rightResultFromIndexQuery(Query query, HugeElement elem) { } ConditionQuery cq = (ConditionQuery) query; - if (uniqueLabel(cq) != null && cq.resultType().isEdge()) { + if (cq.uniqueConditionValue(HugeKeys.LABEL) != null && + cq.resultType().isEdge()) { if (cq.conditions().size() == 1) { // g.E().hasLabel(xxx) return true; @@ -1966,14 +1967,6 @@ private boolean rightResultFromIndexQuery(Query query, HugeElement elem) { return false; } - private static Id uniqueLabel(ConditionQuery query) { - Set labels = query.conditionValues(HugeKeys.LABEL); - if (labels.size() != 1) { - return null; - } - return (Id) labels.iterator().next(); - } - private Iterator filterExpiredResultFromBackend( Query query, Iterator results) { if (this.store().features().supportsTtl() || query.showExpired()) { diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/traversal/algorithm/HugeTraverser.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/traversal/algorithm/HugeTraverser.java index a19c32a04f..0785286d3f 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/traversal/algorithm/HugeTraverser.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/traversal/algorithm/HugeTraverser.java @@ -582,6 +582,8 @@ private void fillFilterBySortKeys(Query query, Id[] edgeLabels, ConditionQuery condQuery = (ConditionQuery) query; if (!GraphTransaction.matchFullEdgeSortKeys(condQuery, this.graph())) { + // Sort-key validation needs one concrete edge label so that the + // error message points to the exact schema label in use. Id label = condQuery.conditionValue(HugeKeys.LABEL); E.checkArgument(false, "The properties %s does not match " + "sort keys of edge label '%s'", diff --git a/hugegraph-server/hugegraph-hstore/src/main/java/org/apache/hugegraph/backend/store/hstore/HstoreStore.java b/hugegraph-server/hugegraph-hstore/src/main/java/org/apache/hugegraph/backend/store/hstore/HstoreStore.java index 713511490d..00bef521d0 100644 --- a/hugegraph-server/hugegraph-hstore/src/main/java/org/apache/hugegraph/backend/store/hstore/HstoreStore.java +++ b/hugegraph-server/hugegraph-hstore/src/main/java/org/apache/hugegraph/backend/store/hstore/HstoreStore.java @@ -403,7 +403,7 @@ public IdPrefixQuery next() { if (hugeGraph != null) { for (ConditionQuery conditionQuery : ConditionQueryFlatten.flatten(cq)) { - Id label = this.uniqueLabel(conditionQuery); + Id label = conditionQuery.uniqueConditionValue(HugeKeys.LABEL); /* Parent type + sortKeys: g.V("V.id").outE("parentLabel") .has("sortKey","value") converted to all subtypes + sortKeys */ if ((this.subEls == null || @@ -455,19 +455,11 @@ public IdPrefixQuery next() { buffer.bytes(), ownerId)); } - private Id uniqueLabel(ConditionQuery query) { - Set labels = query.conditionValues(HugeKeys.LABEL); - if (labels.size() != 1) { - return null; - } - return (Id) labels.iterator().next(); - } - private boolean matchEdgeSortKeys(ConditionQuery query, boolean matchAll, HugeGraph graph) { assert query.resultType().isEdge(); - Id label = this.uniqueLabel(query); + Id label = query.uniqueConditionValue(HugeKeys.LABEL); if (label == null) { return false; } diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/VertexCoreTest.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/VertexCoreTest.java index adbdddc24c..6cb743eab6 100644 --- a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/VertexCoreTest.java +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/VertexCoreTest.java @@ -9077,6 +9077,22 @@ public void testQueryByJointLabels() { Assert.assertEquals(0, vertices.size()); } + @Test + public void testQueryByNonEqLabelAndIndexedProperty() { + HugeGraph graph = graph(); + initPersonIndex(true); + init5Persons(); + + GraphTraversalSource g = graph.traversal(); + + List vertices = g.V().has(T.label, P.neq("author")) + .has("city", "Beijing").toList(); + Assert.assertEquals(3, vertices.size()); + for (Vertex vertex : vertices) { + Assert.assertEquals("person", vertex.label()); + } + } + @Test public void testCollectMatchedIndexesByJointLabelsWithIndexedProperties() { HugeGraph graph = graph(); diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/core/QueryTest.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/core/QueryTest.java index bb2951c8ea..b8b505c3d8 100644 --- a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/core/QueryTest.java +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/core/QueryTest.java @@ -53,8 +53,10 @@ public void testConditionWithoutLabel() { ConditionQuery query = new ConditionQuery(HugeType.EDGE); Assert.assertFalse(query.containsCondition(HugeKeys.LABEL)); + Assert.assertFalse(query.containsConditionValues(HugeKeys.LABEL)); Assert.assertEquals(ImmutableSet.of(), query.conditionValues(HugeKeys.LABEL)); + Assert.assertNull(query.uniqueConditionValue(HugeKeys.LABEL)); Assert.assertNull(query.conditionValue(HugeKeys.LABEL)); Assert.assertNull(query.condition(HugeKeys.LABEL)); } @@ -70,8 +72,10 @@ public void testConditionWithEqAndIn() { ImmutableList.of(label1, label2))); Assert.assertTrue(query.containsCondition(HugeKeys.LABEL)); + Assert.assertTrue(query.containsConditionValues(HugeKeys.LABEL)); Assert.assertEquals(ImmutableSet.of(label1), query.conditionValues(HugeKeys.LABEL)); + Assert.assertEquals(label1, query.uniqueConditionValue(HugeKeys.LABEL)); Assert.assertEquals(label1, query.conditionValue(HugeKeys.LABEL)); Assert.assertEquals(label1, query.condition(HugeKeys.LABEL)); } @@ -86,8 +90,10 @@ public void testConditionWithSingleInValues() { ImmutableList.of(label1, label2))); Assert.assertTrue(query.containsCondition(HugeKeys.LABEL)); + Assert.assertTrue(query.containsConditionValues(HugeKeys.LABEL)); Assert.assertEquals(ImmutableSet.of(label1, label2), query.conditionValues(HugeKeys.LABEL)); + Assert.assertNull(query.uniqueConditionValue(HugeKeys.LABEL)); Assert.assertThrows(IllegalStateException.class, () -> query.conditionValue(HugeKeys.LABEL), e -> Assert.assertContains("Illegal key 'LABEL'", @@ -109,12 +115,43 @@ public void testConditionWithConflictingEqAndIn() { ImmutableList.of(label1, label3))); Assert.assertTrue(query.containsCondition(HugeKeys.LABEL)); + Assert.assertTrue(query.containsConditionValues(HugeKeys.LABEL)); Assert.assertEquals(ImmutableSet.of(), query.conditionValues(HugeKeys.LABEL)); + Assert.assertNull(query.uniqueConditionValue(HugeKeys.LABEL)); Assert.assertNull(query.conditionValue(HugeKeys.LABEL)); Assert.assertNull(query.condition(HugeKeys.LABEL)); } + @Test + public void testConditionWithNonEqInLabel() { + Id label = IdGenerator.of(1); + + ConditionQuery query = new ConditionQuery(HugeType.EDGE); + query.neq(HugeKeys.LABEL, label); + + Assert.assertTrue(query.containsCondition(HugeKeys.LABEL)); + Assert.assertFalse(query.containsConditionValues(HugeKeys.LABEL)); + Assert.assertTrue(query.hasNeqCondition()); + Assert.assertFalse(query.hasUserpropNeqCondition()); + Assert.assertEquals(ImmutableSet.of(), + query.conditionValues(HugeKeys.LABEL)); + Assert.assertNull(query.uniqueConditionValue(HugeKeys.LABEL)); + Assert.assertNull(query.conditionValue(HugeKeys.LABEL)); + Assert.assertNull(query.condition(HugeKeys.LABEL)); + } + + @Test + public void testConditionWithUserpropNeq() { + Id prop = IdGenerator.of(1); + + ConditionQuery query = new ConditionQuery(HugeType.EDGE); + query.query(Condition.neq(prop, "Beijing")); + + Assert.assertTrue(query.hasNeqCondition()); + Assert.assertTrue(query.hasUserpropNeqCondition()); + } + @Test public void testConditionWithMultipleMatchedInValues() { Id label1 = IdGenerator.of(1); @@ -129,8 +166,10 @@ public void testConditionWithMultipleMatchedInValues() { ImmutableList.of(label1, label2, label4))); Assert.assertTrue(query.containsCondition(HugeKeys.LABEL)); + Assert.assertTrue(query.containsConditionValues(HugeKeys.LABEL)); Assert.assertEquals(ImmutableSet.of(label1, label2), query.conditionValues(HugeKeys.LABEL)); + Assert.assertNull(query.uniqueConditionValue(HugeKeys.LABEL)); Assert.assertThrows(IllegalStateException.class, () -> query.conditionValue(HugeKeys.LABEL), e -> Assert.assertContains("Illegal key 'LABEL'", From a326f44c5b7797aed2cd647895e366d5ac3c28c8 Mon Sep 17 00:00:00 2001 From: contrueCT Date: Thu, 4 Jun 2026 20:54:48 +0800 Subject: [PATCH 04/16] test(core): cover edge label query semantics --- .../apache/hugegraph/core/EdgeCoreTest.java | 71 +++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/EdgeCoreTest.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/EdgeCoreTest.java index bbf7db6562..fc7ed5edc6 100644 --- a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/EdgeCoreTest.java +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/EdgeCoreTest.java @@ -3582,6 +3582,43 @@ public void testQueryOutEdgesOfVertexBySortkeyAndProps() { Assert.assertEquals(0, edges.size()); } + @Test + public void testQueryOutEdgesBySingleResolvedLabelAndSortKey() { + HugeGraph graph = graph(); + Vertex reader = initEdgeLabelQueryEdges(); + + List edges = graph.traversal().V(reader.id()) + .outE("reviewed") + .has(T.label, P.within("reviewed", + "recommended")) + .has("time", "2026-1-1") + .toList(); + + Assert.assertEquals(1, edges.size()); + Assert.assertEquals("reviewed", edges.get(0).label()); + Assert.assertEquals("2026-1-1", edges.get(0).value("time")); + } + + @Test + public void testQueryOutEdgesByMultiLabelsAndSortKey() { + HugeGraph graph = graph(); + Vertex reader = initEdgeLabelQueryEdges(); + + List edges = graph.traversal().V(reader.id()) + .outE("reviewed", "recommended") + .has("time", "2026-1-1") + .toList(); + + Set labels = new HashSet<>(); + for (Edge edge : edges) { + labels.add(edge.label()); + Assert.assertEquals("2026-1-1", edge.value("time")); + } + Assert.assertEquals(2, edges.size()); + Assert.assertEquals(ImmutableSet.of("reviewed", "recommended"), + labels); + } + @Test public void testQueryOutEdgesOfVertexBySortkeyWithRange() { // FIXME: skip this test for hstore @@ -7691,6 +7728,40 @@ private void init18Edges(boolean commit) { } } + private Vertex initEdgeLabelQueryEdges() { + HugeGraph graph = graph(); + SchemaManager schema = graph.schema(); + + schema.edgeLabel("reviewed").properties("time", "score") + .multiTimes().sortKeys("time") + .link("person", "book") + .enableLabelIndex(false) + .create(); + schema.edgeLabel("recommended").properties("time", "score") + .multiTimes().sortKeys("time") + .link("person", "book") + .enableLabelIndex(false) + .create(); + + Vertex reader = graph.addVertex(T.label, "person", + "name", "edge-label-reader", + "city", "Beijing", + "age", 29); + Vertex book1 = graph.addVertex(T.label, "book", + "name", "edge-label-book-1"); + Vertex book2 = graph.addVertex(T.label, "book", + "name", "edge-label-book-2"); + Vertex book3 = graph.addVertex(T.label, "book", + "name", "edge-label-book-3"); + + reader.addEdge("reviewed", book1, "time", "2026-1-1", "score", 1); + reader.addEdge("recommended", book2, "time", "2026-1-1", "score", 2); + reader.addEdge("reviewed", book3, "time", "2026-1-2", "score", 3); + + graph.tx().commit(); + return reader; + } + private void init100LookEdges() { HugeGraph graph = graph(); From b10e3c24fe2f24079bddf06ae38385b38b3fbdaf Mon Sep 17 00:00:00 2001 From: contrueCT Date: Fri, 5 Jun 2026 13:08:44 +0800 Subject: [PATCH 05/16] fix(core): tolerate missing related index labels --- .../hugegraph/backend/tx/GraphIndexTransaction.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/tx/GraphIndexTransaction.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/tx/GraphIndexTransaction.java index c6f105a0d9..f24de2faf2 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/tx/GraphIndexTransaction.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/tx/GraphIndexTransaction.java @@ -1564,8 +1564,13 @@ private static Set relatedIndexLabels(HugeElement element) { Set indexLabelIds = element.schemaLabel().indexLabels(); for (Id id : indexLabelIds) { - IndexLabel indexLabel = element.graph().indexLabel(id); - indexLabels.add(indexLabel); + try { + IndexLabel indexLabel = element.graph().indexLabel(id); + indexLabels.add(indexLabel); + } catch (IllegalArgumentException e) { + LOG.debug("Skip missing related index label '{}' of element {}", + id, element.id(), e); + } } return indexLabels; } From a1835719c3382f58c2e85aa46daefa34c9967b2a Mon Sep 17 00:00:00 2001 From: contrueCT Date: Fri, 5 Jun 2026 13:40:13 +0800 Subject: [PATCH 06/16] fix(core): skip stale index entries --- .../backend/tx/GraphIndexTransaction.java | 42 +++++++++++++++++-- 1 file changed, 38 insertions(+), 4 deletions(-) diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/tx/GraphIndexTransaction.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/tx/GraphIndexTransaction.java index f24de2faf2..09e2653237 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/tx/GraphIndexTransaction.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/tx/GraphIndexTransaction.java @@ -685,8 +685,11 @@ private IdHolder doIndexQueryBatch(IndexLabel indexLabel, Set ids = InsertionOrderUtil.newSet(); while ((batch == Query.NO_LIMIT || ids.size() < batch) && entries.hasNext()) { - HugeIndex index = this.serializer.readIndex(graph(), query, - entries.next()); + HugeIndex index = this.readMatchedIndex(indexLabel, query, + entries.next()); + if (index == null) { + continue; + } this.removeExpiredIndexIfNeeded(index, query.showExpired()); ids.addAll(index.elementIds()); Query.checkForceCapacity(ids.size()); @@ -727,8 +730,11 @@ private PageIds doIndexQueryOnce(IndexLabel indexLabel, Set ids = InsertionOrderUtil.newSet(); entries = super.query(query).iterator(); while (entries.hasNext()) { - HugeIndex index = this.serializer.readIndex(graph(), query, - entries.next()); + HugeIndex index = this.readMatchedIndex(indexLabel, query, + entries.next()); + if (index == null) { + continue; + } this.removeExpiredIndexIfNeeded(index, query.showExpired()); ids.addAll(index.elementIds()); if (query.reachLimit(ids.size())) { @@ -756,6 +762,34 @@ private PageIds doIndexQueryOnce(IndexLabel indexLabel, } } + private HugeIndex readMatchedIndex(IndexLabel indexLabel, + ConditionQuery query, + BackendEntry entry) { + HugeIndex index; + try { + index = this.serializer.readIndex(graph(), query, entry); + } catch (IllegalArgumentException e) { + if (!missingIndexLabel(e)) { + throw e; + } + LOG.debug("Skip stale index entry with missing index label while " + + "querying index label '{}'", indexLabel.id(), e); + return null; + } + if (!Objects.equals(index.indexLabelId(), indexLabel.id())) { + LOG.debug("Skip stale index entry of index label '{}' while " + + "querying index label '{}'", + index.indexLabelId(), indexLabel.id()); + return null; + } + return index; + } + + private static boolean missingIndexLabel(IllegalArgumentException e) { + String message = e.getMessage(); + return message != null && message.contains("Undefined index label"); + } + @Watched(prefix = "index") private Set collectMatchedIndexes(ConditionQuery query) { ISchemaTransaction schema = this.params().schemaTransaction(); From ebc31c831ac8d152208f131830ffd76920db9d4b Mon Sep 17 00:00:00 2001 From: contrueCT Date: Sat, 6 Jun 2026 02:04:49 +0800 Subject: [PATCH 07/16] fix(core): stabilize hstore range index ordering --- .../backend/tx/GraphIndexTransaction.java | 185 +++++++++++++++++- 1 file changed, 183 insertions(+), 2 deletions(-) diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/tx/GraphIndexTransaction.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/tx/GraphIndexTransaction.java index 09e2653237..bb0f90f5a0 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/tx/GraphIndexTransaction.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/tx/GraphIndexTransaction.java @@ -651,6 +651,9 @@ private void storeSelectedIndexField(IndexLabel indexLabel, @Watched(prefix = "index") private IdHolder doIndexQuery(IndexLabel indexLabel, ConditionQuery query) { + if (this.needHstoreRangeIndexOrder(indexLabel)) { + return this.doHstoreRangeIndexQuery(indexLabel, query); + } if (!query.paging()) { return this.doIndexQueryBatch(indexLabel, query); } else { @@ -660,6 +663,184 @@ private IdHolder doIndexQuery(IndexLabel indexLabel, ConditionQuery query) { } } + private boolean needHstoreRangeIndexOrder(IndexLabel indexLabel) { + return this.store().provider().isHstore() && + indexLabel.indexType().isRange(); + } + + private IdHolder doHstoreRangeIndexQuery(IndexLabel indexLabel, + ConditionQuery query) { + if (!query.paging()) { + if (query.noLimitAndOffset()) { + return this.doIndexQueryBatch(indexLabel, query); + } + Set ids = this.querySortedRangeIndexIds(indexLabel, query); + return this.newSortedRangeIndexBatchHolder(query, ids); + } + return new PagingIdHolder(query, q -> { + return this.querySortedRangeIndexPage(indexLabel, q); + }); + } + + private BatchIdHolder newSortedRangeIndexBatchHolder(ConditionQuery query, + Set ids) { + List idList = new ArrayList<>(ids); + return new BatchIdHolder(query, Collections.emptyIterator(), batch -> { + throw new IllegalStateException("Unexpected sorted index fetcher"); + }) { + private int offset = 0; + + @Override + public boolean hasNext() { + return this.offset < idList.size(); + } + + @Override + public IdHolder next() { + if (!this.hasNext()) { + throw new java.util.NoSuchElementException(); + } + return this; + } + + @Override + public PageIds fetchNext(String page, long batchSize) { + E.checkArgument(page == null, + "Not support page parameter by BatchIdHolder"); + if (!this.hasNext()) { + return PageIds.EMPTY; + } + + int end; + if (batchSize == Query.NO_LIMIT) { + end = idList.size(); + } else { + end = (int) Math.min((long) idList.size(), + this.offset + batchSize); + } + Set batchIds = InsertionOrderUtil.newSet(); + batchIds.addAll(idList.subList(this.offset, end)); + this.offset = end; + return new PageIds(batchIds, PageState.EMPTY); + } + + @Override + public Set all() { + Set allIds = InsertionOrderUtil.newSet(); + allIds.addAll(idList); + return allIds; + } + + @Override + public void close() { + this.offset = idList.size(); + } + }; + } + + private Set querySortedRangeIndexIds(IndexLabel indexLabel, + ConditionQuery query) { + List indexes = this.querySortedRangeIndexes(indexLabel, + query); + Set ids = InsertionOrderUtil.newSet(); + for (HugeIndex index : indexes) { + ids.addAll(index.elementIds()); + Query.checkForceCapacity(ids.size()); + } + return ids; + } + + private PageIds querySortedRangeIndexPage(IndexLabel indexLabel, + ConditionQuery query) { + List indexes = this.querySortedRangeIndexes(indexLabel, + query); + Set allIds = InsertionOrderUtil.newSet(); + for (HugeIndex index : indexes) { + allIds.addAll(index.elementIds()); + Query.checkForceCapacity(allIds.size()); + } + if (allIds.isEmpty()) { + return PageIds.EMPTY; + } + + int start = 0; + if (!query.page().isEmpty()) { + start = PageState.fromString(query.page()).offset(); + } + if (start >= allIds.size()) { + return PageIds.EMPTY; + } + + long total = allIds.size(); + long end = query.noLimit() ? total : + Math.min(total, (long) start + query.limit()); + Set pageIds = CollectionUtil.subSet(allIds, start, (int) end); + if (pageIds.isEmpty()) { + return PageIds.EMPTY; + } + + int next = (int) end; + PageState pageState; + if (next < total) { + pageState = new PageState(new byte[]{1}, next, pageIds.size()); + } else { + pageState = new PageState(PageState.EMPTY_BYTES, 0, + pageIds.size()); + } + return new PageIds(pageIds, pageState); + } + + private List querySortedRangeIndexes(IndexLabel indexLabel, + ConditionQuery query) { + List indexes = new ArrayList<>(); + Iterator entries = null; + String spaceGraph = this.params() + .graph().spaceGraphName(); + LockUtil.Locks locks = new LockUtil.Locks(spaceGraph); + ConditionQuery scanQuery = query.copy(); + scanQuery.page(null); + scanQuery.limit(Query.NO_LIMIT); + try { + locks.lockReads(LockUtil.INDEX_LABEL_DELETE, indexLabel.id()); + locks.lockReads(LockUtil.INDEX_LABEL_REBUILD, indexLabel.id()); + if (!indexLabel.system()) { + graph().indexLabel(indexLabel.id()); + } + + entries = super.query(scanQuery).iterator(); + while (entries.hasNext()) { + HugeIndex index = this.readMatchedIndex(indexLabel, scanQuery, + entries.next()); + if (index == null) { + continue; + } + this.removeExpiredIndexIfNeeded(index, scanQuery.showExpired()); + this.recordIndexValue(scanQuery, index); + indexes.add(index); + Query.checkForceCapacity(indexes.size()); + } + } finally { + locks.unlock(); + CloseableIterator.closeIterator(entries); + } + + Collections.sort(indexes, (a, b) -> { + return this.compareRangeIndexValues(a, b); + }); + return indexes; + } + + @SuppressWarnings({"rawtypes", "unchecked"}) + private int compareRangeIndexValues(HugeIndex left, HugeIndex right) { + Object leftValue = left.fieldValues(); + Object rightValue = right.fieldValues(); + E.checkArgument(leftValue instanceof Comparable, + "Invalid range index value '%s'", leftValue); + E.checkArgument(rightValue instanceof Comparable, + "Invalid range index value '%s'", rightValue); + return ((Comparable) leftValue).compareTo(rightValue); + } + @Watched(prefix = "index") private IdHolder doIndexQueryBatch(IndexLabel indexLabel, ConditionQuery query) { @@ -772,8 +953,8 @@ private HugeIndex readMatchedIndex(IndexLabel indexLabel, if (!missingIndexLabel(e)) { throw e; } - LOG.debug("Skip stale index entry with missing index label while " + - "querying index label '{}'", indexLabel.id(), e); + LOG.debug("Skip stale index entry with missing index label " + + "while querying index label '{}'", indexLabel.id(), e); return null; } if (!Objects.equals(index.indexLabelId(), indexLabel.id())) { From 2df480248f24f5ecd601d2222c56c05162f9f1f6 Mon Sep 17 00:00:00 2001 From: contrueCT Date: Sun, 7 Jun 2026 11:05:00 +0800 Subject: [PATCH 08/16] fix(core): reset hstore range scan offset --- .../hugegraph/backend/tx/GraphIndexTransaction.java | 1 + .../java/org/apache/hugegraph/core/VertexCoreTest.java | 8 ++++++++ 2 files changed, 9 insertions(+) diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/tx/GraphIndexTransaction.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/tx/GraphIndexTransaction.java index bb0f90f5a0..ca51ec9b6e 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/tx/GraphIndexTransaction.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/tx/GraphIndexTransaction.java @@ -799,6 +799,7 @@ private List querySortedRangeIndexes(IndexLabel indexLabel, LockUtil.Locks locks = new LockUtil.Locks(spaceGraph); ConditionQuery scanQuery = query.copy(); scanQuery.page(null); + scanQuery.offset(0L); scanQuery.limit(Query.NO_LIMIT); try { locks.lockReads(LockUtil.INDEX_LABEL_DELETE, indexLabel.id()); diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/VertexCoreTest.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/VertexCoreTest.java index 6cb743eab6..bf2714aea1 100644 --- a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/VertexCoreTest.java +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/VertexCoreTest.java @@ -4415,6 +4415,14 @@ public void testQueryByDateProperty() { Assert.assertEquals(dates[1], vertices.get(0).value("birth")); Assert.assertEquals(dates[2], vertices.get(1).value("birth")); + // range with offset + vertices = graph.traversal().V().hasLabel("person") + .has("birth", P.between(dates[1], dates[4])) + .range(1, 3).toList(); + Assert.assertEquals(2, vertices.size()); + Assert.assertEquals(dates[2], vertices.get(0).value("birth")); + Assert.assertEquals(dates[3], vertices.get(1).value("birth")); + // limit after delete graph.traversal().V().hasLabel("person") .has("birth", P.between(dates[1], dates[4])) From 939cc12d74e780275644d51e99d439f3334d785c Mon Sep 17 00:00:00 2001 From: contrueCT Date: Mon, 8 Jun 2026 16:49:35 +0800 Subject: [PATCH 09/16] fix(core): preserve sorted hstore range ordering --- .../hugegraph/backend/page/QueryList.java | 5 +- .../hugegraph/backend/query/QueryResults.java | 8 +- .../backend/tx/GraphIndexTransaction.java | 171 ++++++++++++------ .../backend/tx/GraphTransaction.java | 11 +- .../backend/tx/GraphIndexTransactionTest.java | 82 +++++++++ .../apache/hugegraph/unit/UnitTestSuite.java | 4 + .../hugegraph/unit/core/QueryResultsTest.java | 82 +++++++++ 7 files changed, 302 insertions(+), 61 deletions(-) create mode 100644 hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/backend/tx/GraphIndexTransactionTest.java create mode 100644 hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/core/QueryResultsTest.java diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/page/QueryList.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/page/QueryList.java index d1e11e9220..f1930bd55a 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/page/QueryList.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/page/QueryList.java @@ -258,7 +258,7 @@ private QueryResults each(IdHolder holder) { return null; } - return this.queryByIndexIds(ids); + return this.queryByIndexIds(ids, holder.keepOrder()); }); } @@ -275,7 +275,8 @@ public PageResults iterator(int index, String page, long pageSize) { return PageResults.emptyIterator(); } - QueryResults results = this.queryByIndexIds(pageIds.ids()); + QueryResults results = this.queryByIndexIds(pageIds.ids(), + holder.keepOrder()); return new PageResults<>(results, pageIds.pageState()); } diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/query/QueryResults.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/query/QueryResults.java index a03e5c9aee..48e06b2afe 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/query/QueryResults.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/query/QueryResults.java @@ -102,12 +102,12 @@ public Iterator keepInputOrderIfNeeded( return origin; } Collection ids; - if (!this.mustSortByInputIds() || this.paging() || + if (!this.mustSortByInputIds() || (ids = this.queryIds()).size() <= 1) { /* - * Return the original iterator if it's paging query or if the - * query input is less than one id, or don't have to do sort. - * NOTE: queryIds() only return the first batch of index query + * Return the original iterator if the query input is less than one + * id, or don't have to do sort. + * NOTE: queryIds() only return the first batch of index query. */ return origin; } diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/tx/GraphIndexTransaction.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/tx/GraphIndexTransaction.java index ca51ec9b6e..d9ddbd322b 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/tx/GraphIndexTransaction.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/tx/GraphIndexTransaction.java @@ -677,65 +677,14 @@ private IdHolder doHstoreRangeIndexQuery(IndexLabel indexLabel, Set ids = this.querySortedRangeIndexIds(indexLabel, query); return this.newSortedRangeIndexBatchHolder(query, ids); } - return new PagingIdHolder(query, q -> { + return new SortedRangePagingIdHolder(query, q -> { return this.querySortedRangeIndexPage(indexLabel, q); }); } private BatchIdHolder newSortedRangeIndexBatchHolder(ConditionQuery query, Set ids) { - List idList = new ArrayList<>(ids); - return new BatchIdHolder(query, Collections.emptyIterator(), batch -> { - throw new IllegalStateException("Unexpected sorted index fetcher"); - }) { - private int offset = 0; - - @Override - public boolean hasNext() { - return this.offset < idList.size(); - } - - @Override - public IdHolder next() { - if (!this.hasNext()) { - throw new java.util.NoSuchElementException(); - } - return this; - } - - @Override - public PageIds fetchNext(String page, long batchSize) { - E.checkArgument(page == null, - "Not support page parameter by BatchIdHolder"); - if (!this.hasNext()) { - return PageIds.EMPTY; - } - - int end; - if (batchSize == Query.NO_LIMIT) { - end = idList.size(); - } else { - end = (int) Math.min((long) idList.size(), - this.offset + batchSize); - } - Set batchIds = InsertionOrderUtil.newSet(); - batchIds.addAll(idList.subList(this.offset, end)); - this.offset = end; - return new PageIds(batchIds, PageState.EMPTY); - } - - @Override - public Set all() { - Set allIds = InsertionOrderUtil.newSet(); - allIds.addAll(idList); - return allIds; - } - - @Override - public void close() { - this.offset = idList.size(); - } - }; + return new SortedRangeBatchIdHolder(query, ids); } private Set querySortedRangeIndexIds(IndexLabel indexLabel, @@ -842,6 +791,122 @@ private int compareRangeIndexValues(HugeIndex left, HugeIndex right) { return ((Comparable) leftValue).compareTo(rightValue); } + static class SortedRangeBatchIdHolder extends BatchIdHolder { + + private final List idList; + private int offset; + private PageIds pendingBatch; + + SortedRangeBatchIdHolder(ConditionQuery query, Set ids) { + super(query, Collections.emptyIterator(), batch -> { + throw new IllegalStateException("Unexpected sorted index fetcher"); + }); + this.idList = new ArrayList<>(ids); + this.offset = 0; + this.pendingBatch = null; + } + + @Override + public boolean keepOrder() { + return true; + } + + @Override + public boolean hasNext() { + if (this.pendingBatch != null) { + return true; + } + if (this.exhausted) { + return false; + } + return this.offset < this.idList.size(); + } + + @Override + public IdHolder next() { + if (!this.hasNext()) { + throw new java.util.NoSuchElementException(); + } + return this; + } + + @Override + public PageIds fetchNext(String page, long batchSize) { + E.checkArgument(page == null, + "Not support page parameter by BatchIdHolder"); + E.checkArgument(batchSize >= 0L, + "Invalid batch size value: %s", batchSize); + if (this.pendingBatch != null) { + PageIds result = this.pendingBatch; + this.pendingBatch = null; + return result; + } + return this.fetchBatch(batchSize); + } + + @Override + public Set all() { + Set allIds = InsertionOrderUtil.newSet(); + if (this.pendingBatch != null) { + allIds.addAll(this.pendingBatch.ids()); + } + if (this.offset < this.idList.size()) { + allIds.addAll(this.idList.subList(this.offset, + this.idList.size())); + } + this.close(); + return allIds; + } + + @Override + public PageIds peekNext(long size) { + E.checkArgument(this.pendingBatch == null, + "Can't call peekNext() twice"); + this.pendingBatch = this.fetchBatch(size); + return this.pendingBatch; + } + + @Override + public void close() { + this.exhausted = true; + this.pendingBatch = null; + this.offset = this.idList.size(); + } + + private PageIds fetchBatch(long batchSize) { + if (this.offset >= this.idList.size() || batchSize == 0L) { + this.close(); + return PageIds.EMPTY; + } + + int end; + if (batchSize == Query.NO_LIMIT) { + end = this.idList.size(); + } else { + end = (int) Math.min((long) this.idList.size(), + this.offset + batchSize); + } + Set batchIds = InsertionOrderUtil.newSet(); + batchIds.addAll(this.idList.subList(this.offset, end)); + this.offset = end; + this.exhausted = this.offset >= this.idList.size(); + return new PageIds(batchIds, PageState.EMPTY); + } + } + + private static class SortedRangePagingIdHolder extends PagingIdHolder { + + SortedRangePagingIdHolder(ConditionQuery query, + Function fetcher) { + super(query, fetcher); + } + + @Override + public boolean keepOrder() { + return true; + } + } + @Watched(prefix = "index") private IdHolder doIndexQueryBatch(IndexLabel indexLabel, ConditionQuery query) { diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/tx/GraphTransaction.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/tx/GraphTransaction.java index 79e1211908..0a4653c019 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/tx/GraphTransaction.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/tx/GraphTransaction.java @@ -865,7 +865,8 @@ protected Iterator queryVerticesFromBackend(Query query) { this::parseEntry); vertices = this.filterExpiredResultFromBackend(query, vertices); - if (!this.store().features().supportsQuerySortByInputIds()) { + if (!this.store().features().supportsQuerySortByInputIds() || + this.needKeepInputOrder(query)) { // There is no id in BackendEntry, so sort after deserialization vertices = results.keepInputOrderIfNeeded(vertices); } @@ -1104,13 +1105,19 @@ private Iterator queryEdgesFromBackendInternal(Query query) { edges = this.filterExpiredResultFromBackend(query, edges); - if (!this.store().features().supportsQuerySortByInputIds()) { + if (!this.store().features().supportsQuerySortByInputIds() || + this.needKeepInputOrder(query)) { // There is no id in BackendEntry, so sort after deserialization edges = results.keepInputOrderIfNeeded(edges); } return edges; } + private boolean needKeepInputOrder(Query query) { + return query instanceof IdQuery && + ((IdQuery) query).mustSortByInput(); + } + private Iterator parentElQueryWithSortKeys(EdgeLabel label, Collection allEls, ConditionQuery cq) { diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/backend/tx/GraphIndexTransactionTest.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/backend/tx/GraphIndexTransactionTest.java new file mode 100644 index 0000000000..a1b0d4e8c6 --- /dev/null +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/backend/tx/GraphIndexTransactionTest.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.backend.tx; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +import org.apache.hugegraph.backend.id.Id; +import org.apache.hugegraph.backend.id.IdGenerator; +import org.apache.hugegraph.backend.page.PageIds; +import org.apache.hugegraph.backend.query.ConditionQuery; +import org.apache.hugegraph.testutil.Assert; +import org.apache.hugegraph.type.HugeType; +import org.apache.hugegraph.util.InsertionOrderUtil; +import org.junit.Test; + +import com.google.common.collect.ImmutableList; + +public class GraphIndexTransactionTest { + + @Test + public void testSortedRangeBatchHolderKeepsPeekedBatch() { + ConditionQuery query = new ConditionQuery(HugeType.VERTEX); + Id id1 = IdGenerator.of(1); + Id id2 = IdGenerator.of(2); + Id id3 = IdGenerator.of(3); + Set ids = InsertionOrderUtil.newSet(); + ids.add(id1); + ids.add(id2); + ids.add(id3); + + GraphIndexTransaction.SortedRangeBatchIdHolder holder = + new GraphIndexTransaction.SortedRangeBatchIdHolder(query, ids); + + Assert.assertTrue(holder.keepOrder()); + + PageIds peeked = holder.peekNext(2); + Assert.assertEquals(ImmutableList.of(id1, id2), asList(peeked.ids())); + + PageIds firstBatch = holder.fetchNext(null, 2); + Assert.assertEquals(ImmutableList.of(id1, id2), + asList(firstBatch.ids())); + + PageIds secondBatch = holder.fetchNext(null, 2); + Assert.assertEquals(ImmutableList.of(id3), asList(secondBatch.ids())); + Assert.assertFalse(holder.hasNext()); + } + + @Test + public void testSortedRangeBatchHolderClosesOnZeroBatch() { + ConditionQuery query = new ConditionQuery(HugeType.VERTEX); + Set ids = InsertionOrderUtil.newSet(); + ids.add(IdGenerator.of(1)); + + GraphIndexTransaction.SortedRangeBatchIdHolder holder = + new GraphIndexTransaction.SortedRangeBatchIdHolder(query, ids); + + PageIds batch = holder.fetchNext(null, 0); + Assert.assertTrue(batch.empty()); + Assert.assertFalse(holder.hasNext()); + } + + private static List asList(Set ids) { + return new ArrayList<>(ids); + } +} diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/UnitTestSuite.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/UnitTestSuite.java index fb7f0e744b..9935909a02 100644 --- a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/UnitTestSuite.java +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/UnitTestSuite.java @@ -17,6 +17,7 @@ package org.apache.hugegraph.unit; +import org.apache.hugegraph.backend.tx.GraphIndexTransactionTest; import org.apache.hugegraph.core.RoleElectionStateMachineTest; import org.apache.hugegraph.meta.MetaManagerSchemaCacheClearEventTest; import org.apache.hugegraph.traversal.optimize.TraversalUtilOptimizeTest; @@ -40,6 +41,7 @@ import org.apache.hugegraph.unit.core.ExceptionTest; import org.apache.hugegraph.unit.core.LocksTableTest; import org.apache.hugegraph.unit.core.PageStateTest; +import org.apache.hugegraph.unit.core.QueryResultsTest; import org.apache.hugegraph.unit.core.QueryTest; import org.apache.hugegraph.unit.core.RangeTest; import org.apache.hugegraph.unit.core.RolePermissionTest; @@ -121,7 +123,9 @@ BackendMutationTest.class, ConditionTest.class, ConditionQueryFlattenTest.class, + GraphIndexTransactionTest.class, QueryTest.class, + QueryResultsTest.class, RangeTest.class, SecurityManagerTest.class, RolePermissionTest.class, diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/core/QueryResultsTest.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/core/QueryResultsTest.java new file mode 100644 index 0000000000..3f1df40728 --- /dev/null +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/core/QueryResultsTest.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.unit.core; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Set; + +import org.apache.hugegraph.backend.id.Id; +import org.apache.hugegraph.backend.id.IdGenerator; +import org.apache.hugegraph.backend.query.IdQuery; +import org.apache.hugegraph.backend.query.Query; +import org.apache.hugegraph.backend.query.QueryResults; +import org.apache.hugegraph.testutil.Assert; +import org.apache.hugegraph.type.HugeType; +import org.apache.hugegraph.type.Idfiable; +import org.apache.hugegraph.util.InsertionOrderUtil; +import org.junit.Test; + +import com.google.common.collect.ImmutableList; + +public class QueryResultsTest { + + @Test + public void testKeepInputOrderForPagingIdQuery() { + Id id1 = IdGenerator.of(1); + Id id2 = IdGenerator.of(2); + Query pagingQuery = new Query(HugeType.VERTEX); + pagingQuery.page("page-1"); + pagingQuery.limit(2L); + + Set ids = InsertionOrderUtil.newSet(); + ids.add(id2); + ids.add(id1); + + IdQuery idQuery = new IdQuery(pagingQuery, ids); + idQuery.mustSortByInput(true); + + QueryResults results = new QueryResults<>( + Arrays.asList(new TestIdfiable(id1), + new TestIdfiable(id2)).iterator(), + idQuery); + + List orderedIds = new ArrayList<>(); + results.keepInputOrderIfNeeded(Arrays.asList(new TestIdfiable(id1), + new TestIdfiable(id2)) + .iterator()) + .forEachRemaining(item -> orderedIds.add(item.id())); + + Assert.assertEquals(ImmutableList.of(id2, id1), orderedIds); + } + + private static final class TestIdfiable implements Idfiable { + + private final Id id; + + private TestIdfiable(Id id) { + this.id = id; + } + + @Override + public Id id() { + return this.id; + } + } +} From 9300691ada8fada31296ca7124448d7716850f99 Mon Sep 17 00:00:00 2001 From: contrueCT Date: Thu, 11 Jun 2026 19:33:45 +0800 Subject: [PATCH 10/16] fix(core): handle non-eq label-only traversals --- .../backend/tx/GraphIndexTransaction.java | 5 +++-- .../traversal/optimize/TraversalUtil.java | 22 +++++++++++++++++++ .../apache/hugegraph/core/EdgeCoreTest.java | 14 ++++++++++++ .../apache/hugegraph/core/VertexCoreTest.java | 14 ++++++++++++ 4 files changed, 53 insertions(+), 2 deletions(-) diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/tx/GraphIndexTransaction.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/tx/GraphIndexTransaction.java index d9ddbd322b..389904392a 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/tx/GraphIndexTransaction.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/tx/GraphIndexTransaction.java @@ -400,9 +400,10 @@ public IdHolderList queryIndex(ConditionQuery query) { // Query by index query.optimized(OptimizedType.INDEX); + Id label = query.uniqueConditionValue(HugeKeys.LABEL); if (query.allSysprop() && conds.size() == 1 && - query.containsCondition(HugeKeys.LABEL)) { - // Query only by label + label != null) { + // Query only by one EQ/IN-resolved label return this.queryByLabel(query); } else { // Query by userprops (or userprops + label) diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/traversal/optimize/TraversalUtil.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/traversal/optimize/TraversalUtil.java index 7b68f71778..9648a0f8a3 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/traversal/optimize/TraversalUtil.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/traversal/optimize/TraversalUtil.java @@ -445,6 +445,12 @@ private static boolean extractHasContainers(HugeVertexStep newStep, private static boolean canExtractHasContainers(HugeGraph graph, HasContainerHolder holder) { + List hasContainers = holder.getHasContainers(); + // Keep pure label non-EQ/IN predicates on GraphStep for TinkerPop filtering. + if (hasContainers.size() == 1 && + isOnlyNonEqInLabelPredicate(hasContainers.get(0))) { + return false; + } for (HasContainer has : holder.getHasContainers()) { if (!canExtractHasContainer(graph, has)) { return false; @@ -453,6 +459,22 @@ private static boolean canExtractHasContainers(HugeGraph graph, return true; } + private static boolean isOnlyNonEqInLabelPredicate(HasContainer has) { + if (!has.getKey().equals(T.label.getAccessor())) { + return false; + } + + List> predicates = new ArrayList<>(); + collectPredicates(predicates, ImmutableList.of(has.getPredicate())); + for (P predicate : predicates) { + BiPredicate bp = predicate.getBiPredicate(); + if (bp == Compare.eq || bp == Contains.within) { + return false; + } + } + return true; + } + static boolean canExtractHasContainer(HugeGraph graph, HasContainer has) { if (isSysProp(has.getKey())) { diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/EdgeCoreTest.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/EdgeCoreTest.java index fc7ed5edc6..a683486e4c 100644 --- a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/EdgeCoreTest.java +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/EdgeCoreTest.java @@ -3619,6 +3619,20 @@ public void testQueryOutEdgesByMultiLabelsAndSortKey() { labels); } + @Test + public void testQueryEdgesByNonEqLabel() { + HugeGraph graph = graph(); + init18Edges(); + + List edges = graph.traversal().E() + .has(T.label, P.neq("created")) + .toList(); + Assert.assertEquals(16, edges.size()); + for (Edge edge : edges) { + Assert.assertNotEquals("created", edge.label()); + } + } + @Test public void testQueryOutEdgesOfVertexBySortkeyWithRange() { // FIXME: skip this test for hstore diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/VertexCoreTest.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/VertexCoreTest.java index bf2714aea1..d8f0b4c35b 100644 --- a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/VertexCoreTest.java +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/VertexCoreTest.java @@ -9101,6 +9101,20 @@ public void testQueryByNonEqLabelAndIndexedProperty() { } } + @Test + public void testQueryByNonEqLabel() { + HugeGraph graph = graph(); + init10Vertices(); + + GraphTraversalSource g = graph.traversal(); + + List vertices = g.V().has(T.label, P.neq("author")).toList(); + Assert.assertEquals(8, vertices.size()); + for (Vertex vertex : vertices) { + Assert.assertNotEquals("author", vertex.label()); + } + } + @Test public void testCollectMatchedIndexesByJointLabelsWithIndexedProperties() { HugeGraph graph = graph(); From 7c85ccad14009eef37573d1c0508a962022e102d Mon Sep 17 00:00:00 2001 From: contrueCT Date: Sat, 13 Jun 2026 23:53:25 +0800 Subject: [PATCH 11/16] fix(hstore): preserve range index paging order --- .../hugegraph/backend/page/IdHolder.java | 25 +- .../backend/tx/GraphIndexTransaction.java | 248 +----------------- .../backend/store/hstore/HstoreSessions.java | 25 ++ .../store/hstore/HstoreSessionsImpl.java | 36 ++- .../backend/store/hstore/HstoreTable.java | 19 +- .../backend/tx/GraphIndexTransactionTest.java | 82 ------ .../apache/hugegraph/core/VertexCoreTest.java | 30 +++ .../apache/hugegraph/unit/UnitTestSuite.java | 2 - .../store/client/NodeTxSessionProxy.java | 15 +- .../store/client/OrderedKvIterator.java | 150 +++++++++++ .../store/client/OrderedKvIteratorTest.java | 173 ++++++++++++ 11 files changed, 461 insertions(+), 344 deletions(-) delete mode 100644 hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/backend/tx/GraphIndexTransactionTest.java create mode 100644 hugegraph-store/hg-store-client/src/main/java/org/apache/hugegraph/store/client/OrderedKvIterator.java create mode 100644 hugegraph-store/hg-store-client/src/test/java/org/apache/hugegraph/store/client/OrderedKvIteratorTest.java diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/page/IdHolder.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/page/IdHolder.java index b420648767..ab64959a01 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/page/IdHolder.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/page/IdHolder.java @@ -35,11 +35,17 @@ public abstract class IdHolder { protected final Query query; + private final boolean keepOrder; protected boolean exhausted; public IdHolder(Query query) { + this(query, false); + } + + public IdHolder(Query query, boolean keepOrder) { E.checkNotNull(query, "query"); this.query = query; + this.keepOrder = keepOrder; this.exhausted = false; } @@ -48,7 +54,7 @@ public Query query() { } public boolean keepOrder() { - return false; + return this.keepOrder; } @Override @@ -97,7 +103,13 @@ public static class PagingIdHolder extends IdHolder { public PagingIdHolder(ConditionQuery query, Function fetcher) { - super(query.copy()); + this(query, fetcher, false); + } + + public PagingIdHolder(ConditionQuery query, + Function fetcher, + boolean keepOrder) { + super(query.copy(), keepOrder); E.checkArgument(query.paging(), "Query '%s' must include page info", query); this.fetcher = fetcher; @@ -142,7 +154,14 @@ public static class BatchIdHolder extends IdHolder public BatchIdHolder(ConditionQuery query, Iterator entries, Function> fetcher) { - super(query); + this(query, entries, fetcher, false); + } + + public BatchIdHolder(ConditionQuery query, + Iterator entries, + Function> fetcher, + boolean keepOrder) { + super(query, keepOrder); this.entries = entries; this.fetcher = fetcher; this.count = 0L; diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/tx/GraphIndexTransaction.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/tx/GraphIndexTransaction.java index 389904392a..7eeed8d081 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/tx/GraphIndexTransaction.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/tx/GraphIndexTransaction.java @@ -652,262 +652,20 @@ private void storeSelectedIndexField(IndexLabel indexLabel, @Watched(prefix = "index") private IdHolder doIndexQuery(IndexLabel indexLabel, ConditionQuery query) { - if (this.needHstoreRangeIndexOrder(indexLabel)) { - return this.doHstoreRangeIndexQuery(indexLabel, query); - } if (!query.paging()) { return this.doIndexQueryBatch(indexLabel, query); } else { return new PagingIdHolder(query, q -> { return this.doIndexQueryOnce(indexLabel, q); - }); + }, this.keepBackendIndexOrder(indexLabel)); } } - private boolean needHstoreRangeIndexOrder(IndexLabel indexLabel) { + private boolean keepBackendIndexOrder(IndexLabel indexLabel) { return this.store().provider().isHstore() && indexLabel.indexType().isRange(); } - private IdHolder doHstoreRangeIndexQuery(IndexLabel indexLabel, - ConditionQuery query) { - if (!query.paging()) { - if (query.noLimitAndOffset()) { - return this.doIndexQueryBatch(indexLabel, query); - } - Set ids = this.querySortedRangeIndexIds(indexLabel, query); - return this.newSortedRangeIndexBatchHolder(query, ids); - } - return new SortedRangePagingIdHolder(query, q -> { - return this.querySortedRangeIndexPage(indexLabel, q); - }); - } - - private BatchIdHolder newSortedRangeIndexBatchHolder(ConditionQuery query, - Set ids) { - return new SortedRangeBatchIdHolder(query, ids); - } - - private Set querySortedRangeIndexIds(IndexLabel indexLabel, - ConditionQuery query) { - List indexes = this.querySortedRangeIndexes(indexLabel, - query); - Set ids = InsertionOrderUtil.newSet(); - for (HugeIndex index : indexes) { - ids.addAll(index.elementIds()); - Query.checkForceCapacity(ids.size()); - } - return ids; - } - - private PageIds querySortedRangeIndexPage(IndexLabel indexLabel, - ConditionQuery query) { - List indexes = this.querySortedRangeIndexes(indexLabel, - query); - Set allIds = InsertionOrderUtil.newSet(); - for (HugeIndex index : indexes) { - allIds.addAll(index.elementIds()); - Query.checkForceCapacity(allIds.size()); - } - if (allIds.isEmpty()) { - return PageIds.EMPTY; - } - - int start = 0; - if (!query.page().isEmpty()) { - start = PageState.fromString(query.page()).offset(); - } - if (start >= allIds.size()) { - return PageIds.EMPTY; - } - - long total = allIds.size(); - long end = query.noLimit() ? total : - Math.min(total, (long) start + query.limit()); - Set pageIds = CollectionUtil.subSet(allIds, start, (int) end); - if (pageIds.isEmpty()) { - return PageIds.EMPTY; - } - - int next = (int) end; - PageState pageState; - if (next < total) { - pageState = new PageState(new byte[]{1}, next, pageIds.size()); - } else { - pageState = new PageState(PageState.EMPTY_BYTES, 0, - pageIds.size()); - } - return new PageIds(pageIds, pageState); - } - - private List querySortedRangeIndexes(IndexLabel indexLabel, - ConditionQuery query) { - List indexes = new ArrayList<>(); - Iterator entries = null; - String spaceGraph = this.params() - .graph().spaceGraphName(); - LockUtil.Locks locks = new LockUtil.Locks(spaceGraph); - ConditionQuery scanQuery = query.copy(); - scanQuery.page(null); - scanQuery.offset(0L); - scanQuery.limit(Query.NO_LIMIT); - try { - locks.lockReads(LockUtil.INDEX_LABEL_DELETE, indexLabel.id()); - locks.lockReads(LockUtil.INDEX_LABEL_REBUILD, indexLabel.id()); - if (!indexLabel.system()) { - graph().indexLabel(indexLabel.id()); - } - - entries = super.query(scanQuery).iterator(); - while (entries.hasNext()) { - HugeIndex index = this.readMatchedIndex(indexLabel, scanQuery, - entries.next()); - if (index == null) { - continue; - } - this.removeExpiredIndexIfNeeded(index, scanQuery.showExpired()); - this.recordIndexValue(scanQuery, index); - indexes.add(index); - Query.checkForceCapacity(indexes.size()); - } - } finally { - locks.unlock(); - CloseableIterator.closeIterator(entries); - } - - Collections.sort(indexes, (a, b) -> { - return this.compareRangeIndexValues(a, b); - }); - return indexes; - } - - @SuppressWarnings({"rawtypes", "unchecked"}) - private int compareRangeIndexValues(HugeIndex left, HugeIndex right) { - Object leftValue = left.fieldValues(); - Object rightValue = right.fieldValues(); - E.checkArgument(leftValue instanceof Comparable, - "Invalid range index value '%s'", leftValue); - E.checkArgument(rightValue instanceof Comparable, - "Invalid range index value '%s'", rightValue); - return ((Comparable) leftValue).compareTo(rightValue); - } - - static class SortedRangeBatchIdHolder extends BatchIdHolder { - - private final List idList; - private int offset; - private PageIds pendingBatch; - - SortedRangeBatchIdHolder(ConditionQuery query, Set ids) { - super(query, Collections.emptyIterator(), batch -> { - throw new IllegalStateException("Unexpected sorted index fetcher"); - }); - this.idList = new ArrayList<>(ids); - this.offset = 0; - this.pendingBatch = null; - } - - @Override - public boolean keepOrder() { - return true; - } - - @Override - public boolean hasNext() { - if (this.pendingBatch != null) { - return true; - } - if (this.exhausted) { - return false; - } - return this.offset < this.idList.size(); - } - - @Override - public IdHolder next() { - if (!this.hasNext()) { - throw new java.util.NoSuchElementException(); - } - return this; - } - - @Override - public PageIds fetchNext(String page, long batchSize) { - E.checkArgument(page == null, - "Not support page parameter by BatchIdHolder"); - E.checkArgument(batchSize >= 0L, - "Invalid batch size value: %s", batchSize); - if (this.pendingBatch != null) { - PageIds result = this.pendingBatch; - this.pendingBatch = null; - return result; - } - return this.fetchBatch(batchSize); - } - - @Override - public Set all() { - Set allIds = InsertionOrderUtil.newSet(); - if (this.pendingBatch != null) { - allIds.addAll(this.pendingBatch.ids()); - } - if (this.offset < this.idList.size()) { - allIds.addAll(this.idList.subList(this.offset, - this.idList.size())); - } - this.close(); - return allIds; - } - - @Override - public PageIds peekNext(long size) { - E.checkArgument(this.pendingBatch == null, - "Can't call peekNext() twice"); - this.pendingBatch = this.fetchBatch(size); - return this.pendingBatch; - } - - @Override - public void close() { - this.exhausted = true; - this.pendingBatch = null; - this.offset = this.idList.size(); - } - - private PageIds fetchBatch(long batchSize) { - if (this.offset >= this.idList.size() || batchSize == 0L) { - this.close(); - return PageIds.EMPTY; - } - - int end; - if (batchSize == Query.NO_LIMIT) { - end = this.idList.size(); - } else { - end = (int) Math.min((long) this.idList.size(), - this.offset + batchSize); - } - Set batchIds = InsertionOrderUtil.newSet(); - batchIds.addAll(this.idList.subList(this.offset, end)); - this.offset = end; - this.exhausted = this.offset >= this.idList.size(); - return new PageIds(batchIds, PageState.EMPTY); - } - } - - private static class SortedRangePagingIdHolder extends PagingIdHolder { - - SortedRangePagingIdHolder(ConditionQuery query, - Function fetcher) { - super(query, fetcher); - } - - @Override - public boolean keepOrder() { - return true; - } - } - @Watched(prefix = "index") private IdHolder doIndexQueryBatch(IndexLabel indexLabel, ConditionQuery query) { @@ -947,7 +705,7 @@ private IdHolder doIndexQueryBatch(IndexLabel indexLabel, } finally { locks.unlock(); } - }); + }, this.keepBackendIndexOrder(indexLabel)); } private void recordIndexValue(ConditionQuery query, HugeIndex index) { diff --git a/hugegraph-server/hugegraph-hstore/src/main/java/org/apache/hugegraph/backend/store/hstore/HstoreSessions.java b/hugegraph-server/hugegraph-hstore/src/main/java/org/apache/hugegraph/backend/store/hstore/HstoreSessions.java index 0abb6458b9..b85f01bb47 100755 --- a/hugegraph-server/hugegraph-hstore/src/main/java/org/apache/hugegraph/backend/store/hstore/HstoreSessions.java +++ b/hugegraph-server/hugegraph-hstore/src/main/java/org/apache/hugegraph/backend/store/hstore/HstoreSessions.java @@ -150,6 +150,18 @@ public abstract BackendColumnIterator scan(String table, int scanType, byte[] query); + public BackendColumnIterator scan(String table, + byte[] ownerKeyFrom, + byte[] ownerKeyTo, + byte[] keyFrom, + byte[] keyTo, + int scanType, + byte[] query, + long limit) { + return this.scan(table, ownerKeyFrom, ownerKeyTo, keyFrom, keyTo, + scanType, query); + } + public abstract BackendColumnIterator scan(String table, byte[] ownerKeyFrom, byte[] ownerKeyTo, @@ -159,6 +171,19 @@ public abstract BackendColumnIterator scan(String table, byte[] query, byte[] position); + public BackendColumnIterator scan(String table, + byte[] ownerKeyFrom, + byte[] ownerKeyTo, + byte[] keyFrom, + byte[] keyTo, + int scanType, + byte[] query, + byte[] position, + long limit) { + return this.scan(table, ownerKeyFrom, ownerKeyTo, keyFrom, keyTo, + scanType, query, position); + } + public abstract BackendColumnIterator scan(String table, int codeFrom, int codeTo, diff --git a/hugegraph-server/hugegraph-hstore/src/main/java/org/apache/hugegraph/backend/store/hstore/HstoreSessionsImpl.java b/hugegraph-server/hugegraph-hstore/src/main/java/org/apache/hugegraph/backend/store/hstore/HstoreSessionsImpl.java index 2f98d03745..0e53786a79 100755 --- a/hugegraph-server/hugegraph-hstore/src/main/java/org/apache/hugegraph/backend/store/hstore/HstoreSessionsImpl.java +++ b/hugegraph-server/hugegraph-hstore/src/main/java/org/apache/hugegraph/backend/store/hstore/HstoreSessionsImpl.java @@ -691,6 +691,16 @@ public BackendColumnIterator scan(String table, byte[] ownerKeyFrom, byte[] ownerKeyTo, byte[] keyFrom, byte[] keyTo, int scanType, byte[] query) { + return this.scan(table, ownerKeyFrom, ownerKeyTo, keyFrom, keyTo, + scanType, query, HgStoreClientConst.NO_LIMIT); + } + + @Override + public BackendColumnIterator scan(String table, byte[] ownerKeyFrom, + byte[] ownerKeyTo, + byte[] keyFrom, byte[] keyTo, + int scanType, byte[] query, + long limit) { assert !this.hasChanges(); HgKvIterator result = this.graph.scanIterator(table, HgOwnerKey.of( @@ -699,7 +709,7 @@ public BackendColumnIterator scan(String table, byte[] ownerKeyFrom, HgOwnerKey.of( ownerKeyTo, keyTo), - 0, + toHstoreLimit(limit), scanType, query); return new ColumnIterator<>(table, result, keyFrom, keyTo, @@ -712,6 +722,17 @@ public BackendColumnIterator scan(String table, byte[] ownerKeyFrom, byte[] keyFrom, byte[] keyTo, int scanType, byte[] query, byte[] position) { + return this.scan(table, ownerKeyFrom, ownerKeyTo, keyFrom, keyTo, + scanType, query, position, + HgStoreClientConst.NO_LIMIT); + } + + @Override + public BackendColumnIterator scan(String table, byte[] ownerKeyFrom, + byte[] ownerKeyTo, + byte[] keyFrom, byte[] keyTo, + int scanType, byte[] query, + byte[] position, long limit) { assert !this.hasChanges(); HgKvIterator result = this.graph.scanIterator(table, HgOwnerKey.of( @@ -720,14 +741,23 @@ public BackendColumnIterator scan(String table, byte[] ownerKeyFrom, HgOwnerKey.of( ownerKeyTo, keyTo), - 0, + toHstoreLimit(limit), scanType, query); - result.seek(position); + if (position != null && position.length > 0) { + result.seek(position); + } return new ColumnIterator<>(table, result, keyFrom, keyTo, scanType); } + private long toHstoreLimit(long limit) { + if (limit <= 0L || limit == Query.NO_LIMIT) { + return HgStoreClientConst.NO_LIMIT; + } + return limit; + } + @Override public BackendColumnIterator scan(String table, int codeFrom, int codeTo, int scanType, diff --git a/hugegraph-server/hugegraph-hstore/src/main/java/org/apache/hugegraph/backend/store/hstore/HstoreTable.java b/hugegraph-server/hugegraph-hstore/src/main/java/org/apache/hugegraph/backend/store/hstore/HstoreTable.java index e1830111c3..4ac885b177 100755 --- a/hugegraph-server/hugegraph-hstore/src/main/java/org/apache/hugegraph/backend/store/hstore/HstoreTable.java +++ b/hugegraph-server/hugegraph-hstore/src/main/java/org/apache/hugegraph/backend/store/hstore/HstoreTable.java @@ -625,9 +625,8 @@ protected BackendColumnIterator queryByRange(Session session, } ConditionQuery cq; Query origin = query.originQuery(); - byte[] position = null; if (query.paging() && !query.page().isEmpty()) { - position = PageState.fromString(query.page()).position(); + type = (type & ~Session.SCAN_GTE_BEGIN) | Session.SCAN_GT_BEGIN; } byte[] ownerStart = this.ownerByQueryDelegate.apply(query.resultType(), query.start()); @@ -636,6 +635,7 @@ protected BackendColumnIterator queryByRange(Session session, if (origin instanceof ConditionQuery && (query.resultType().isEdge() || query.resultType().isVertex())) { cq = (ConditionQuery) query.originQuery(); + long limit = rangeScanLimit(query); // LOG.debug("query {} with ownerKeyFrom: {}, ownerKeyTo: {}, " + // "keyFrom: {}, keyTo: {}, " + @@ -643,11 +643,18 @@ protected BackendColumnIterator queryByRange(Session session, // this.table(), bytes2String(ownerStart), // bytes2String(ownerEnd), bytes2String(start), // bytes2String(end), type, cq.bytes()); - return session.scan(this.table(), ownerStart, - ownerEnd, start, end, type, cq.bytes(), position); + return session.scan(this.table(), ownerStart, ownerEnd, start, + end, type, cq.bytes(), null, limit); + } + return session.scan(this.table(), ownerStart, ownerEnd, start, end, + type, null, null, rangeScanLimit(query)); + } + + private static long rangeScanLimit(IdRangeQuery query) { + if (query.noLimit()) { + return HgStoreClientConst.NO_LIMIT; } - return session.scan(this.table(), ownerStart, - ownerEnd, start, end, type, null, position); + return query.total(); } protected BackendColumnIterator queryByCond(Session session, diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/backend/tx/GraphIndexTransactionTest.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/backend/tx/GraphIndexTransactionTest.java deleted file mode 100644 index a1b0d4e8c6..0000000000 --- a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/backend/tx/GraphIndexTransactionTest.java +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hugegraph.backend.tx; - -import java.util.ArrayList; -import java.util.List; -import java.util.Set; - -import org.apache.hugegraph.backend.id.Id; -import org.apache.hugegraph.backend.id.IdGenerator; -import org.apache.hugegraph.backend.page.PageIds; -import org.apache.hugegraph.backend.query.ConditionQuery; -import org.apache.hugegraph.testutil.Assert; -import org.apache.hugegraph.type.HugeType; -import org.apache.hugegraph.util.InsertionOrderUtil; -import org.junit.Test; - -import com.google.common.collect.ImmutableList; - -public class GraphIndexTransactionTest { - - @Test - public void testSortedRangeBatchHolderKeepsPeekedBatch() { - ConditionQuery query = new ConditionQuery(HugeType.VERTEX); - Id id1 = IdGenerator.of(1); - Id id2 = IdGenerator.of(2); - Id id3 = IdGenerator.of(3); - Set ids = InsertionOrderUtil.newSet(); - ids.add(id1); - ids.add(id2); - ids.add(id3); - - GraphIndexTransaction.SortedRangeBatchIdHolder holder = - new GraphIndexTransaction.SortedRangeBatchIdHolder(query, ids); - - Assert.assertTrue(holder.keepOrder()); - - PageIds peeked = holder.peekNext(2); - Assert.assertEquals(ImmutableList.of(id1, id2), asList(peeked.ids())); - - PageIds firstBatch = holder.fetchNext(null, 2); - Assert.assertEquals(ImmutableList.of(id1, id2), - asList(firstBatch.ids())); - - PageIds secondBatch = holder.fetchNext(null, 2); - Assert.assertEquals(ImmutableList.of(id3), asList(secondBatch.ids())); - Assert.assertFalse(holder.hasNext()); - } - - @Test - public void testSortedRangeBatchHolderClosesOnZeroBatch() { - ConditionQuery query = new ConditionQuery(HugeType.VERTEX); - Set ids = InsertionOrderUtil.newSet(); - ids.add(IdGenerator.of(1)); - - GraphIndexTransaction.SortedRangeBatchIdHolder holder = - new GraphIndexTransaction.SortedRangeBatchIdHolder(query, ids); - - PageIds batch = holder.fetchNext(null, 0); - Assert.assertTrue(batch.empty()); - Assert.assertFalse(holder.hasNext()); - } - - private static List asList(Set ids) { - return new ArrayList<>(ids); - } -} diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/VertexCoreTest.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/VertexCoreTest.java index d8f0b4c35b..967ca0049a 100644 --- a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/VertexCoreTest.java +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/VertexCoreTest.java @@ -8775,6 +8775,27 @@ public void testQueryByRangeIndexInPage() { Assert.assertEquals(3, vertices2.size()); Assert.assertTrue(CollectionUtil.intersect(vertices1, vertices2) .isEmpty()); + + // There are 12 vertices matched, ordered by range-index key + iter = g.V().hasLabel("software") + .has("price", P.between(100, 400)) + .has("~page", "") + .limit(5); + + vertices1 = IteratorUtils.list(iter); + Assert.assertEquals(5, vertices1.size()); + assertSortedByPrice(vertices1); + page = TraversalUtil.page(iter); + vertices2 = g.V().hasLabel("software") + .has("price", P.between(100, 400)) + .has("~page", page).limit(5).toList(); + Assert.assertEquals(5, vertices2.size()); + assertSortedByPrice(vertices2); + Assert.assertTrue((int) vertices1.get(vertices1.size() - 1) + .value("price") <= + (int) vertices2.get(0).value("price")); + Assert.assertTrue(CollectionUtil.intersect(vertices1, vertices2) + .isEmpty()); } @Test @@ -9603,6 +9624,15 @@ private Vertex vertex(String label, String pkName, Object pkValue) { return vertices.size() == 1 ? vertices.get(0) : null; } + private static void assertSortedByPrice(List vertices) { + int previous = Integer.MIN_VALUE; + for (Vertex vertex : vertices) { + int current = vertex.value("price"); + Assert.assertTrue(previous <= current); + previous = current; + } + } + private static void assertContains(List vertices, Object... keyValues) { Assert.assertTrue(Utils.contains(vertices, new FakeObjects.FakeVertex(keyValues))); diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/UnitTestSuite.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/UnitTestSuite.java index 9935909a02..3c856937bd 100644 --- a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/UnitTestSuite.java +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/UnitTestSuite.java @@ -17,7 +17,6 @@ package org.apache.hugegraph.unit; -import org.apache.hugegraph.backend.tx.GraphIndexTransactionTest; import org.apache.hugegraph.core.RoleElectionStateMachineTest; import org.apache.hugegraph.meta.MetaManagerSchemaCacheClearEventTest; import org.apache.hugegraph.traversal.optimize.TraversalUtilOptimizeTest; @@ -123,7 +122,6 @@ BackendMutationTest.class, ConditionTest.class, ConditionQueryFlattenTest.class, - GraphIndexTransactionTest.class, QueryTest.class, QueryResultsTest.class, RangeTest.class, diff --git a/hugegraph-store/hg-store-client/src/main/java/org/apache/hugegraph/store/client/NodeTxSessionProxy.java b/hugegraph-store/hg-store-client/src/main/java/org/apache/hugegraph/store/client/NodeTxSessionProxy.java index 65e3b76ec0..401e415e8b 100644 --- a/hugegraph-store/hg-store-client/src/main/java/org/apache/hugegraph/store/client/NodeTxSessionProxy.java +++ b/hugegraph-store/hg-store-client/src/main/java/org/apache/hugegraph/store/client/NodeTxSessionProxy.java @@ -434,7 +434,7 @@ public HgKvIterator scanIterator(String table, HgOwnerKey startKey, HgAssert.isFalse(startKey == null, "The argument is invalid: startKey"); HgAssert.isFalse(endKey == null, "The argument is invalid: endKey"); - return this.toHgKvIteratorProxy( + return this.toOrderedHgKvIteratorProxy( this.toNodeTkvList(table, startKey, endKey) .parallelStream() .map( @@ -452,7 +452,7 @@ public HgKvIterator scanIterator(String table, HgOwnerKey startKey, H HgAssert.isFalse(startKey == null, "The argument is invalid: startKey"); HgAssert.isFalse(endKey == null, "The argument is invalid: endKey"); - return this.toHgKvIteratorProxy( + return this.toOrderedHgKvIteratorProxy( this.toNodeTkvList(table, startKey, endKey) .parallelStream() .map( @@ -472,7 +472,7 @@ public HgKvIterator scanIterator(String table, HgOwnerKey startKey, H HgAssert.isFalse(startKey == null, "The argument is invalid: startKey"); HgAssert.isFalse(endKey == null, "The argument is invalid: endKey"); - return this.toHgKvIteratorProxy( + return this.toOrderedHgKvIteratorProxy( this.toNodeTkvList(table, startKey, endKey) .parallelStream() .map( @@ -642,6 +642,15 @@ private BiFunction, HgScanQuery.ScanBuilder> toScanQueryFu } /*-- common --*/ + @SuppressWarnings("unchecked") + private HgKvIterator toOrderedHgKvIteratorProxy(List iteratorList, + long limit) { + List> iterators = + (List>) + (List) iteratorList; + return new OrderedKvIterator(iterators, limit); + } + private HgKvIterator toHgKvIteratorProxy(List iteratorList, long limit) { boolean isAllOrderedLimiter = iteratorList.stream() .allMatch( diff --git a/hugegraph-store/hg-store-client/src/main/java/org/apache/hugegraph/store/client/OrderedKvIterator.java b/hugegraph-store/hg-store-client/src/main/java/org/apache/hugegraph/store/client/OrderedKvIterator.java new file mode 100644 index 0000000000..734829401b --- /dev/null +++ b/hugegraph-store/hg-store-client/src/main/java/org/apache/hugegraph/store/client/OrderedKvIterator.java @@ -0,0 +1,150 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store.client; + +import java.util.Arrays; +import java.util.List; +import java.util.NoSuchElementException; +import java.util.PriorityQueue; + +import org.apache.hugegraph.store.HgKvEntry; +import org.apache.hugegraph.store.HgKvIterator; +import org.apache.hugegraph.store.client.util.HgStoreClientConst; + +final class OrderedKvIterator implements HgKvIterator { + + private final List> iterators; + private final PriorityQueue queue; + private final long limit; + + private boolean initialized; + private long count; + private HgKvEntry current; + private byte[] position; + private byte[] seekPosition; + + OrderedKvIterator(List> iterators, + long limit) { + this.iterators = iterators; + this.queue = new PriorityQueue<>((left, right) -> { + int result = Arrays.compareUnsigned(left.entry.key(), + right.entry.key()); + if (result != 0) { + return result; + } + return Integer.compare(left.source, right.source); + }); + this.limit = limit <= HgStoreClientConst.NO_LIMIT ? Long.MAX_VALUE : + limit; + this.initialized = false; + this.count = 0L; + this.current = null; + this.position = HgStoreClientConst.EMPTY_BYTES; + this.seekPosition = HgStoreClientConst.EMPTY_BYTES; + } + + @Override + public boolean hasNext() { + this.initialize(); + return this.count < this.limit && !this.queue.isEmpty(); + } + + @Override + public HgKvEntry next() { + if (!this.hasNext()) { + throw new NoSuchElementException(); + } + + SourceEntry entry = this.queue.poll(); + this.current = entry.entry; + this.position = entry.entry.key(); + this.count++; + + HgKvIterator iterator = + this.iterators.get(entry.source); + this.addNext(entry.source, iterator); + return this.current; + } + + @Override + public byte[] key() { + return this.current == null ? null : this.current.key(); + } + + @Override + public byte[] value() { + return this.current == null ? null : this.current.value(); + } + + @Override + public byte[] position() { + return this.position; + } + + @Override + public void seek(byte[] position) { + if (this.initialized) { + throw new IllegalStateException("Can't seek after reading"); + } + this.seekPosition = position == null ? HgStoreClientConst.EMPTY_BYTES : + position; + } + + @Override + public void close() { + for (HgKvIterator iterator : this.iterators) { + iterator.close(); + } + this.queue.clear(); + } + + private void initialize() { + if (this.initialized) { + return; + } + for (int i = 0; i < this.iterators.size(); i++) { + HgKvIterator iterator = + this.iterators.get(i); + this.addNext(i, iterator); + } + this.initialized = true; + } + + private void addNext(int source, + HgKvIterator iterator) { + while (iterator.hasNext()) { + HgKvEntry entry = iterator.next(); + if (this.seekPosition.length == 0 || + Arrays.compareUnsigned(entry.key(), this.seekPosition) > 0) { + this.queue.add(new SourceEntry(source, entry)); + break; + } + } + } + + private static final class SourceEntry { + + private final int source; + private final HgKvEntry entry; + + private SourceEntry(int source, HgKvEntry entry) { + this.source = source; + this.entry = entry; + } + } +} diff --git a/hugegraph-store/hg-store-client/src/test/java/org/apache/hugegraph/store/client/OrderedKvIteratorTest.java b/hugegraph-store/hg-store-client/src/test/java/org/apache/hugegraph/store/client/OrderedKvIteratorTest.java new file mode 100644 index 0000000000..742e3e8a5f --- /dev/null +++ b/hugegraph-store/hg-store-client/src/test/java/org/apache/hugegraph/store/client/OrderedKvIteratorTest.java @@ -0,0 +1,173 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store.client; + +import java.util.Arrays; +import java.util.List; +import java.util.NoSuchElementException; + +import org.apache.hugegraph.store.HgKvEntry; +import org.apache.hugegraph.store.HgKvIterator; +import org.junit.Assert; +import org.junit.Test; + +public class OrderedKvIteratorTest { + + @Test + public void testMergeByKeyWithLimit() { + OrderedKvIterator iterator = new OrderedKvIterator(Arrays.asList( + new TestIterator(1, 4), + new TestIterator(2, 3) + ), 3L); + + Assert.assertTrue(iterator.hasNext()); + Assert.assertEquals(1, key(iterator.next())); + Assert.assertArrayEquals(keyBytes(1), iterator.position()); + + Assert.assertTrue(iterator.hasNext()); + Assert.assertEquals(2, key(iterator.next())); + Assert.assertArrayEquals(keyBytes(2), iterator.position()); + + Assert.assertTrue(iterator.hasNext()); + Assert.assertEquals(3, key(iterator.next())); + Assert.assertArrayEquals(keyBytes(3), iterator.position()); + + Assert.assertFalse(iterator.hasNext()); + } + + @Test + public void testNoLimitReturnsAllKeysInOrder() { + OrderedKvIterator iterator = new OrderedKvIterator(Arrays.asList( + new TestIterator(1, 5), + new TestIterator(2, 4) + ), 0L); + + Assert.assertEquals(1, key(iterator.next())); + Assert.assertEquals(2, key(iterator.next())); + Assert.assertEquals(4, key(iterator.next())); + Assert.assertEquals(5, key(iterator.next())); + Assert.assertFalse(iterator.hasNext()); + } + + @Test + public void testSeekBeforeReadSkipsBoundaryKey() { + OrderedKvIterator iterator = new OrderedKvIterator(Arrays.asList( + new TestIterator(1, 4), + new TestIterator(2, 3) + ), 0L); + + iterator.seek(keyBytes(3)); + + Assert.assertEquals(4, key(iterator.next())); + Assert.assertFalse(iterator.hasNext()); + } + + @Test + public void testSeekDoesNotSkipLowerSourceWithLargerKeys() { + OrderedKvIterator iterator = new OrderedKvIterator(Arrays.asList( + new TestIterator(1, 100), + new TestIterator(2, 3) + ), 0L); + + iterator.seek(keyBytes(2)); + + Assert.assertEquals(3, key(iterator.next())); + Assert.assertEquals(100, key(iterator.next())); + Assert.assertFalse(iterator.hasNext()); + } + + private static int key(HgKvEntry entry) { + return entry.key()[0] & 0xFF; + } + + private static byte[] keyBytes(int key) { + return new byte[]{(byte) key}; + } + + private static final class TestIterator implements HgKvIterator { + + private final List keys; + private int offset; + private HgKvEntry current; + + private TestIterator(Integer... keys) { + this.keys = Arrays.asList(keys); + this.offset = 0; + this.current = null; + } + + @Override + public boolean hasNext() { + return this.offset < this.keys.size(); + } + + @Override + public HgKvEntry next() { + if (!this.hasNext()) { + throw new NoSuchElementException(); + } + int key = this.keys.get(this.offset++); + this.current = new TestEntry(keyBytes(key)); + return this.current; + } + + @Override + public byte[] key() { + return this.current == null ? null : this.current.key(); + } + + @Override + public byte[] value() { + return this.current == null ? null : this.current.value(); + } + + @Override + public byte[] position() { + byte[] key = this.key(); + return key == null ? null : new byte[]{(byte) 0xFF, key[0]}; + } + + @Override + public void seek(byte[] position) { + int target = position[0] & 0xFF; + while (this.offset < this.keys.size() && + this.keys.get(this.offset) < target) { + this.offset++; + } + } + } + + private static final class TestEntry implements HgKvEntry { + + private final byte[] key; + + private TestEntry(byte[] key) { + this.key = key; + } + + @Override + public byte[] key() { + return this.key; + } + + @Override + public byte[] value() { + return this.key; + } + } +} From 9e24432f0ffcd57b4b1b03a712f4abaad21796e5 Mon Sep 17 00:00:00 2001 From: contrueCT Date: Sun, 14 Jun 2026 22:11:12 +0800 Subject: [PATCH 12/16] fix(hstore): scope ordered range scan to indexes --- .../hugegraph/backend/page/IdHolder.java | 37 ++- .../backend/page/PageEntryIterator.java | 8 +- .../hugegraph/backend/page/QueryList.java | 28 +- .../backend/tx/GraphIndexTransaction.java | 24 +- .../backend/store/hstore/HstoreSessions.java | 15 + .../store/hstore/HstoreSessionsImpl.java | 49 +++- .../backend/store/hstore/HstoreTable.java | 88 +++++- .../store/hstore/HstoreSessionsImplTest.java | 181 ++++++++++++ .../backend/store/hstore/HstoreTableTest.java | 184 +++++++++++++ .../apache/hugegraph/unit/UnitTestSuite.java | 2 + .../hugegraph/unit/core/IdHolderTest.java | 254 +++++++++++++++++ .../hugegraph/store/client/NodeTkv.java | 23 +- .../store/client/NodeTxSessionProxy.java | 63 ++++- .../store/client/OrderedKvIterator.java | 25 +- .../store/client/NodeTxSessionProxyTest.java | 260 ++++++++++++++++++ .../store/client/OrderedKvIteratorTest.java | 35 +++ 16 files changed, 1230 insertions(+), 46 deletions(-) create mode 100644 hugegraph-server/hugegraph-hstore/src/test/java/org/apache/hugegraph/backend/store/hstore/HstoreSessionsImplTest.java create mode 100644 hugegraph-server/hugegraph-hstore/src/test/java/org/apache/hugegraph/backend/store/hstore/HstoreTableTest.java create mode 100644 hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/core/IdHolderTest.java create mode 100644 hugegraph-store/hg-store-client/src/test/java/org/apache/hugegraph/store/client/NodeTxSessionProxyTest.java diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/page/IdHolder.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/page/IdHolder.java index ab64959a01..caffa30d8e 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/page/IdHolder.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/page/IdHolder.java @@ -57,6 +57,10 @@ public boolean keepOrder() { return this.keepOrder; } + public boolean continueEmptyPage() { + return false; + } + @Override public String toString() { return String.format("%s{origin:%s,final:%s}", @@ -100,6 +104,8 @@ public PageIds fetchNext(String page, long pageSize) { public static class PagingIdHolder extends IdHolder { private final Function fetcher; + private final boolean continueEmptyPage; + private final boolean continuePartialPage; public PagingIdHolder(ConditionQuery query, Function fetcher) { @@ -109,10 +115,28 @@ public PagingIdHolder(ConditionQuery query, public PagingIdHolder(ConditionQuery query, Function fetcher, boolean keepOrder) { + this(query, fetcher, keepOrder, false, false); + } + + public PagingIdHolder(ConditionQuery query, + Function fetcher, + boolean keepOrder, + boolean continueEmptyPage) { + this(query, fetcher, keepOrder, continueEmptyPage, + continueEmptyPage); + } + + public PagingIdHolder(ConditionQuery query, + Function fetcher, + boolean keepOrder, + boolean continueEmptyPage, + boolean continuePartialPage) { super(query.copy(), keepOrder); E.checkArgument(query.paging(), "Query '%s' must include page info", query); this.fetcher = fetcher; + this.continueEmptyPage = continueEmptyPage; + this.continuePartialPage = continuePartialPage; } @Override @@ -131,12 +155,23 @@ public PageIds fetchNext(String page, long pageSize) { PageIds result = this.fetcher.apply((ConditionQuery) this.query); assert result != null; - if (result.ids().size() < pageSize || result.page() == null) { + if (result.empty() && !this.continueEmptyPage) { + this.exhausted = true; + return PageIds.EMPTY; + } + if (result.page() == null || + (!this.continuePartialPage && + result.ids().size() < pageSize)) { this.exhausted = true; } return result; } + @Override + public boolean continueEmptyPage() { + return this.continueEmptyPage; + } + @Override public Set all() { throw new NotImplementedException("PagingIdHolder.all"); diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/page/PageEntryIterator.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/page/PageEntryIterator.java index bbc93c79b6..0a0191c041 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/page/PageEntryIterator.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/page/PageEntryIterator.java @@ -87,7 +87,13 @@ private boolean fetch() { this.remaining -= this.pageResults.total(); return true; } else { - this.pageInfo.increase(); + if (this.pageResults.continueOnEmpty() && + this.pageResults.hasNextPage() && + !this.pageResults.page().equals(this.pageInfo.page())) { + this.pageInfo.page(this.pageResults.page()); + } else { + this.pageInfo.increase(); + } return this.fetch(); } } diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/page/QueryList.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/page/QueryList.java index f1930bd55a..2392516f25 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/page/QueryList.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/page/QueryList.java @@ -272,7 +272,9 @@ public PageResults iterator(int index, String page, long pageSize) { this.updateResultsFilter(bindQuery); PageIds pageIds = holder.fetchNext(page, pageSize); if (pageIds.empty()) { - return PageResults.emptyIterator(); + return PageResults.emptyIterator(bindQuery, + pageIds.pageState(), + holder.continueEmptyPage()); } QueryResults results = this.queryByIndexIds(pageIds.ids(), @@ -331,10 +333,17 @@ public static class PageResults { private final QueryResults results; private final PageState pageState; + private final boolean continueOnEmpty; public PageResults(QueryResults results, PageState pageState) { + this(results, pageState, false); + } + + public PageResults(QueryResults results, PageState pageState, + boolean continueOnEmpty) { this.results = results; this.pageState = pageState; + this.continueOnEmpty = continueOnEmpty; } public Iterator get() { @@ -346,6 +355,10 @@ public boolean hasNextPage() { PageState.EMPTY_BYTES); } + public boolean continueOnEmpty() { + return this.continueOnEmpty; + } + public Query query() { List queries = this.results.queries(); E.checkState(queries.size() == 1, @@ -365,5 +378,18 @@ public long total() { public static PageResults emptyIterator() { return (PageResults) EMPTY; } + + public static PageResults emptyIterator(Query query, + PageState pageState) { + return emptyIterator(query, pageState, false); + } + + public static PageResults emptyIterator(Query query, + PageState pageState, + boolean continueOnEmpty) { + return new PageResults<>( + new QueryResults<>(QueryResults.emptyIterator(), query), + pageState, continueOnEmpty); + } } } diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/tx/GraphIndexTransaction.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/tx/GraphIndexTransaction.java index 7eeed8d081..9118c27a83 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/tx/GraphIndexTransaction.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/tx/GraphIndexTransaction.java @@ -657,13 +657,15 @@ private IdHolder doIndexQuery(IndexLabel indexLabel, ConditionQuery query) { } else { return new PagingIdHolder(query, q -> { return this.doIndexQueryOnce(indexLabel, q); - }, this.keepBackendIndexOrder(indexLabel)); + }, this.keepBackendIndexOrder(indexLabel, query)); } } - private boolean keepBackendIndexOrder(IndexLabel indexLabel) { + private boolean keepBackendIndexOrder(IndexLabel indexLabel, + ConditionQuery query) { return this.store().provider().isHstore() && - indexLabel.indexType().isRange(); + indexLabel.indexType().isRange() && + !query.noLimit(); } @Watched(prefix = "index") @@ -705,7 +707,7 @@ private IdHolder doIndexQueryBatch(IndexLabel indexLabel, } finally { locks.unlock(); } - }, this.keepBackendIndexOrder(indexLabel)); + }, this.keepBackendIndexOrder(indexLabel, query)); } private void recordIndexValue(ConditionQuery query, HugeIndex index) { @@ -749,8 +751,15 @@ private PageIds doIndexQueryOnce(IndexLabel indexLabel, Query.checkForceCapacity(ids.size()); this.recordIndexValue(query, index); } - // If there is no data, the entries is not a Metadatable object if (ids.isEmpty()) { + if (query.paging() && entries instanceof Metadatable) { + PageState pageState = PageInfo.pageState(entries); + if (pageState.position().length > 0) { + pageState = new PageState(pageState.position(), + pageState.offset(), 0); + return new PageIds(ids, pageState); + } + } return PageIds.EMPTY; } // NOTE: Memory backend's iterator is not Metadatable @@ -761,7 +770,10 @@ private PageIds doIndexQueryOnce(IndexLabel indexLabel, "The entries must be Metadatable when query " + "in paging, but got '%s'", entries.getClass().getName()); - return new PageIds(ids, PageInfo.pageState(entries)); + PageState pageState = PageInfo.pageState(entries); + pageState = new PageState(pageState.position(), + pageState.offset(), ids.size()); + return new PageIds(ids, pageState); } finally { locks.unlock(); CloseableIterator.closeIterator(entries); diff --git a/hugegraph-server/hugegraph-hstore/src/main/java/org/apache/hugegraph/backend/store/hstore/HstoreSessions.java b/hugegraph-server/hugegraph-hstore/src/main/java/org/apache/hugegraph/backend/store/hstore/HstoreSessions.java index b85f01bb47..563640d4df 100755 --- a/hugegraph-server/hugegraph-hstore/src/main/java/org/apache/hugegraph/backend/store/hstore/HstoreSessions.java +++ b/hugegraph-server/hugegraph-hstore/src/main/java/org/apache/hugegraph/backend/store/hstore/HstoreSessions.java @@ -162,6 +162,15 @@ public BackendColumnIterator scan(String table, scanType, query); } + public abstract BackendColumnIterator scanOrdered(String table, + byte[] ownerKeyFrom, + byte[] ownerKeyTo, + byte[] keyFrom, + byte[] keyTo, + int scanType, + byte[] query, + long limit); + public abstract BackendColumnIterator scan(String table, byte[] ownerKeyFrom, byte[] ownerKeyTo, @@ -210,6 +219,12 @@ public abstract void merge(String table, byte[] ownerKey, public abstract BackendColumnIterator scan(String table, byte[] conditionQueryToByte); + public BackendColumnIterator scan(String table, + byte[] conditionQueryToByte, + long limit) { + return this.scan(table, conditionQueryToByte); + } + public HugeConfig getConf() { return conf; } diff --git a/hugegraph-server/hugegraph-hstore/src/main/java/org/apache/hugegraph/backend/store/hstore/HstoreSessionsImpl.java b/hugegraph-server/hugegraph-hstore/src/main/java/org/apache/hugegraph/backend/store/hstore/HstoreSessionsImpl.java index 0e53786a79..5887a4f093 100755 --- a/hugegraph-server/hugegraph-hstore/src/main/java/org/apache/hugegraph/backend/store/hstore/HstoreSessionsImpl.java +++ b/hugegraph-server/hugegraph-hstore/src/main/java/org/apache/hugegraph/backend/store/hstore/HstoreSessionsImpl.java @@ -50,6 +50,7 @@ import org.apache.hugegraph.store.HgScanQuery; import org.apache.hugegraph.store.HgStoreClient; import org.apache.hugegraph.store.HgStoreSession; +import org.apache.hugegraph.store.client.NodeTxSessionProxy; import org.apache.hugegraph.store.client.grpc.KvCloseableIterator; import org.apache.hugegraph.store.client.util.HgStoreClientConst; import org.apache.hugegraph.store.grpc.common.ScanOrderType; @@ -226,15 +227,22 @@ private static class ColumnIterator implements private final int scanType; private final String table; private final byte[] value; + private final boolean keepPositionAfterExhausted; private boolean gotNext; private byte[] position; public ColumnIterator(String table, T results) { - this(table, results, null, null, 0); + this(table, results, null, null, 0, false); } public ColumnIterator(String table, T results, byte[] keyBegin, byte[] keyEnd, int scanType) { + this(table, results, keyBegin, keyEnd, scanType, false); + } + + public ColumnIterator(String table, T results, byte[] keyBegin, + byte[] keyEnd, int scanType, + boolean keepPositionAfterExhausted) { E.checkNotNull(results, "results"); this.table = table; this.iter = results; @@ -242,6 +250,7 @@ public ColumnIterator(String table, T results, byte[] keyBegin, this.keyEnd = keyEnd; this.scanType = scanType; this.value = null; + this.keepPositionAfterExhausted = keepPositionAfterExhausted; if (this.iter.hasNext()) { this.iter.next(); this.gotNext = true; @@ -317,11 +326,9 @@ private boolean match(int expected) { @Override public boolean hasNext() { - if (gotNext) { + if (gotNext && !this.keepPositionAfterExhausted) { this.position = this.iter.position(); - } else { - // QUESTION: Resetting the position may result in the caller being unable to - // retrieve the corresponding position. + } else if (!this.keepPositionAfterExhausted) { this.position = null; } return gotNext; @@ -376,6 +383,9 @@ public BackendColumn next() { BackendColumn col = BackendColumn.of(this.iter.key(), this.iter.value()); + if (this.keepPositionAfterExhausted) { + this.position = col.name; + } if (this.iter.hasNext()) { gotNext = true; this.iter.next(); @@ -587,6 +597,16 @@ public BackendColumnIterator scan(String table, return new ColumnIterator<>(table, results); } + @Override + public BackendColumnIterator scan(String table, + byte[] conditionQueryToByte, + long limit) { + assert !this.hasChanges(); + HgKvIterator results = this.graph.scanIterator( + table, toHstoreLimit(limit), conditionQueryToByte); + return new ColumnIterator<>(table, results); + } + @Override public BackendColumnIterator scan(String table, byte[] ownerKey, byte[] prefix) { @@ -716,6 +736,25 @@ public BackendColumnIterator scan(String table, byte[] ownerKeyFrom, scanType); } + @Override + public BackendColumnIterator scanOrdered(String table, + byte[] ownerKeyFrom, + byte[] ownerKeyTo, + byte[] keyFrom, + byte[] keyTo, + int scanType, + byte[] query, + long limit) { + assert !this.hasChanges(); + HgKvIterator result = + ((NodeTxSessionProxy) this.graph).scanIteratorOrdered( + table, HgOwnerKey.of(ownerKeyFrom, keyFrom), + HgOwnerKey.of(ownerKeyTo, keyTo), + toHstoreLimit(limit), scanType, query); + return new ColumnIterator<>(table, result, keyFrom, keyTo, + scanType, true); + } + @Override public BackendColumnIterator scan(String table, byte[] ownerKeyFrom, byte[] ownerKeyTo, diff --git a/hugegraph-server/hugegraph-hstore/src/main/java/org/apache/hugegraph/backend/store/hstore/HstoreTable.java b/hugegraph-server/hugegraph-hstore/src/main/java/org/apache/hugegraph/backend/store/hstore/HstoreTable.java index 4ac885b177..c87d48695d 100755 --- a/hugegraph-server/hugegraph-hstore/src/main/java/org/apache/hugegraph/backend/store/hstore/HstoreTable.java +++ b/hugegraph-server/hugegraph-hstore/src/main/java/org/apache/hugegraph/backend/store/hstore/HstoreTable.java @@ -55,6 +55,7 @@ import org.apache.hugegraph.pd.common.PDException; import org.apache.hugegraph.pd.grpc.Metapb; import org.apache.hugegraph.store.HgOwnerKey; +import org.apache.hugegraph.store.client.util.HgStoreClientConfig; import org.apache.hugegraph.store.client.util.HgStoreClientConst; import org.apache.hugegraph.type.HugeType; import org.apache.hugegraph.type.define.HugeKeys; @@ -114,15 +115,12 @@ private static boolean direction(Condition condition) { protected static BackendEntryIterator newEntryIterator( BackendColumnIterator cols, Query query) { - return new BinaryEntryIterator<>(cols, query, (entry, col) -> { - if (entry == null || !entry.belongToMe(col)) { - HugeType type = query.resultType(); - // NOTE: only support BinaryBackendEntry currently - entry = new BinaryBackendEntry(type, col.name); - } - entry.columns(col); - return entry; - }); + BiFunction merger = + (entry, col) -> mergeColumn(query, entry, col); + if (query.resultType().isRangeIndex()) { + return new RangeIndexEntryIterator(cols, query, merger); + } + return new BinaryEntryIterator<>(cols, query, merger); } protected static BackendEntryIterator newEntryIteratorOlap( @@ -138,6 +136,42 @@ protected static BackendEntryIterator newEntryIteratorOlap( }); } + private static BackendEntry mergeColumn(Query query, BackendEntry entry, + BackendColumn col) { + if (entry == null || !entry.belongToMe(col)) { + HugeType type = query.resultType(); + // NOTE: only support BinaryBackendEntry currently + entry = new BinaryBackendEntry(type, col.name); + } + entry.columns(col); + return entry; + } + + private static final class RangeIndexEntryIterator + extends BinaryEntryIterator { + + private byte[] lastPosition; + + private RangeIndexEntryIterator( + BackendColumnIterator cols, Query query, + BiFunction merger) { + super(cols, query, merger); + this.lastPosition = PageState.EMPTY_BYTES; + } + + @Override + public BackendEntry next() { + BackendEntry entry = super.next(); + this.lastPosition = entry.id().asBytes(); + return entry; + } + + @Override + protected PageState pageState() { + return new PageState(this.lastPosition, 0, (int) this.count()); + } + } + public static String bytes2String(byte[] bytes) { StringBuilder result = new StringBuilder(); for (byte b : bytes) { @@ -625,17 +659,26 @@ protected BackendColumnIterator queryByRange(Session session, } ConditionQuery cq; Query origin = query.originQuery(); - if (query.paging() && !query.page().isEmpty()) { - type = (type & ~Session.SCAN_GTE_BEGIN) | Session.SCAN_GT_BEGIN; - } + byte[] position = null; byte[] ownerStart = this.ownerByQueryDelegate.apply(query.resultType(), query.start()); byte[] ownerEnd = this.ownerByQueryDelegate.apply(query.resultType(), query.end()); + if (shouldUseOrderedRangeScan(query)) { + if (query.paging() && !query.page().isEmpty()) { + start = rangeIndexScanStart(query, start); + type = (type & ~Session.SCAN_GTE_BEGIN) | Session.SCAN_GT_BEGIN; + } + return session.scanOrdered(this.table(), ownerStart, ownerEnd, + start, end, type, null, + rangeScanLimit(query)); + } + if (query.paging() && !query.page().isEmpty()) { + position = PageState.fromString(query.page()).position(); + } if (origin instanceof ConditionQuery && (query.resultType().isEdge() || query.resultType().isVertex())) { cq = (ConditionQuery) query.originQuery(); - long limit = rangeScanLimit(query); // LOG.debug("query {} with ownerKeyFrom: {}, ownerKeyTo: {}, " + // "keyFrom: {}, keyTo: {}, " + @@ -644,10 +687,25 @@ protected BackendColumnIterator queryByRange(Session session, // bytes2String(ownerEnd), bytes2String(start), // bytes2String(end), type, cq.bytes()); return session.scan(this.table(), ownerStart, ownerEnd, start, - end, type, cq.bytes(), null, limit); + end, type, cq.bytes(), position); } return session.scan(this.table(), ownerStart, ownerEnd, start, end, - type, null, null, rangeScanLimit(query)); + type, null, position); + } + + static boolean shouldUseOrderedRangeScan(IdRangeQuery query) { + long limit = rangeScanLimit(query); + return query.resultType().isRangeIndex() && + query.paging() && + limit > HgStoreClientConst.NO_LIMIT && + limit <= HgStoreClientConfig.of().getNetKvScannerPageSize(); + } + + static byte[] rangeIndexScanStart(IdRangeQuery query, byte[] start) { + if (query.paging() && !query.page().isEmpty()) { + return PageState.fromString(query.page()).position(); + } + return start; } private static long rangeScanLimit(IdRangeQuery query) { diff --git a/hugegraph-server/hugegraph-hstore/src/test/java/org/apache/hugegraph/backend/store/hstore/HstoreSessionsImplTest.java b/hugegraph-server/hugegraph-hstore/src/test/java/org/apache/hugegraph/backend/store/hstore/HstoreSessionsImplTest.java new file mode 100644 index 0000000000..9b7e052012 --- /dev/null +++ b/hugegraph-server/hugegraph-hstore/src/test/java/org/apache/hugegraph/backend/store/hstore/HstoreSessionsImplTest.java @@ -0,0 +1,181 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.backend.store.hstore; + +import java.lang.reflect.Constructor; +import java.util.Arrays; +import java.util.List; +import java.util.NoSuchElementException; + +import org.apache.hugegraph.backend.store.BackendEntry.BackendColumn; +import org.apache.hugegraph.backend.store.BackendEntry.BackendColumnIterator; +import org.apache.hugegraph.store.HgKvEntry; +import org.apache.hugegraph.store.HgKvIterator; +import org.junit.Assert; +import org.junit.Test; + +public class HstoreSessionsImplTest { + + @Test + public void testColumnIteratorClearsPositionAfterExhaustedByDefault() + throws Exception { + BackendColumnIterator iterator = newColumnIterator( + new TestIterator(1, 2) + ); + + Assert.assertTrue(iterator.hasNext()); + BackendColumn first = iterator.next(); + Assert.assertArrayEquals(keyBytes(1), first.name); + + Assert.assertTrue(iterator.hasNext()); + BackendColumn second = iterator.next(); + Assert.assertArrayEquals(keyBytes(2), second.name); + + Assert.assertFalse(iterator.hasNext()); + Assert.assertNull(iterator.position()); + } + + @Test + public void testColumnIteratorKeepsLastPositionForOrderedScan() + throws Exception { + BackendColumnIterator iterator = newColumnIterator( + new TestIterator(1, 2), true + ); + + Assert.assertTrue(iterator.hasNext()); + BackendColumn first = iterator.next(); + Assert.assertArrayEquals(keyBytes(1), first.name); + + Assert.assertTrue(iterator.hasNext()); + BackendColumn second = iterator.next(); + Assert.assertArrayEquals(keyBytes(2), second.name); + + Assert.assertFalse(iterator.hasNext()); + Assert.assertArrayEquals(keyBytes(2), iterator.position()); + } + + @Test + public void testOrderedColumnIteratorPositionIgnoresPrefetchedKey() + throws Exception { + BackendColumnIterator iterator = newColumnIterator( + new TestIterator(1, 2, 3), true + ); + + Assert.assertTrue(iterator.hasNext()); + BackendColumn first = iterator.next(); + Assert.assertArrayEquals(keyBytes(1), first.name); + + Assert.assertTrue(iterator.hasNext()); + BackendColumn second = iterator.next(); + Assert.assertArrayEquals(keyBytes(2), second.name); + + Assert.assertTrue(iterator.hasNext()); + Assert.assertArrayEquals(keyBytes(2), iterator.position()); + } + + private static BackendColumnIterator newColumnIterator( + HgKvIterator iterator) throws Exception { + return newColumnIterator(iterator, false); + } + + private static BackendColumnIterator newColumnIterator( + HgKvIterator iterator, + boolean keepPositionAfterExhausted) throws Exception { + Class clazz = Class.forName(HstoreSessionsImpl.class.getName() + + "$ColumnIterator"); + Constructor constructor = clazz.getDeclaredConstructor( + String.class, HgKvIterator.class, byte[].class, byte[].class, + int.class, boolean.class + ); + constructor.setAccessible(true); + return (BackendColumnIterator) constructor.newInstance("test", iterator, + null, null, 0, + keepPositionAfterExhausted); + } + + private static byte[] keyBytes(int key) { + return new byte[]{(byte) key}; + } + + private static final class TestIterator implements HgKvIterator { + + private final List keys; + private int offset; + private HgKvEntry current; + + private TestIterator(Integer... keys) { + this.keys = Arrays.asList(keys); + this.offset = 0; + this.current = null; + } + + @Override + public boolean hasNext() { + return this.offset < this.keys.size(); + } + + @Override + public HgKvEntry next() { + if (!this.hasNext()) { + throw new NoSuchElementException(); + } + int key = this.keys.get(this.offset++); + this.current = new TestEntry(keyBytes(key)); + return this.current; + } + + @Override + public byte[] key() { + return this.current == null ? null : this.current.key(); + } + + @Override + public byte[] value() { + return this.current == null ? null : this.current.value(); + } + + @Override + public byte[] position() { + return this.key(); + } + } + + private static final class TestEntry implements HgKvEntry { + + private final byte[] key; + + private TestEntry(byte[] key) { + this.key = key; + } + + @Override + public byte[] key() { + return this.key; + } + + @Override + public byte[] value() { + return this.key; + } + + @Override + public int code() { + return 0; + } + } +} diff --git a/hugegraph-server/hugegraph-hstore/src/test/java/org/apache/hugegraph/backend/store/hstore/HstoreTableTest.java b/hugegraph-server/hugegraph-hstore/src/test/java/org/apache/hugegraph/backend/store/hstore/HstoreTableTest.java new file mode 100644 index 0000000000..edd888492d --- /dev/null +++ b/hugegraph-server/hugegraph-hstore/src/test/java/org/apache/hugegraph/backend/store/hstore/HstoreTableTest.java @@ -0,0 +1,184 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.backend.store.hstore; + +import java.util.Arrays; +import java.util.List; +import java.util.NoSuchElementException; + +import org.apache.hugegraph.backend.id.Id.IdType; +import org.apache.hugegraph.backend.id.IdGenerator; +import org.apache.hugegraph.backend.page.PageInfo; +import org.apache.hugegraph.backend.page.PageState; +import org.apache.hugegraph.backend.query.IdRangeQuery; +import org.apache.hugegraph.backend.query.Query; +import org.apache.hugegraph.backend.store.BackendEntry; +import org.apache.hugegraph.backend.store.BackendEntry.BackendColumn; +import org.apache.hugegraph.backend.store.BackendEntry.BackendColumnIterator; +import org.apache.hugegraph.backend.store.BackendEntryIterator; +import org.apache.hugegraph.store.client.util.HgStoreClientConfig; +import org.apache.hugegraph.type.HugeType; +import org.junit.Assert; +import org.junit.Test; + +public class HstoreTableTest { + + @Test + public void testRangeIndexPageStateIgnoresPrefetchedColumn() { + Query query = new Query(HugeType.RANGE_INT_INDEX); + query.page(""); + query.limit(1L); + + BackendEntryIterator iterator = HstoreTable.newEntryIterator( + new TestColumnIterator(1, 2), query); + + Assert.assertTrue(iterator.hasNext()); + BackendEntry entry = iterator.next(); + Assert.assertArrayEquals(keyBytes(1), entry.id().asBytes()); + + PageState pageState = PageInfo.pageState(iterator); + Assert.assertArrayEquals(keyBytes(1), pageState.position()); + Assert.assertEquals(1L, pageState.total()); + } + + @Test + public void testRangeIndexPagingUsesPagePositionAsScanStart() { + byte[] originalStart = keyBytes(1); + byte[] pagePosition = keyBytes(2); + IdRangeQuery query = new IdRangeQuery(HugeType.RANGE_INT_INDEX, null, + IdGenerator.of(originalStart, + IdType.STRING), + true, + IdGenerator.of(keyBytes(9), + IdType.STRING), + false); + + query.page(""); + Assert.assertArrayEquals(originalStart, + HstoreTable.rangeIndexScanStart( + query, originalStart)); + + query.page(new PageState(pagePosition, 0, 1).toString()); + Assert.assertArrayEquals(pagePosition, + HstoreTable.rangeIndexScanStart( + query, originalStart)); + } + + @Test + public void testRangeIndexOrderedScanOnlyForBoundedPagingQuery() { + IdRangeQuery query = rangeIndexQuery(); + Assert.assertFalse(HstoreTable.shouldUseOrderedRangeScan(query)); + + query.limit(10L); + Assert.assertFalse(HstoreTable.shouldUseOrderedRangeScan(query)); + + query = rangeIndexQuery(); + query.offset(1L); + Assert.assertFalse(HstoreTable.shouldUseOrderedRangeScan(query)); + + query = rangeIndexQuery(); + query.offset(1L); + query.limit(10L); + Assert.assertFalse(HstoreTable.shouldUseOrderedRangeScan(query)); + + query = rangeIndexQuery(); + query.page(""); + Assert.assertFalse(HstoreTable.shouldUseOrderedRangeScan(query)); + + query = rangeIndexQuery(); + query.page(""); + query.limit(10L); + Assert.assertTrue(HstoreTable.shouldUseOrderedRangeScan(query)); + + query = rangeIndexQuery(); + query.page(new PageState(keyBytes(2), 0, 1).toString()); + query.limit(10L); + Assert.assertTrue(HstoreTable.shouldUseOrderedRangeScan(query)); + + query = rangeIndexQuery(); + query.limit(HgStoreClientConfig.of().getNetKvScannerPageSize() + 1L); + Assert.assertFalse(HstoreTable.shouldUseOrderedRangeScan(query)); + + query = rangeIndexQuery(); + query.page(""); + query.limit(HgStoreClientConfig.of().getNetKvScannerPageSize() + 1L); + Assert.assertFalse(HstoreTable.shouldUseOrderedRangeScan(query)); + + query = new IdRangeQuery(HugeType.VERTEX, null, + IdGenerator.of(keyBytes(1), IdType.STRING), + true, + IdGenerator.of(keyBytes(9), IdType.STRING), + false); + query.limit(10L); + Assert.assertFalse(HstoreTable.shouldUseOrderedRangeScan(query)); + } + + private static IdRangeQuery rangeIndexQuery() { + return new IdRangeQuery(HugeType.RANGE_INT_INDEX, null, + IdGenerator.of(keyBytes(1), IdType.STRING), + true, + IdGenerator.of(keyBytes(9), IdType.STRING), + false); + } + + private static byte[] keyBytes(int key) { + byte[] bytes = new byte[9]; + bytes[0] = HugeType.RANGE_INT_INDEX.code(); + bytes[8] = (byte) key; + return bytes; + } + + private static final class TestColumnIterator + implements BackendColumnIterator { + + private final List keys; + private int offset; + private byte[] position; + + private TestColumnIterator(Integer... keys) { + this.keys = Arrays.asList(keys); + this.offset = 0; + this.position = null; + } + + @Override + public boolean hasNext() { + return this.offset < this.keys.size(); + } + + @Override + public BackendColumn next() { + if (!this.hasNext()) { + throw new NoSuchElementException(); + } + byte[] key = keyBytes(this.keys.get(this.offset++)); + this.position = key; + return BackendColumn.of(key, key); + } + + @Override + public void close() { + // pass + } + + @Override + public byte[] position() { + return this.position; + } + } +} diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/UnitTestSuite.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/UnitTestSuite.java index 3c856937bd..11f8111949 100644 --- a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/UnitTestSuite.java +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/UnitTestSuite.java @@ -38,6 +38,7 @@ import org.apache.hugegraph.unit.core.DataTypeTest; import org.apache.hugegraph.unit.core.DirectionsTest; import org.apache.hugegraph.unit.core.ExceptionTest; +import org.apache.hugegraph.unit.core.IdHolderTest; import org.apache.hugegraph.unit.core.LocksTableTest; import org.apache.hugegraph.unit.core.PageStateTest; import org.apache.hugegraph.unit.core.QueryResultsTest; @@ -131,6 +132,7 @@ BackendStoreInfoTest.class, TraversalUtilTest.class, TraversalUtilOptimizeTest.class, + IdHolderTest.class, PageStateTest.class, SystemSchemaStoreTest.class, ServerInfoManagerTest.class, diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/core/IdHolderTest.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/core/IdHolderTest.java new file mode 100644 index 0000000000..f6bcd22b77 --- /dev/null +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/core/IdHolderTest.java @@ -0,0 +1,254 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.unit.core; + +import java.util.Collections; +import java.util.Iterator; +import java.util.NoSuchElementException; +import java.util.concurrent.atomic.AtomicInteger; + +import org.apache.hugegraph.backend.id.Id; +import org.apache.hugegraph.backend.id.IdGenerator; +import org.apache.hugegraph.backend.page.IdHolder.PagingIdHolder; +import org.apache.hugegraph.backend.page.IdHolderList; +import org.apache.hugegraph.backend.page.PageIds; +import org.apache.hugegraph.backend.page.PageState; +import org.apache.hugegraph.backend.page.QueryList; +import org.apache.hugegraph.backend.query.ConditionQuery; +import org.apache.hugegraph.backend.query.Query; +import org.apache.hugegraph.backend.query.QueryResults; +import org.apache.hugegraph.iterator.CIter; +import org.apache.hugegraph.testutil.Assert; +import org.apache.hugegraph.type.HugeType; +import org.junit.Test; + +public class IdHolderTest { + + @Test + public void testPagingHolderStopsShortPageByDefault() { + ConditionQuery query = new ConditionQuery(HugeType.SECONDARY_INDEX); + query.page(""); + + AtomicInteger calls = new AtomicInteger(); + PagingIdHolder holder = new PagingIdHolder(query, q -> { + calls.incrementAndGet(); + return new PageIds(Collections.singleton(IdGenerator.of(1L)), + new PageState(new byte[]{1}, 0, 1)); + }); + + PageIds first = holder.fetchNext("", 2L); + Assert.assertEquals(1, first.ids().size()); + + PageIds second = holder.fetchNext(first.page(), 2L); + Assert.assertTrue(second.empty()); + Assert.assertEquals(1, calls.get()); + } + + @Test + public void testOrderedPagingHolderStopsShortPageByDefault() { + ConditionQuery query = new ConditionQuery(HugeType.SECONDARY_INDEX); + query.page(""); + + AtomicInteger calls = new AtomicInteger(); + PagingIdHolder holder = new PagingIdHolder(query, q -> { + calls.incrementAndGet(); + return new PageIds(Collections.singleton(IdGenerator.of(1L)), + new PageState(new byte[]{1}, 0, 1)); + }, true); + + Assert.assertTrue(holder.keepOrder()); + + PageIds first = holder.fetchNext("", 2L); + Assert.assertEquals(1, first.ids().size()); + + PageIds second = holder.fetchNext(first.page(), 2L); + Assert.assertTrue(second.empty()); + Assert.assertEquals(1, calls.get()); + } + + @Test + public void testPagingHolderCanExplicitlyContinueShortPage() { + ConditionQuery query = new ConditionQuery(HugeType.SECONDARY_INDEX); + query.page(""); + + AtomicInteger calls = new AtomicInteger(); + PagingIdHolder holder = new PagingIdHolder(query, q -> { + int call = calls.incrementAndGet(); + if (call == 1) { + return new PageIds(Collections.singleton(IdGenerator.of(1L)), + new PageState(new byte[]{1}, 0, 1)); + } + if (call == 2) { + return new PageIds(Collections.singleton(IdGenerator.of(2L)), + PageState.EMPTY); + } + return PageIds.EMPTY; + }, true, false, true); + + PageIds first = holder.fetchNext("", 2L); + Assert.assertEquals(1, first.ids().size()); + + PageIds second = holder.fetchNext(first.page(), 2L); + Assert.assertEquals(1, second.ids().size()); + Assert.assertEquals(2, calls.get()); + } + + @Test + public void testPagingHolderStopsEmptyPageByDefault() { + ConditionQuery query = new ConditionQuery(HugeType.SECONDARY_INDEX); + query.page(""); + + AtomicInteger calls = new AtomicInteger(); + PagingIdHolder holder = new PagingIdHolder(query, q -> { + calls.incrementAndGet(); + return new PageIds(Collections.emptySet(), + new PageState(new byte[]{1}, 0, 0)); + }); + + PageIds first = holder.fetchNext("", 1L); + Assert.assertTrue(first.empty()); + Assert.assertNull(first.page()); + + PageIds second = holder.fetchNext(first.page(), 1L); + Assert.assertTrue(second.empty()); + Assert.assertEquals(1, calls.get()); + } + + @Test + public void testOrderedPagingIndexQueryStopsEmptyPageByDefault() { + ConditionQuery parent = new ConditionQuery(HugeType.VERTEX); + parent.page(""); + + ConditionQuery indexQuery = new ConditionQuery(HugeType.RANGE_INT_INDEX); + indexQuery.page(""); + + AtomicInteger calls = new AtomicInteger(); + PagingIdHolder holder = new PagingIdHolder(indexQuery, q -> { + int call = calls.incrementAndGet(); + if (call == 1) { + return new PageIds(Collections.emptySet(), + new PageState(new byte[]{1}, 0, 0)); + } + if (call == 2) { + return new PageIds(Collections.singleton(IdGenerator.of(2L)), + new PageState(PageState.EMPTY_BYTES, 0, 1)); + } + return PageIds.EMPTY; + }, true); + + IdHolderList holders = new IdHolderList(true); + holders.add(holder); + + QueryList queries = new QueryList<>(parent, q -> { + return new QueryResults<>(q.ids().iterator(), q); + }); + queries.add(holders, Query.QUERY_BATCH); + + QueryResults results = queries.fetch(1); + Iterator iterator = results.iterator(); + Assert.assertFalse(iterator.hasNext()); + Assert.assertEquals(1, calls.get()); + } + + @Test + public void testPagingIndexQueryCanExplicitlyContinueEmptyPage() { + ConditionQuery parent = new ConditionQuery(HugeType.VERTEX); + parent.page(""); + + ConditionQuery indexQuery = new ConditionQuery(HugeType.RANGE_INT_INDEX); + indexQuery.page(""); + + AtomicInteger calls = new AtomicInteger(); + PagingIdHolder holder = new PagingIdHolder(indexQuery, q -> { + int call = calls.incrementAndGet(); + if (call == 1) { + return new PageIds(Collections.emptySet(), + new PageState(new byte[]{1}, 0, 0)); + } + if (call == 2) { + return new PageIds(Collections.singleton(IdGenerator.of(2L)), + new PageState(PageState.EMPTY_BYTES, 0, 1)); + } + return PageIds.EMPTY; + }, true, true, true); + + IdHolderList holders = new IdHolderList(true); + holders.add(holder); + + QueryList queries = new QueryList<>(parent, q -> { + return new QueryResults<>(q.ids().iterator(), q); + }); + queries.add(holders, Query.QUERY_BATCH); + + QueryResults results = queries.fetch(1); + Iterator iterator = results.iterator(); + Assert.assertTrue(iterator.hasNext()); + Assert.assertEquals(IdGenerator.of(2L), iterator.next()); + Assert.assertFalse(iterator.hasNext()); + Assert.assertEquals(2, calls.get()); + } + + @Test + public void testOptimizedQueryStopsEmptyPageWithPageState() { + ConditionQuery parent = new ConditionQuery(HugeType.VERTEX); + parent.page(""); + + AtomicInteger calls = new AtomicInteger(); + QueryList queries = new QueryList<>(parent, q -> { + calls.incrementAndGet(); + return new QueryResults<>( + new EmptyPageIterator<>(new PageState(new byte[]{1}, 0, 0)), + q); + }); + queries.add(new ConditionQuery(HugeType.VERTEX)); + + QueryResults results = queries.fetch(1); + Iterator iterator = results.iterator(); + Assert.assertFalse(iterator.hasNext()); + Assert.assertEquals(1, calls.get()); + } + + private static class EmptyPageIterator implements CIter { + + private final PageState pageState; + + EmptyPageIterator(PageState pageState) { + this.pageState = pageState; + } + + @Override + public boolean hasNext() { + return false; + } + + @Override + public T next() { + throw new NoSuchElementException(); + } + + @Override + public Object metadata(String meta, Object... args) { + return this.pageState; + } + + @Override + public void close() throws Exception { + // pass + } + } +} diff --git a/hugegraph-store/hg-store-client/src/main/java/org/apache/hugegraph/store/client/NodeTkv.java b/hugegraph-store/hg-store-client/src/main/java/org/apache/hugegraph/store/client/NodeTkv.java index e78ced4c10..b9c28bcd9d 100644 --- a/hugegraph-store/hg-store-client/src/main/java/org/apache/hugegraph/store/client/NodeTkv.java +++ b/hugegraph-store/hg-store-client/src/main/java/org/apache/hugegraph/store/client/NodeTkv.java @@ -17,6 +17,7 @@ package org.apache.hugegraph.store.client; +import java.util.Arrays; import java.util.Objects; import javax.annotation.concurrent.ThreadSafe; @@ -39,16 +40,16 @@ class NodeTkv { NodeTkv(HgNodePartition nodePartition, String table, HgOwnerKey key) { this.nodePartition = nodePartition; this.table = table; - this.key = key; - this.endKey = key; + this.key = copyOf(key); + this.endKey = this.key; this.key.setKeyCode(this.nodePartition.getKeyCode()); } NodeTkv(HgNodePartition nodePartition, String table, HgOwnerKey key, int keyCode) { this.nodePartition = nodePartition; this.table = table; - this.key = key; - this.endKey = key; + this.key = copyOf(key); + this.endKey = this.key; this.key.setKeyCode(keyCode); } @@ -57,8 +58,8 @@ class NodeTkv { HgOwnerKey endKey) { this.nodePartition = nodePartition; this.table = table; - this.key = startKey; - this.endKey = endKey; + this.key = copyOf(startKey); + this.endKey = copyOf(endKey); this.key.setKeyCode(nodePartition.getStartKey()); this.endKey.setKeyCode(nodePartition.getEndKey()); } @@ -122,4 +123,14 @@ public HgStoreSession getSession() { public void setSession(HgStoreSession session) { this.session = session; } + + private static HgOwnerKey copyOf(HgOwnerKey key) { + HgOwnerKey copy = HgOwnerKey.of(Arrays.copyOf(key.getOwner(), + key.getOwner().length), + Arrays.copyOf(key.getKey(), + key.getKey().length)); + copy.setKeyCode(key.getKeyCode()); + copy.setSerialNo(key.getSerialNo()); + return copy; + } } diff --git a/hugegraph-store/hg-store-client/src/main/java/org/apache/hugegraph/store/client/NodeTxSessionProxy.java b/hugegraph-store/hg-store-client/src/main/java/org/apache/hugegraph/store/client/NodeTxSessionProxy.java index 401e415e8b..fad184ae9c 100644 --- a/hugegraph-store/hg-store-client/src/main/java/org/apache/hugegraph/store/client/NodeTxSessionProxy.java +++ b/hugegraph-store/hg-store-client/src/main/java/org/apache/hugegraph/store/client/NodeTxSessionProxy.java @@ -434,7 +434,7 @@ public HgKvIterator scanIterator(String table, HgOwnerKey startKey, HgAssert.isFalse(startKey == null, "The argument is invalid: startKey"); HgAssert.isFalse(endKey == null, "The argument is invalid: endKey"); - return this.toOrderedHgKvIteratorProxy( + return this.toHgKvIteratorProxy( this.toNodeTkvList(table, startKey, endKey) .parallelStream() .map( @@ -452,7 +452,7 @@ public HgKvIterator scanIterator(String table, HgOwnerKey startKey, H HgAssert.isFalse(startKey == null, "The argument is invalid: startKey"); HgAssert.isFalse(endKey == null, "The argument is invalid: endKey"); - return this.toOrderedHgKvIteratorProxy( + return this.toHgKvIteratorProxy( this.toNodeTkvList(table, startKey, endKey) .parallelStream() .map( @@ -472,7 +472,7 @@ public HgKvIterator scanIterator(String table, HgOwnerKey startKey, H HgAssert.isFalse(startKey == null, "The argument is invalid: startKey"); HgAssert.isFalse(endKey == null, "The argument is invalid: endKey"); - return this.toOrderedHgKvIteratorProxy( + return this.toHgKvIteratorProxy( this.toNodeTkvList(table, startKey, endKey) .parallelStream() .map( @@ -485,6 +485,26 @@ public HgKvIterator scanIterator(String table, HgOwnerKey startKey, H } + public HgKvIterator scanIteratorOrdered(String table, HgOwnerKey startKey, + HgOwnerKey endKey, long limit, + int scanType, byte[] query) { + HgAssert.isFalse(HgAssert.isInvalid(table), "The argument is invalid: table"); + HgAssert.isFalse(startKey == null, "The argument is invalid: startKey"); + HgAssert.isFalse(endKey == null, "The argument is invalid: endKey"); + + List iterators = + this.toOrderedRangeNodeTkvList(table, startKey, endKey) + .parallelStream() + .map(e -> this.getStoreNode(e.getNodeId()) + .openSession(this.graphName) + .scanIterator(e.getTable(), e.getKey(), + e.getEndKey(), limit, + scanType, query)) + .collect(Collectors.toList()); + prefetchOrderedRangeScanIterators(iterators); + return this.toOrderedHgKvIteratorProxy(iterators, limit); + } + @Override public HgKvIterator scanIterator(String table, int codeFrom, int codeTo, int scanType, byte[] query) { @@ -645,24 +665,47 @@ private BiFunction, HgScanQuery.ScanBuilder> toScanQueryFu @SuppressWarnings("unchecked") private HgKvIterator toOrderedHgKvIteratorProxy(List iteratorList, long limit) { + return mergeOrderedRangeScanIterators(iteratorList, limit); + } + + @SuppressWarnings("unchecked") + static HgKvIterator mergeOrderedRangeScanIterators( + List iteratorList, long limit) { List> iterators = (List>) (List) iteratorList; return new OrderedKvIterator(iterators, limit); } + static void prefetchOrderedRangeScanIterators( + List iteratorList) { + try { + iteratorList.parallelStream().forEach(iterator -> iterator.hasNext()); + } catch (RuntimeException | Error e) { + iteratorList.forEach(HgKvIterator::close); + throw e; + } + } + private HgKvIterator toHgKvIteratorProxy(List iteratorList, long limit) { + return mergeRangeScanIterators(iteratorList, limit); + } + + @SuppressWarnings({"unchecked", "rawtypes"}) + static HgKvIterator mergeRangeScanIterators( + List iteratorList, long limit) { + List iterators = (List) (List) iteratorList; boolean isAllOrderedLimiter = iteratorList.stream() .allMatch( e -> e instanceof HgKvOrderedIterator); HgKvIterator iterator; if (isAllOrderedLimiter) { - iterator = new SequencedIterator(iteratorList.stream() - .map(e -> (HgKvOrderedIterator) e) - .collect(Collectors.toList()), limit); + iterator = new SequencedIterator(iterators.stream() + .map(e -> (HgKvOrderedIterator) e) + .collect(Collectors.toList()), limit); } else { - iterator = new TopWorkIteratorProxy(iteratorList, limit); + iterator = new TopWorkIteratorProxy(iterators, limit); } return iterator; @@ -757,6 +800,12 @@ private List toNodeTkvList(String table, HgOwnerKey startKey, HgOwnerKe return nodeTkvs; } + private List toOrderedRangeNodeTkvList(String table, + HgOwnerKey startKey, + HgOwnerKey endKey) { + return this.toNodeTkvList(table, startKey, endKey); + } + private List toNodeTkvList(String table, int startCode, int endCode) { Collection partitions = this.doPartition(table, startCode, endCode); ArrayList nodeTkvs = new ArrayList<>(partitions.size()); diff --git a/hugegraph-store/hg-store-client/src/main/java/org/apache/hugegraph/store/client/OrderedKvIterator.java b/hugegraph-store/hg-store-client/src/main/java/org/apache/hugegraph/store/client/OrderedKvIterator.java index 734829401b..b537a15b82 100644 --- a/hugegraph-store/hg-store-client/src/main/java/org/apache/hugegraph/store/client/OrderedKvIterator.java +++ b/hugegraph-store/hg-store-client/src/main/java/org/apache/hugegraph/store/client/OrderedKvIterator.java @@ -37,6 +37,7 @@ final class OrderedKvIterator implements HgKvIterator { private HgKvEntry current; private byte[] position; private byte[] seekPosition; + private boolean closed; OrderedKvIterator(List> iterators, long limit) { @@ -56,12 +57,17 @@ final class OrderedKvIterator implements HgKvIterator { this.current = null; this.position = HgStoreClientConst.EMPTY_BYTES; this.seekPosition = HgStoreClientConst.EMPTY_BYTES; + this.closed = false; } @Override public boolean hasNext() { this.initialize(); - return this.count < this.limit && !this.queue.isEmpty(); + boolean hasNext = this.count < this.limit && !this.queue.isEmpty(); + if (!hasNext) { + this.close(); + } + return hasNext; } @Override @@ -75,9 +81,13 @@ public HgKvEntry next() { this.position = entry.entry.key(); this.count++; - HgKvIterator iterator = - this.iterators.get(entry.source); - this.addNext(entry.source, iterator); + if (this.count < this.limit) { + HgKvIterator iterator = + this.iterators.get(entry.source); + this.addNext(entry.source, iterator); + } else { + this.close(); + } return this.current; } @@ -107,6 +117,10 @@ public void seek(byte[] position) { @Override public void close() { + if (this.closed) { + return; + } + this.closed = true; for (HgKvIterator iterator : this.iterators) { iterator.close(); } @@ -135,6 +149,9 @@ private void addNext(int source, break; } } + if (this.queue.stream().noneMatch(entry -> entry.source == source)) { + iterator.close(); + } } private static final class SourceEntry { diff --git a/hugegraph-store/hg-store-client/src/test/java/org/apache/hugegraph/store/client/NodeTxSessionProxyTest.java b/hugegraph-store/hg-store-client/src/test/java/org/apache/hugegraph/store/client/NodeTxSessionProxyTest.java new file mode 100644 index 0000000000..e0811b4093 --- /dev/null +++ b/hugegraph-store/hg-store-client/src/test/java/org/apache/hugegraph/store/client/NodeTxSessionProxyTest.java @@ -0,0 +1,260 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store.client; + +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.util.Arrays; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.NoSuchElementException; +import java.util.Set; + +import org.apache.hugegraph.store.HgKvEntry; +import org.apache.hugegraph.store.HgKvIterator; +import org.apache.hugegraph.store.HgOwnerKey; +import org.junit.Assert; +import org.junit.Test; + +public class NodeTxSessionProxyTest { + + @Test + public void testNodeTkvDoesNotMutateSharedOwnerKeys() { + HgOwnerKey start = HgOwnerKey.of(keyBytes(9), keyBytes(1)); + HgOwnerKey end = HgOwnerKey.of(keyBytes(9), keyBytes(5)); + + NodeTkv first = new NodeTkv(HgNodePartition.of(1L, 0, 10, 20), + "g+index", start, end); + NodeTkv second = new NodeTkv(HgNodePartition.of(2L, 0, 30, 40), + "g+index", start, end); + + Assert.assertEquals(0, start.getKeyCode()); + Assert.assertEquals(0, end.getKeyCode()); + Assert.assertEquals(10, first.getKey().getKeyCode()); + Assert.assertEquals(20, first.getEndKey().getKeyCode()); + Assert.assertEquals(30, second.getKey().getKeyCode()); + Assert.assertEquals(40, second.getEndKey().getKeyCode()); + } + + @Test + public void testMergeRangeScanIteratorsKeepsLegacyTopWorkOrder() { + HgKvIterator iterator = NodeTxSessionProxy.mergeRangeScanIterators( + Arrays.asList(new TestIterator(3, 4), new TestIterator(1, 2)), 0L); + + Assert.assertEquals(3, key(iterator.next())); + Assert.assertEquals(1, key(iterator.next())); + Assert.assertEquals(4, key(iterator.next())); + Assert.assertEquals(2, key(iterator.next())); + Assert.assertFalse(iterator.hasNext()); + } + + @Test + public void testMergeOrderedRangeScanIteratorsSortsByKey() { + HgKvIterator iterator = + NodeTxSessionProxy.mergeOrderedRangeScanIterators( + Arrays.asList(new TestIterator(3, 4), + new TestIterator(1, 2)), 0L); + + Assert.assertEquals(1, key(iterator.next())); + Assert.assertEquals(2, key(iterator.next())); + Assert.assertEquals(3, key(iterator.next())); + Assert.assertEquals(4, key(iterator.next())); + Assert.assertFalse(iterator.hasNext()); + } + + @Test + public void testPrefetchOrderedRangeScanIteratorsDoesNotConsumeEntries() { + TestIterator first = new TestIterator(3, 4); + TestIterator second = new TestIterator(1, 2); + NodeTxSessionProxy.prefetchOrderedRangeScanIterators( + Arrays.asList(first, second)); + + Assert.assertTrue(first.hasNextCalls > 0); + Assert.assertTrue(second.hasNextCalls > 0); + Assert.assertEquals(0, first.nextCalls); + Assert.assertEquals(0, second.nextCalls); + + HgKvIterator iterator = + NodeTxSessionProxy.mergeOrderedRangeScanIterators( + Arrays.asList(first, second), 0L); + Assert.assertEquals(1, key(iterator.next())); + Assert.assertEquals(2, key(iterator.next())); + Assert.assertEquals(3, key(iterator.next())); + Assert.assertEquals(4, key(iterator.next())); + Assert.assertFalse(iterator.hasNext()); + } + + @Test + public void testOrderedRangeNodeTkvListKeepsLegacyOwnerPartitioning() + throws Exception { + HgStoreNodeManager manager = HgStoreNodeManager.getInstance(); + HgStoreNodePartitioner oldPartitioner = manager.getNodePartitioner(); + RecordingPartitioner partitioner = new RecordingPartitioner(); + manager.setNodePartitioner(partitioner); + try { + NodeTxSessionProxy proxy = new NodeTxSessionProxy("graph", manager); + List nodeTkvs = toOrderedRangeNodeTkvList( + proxy, HgOwnerKey.of(keyBytes(9), keyBytes(1)), + HgOwnerKey.of(keyBytes(9), keyBytes(5))); + + Assert.assertEquals(1, partitioner.byteRangeCalls); + Assert.assertEquals(0, partitioner.codeRangeCalls); + Assert.assertArrayEquals(keyBytes(9), partitioner.lastStartKey); + Assert.assertArrayEquals(keyBytes(9), partitioner.lastEndKey); + Assert.assertEquals(2, nodeTkvs.size()); + Assert.assertEquals(10, nodeTkvs.get(0).getKey().getKeyCode()); + Assert.assertEquals(20, nodeTkvs.get(0).getEndKey().getKeyCode()); + Assert.assertEquals(30, nodeTkvs.get(1).getKey().getKeyCode()); + Assert.assertEquals(40, nodeTkvs.get(1).getEndKey().getKeyCode()); + } finally { + restoreNodePartitioner(manager, oldPartitioner); + } + } + + @SuppressWarnings("unchecked") + private static List toOrderedRangeNodeTkvList( + NodeTxSessionProxy proxy, HgOwnerKey startKey, + HgOwnerKey endKey) throws Exception { + Method method = NodeTxSessionProxy.class.getDeclaredMethod( + "toOrderedRangeNodeTkvList", String.class, HgOwnerKey.class, + HgOwnerKey.class); + method.setAccessible(true); + return (List) method.invoke(proxy, "table", startKey, endKey); + } + + private static void restoreNodePartitioner(HgStoreNodeManager manager, + HgStoreNodePartitioner old) + throws Exception { + Field field = HgStoreNodeManager.class.getDeclaredField( + "nodePartitioner"); + field.setAccessible(true); + field.set(manager, old); + } + + private static int key(HgKvEntry entry) { + return entry.key()[0] & 0xFF; + } + + private static byte[] keyBytes(int key) { + return new byte[]{(byte) key}; + } + + private static final class TestIterator implements HgKvIterator { + + private final List keys; + private int offset; + private HgKvEntry current; + private int hasNextCalls; + private int nextCalls; + + private TestIterator(Integer... keys) { + this.keys = Arrays.asList(keys); + this.offset = 0; + this.current = null; + this.hasNextCalls = 0; + this.nextCalls = 0; + } + + @Override + public boolean hasNext() { + this.hasNextCalls++; + return this.offset < this.keys.size(); + } + + @Override + public HgKvEntry next() { + if (!this.hasNext()) { + throw new NoSuchElementException(); + } + this.nextCalls++; + int key = this.keys.get(this.offset++); + this.current = new TestEntry(keyBytes(key)); + return this.current; + } + + @Override + public byte[] key() { + return this.current == null ? null : this.current.key(); + } + + @Override + public byte[] value() { + return this.current == null ? null : this.current.value(); + } + + @Override + public byte[] position() { + return this.key(); + } + } + + private static final class TestEntry implements HgKvEntry { + + private final byte[] key; + + private TestEntry(byte[] key) { + this.key = key; + } + + @Override + public byte[] key() { + return this.key; + } + + @Override + public byte[] value() { + return this.key; + } + } + + private static final class RecordingPartitioner + implements HgStoreNodePartitioner { + + private int byteRangeCalls; + private int codeRangeCalls; + private byte[] lastStartKey; + private byte[] lastEndKey; + + @Override + public int partition(HgNodePartitionerBuilder builder, + String graphName, byte[] startKey, + byte[] endKey) { + this.byteRangeCalls++; + this.lastStartKey = startKey; + this.lastEndKey = endKey; + builder.setPartitions(partitions()); + return 0; + } + + @Override + public int partition(HgNodePartitionerBuilder builder, + String graphName, int startCode, + int endCode) { + this.codeRangeCalls++; + builder.setPartitions(partitions()); + return 0; + } + + private static Set partitions() { + Set partitions = new LinkedHashSet<>(); + partitions.add(HgNodePartition.of(1L, 10, 10, 20)); + partitions.add(HgNodePartition.of(2L, 30, 30, 40)); + return partitions; + } + } +} diff --git a/hugegraph-store/hg-store-client/src/test/java/org/apache/hugegraph/store/client/OrderedKvIteratorTest.java b/hugegraph-store/hg-store-client/src/test/java/org/apache/hugegraph/store/client/OrderedKvIteratorTest.java index 742e3e8a5f..9a1aae631f 100644 --- a/hugegraph-store/hg-store-client/src/test/java/org/apache/hugegraph/store/client/OrderedKvIteratorTest.java +++ b/hugegraph-store/hg-store-client/src/test/java/org/apache/hugegraph/store/client/OrderedKvIteratorTest.java @@ -91,6 +91,30 @@ public void testSeekDoesNotSkipLowerSourceWithLargerKeys() { Assert.assertFalse(iterator.hasNext()); } + @Test + public void testCloseUnderlyingIteratorsWhenLimitReached() { + TestIterator first = new TestIterator(1, 4); + TestIterator second = new TestIterator(2, 3); + OrderedKvIterator iterator = new OrderedKvIterator(Arrays.asList(first, second), 1L); + + Assert.assertEquals(1, key(iterator.next())); + Assert.assertFalse(iterator.hasNext()); + Assert.assertTrue(first.closed()); + Assert.assertTrue(second.closed()); + } + + @Test + public void testCloseUnderlyingIteratorsWhenExhausted() { + TestIterator first = new TestIterator(1); + TestIterator second = new TestIterator(); + OrderedKvIterator iterator = new OrderedKvIterator(Arrays.asList(first, second), 0L); + + Assert.assertEquals(1, key(iterator.next())); + Assert.assertFalse(iterator.hasNext()); + Assert.assertTrue(first.closed()); + Assert.assertTrue(second.closed()); + } + private static int key(HgKvEntry entry) { return entry.key()[0] & 0xFF; } @@ -104,11 +128,13 @@ private static final class TestIterator implements HgKvIterator { private final List keys; private int offset; private HgKvEntry current; + private boolean closed; private TestIterator(Integer... keys) { this.keys = Arrays.asList(keys); this.offset = 0; this.current = null; + this.closed = false; } @Override @@ -150,6 +176,15 @@ public void seek(byte[] position) { this.offset++; } } + + @Override + public void close() { + this.closed = true; + } + + private boolean closed() { + return this.closed; + } } private static final class TestEntry implements HgKvEntry { From 3646a144fe3392e0edd67ed0186b3c486d4a078a Mon Sep 17 00:00:00 2001 From: contrueCT Date: Mon, 15 Jun 2026 21:47:29 +0800 Subject: [PATCH 13/16] fix(hstore): route order-sensitive range scans through ordered merge --- .../backend/tx/GraphIndexTransaction.java | 2 +- .../backend/store/hstore/HstoreTable.java | 6 +- .../backend/store/hstore/HstoreTableTest.java | 137 +++++++++++++++++- 3 files changed, 132 insertions(+), 13 deletions(-) diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/tx/GraphIndexTransaction.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/tx/GraphIndexTransaction.java index 9118c27a83..6b9c993242 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/tx/GraphIndexTransaction.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/tx/GraphIndexTransaction.java @@ -665,7 +665,7 @@ private boolean keepBackendIndexOrder(IndexLabel indexLabel, ConditionQuery query) { return this.store().provider().isHstore() && indexLabel.indexType().isRange() && - !query.noLimit(); + (query.paging() || !query.noLimitAndOffset()); } @Watched(prefix = "index") diff --git a/hugegraph-server/hugegraph-hstore/src/main/java/org/apache/hugegraph/backend/store/hstore/HstoreTable.java b/hugegraph-server/hugegraph-hstore/src/main/java/org/apache/hugegraph/backend/store/hstore/HstoreTable.java index c87d48695d..3d3cffe65a 100755 --- a/hugegraph-server/hugegraph-hstore/src/main/java/org/apache/hugegraph/backend/store/hstore/HstoreTable.java +++ b/hugegraph-server/hugegraph-hstore/src/main/java/org/apache/hugegraph/backend/store/hstore/HstoreTable.java @@ -55,7 +55,6 @@ import org.apache.hugegraph.pd.common.PDException; import org.apache.hugegraph.pd.grpc.Metapb; import org.apache.hugegraph.store.HgOwnerKey; -import org.apache.hugegraph.store.client.util.HgStoreClientConfig; import org.apache.hugegraph.store.client.util.HgStoreClientConst; import org.apache.hugegraph.type.HugeType; import org.apache.hugegraph.type.define.HugeKeys; @@ -694,11 +693,8 @@ protected BackendColumnIterator queryByRange(Session session, } static boolean shouldUseOrderedRangeScan(IdRangeQuery query) { - long limit = rangeScanLimit(query); return query.resultType().isRangeIndex() && - query.paging() && - limit > HgStoreClientConst.NO_LIMIT && - limit <= HgStoreClientConfig.of().getNetKvScannerPageSize(); + (query.paging() || !query.noLimitAndOffset()); } static byte[] rangeIndexScanStart(IdRangeQuery query, byte[] start) { diff --git a/hugegraph-server/hugegraph-hstore/src/test/java/org/apache/hugegraph/backend/store/hstore/HstoreTableTest.java b/hugegraph-server/hugegraph-hstore/src/test/java/org/apache/hugegraph/backend/store/hstore/HstoreTableTest.java index edd888492d..e85c3ffe7c 100644 --- a/hugegraph-server/hugegraph-hstore/src/test/java/org/apache/hugegraph/backend/store/hstore/HstoreTableTest.java +++ b/hugegraph-server/hugegraph-hstore/src/test/java/org/apache/hugegraph/backend/store/hstore/HstoreTableTest.java @@ -32,9 +32,12 @@ import org.apache.hugegraph.backend.store.BackendEntry.BackendColumnIterator; import org.apache.hugegraph.backend.store.BackendEntryIterator; import org.apache.hugegraph.store.client.util.HgStoreClientConfig; +import org.apache.hugegraph.store.client.util.HgStoreClientConst; import org.apache.hugegraph.type.HugeType; import org.junit.Assert; import org.junit.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.Mockito; public class HstoreTableTest { @@ -80,25 +83,25 @@ public void testRangeIndexPagingUsesPagePositionAsScanStart() { } @Test - public void testRangeIndexOrderedScanOnlyForBoundedPagingQuery() { + public void testRangeIndexOrderedScanForOrderSensitiveQuery() { IdRangeQuery query = rangeIndexQuery(); Assert.assertFalse(HstoreTable.shouldUseOrderedRangeScan(query)); query.limit(10L); - Assert.assertFalse(HstoreTable.shouldUseOrderedRangeScan(query)); + Assert.assertTrue(HstoreTable.shouldUseOrderedRangeScan(query)); query = rangeIndexQuery(); query.offset(1L); - Assert.assertFalse(HstoreTable.shouldUseOrderedRangeScan(query)); + Assert.assertTrue(HstoreTable.shouldUseOrderedRangeScan(query)); query = rangeIndexQuery(); query.offset(1L); query.limit(10L); - Assert.assertFalse(HstoreTable.shouldUseOrderedRangeScan(query)); + Assert.assertTrue(HstoreTable.shouldUseOrderedRangeScan(query)); query = rangeIndexQuery(); query.page(""); - Assert.assertFalse(HstoreTable.shouldUseOrderedRangeScan(query)); + Assert.assertTrue(HstoreTable.shouldUseOrderedRangeScan(query)); query = rangeIndexQuery(); query.page(""); @@ -112,12 +115,12 @@ public void testRangeIndexOrderedScanOnlyForBoundedPagingQuery() { query = rangeIndexQuery(); query.limit(HgStoreClientConfig.of().getNetKvScannerPageSize() + 1L); - Assert.assertFalse(HstoreTable.shouldUseOrderedRangeScan(query)); + Assert.assertTrue(HstoreTable.shouldUseOrderedRangeScan(query)); query = rangeIndexQuery(); query.page(""); query.limit(HgStoreClientConfig.of().getNetKvScannerPageSize() + 1L); - Assert.assertFalse(HstoreTable.shouldUseOrderedRangeScan(query)); + Assert.assertTrue(HstoreTable.shouldUseOrderedRangeScan(query)); query = new IdRangeQuery(HugeType.VERTEX, null, IdGenerator.of(keyBytes(1), IdType.STRING), @@ -128,6 +131,54 @@ public void testRangeIndexOrderedScanOnlyForBoundedPagingQuery() { Assert.assertFalse(HstoreTable.shouldUseOrderedRangeScan(query)); } + @Test + public void testQueryByRangeUsesOrderedScanForOrderSensitiveRangeIndex() { + HstoreTable table = new HstoreTable("graph", "index"); + HstoreSessions.Session session = Mockito.mock( + HstoreSessions.Session.class); + + IdRangeQuery query = rangeIndexQuery(); + table.queryByRange(session, query); + verifyLegacyRangeScan(session); + + Mockito.reset(session); + query = rangeIndexQuery(); + query.limit(10L); + table.queryByRange(session, query); + OrderedScan orderedScan = verifyOrderedRangeScan(session); + Assert.assertEquals(10L, orderedScan.limit); + + Mockito.reset(session); + query = rangeIndexQuery(); + query.offset(1L); + table.queryByRange(session, query); + orderedScan = verifyOrderedRangeScan(session); + Assert.assertEquals(HgStoreClientConst.NO_LIMIT, + orderedScan.limit); + + Mockito.reset(session); + byte[] pagePosition = keyBytes(3); + query = rangeIndexQuery(); + query.page(new PageState(pagePosition, 0, 1).toString()); + query.limit(10L); + table.queryByRange(session, query); + orderedScan = verifyOrderedRangeScan(session); + Assert.assertArrayEquals(pagePosition, orderedScan.keyFrom); + Assert.assertTrue(HstoreSessions.Session.matchScanType( + HstoreSessions.Session.SCAN_GT_BEGIN, + orderedScan.scanType)); + Assert.assertFalse(HstoreSessions.Session.matchScanType( + HstoreSessions.Session.SCAN_GTE_BEGIN, + orderedScan.scanType)); + + Mockito.reset(session); + query = rangeIndexQuery(); + query.limit(HgStoreClientConfig.of().getNetKvScannerPageSize() + 1L); + table.queryByRange(session, query); + orderedScan = verifyOrderedRangeScan(session); + Assert.assertEquals(query.total(), orderedScan.limit); + } + private static IdRangeQuery rangeIndexQuery() { return new IdRangeQuery(HugeType.RANGE_INT_INDEX, null, IdGenerator.of(keyBytes(1), IdType.STRING), @@ -181,4 +232,76 @@ public byte[] position() { return this.position; } } + + private static void verifyLegacyRangeScan(HstoreSessions.Session session) { + Mockito.verify(session).scan(Mockito.anyString(), + Mockito.any(byte[].class), + Mockito.any(byte[].class), + Mockito.any(byte[].class), + Mockito.any(byte[].class), + Mockito.anyInt(), + Mockito.isNull(), + Mockito.isNull()); + verifyNoLegacyOrderedScan(session); + } + + private static OrderedScan verifyOrderedRangeScan( + HstoreSessions.Session session) { + ArgumentCaptor keyFrom = ArgumentCaptor.forClass(byte[].class); + ArgumentCaptor scanType = + ArgumentCaptor.forClass(Integer.class); + ArgumentCaptor limit = ArgumentCaptor.forClass(Long.class); + + Mockito.verify(session).scanOrdered(Mockito.anyString(), + Mockito.any(byte[].class), + Mockito.any(byte[].class), + keyFrom.capture(), + Mockito.any(byte[].class), + scanType.capture(), + Mockito.isNull(), + limit.capture()); + verifyNoLegacyRangeScan(session); + + return new OrderedScan(keyFrom.getValue(), scanType.getValue(), + limit.getValue()); + } + + private static void verifyNoLegacyRangeScan( + HstoreSessions.Session session) { + Mockito.verify(session, Mockito.never()) + .scan(Mockito.anyString(), + Mockito.any(byte[].class), + Mockito.any(byte[].class), + Mockito.any(byte[].class), + Mockito.any(byte[].class), + Mockito.anyInt(), + Mockito.isNull(), + Mockito.isNull()); + } + + private static void verifyNoLegacyOrderedScan( + HstoreSessions.Session session) { + Mockito.verify(session, Mockito.never()) + .scanOrdered(Mockito.anyString(), + Mockito.any(byte[].class), + Mockito.any(byte[].class), + Mockito.any(byte[].class), + Mockito.any(byte[].class), + Mockito.anyInt(), + Mockito.isNull(), + Mockito.anyLong()); + } + + private static final class OrderedScan { + + private final byte[] keyFrom; + private final int scanType; + private final long limit; + + private OrderedScan(byte[] keyFrom, int scanType, long limit) { + this.keyFrom = keyFrom; + this.scanType = scanType; + this.limit = limit; + } + } } From 9eb75e805d42d57bf37bacd3426bad7140f917f2 Mon Sep 17 00:00:00 2001 From: contrueCT Date: Mon, 15 Jun 2026 22:42:12 +0800 Subject: [PATCH 14/16] fix(hstore): stream ordered range scans lazily --- .../store/client/NodeTxSessionProxy.java | 46 +++-- .../store/client/NodeTxSessionProxyTest.java | 179 +++++++++++++++++- 2 files changed, 201 insertions(+), 24 deletions(-) diff --git a/hugegraph-store/hg-store-client/src/main/java/org/apache/hugegraph/store/client/NodeTxSessionProxy.java b/hugegraph-store/hg-store-client/src/main/java/org/apache/hugegraph/store/client/NodeTxSessionProxy.java index fad184ae9c..f33dae1778 100644 --- a/hugegraph-store/hg-store-client/src/main/java/org/apache/hugegraph/store/client/NodeTxSessionProxy.java +++ b/hugegraph-store/hg-store-client/src/main/java/org/apache/hugegraph/store/client/NodeTxSessionProxy.java @@ -54,12 +54,17 @@ import org.apache.hugegraph.store.client.util.HgAssert; import org.apache.hugegraph.store.client.util.HgStoreClientConst; import org.apache.hugegraph.store.client.util.HgStoreClientUtil; +import org.apache.hugegraph.store.grpc.common.Header; +import org.apache.hugegraph.store.grpc.common.ScanMethod; +import org.apache.hugegraph.store.grpc.stream.ScanStreamReq; import org.apache.hugegraph.store.grpc.stream.ScanStreamReq.Builder; import org.apache.hugegraph.store.query.StoreQueryParam; import org.apache.hugegraph.store.term.HgPair; import org.apache.hugegraph.store.term.HgTriple; import org.apache.hugegraph.structure.BaseElement; +import com.google.protobuf.ByteString; + import lombok.extern.slf4j.Slf4j; /** @@ -497,11 +502,11 @@ public HgKvIterator scanIteratorOrdered(String table, HgOwnerKey star .parallelStream() .map(e -> this.getStoreNode(e.getNodeId()) .openSession(this.graphName) - .scanIterator(e.getTable(), e.getKey(), - e.getEndKey(), limit, - scanType, query)) + .scanIterator(this.orderedRangeScanBuilder(e, + limit, + scanType, + query))) .collect(Collectors.toList()); - prefetchOrderedRangeScanIterators(iterators); return this.toOrderedHgKvIteratorProxy(iterators, limit); } @@ -677,16 +682,6 @@ static HgKvIterator mergeOrderedRangeScanIterators( return new OrderedKvIterator(iterators, limit); } - static void prefetchOrderedRangeScanIterators( - List iteratorList) { - try { - iteratorList.parallelStream().forEach(iterator -> iterator.hasNext()); - } catch (RuntimeException | Error e) { - iteratorList.forEach(HgKvIterator::close); - throw e; - } - } - private HgKvIterator toHgKvIteratorProxy(List iteratorList, long limit) { return mergeRangeScanIterators(iteratorList, limit); } @@ -806,6 +801,29 @@ private List toOrderedRangeNodeTkvList(String table, return this.toNodeTkvList(table, startKey, endKey); } + private Builder orderedRangeScanBuilder(NodeTkv nodeTkv, long limit, + int scanType, byte[] query) { + long scanLimit = limit <= HgStoreClientConst.NO_LIMIT ? + Integer.MAX_VALUE : limit; + return ScanStreamReq.newBuilder() + .setHeader(Header.newBuilder() + .setGraph(this.graphName) + .build()) + .setMethod(ScanMethod.RANGE) + .setTable(nodeTkv.getTable()) + .setStart(toByteString(nodeTkv.getKey().getKey())) + .setEnd(toByteString(nodeTkv.getEndKey().getKey())) + .setLimit(scanLimit) + .setCode(nodeTkv.getKey().getKeyCode()) + .setScanType(scanType) + .setQuery(toByteString(query)); + } + + private static ByteString toByteString(byte[] bytes) { + return ByteString.copyFrom(bytes != null ? bytes : + HgStoreClientConst.EMPTY_BYTES); + } + private List toNodeTkvList(String table, int startCode, int endCode) { Collection partitions = this.doPartition(table, startCode, endCode); ArrayList nodeTkvs = new ArrayList<>(partitions.size()); diff --git a/hugegraph-store/hg-store-client/src/test/java/org/apache/hugegraph/store/client/NodeTxSessionProxyTest.java b/hugegraph-store/hg-store-client/src/test/java/org/apache/hugegraph/store/client/NodeTxSessionProxyTest.java index e0811b4093..188e6e7ef6 100644 --- a/hugegraph-store/hg-store-client/src/test/java/org/apache/hugegraph/store/client/NodeTxSessionProxyTest.java +++ b/hugegraph-store/hg-store-client/src/test/java/org/apache/hugegraph/store/client/NodeTxSessionProxyTest.java @@ -18,8 +18,12 @@ package org.apache.hugegraph.store.client; import java.lang.reflect.Field; +import java.lang.reflect.InvocationHandler; import java.lang.reflect.Method; +import java.lang.reflect.Proxy; +import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.LinkedHashSet; import java.util.List; import java.util.NoSuchElementException; @@ -28,6 +32,9 @@ import org.apache.hugegraph.store.HgKvEntry; import org.apache.hugegraph.store.HgKvIterator; import org.apache.hugegraph.store.HgOwnerKey; +import org.apache.hugegraph.store.HgStoreSession; +import org.apache.hugegraph.store.grpc.common.ScanMethod; +import org.apache.hugegraph.store.grpc.stream.ScanStreamReq.Builder; import org.junit.Assert; import org.junit.Test; @@ -78,17 +85,92 @@ public void testMergeOrderedRangeScanIteratorsSortsByKey() { } @Test - public void testPrefetchOrderedRangeScanIteratorsDoesNotConsumeEntries() { + public void testMergeOrderedRangeScanIteratorsIsLazy() { TestIterator first = new TestIterator(3, 4); TestIterator second = new TestIterator(1, 2); - NodeTxSessionProxy.prefetchOrderedRangeScanIterators( - Arrays.asList(first, second)); - Assert.assertTrue(first.hasNextCalls > 0); - Assert.assertTrue(second.hasNextCalls > 0); + HgKvIterator iterator = + NodeTxSessionProxy.mergeOrderedRangeScanIterators( + Arrays.asList(first, second), 0L); + + Assert.assertEquals(0, first.hasNextCalls); + Assert.assertEquals(0, second.hasNextCalls); Assert.assertEquals(0, first.nextCalls); Assert.assertEquals(0, second.nextCalls); + Assert.assertEquals(1, key(iterator.next())); + Assert.assertEquals(2, key(iterator.next())); + Assert.assertEquals(3, key(iterator.next())); + Assert.assertEquals(4, key(iterator.next())); + Assert.assertFalse(iterator.hasNext()); + } + + @Test + public void testScanIteratorOrderedUsesStreamingBuildersLazily() + throws Exception { + HgStoreNodeManager manager = HgStoreNodeManager.getInstance(); + HgStoreNodePartitioner oldPartitioner = manager.getNodePartitioner(); + long firstNodeId = System.nanoTime(); + long secondNodeId = firstNodeId + 1L; + RecordingPartitioner partitioner = + new RecordingPartitioner(firstNodeId, secondNodeId); + TestIterator firstIterator = new TestIterator(3, 4); + TestIterator secondIterator = new TestIterator(1, 2); + RecordingSession firstSession = + new RecordingSession(firstIterator); + RecordingSession secondSession = + new RecordingSession(secondIterator); + String graph = "graph-" + firstNodeId; + manager.addNode(graph, new RecordingStoreNode(firstNodeId, + firstSession.proxy())); + manager.addNode(graph, new RecordingStoreNode(secondNodeId, + secondSession.proxy())); + manager.setNodePartitioner(partitioner); + try { + NodeTxSessionProxy proxy = new NodeTxSessionProxy(graph, manager); + HgKvIterator iterator = proxy.scanIteratorOrdered( + "table", HgOwnerKey.of(keyBytes(9), keyBytes(1)), + HgOwnerKey.of(keyBytes(9), keyBytes(5)), 5L, 123, + keyBytes(7)); + + Assert.assertEquals(1, firstSession.builders.size()); + Assert.assertEquals(1, secondSession.builders.size()); + Assert.assertEquals(0, firstSession.rangeScanCalls); + Assert.assertEquals(0, secondSession.rangeScanCalls); + Assert.assertEquals(0, firstIterator.hasNextCalls); + Assert.assertEquals(0, secondIterator.hasNextCalls); + assertOrderedRangeBuilder(firstSession.builders.get(0), 5L, 123, + keyBytes(7)); + assertOrderedRangeBuilder(secondSession.builders.get(0), 5L, 123, + keyBytes(7)); + + Assert.assertEquals(1, key(iterator.next())); + Assert.assertEquals(2, key(iterator.next())); + Assert.assertEquals(3, key(iterator.next())); + Assert.assertEquals(4, key(iterator.next())); + Assert.assertFalse(iterator.hasNext()); + } finally { + restoreNodePartitioner(manager, oldPartitioner); + } + } + + private static void assertOrderedRangeBuilder(Builder builder, long limit, + int scanType, + byte[] query) { + Assert.assertEquals(ScanMethod.RANGE, builder.getMethod()); + Assert.assertEquals("table", builder.getTable()); + Assert.assertEquals(limit, builder.getLimit()); + Assert.assertEquals(scanType, builder.getScanType()); + Assert.assertArrayEquals(query, builder.getQuery().toByteArray()); + } + + @Test + public void testMergeOrderedRangeScanIteratorsSortsPrefetchedSources() { + TestIterator first = new TestIterator(3, 4); + TestIterator second = new TestIterator(1, 2); + first.hasNext(); + second.hasNext(); + HgKvIterator iterator = NodeTxSessionProxy.mergeOrderedRangeScanIterators( Arrays.asList(first, second), 0L); @@ -225,11 +307,22 @@ public byte[] value() { private static final class RecordingPartitioner implements HgStoreNodePartitioner { + private final long firstNodeId; + private final long secondNodeId; private int byteRangeCalls; private int codeRangeCalls; private byte[] lastStartKey; private byte[] lastEndKey; + private RecordingPartitioner() { + this(1L, 2L); + } + + private RecordingPartitioner(long firstNodeId, long secondNodeId) { + this.firstNodeId = firstNodeId; + this.secondNodeId = secondNodeId; + } + @Override public int partition(HgNodePartitionerBuilder builder, String graphName, byte[] startKey, @@ -237,7 +330,7 @@ public int partition(HgNodePartitionerBuilder builder, this.byteRangeCalls++; this.lastStartKey = startKey; this.lastEndKey = endKey; - builder.setPartitions(partitions()); + builder.setPartitions(this.partitions()); return 0; } @@ -246,15 +339,81 @@ public int partition(HgNodePartitionerBuilder builder, String graphName, int startCode, int endCode) { this.codeRangeCalls++; - builder.setPartitions(partitions()); + builder.setPartitions(this.partitions()); return 0; } - private static Set partitions() { + private Set partitions() { Set partitions = new LinkedHashSet<>(); - partitions.add(HgNodePartition.of(1L, 10, 10, 20)); - partitions.add(HgNodePartition.of(2L, 30, 30, 40)); + partitions.add(HgNodePartition.of(this.firstNodeId, 10, 10, 20)); + partitions.add(HgNodePartition.of(this.secondNodeId, 30, 30, 40)); return partitions; } } + + private static final class RecordingStoreNode implements HgStoreNode { + + private final Long nodeId; + private final HgStoreSession session; + + private RecordingStoreNode(Long nodeId, HgStoreSession session) { + this.nodeId = nodeId; + this.session = session; + } + + @Override + public Long getNodeId() { + return this.nodeId; + } + + @Override + public String getAddress() { + return "127.0.0.1:" + this.nodeId; + } + + @Override + public HgStoreSession openSession(String graphName) { + return this.session; + } + } + + private static final class RecordingSession + implements InvocationHandler { + + private final List builders; + private final TestIterator iterator; + private int rangeScanCalls; + + private RecordingSession(TestIterator iterator) { + this.builders = Collections.synchronizedList(new ArrayList<>()); + this.iterator = iterator; + this.rangeScanCalls = 0; + } + + private HgStoreSession proxy() { + return (HgStoreSession) Proxy.newProxyInstance( + HgStoreSession.class.getClassLoader(), + new Class[]{HgStoreSession.class}, this); + } + + @Override + public Object invoke(Object proxy, Method method, Object[] args) { + if ("scanIterator".equals(method.getName())) { + if (args != null && args.length == 1 && + args[0] instanceof Builder) { + this.builders.add(((Builder) args[0]).clone()); + return this.iterator; + } + this.rangeScanCalls++; + return this.iterator; + } + if ("isTx".equals(method.getName())) { + return false; + } + if ("toString".equals(method.getName())) { + return "RecordingSession"; + } + throw new UnsupportedOperationException(method.toString()); + } + } } From 212e89affb739a8980e5dad24c88f13d2fd3902d Mon Sep 17 00:00:00 2001 From: contrueCT Date: Mon, 15 Jun 2026 23:43:58 +0800 Subject: [PATCH 15/16] fix(hstore): keep range ordering in index layer --- .../hugegraph/backend/page/IdHolder.java | 60 +-- .../backend/page/PageEntryIterator.java | 8 +- .../hugegraph/backend/page/QueryList.java | 28 +- .../backend/tx/GraphIndexTransaction.java | 278 +++++++++++- .../backend/store/hstore/HstoreSessions.java | 40 -- .../store/hstore/HstoreSessionsImpl.java | 85 +--- .../backend/store/hstore/HstoreTable.java | 28 -- .../store/hstore/HstoreSessionsImplTest.java | 181 -------- .../backend/store/hstore/HstoreTableTest.java | 169 +------ .../backend/tx/GraphIndexTransactionTest.java | 82 ++++ .../apache/hugegraph/unit/UnitTestSuite.java | 4 +- .../hugegraph/unit/core/IdHolderTest.java | 254 ----------- .../hugegraph/store/client/NodeTkv.java | 23 +- .../store/client/NodeTxSessionProxy.java | 84 +--- .../store/client/OrderedKvIterator.java | 167 ------- .../store/client/NodeTxSessionProxyTest.java | 419 ------------------ .../store/client/OrderedKvIteratorTest.java | 208 --------- 17 files changed, 383 insertions(+), 1735 deletions(-) delete mode 100644 hugegraph-server/hugegraph-hstore/src/test/java/org/apache/hugegraph/backend/store/hstore/HstoreSessionsImplTest.java create mode 100644 hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/backend/tx/GraphIndexTransactionTest.java delete mode 100644 hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/core/IdHolderTest.java delete mode 100644 hugegraph-store/hg-store-client/src/main/java/org/apache/hugegraph/store/client/OrderedKvIterator.java delete mode 100644 hugegraph-store/hg-store-client/src/test/java/org/apache/hugegraph/store/client/NodeTxSessionProxyTest.java delete mode 100644 hugegraph-store/hg-store-client/src/test/java/org/apache/hugegraph/store/client/OrderedKvIteratorTest.java diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/page/IdHolder.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/page/IdHolder.java index caffa30d8e..b420648767 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/page/IdHolder.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/page/IdHolder.java @@ -35,17 +35,11 @@ public abstract class IdHolder { protected final Query query; - private final boolean keepOrder; protected boolean exhausted; public IdHolder(Query query) { - this(query, false); - } - - public IdHolder(Query query, boolean keepOrder) { E.checkNotNull(query, "query"); this.query = query; - this.keepOrder = keepOrder; this.exhausted = false; } @@ -54,10 +48,6 @@ public Query query() { } public boolean keepOrder() { - return this.keepOrder; - } - - public boolean continueEmptyPage() { return false; } @@ -104,39 +94,13 @@ public PageIds fetchNext(String page, long pageSize) { public static class PagingIdHolder extends IdHolder { private final Function fetcher; - private final boolean continueEmptyPage; - private final boolean continuePartialPage; public PagingIdHolder(ConditionQuery query, Function fetcher) { - this(query, fetcher, false); - } - - public PagingIdHolder(ConditionQuery query, - Function fetcher, - boolean keepOrder) { - this(query, fetcher, keepOrder, false, false); - } - - public PagingIdHolder(ConditionQuery query, - Function fetcher, - boolean keepOrder, - boolean continueEmptyPage) { - this(query, fetcher, keepOrder, continueEmptyPage, - continueEmptyPage); - } - - public PagingIdHolder(ConditionQuery query, - Function fetcher, - boolean keepOrder, - boolean continueEmptyPage, - boolean continuePartialPage) { - super(query.copy(), keepOrder); + super(query.copy()); E.checkArgument(query.paging(), "Query '%s' must include page info", query); this.fetcher = fetcher; - this.continueEmptyPage = continueEmptyPage; - this.continuePartialPage = continuePartialPage; } @Override @@ -155,23 +119,12 @@ public PageIds fetchNext(String page, long pageSize) { PageIds result = this.fetcher.apply((ConditionQuery) this.query); assert result != null; - if (result.empty() && !this.continueEmptyPage) { - this.exhausted = true; - return PageIds.EMPTY; - } - if (result.page() == null || - (!this.continuePartialPage && - result.ids().size() < pageSize)) { + if (result.ids().size() < pageSize || result.page() == null) { this.exhausted = true; } return result; } - @Override - public boolean continueEmptyPage() { - return this.continueEmptyPage; - } - @Override public Set all() { throw new NotImplementedException("PagingIdHolder.all"); @@ -189,14 +142,7 @@ public static class BatchIdHolder extends IdHolder public BatchIdHolder(ConditionQuery query, Iterator entries, Function> fetcher) { - this(query, entries, fetcher, false); - } - - public BatchIdHolder(ConditionQuery query, - Iterator entries, - Function> fetcher, - boolean keepOrder) { - super(query, keepOrder); + super(query); this.entries = entries; this.fetcher = fetcher; this.count = 0L; diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/page/PageEntryIterator.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/page/PageEntryIterator.java index 0a0191c041..bbc93c79b6 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/page/PageEntryIterator.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/page/PageEntryIterator.java @@ -87,13 +87,7 @@ private boolean fetch() { this.remaining -= this.pageResults.total(); return true; } else { - if (this.pageResults.continueOnEmpty() && - this.pageResults.hasNextPage() && - !this.pageResults.page().equals(this.pageInfo.page())) { - this.pageInfo.page(this.pageResults.page()); - } else { - this.pageInfo.increase(); - } + this.pageInfo.increase(); return this.fetch(); } } diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/page/QueryList.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/page/QueryList.java index 2392516f25..f1930bd55a 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/page/QueryList.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/page/QueryList.java @@ -272,9 +272,7 @@ public PageResults iterator(int index, String page, long pageSize) { this.updateResultsFilter(bindQuery); PageIds pageIds = holder.fetchNext(page, pageSize); if (pageIds.empty()) { - return PageResults.emptyIterator(bindQuery, - pageIds.pageState(), - holder.continueEmptyPage()); + return PageResults.emptyIterator(); } QueryResults results = this.queryByIndexIds(pageIds.ids(), @@ -333,17 +331,10 @@ public static class PageResults { private final QueryResults results; private final PageState pageState; - private final boolean continueOnEmpty; public PageResults(QueryResults results, PageState pageState) { - this(results, pageState, false); - } - - public PageResults(QueryResults results, PageState pageState, - boolean continueOnEmpty) { this.results = results; this.pageState = pageState; - this.continueOnEmpty = continueOnEmpty; } public Iterator get() { @@ -355,10 +346,6 @@ public boolean hasNextPage() { PageState.EMPTY_BYTES); } - public boolean continueOnEmpty() { - return this.continueOnEmpty; - } - public Query query() { List queries = this.results.queries(); E.checkState(queries.size() == 1, @@ -378,18 +365,5 @@ public long total() { public static PageResults emptyIterator() { return (PageResults) EMPTY; } - - public static PageResults emptyIterator(Query query, - PageState pageState) { - return emptyIterator(query, pageState, false); - } - - public static PageResults emptyIterator(Query query, - PageState pageState, - boolean continueOnEmpty) { - return new PageResults<>( - new QueryResults<>(QueryResults.emptyIterator(), query), - pageState, continueOnEmpty); - } } } diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/tx/GraphIndexTransaction.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/tx/GraphIndexTransaction.java index 6b9c993242..048df9834a 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/tx/GraphIndexTransaction.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/tx/GraphIndexTransaction.java @@ -652,22 +652,278 @@ private void storeSelectedIndexField(IndexLabel indexLabel, @Watched(prefix = "index") private IdHolder doIndexQuery(IndexLabel indexLabel, ConditionQuery query) { + if (this.needMaterializedHstoreRangeOrder(indexLabel, query)) { + return this.doMaterializedHstoreRangeIndexQuery(indexLabel, query); + } if (!query.paging()) { return this.doIndexQueryBatch(indexLabel, query); } else { return new PagingIdHolder(query, q -> { return this.doIndexQueryOnce(indexLabel, q); - }, this.keepBackendIndexOrder(indexLabel, query)); + }); } } - private boolean keepBackendIndexOrder(IndexLabel indexLabel, - ConditionQuery query) { + private boolean needMaterializedHstoreRangeOrder(IndexLabel indexLabel, + ConditionQuery query) { return this.store().provider().isHstore() && indexLabel.indexType().isRange() && (query.paging() || !query.noLimitAndOffset()); } + private IdHolder doMaterializedHstoreRangeIndexQuery(IndexLabel indexLabel, + ConditionQuery query) { + if (!query.paging()) { + Set ids = this.querySortedRangeIndexIds(indexLabel, query); + return this.newSortedRangeIndexBatchHolder(query, ids); + } + return new SortedRangePagingIdHolder(query, q -> { + return this.querySortedRangeIndexPage(indexLabel, q); + }); + } + + private BatchIdHolder newSortedRangeIndexBatchHolder(ConditionQuery query, + Set ids) { + return new SortedRangeBatchIdHolder(query, ids); + } + + private Set querySortedRangeIndexIds(IndexLabel indexLabel, + ConditionQuery query) { + List indexes = this.querySortedRangeIndexes(indexLabel, + query); + Set ids = InsertionOrderUtil.newSet(); + for (HugeIndex index : indexes) { + ids.addAll(index.elementIds()); + Query.checkForceCapacity(ids.size()); + } + return ids; + } + + private PageIds querySortedRangeIndexPage(IndexLabel indexLabel, + ConditionQuery query) { + List indexes = this.querySortedRangeIndexes(indexLabel, + query); + Set allIds = InsertionOrderUtil.newSet(); + for (HugeIndex index : indexes) { + allIds.addAll(index.elementIds()); + Query.checkForceCapacity(allIds.size()); + } + if (allIds.isEmpty()) { + return PageIds.EMPTY; + } + + int start = 0; + if (!query.page().isEmpty()) { + start = PageState.fromString(query.page()).offset(); + } + if (start >= allIds.size()) { + return PageIds.EMPTY; + } + + long total = allIds.size(); + long end = query.noLimit() ? total : + Math.min(total, (long) start + query.limit()); + Set pageIds = CollectionUtil.subSet(allIds, start, (int) end); + if (pageIds.isEmpty()) { + return PageIds.EMPTY; + } + + int next = (int) end; + PageState pageState; + if (next < total) { + pageState = new PageState(new byte[]{1}, next, pageIds.size()); + } else { + pageState = new PageState(PageState.EMPTY_BYTES, 0, + pageIds.size()); + } + return new PageIds(pageIds, pageState); + } + + private List querySortedRangeIndexes(IndexLabel indexLabel, + ConditionQuery query) { + List indexes = new ArrayList<>(); + Iterator entries = null; + String spaceGraph = this.params() + .graph().spaceGraphName(); + LockUtil.Locks locks = new LockUtil.Locks(spaceGraph); + ConditionQuery scanQuery = query.copy(); + scanQuery.page(null); + scanQuery.offset(0L); + scanQuery.limit(Query.NO_LIMIT); + try { + locks.lockReads(LockUtil.INDEX_LABEL_DELETE, indexLabel.id()); + locks.lockReads(LockUtil.INDEX_LABEL_REBUILD, indexLabel.id()); + if (!indexLabel.system()) { + graph().indexLabel(indexLabel.id()); + } + + entries = super.query(scanQuery).iterator(); + while (entries.hasNext()) { + HugeIndex index = this.readMatchedIndex(indexLabel, scanQuery, + entries.next()); + if (index == null) { + continue; + } + this.removeExpiredIndexIfNeeded(index, scanQuery.showExpired()); + this.recordIndexValue(scanQuery, index); + indexes.add(index); + Query.checkForceCapacity(indexes.size()); + } + } finally { + locks.unlock(); + CloseableIterator.closeIterator(entries); + } + + Collections.sort(indexes, this::compareRangeIndexValues); + return indexes; + } + + @SuppressWarnings({"rawtypes", "unchecked"}) + private int compareRangeIndexValues(HugeIndex left, HugeIndex right) { + Object leftValue = left.fieldValues(); + Object rightValue = right.fieldValues(); + E.checkArgument(leftValue instanceof Comparable, + "Invalid range index value '%s'", leftValue); + E.checkArgument(rightValue instanceof Comparable, + "Invalid range index value '%s'", rightValue); + + int result = ((Comparable) leftValue).compareTo(rightValue); + if (result != 0) { + return result; + } + return compareRangeIndexElementIds(left.elementIds(), + right.elementIds()); + } + + private static int compareRangeIndexElementIds(Set leftIds, + Set rightIds) { + Iterator left = leftIds.iterator(); + Iterator right = rightIds.iterator(); + while (left.hasNext() && right.hasNext()) { + int result = left.next().compareTo(right.next()); + if (result != 0) { + return result; + } + } + return Boolean.compare(left.hasNext(), right.hasNext()); + } + + static class SortedRangeBatchIdHolder extends BatchIdHolder { + + private final List idList; + private int offset; + private PageIds pendingBatch; + + SortedRangeBatchIdHolder(ConditionQuery query, Set ids) { + super(query, Collections.emptyIterator(), batch -> { + throw new IllegalStateException("Unexpected sorted index fetcher"); + }); + this.idList = new ArrayList<>(ids); + this.offset = 0; + this.pendingBatch = null; + } + + @Override + public boolean keepOrder() { + return true; + } + + @Override + public boolean hasNext() { + if (this.pendingBatch != null) { + return true; + } + if (this.exhausted) { + return false; + } + return this.offset < this.idList.size(); + } + + @Override + public IdHolder next() { + if (!this.hasNext()) { + throw new java.util.NoSuchElementException(); + } + return this; + } + + @Override + public PageIds fetchNext(String page, long batchSize) { + E.checkArgument(page == null, + "Not support page parameter by BatchIdHolder"); + E.checkArgument(batchSize >= 0L, + "Invalid batch size value: %s", batchSize); + if (this.pendingBatch != null) { + PageIds result = this.pendingBatch; + this.pendingBatch = null; + return result; + } + return this.fetchBatch(batchSize); + } + + @Override + public Set all() { + Set allIds = InsertionOrderUtil.newSet(); + if (this.pendingBatch != null) { + allIds.addAll(this.pendingBatch.ids()); + } + if (this.offset < this.idList.size()) { + allIds.addAll(this.idList.subList(this.offset, + this.idList.size())); + } + this.close(); + return allIds; + } + + @Override + public PageIds peekNext(long size) { + E.checkArgument(this.pendingBatch == null, + "Can't call peekNext() twice"); + this.pendingBatch = this.fetchBatch(size); + return this.pendingBatch; + } + + @Override + public void close() { + this.exhausted = true; + this.pendingBatch = null; + this.offset = this.idList.size(); + } + + private PageIds fetchBatch(long batchSize) { + if (this.offset >= this.idList.size() || batchSize == 0L) { + this.close(); + return PageIds.EMPTY; + } + + int end; + if (batchSize == Query.NO_LIMIT) { + end = this.idList.size(); + } else { + end = (int) Math.min((long) this.idList.size(), + this.offset + batchSize); + } + Set batchIds = InsertionOrderUtil.newSet(); + batchIds.addAll(this.idList.subList(this.offset, end)); + this.offset = end; + this.exhausted = this.offset >= this.idList.size(); + return new PageIds(batchIds, PageState.EMPTY); + } + } + + private static class SortedRangePagingIdHolder extends PagingIdHolder { + + SortedRangePagingIdHolder(ConditionQuery query, + Function fetcher) { + super(query, fetcher); + } + + @Override + public boolean keepOrder() { + return true; + } + } + @Watched(prefix = "index") private IdHolder doIndexQueryBatch(IndexLabel indexLabel, ConditionQuery query) { @@ -707,7 +963,7 @@ private IdHolder doIndexQueryBatch(IndexLabel indexLabel, } finally { locks.unlock(); } - }, this.keepBackendIndexOrder(indexLabel, query)); + }); } private void recordIndexValue(ConditionQuery query, HugeIndex index) { @@ -751,15 +1007,8 @@ private PageIds doIndexQueryOnce(IndexLabel indexLabel, Query.checkForceCapacity(ids.size()); this.recordIndexValue(query, index); } + // If there is no data, the entries is not a Metadatable object if (ids.isEmpty()) { - if (query.paging() && entries instanceof Metadatable) { - PageState pageState = PageInfo.pageState(entries); - if (pageState.position().length > 0) { - pageState = new PageState(pageState.position(), - pageState.offset(), 0); - return new PageIds(ids, pageState); - } - } return PageIds.EMPTY; } // NOTE: Memory backend's iterator is not Metadatable @@ -770,10 +1019,7 @@ private PageIds doIndexQueryOnce(IndexLabel indexLabel, "The entries must be Metadatable when query " + "in paging, but got '%s'", entries.getClass().getName()); - PageState pageState = PageInfo.pageState(entries); - pageState = new PageState(pageState.position(), - pageState.offset(), ids.size()); - return new PageIds(ids, pageState); + return new PageIds(ids, PageInfo.pageState(entries)); } finally { locks.unlock(); CloseableIterator.closeIterator(entries); diff --git a/hugegraph-server/hugegraph-hstore/src/main/java/org/apache/hugegraph/backend/store/hstore/HstoreSessions.java b/hugegraph-server/hugegraph-hstore/src/main/java/org/apache/hugegraph/backend/store/hstore/HstoreSessions.java index 563640d4df..0abb6458b9 100755 --- a/hugegraph-server/hugegraph-hstore/src/main/java/org/apache/hugegraph/backend/store/hstore/HstoreSessions.java +++ b/hugegraph-server/hugegraph-hstore/src/main/java/org/apache/hugegraph/backend/store/hstore/HstoreSessions.java @@ -150,27 +150,6 @@ public abstract BackendColumnIterator scan(String table, int scanType, byte[] query); - public BackendColumnIterator scan(String table, - byte[] ownerKeyFrom, - byte[] ownerKeyTo, - byte[] keyFrom, - byte[] keyTo, - int scanType, - byte[] query, - long limit) { - return this.scan(table, ownerKeyFrom, ownerKeyTo, keyFrom, keyTo, - scanType, query); - } - - public abstract BackendColumnIterator scanOrdered(String table, - byte[] ownerKeyFrom, - byte[] ownerKeyTo, - byte[] keyFrom, - byte[] keyTo, - int scanType, - byte[] query, - long limit); - public abstract BackendColumnIterator scan(String table, byte[] ownerKeyFrom, byte[] ownerKeyTo, @@ -180,19 +159,6 @@ public abstract BackendColumnIterator scan(String table, byte[] query, byte[] position); - public BackendColumnIterator scan(String table, - byte[] ownerKeyFrom, - byte[] ownerKeyTo, - byte[] keyFrom, - byte[] keyTo, - int scanType, - byte[] query, - byte[] position, - long limit) { - return this.scan(table, ownerKeyFrom, ownerKeyTo, keyFrom, keyTo, - scanType, query, position); - } - public abstract BackendColumnIterator scan(String table, int codeFrom, int codeTo, @@ -219,12 +185,6 @@ public abstract void merge(String table, byte[] ownerKey, public abstract BackendColumnIterator scan(String table, byte[] conditionQueryToByte); - public BackendColumnIterator scan(String table, - byte[] conditionQueryToByte, - long limit) { - return this.scan(table, conditionQueryToByte); - } - public HugeConfig getConf() { return conf; } diff --git a/hugegraph-server/hugegraph-hstore/src/main/java/org/apache/hugegraph/backend/store/hstore/HstoreSessionsImpl.java b/hugegraph-server/hugegraph-hstore/src/main/java/org/apache/hugegraph/backend/store/hstore/HstoreSessionsImpl.java index 5887a4f093..2f98d03745 100755 --- a/hugegraph-server/hugegraph-hstore/src/main/java/org/apache/hugegraph/backend/store/hstore/HstoreSessionsImpl.java +++ b/hugegraph-server/hugegraph-hstore/src/main/java/org/apache/hugegraph/backend/store/hstore/HstoreSessionsImpl.java @@ -50,7 +50,6 @@ import org.apache.hugegraph.store.HgScanQuery; import org.apache.hugegraph.store.HgStoreClient; import org.apache.hugegraph.store.HgStoreSession; -import org.apache.hugegraph.store.client.NodeTxSessionProxy; import org.apache.hugegraph.store.client.grpc.KvCloseableIterator; import org.apache.hugegraph.store.client.util.HgStoreClientConst; import org.apache.hugegraph.store.grpc.common.ScanOrderType; @@ -227,22 +226,15 @@ private static class ColumnIterator implements private final int scanType; private final String table; private final byte[] value; - private final boolean keepPositionAfterExhausted; private boolean gotNext; private byte[] position; public ColumnIterator(String table, T results) { - this(table, results, null, null, 0, false); + this(table, results, null, null, 0); } public ColumnIterator(String table, T results, byte[] keyBegin, byte[] keyEnd, int scanType) { - this(table, results, keyBegin, keyEnd, scanType, false); - } - - public ColumnIterator(String table, T results, byte[] keyBegin, - byte[] keyEnd, int scanType, - boolean keepPositionAfterExhausted) { E.checkNotNull(results, "results"); this.table = table; this.iter = results; @@ -250,7 +242,6 @@ public ColumnIterator(String table, T results, byte[] keyBegin, this.keyEnd = keyEnd; this.scanType = scanType; this.value = null; - this.keepPositionAfterExhausted = keepPositionAfterExhausted; if (this.iter.hasNext()) { this.iter.next(); this.gotNext = true; @@ -326,9 +317,11 @@ private boolean match(int expected) { @Override public boolean hasNext() { - if (gotNext && !this.keepPositionAfterExhausted) { + if (gotNext) { this.position = this.iter.position(); - } else if (!this.keepPositionAfterExhausted) { + } else { + // QUESTION: Resetting the position may result in the caller being unable to + // retrieve the corresponding position. this.position = null; } return gotNext; @@ -383,9 +376,6 @@ public BackendColumn next() { BackendColumn col = BackendColumn.of(this.iter.key(), this.iter.value()); - if (this.keepPositionAfterExhausted) { - this.position = col.name; - } if (this.iter.hasNext()) { gotNext = true; this.iter.next(); @@ -597,16 +587,6 @@ public BackendColumnIterator scan(String table, return new ColumnIterator<>(table, results); } - @Override - public BackendColumnIterator scan(String table, - byte[] conditionQueryToByte, - long limit) { - assert !this.hasChanges(); - HgKvIterator results = this.graph.scanIterator( - table, toHstoreLimit(limit), conditionQueryToByte); - return new ColumnIterator<>(table, results); - } - @Override public BackendColumnIterator scan(String table, byte[] ownerKey, byte[] prefix) { @@ -711,16 +691,6 @@ public BackendColumnIterator scan(String table, byte[] ownerKeyFrom, byte[] ownerKeyTo, byte[] keyFrom, byte[] keyTo, int scanType, byte[] query) { - return this.scan(table, ownerKeyFrom, ownerKeyTo, keyFrom, keyTo, - scanType, query, HgStoreClientConst.NO_LIMIT); - } - - @Override - public BackendColumnIterator scan(String table, byte[] ownerKeyFrom, - byte[] ownerKeyTo, - byte[] keyFrom, byte[] keyTo, - int scanType, byte[] query, - long limit) { assert !this.hasChanges(); HgKvIterator result = this.graph.scanIterator(table, HgOwnerKey.of( @@ -729,49 +699,19 @@ public BackendColumnIterator scan(String table, byte[] ownerKeyFrom, HgOwnerKey.of( ownerKeyTo, keyTo), - toHstoreLimit(limit), + 0, scanType, query); return new ColumnIterator<>(table, result, keyFrom, keyTo, scanType); } - @Override - public BackendColumnIterator scanOrdered(String table, - byte[] ownerKeyFrom, - byte[] ownerKeyTo, - byte[] keyFrom, - byte[] keyTo, - int scanType, - byte[] query, - long limit) { - assert !this.hasChanges(); - HgKvIterator result = - ((NodeTxSessionProxy) this.graph).scanIteratorOrdered( - table, HgOwnerKey.of(ownerKeyFrom, keyFrom), - HgOwnerKey.of(ownerKeyTo, keyTo), - toHstoreLimit(limit), scanType, query); - return new ColumnIterator<>(table, result, keyFrom, keyTo, - scanType, true); - } - @Override public BackendColumnIterator scan(String table, byte[] ownerKeyFrom, byte[] ownerKeyTo, byte[] keyFrom, byte[] keyTo, int scanType, byte[] query, byte[] position) { - return this.scan(table, ownerKeyFrom, ownerKeyTo, keyFrom, keyTo, - scanType, query, position, - HgStoreClientConst.NO_LIMIT); - } - - @Override - public BackendColumnIterator scan(String table, byte[] ownerKeyFrom, - byte[] ownerKeyTo, - byte[] keyFrom, byte[] keyTo, - int scanType, byte[] query, - byte[] position, long limit) { assert !this.hasChanges(); HgKvIterator result = this.graph.scanIterator(table, HgOwnerKey.of( @@ -780,23 +720,14 @@ public BackendColumnIterator scan(String table, byte[] ownerKeyFrom, HgOwnerKey.of( ownerKeyTo, keyTo), - toHstoreLimit(limit), + 0, scanType, query); - if (position != null && position.length > 0) { - result.seek(position); - } + result.seek(position); return new ColumnIterator<>(table, result, keyFrom, keyTo, scanType); } - private long toHstoreLimit(long limit) { - if (limit <= 0L || limit == Query.NO_LIMIT) { - return HgStoreClientConst.NO_LIMIT; - } - return limit; - } - @Override public BackendColumnIterator scan(String table, int codeFrom, int codeTo, int scanType, diff --git a/hugegraph-server/hugegraph-hstore/src/main/java/org/apache/hugegraph/backend/store/hstore/HstoreTable.java b/hugegraph-server/hugegraph-hstore/src/main/java/org/apache/hugegraph/backend/store/hstore/HstoreTable.java index 3d3cffe65a..415a2577da 100755 --- a/hugegraph-server/hugegraph-hstore/src/main/java/org/apache/hugegraph/backend/store/hstore/HstoreTable.java +++ b/hugegraph-server/hugegraph-hstore/src/main/java/org/apache/hugegraph/backend/store/hstore/HstoreTable.java @@ -663,15 +663,6 @@ protected BackendColumnIterator queryByRange(Session session, query.start()); byte[] ownerEnd = this.ownerByQueryDelegate.apply(query.resultType(), query.end()); - if (shouldUseOrderedRangeScan(query)) { - if (query.paging() && !query.page().isEmpty()) { - start = rangeIndexScanStart(query, start); - type = (type & ~Session.SCAN_GTE_BEGIN) | Session.SCAN_GT_BEGIN; - } - return session.scanOrdered(this.table(), ownerStart, ownerEnd, - start, end, type, null, - rangeScanLimit(query)); - } if (query.paging() && !query.page().isEmpty()) { position = PageState.fromString(query.page()).position(); } @@ -692,25 +683,6 @@ protected BackendColumnIterator queryByRange(Session session, type, null, position); } - static boolean shouldUseOrderedRangeScan(IdRangeQuery query) { - return query.resultType().isRangeIndex() && - (query.paging() || !query.noLimitAndOffset()); - } - - static byte[] rangeIndexScanStart(IdRangeQuery query, byte[] start) { - if (query.paging() && !query.page().isEmpty()) { - return PageState.fromString(query.page()).position(); - } - return start; - } - - private static long rangeScanLimit(IdRangeQuery query) { - if (query.noLimit()) { - return HgStoreClientConst.NO_LIMIT; - } - return query.total(); - } - protected BackendColumnIterator queryByCond(Session session, ConditionQuery query) { if (query.containsScanCondition()) { diff --git a/hugegraph-server/hugegraph-hstore/src/test/java/org/apache/hugegraph/backend/store/hstore/HstoreSessionsImplTest.java b/hugegraph-server/hugegraph-hstore/src/test/java/org/apache/hugegraph/backend/store/hstore/HstoreSessionsImplTest.java deleted file mode 100644 index 9b7e052012..0000000000 --- a/hugegraph-server/hugegraph-hstore/src/test/java/org/apache/hugegraph/backend/store/hstore/HstoreSessionsImplTest.java +++ /dev/null @@ -1,181 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to You under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hugegraph.backend.store.hstore; - -import java.lang.reflect.Constructor; -import java.util.Arrays; -import java.util.List; -import java.util.NoSuchElementException; - -import org.apache.hugegraph.backend.store.BackendEntry.BackendColumn; -import org.apache.hugegraph.backend.store.BackendEntry.BackendColumnIterator; -import org.apache.hugegraph.store.HgKvEntry; -import org.apache.hugegraph.store.HgKvIterator; -import org.junit.Assert; -import org.junit.Test; - -public class HstoreSessionsImplTest { - - @Test - public void testColumnIteratorClearsPositionAfterExhaustedByDefault() - throws Exception { - BackendColumnIterator iterator = newColumnIterator( - new TestIterator(1, 2) - ); - - Assert.assertTrue(iterator.hasNext()); - BackendColumn first = iterator.next(); - Assert.assertArrayEquals(keyBytes(1), first.name); - - Assert.assertTrue(iterator.hasNext()); - BackendColumn second = iterator.next(); - Assert.assertArrayEquals(keyBytes(2), second.name); - - Assert.assertFalse(iterator.hasNext()); - Assert.assertNull(iterator.position()); - } - - @Test - public void testColumnIteratorKeepsLastPositionForOrderedScan() - throws Exception { - BackendColumnIterator iterator = newColumnIterator( - new TestIterator(1, 2), true - ); - - Assert.assertTrue(iterator.hasNext()); - BackendColumn first = iterator.next(); - Assert.assertArrayEquals(keyBytes(1), first.name); - - Assert.assertTrue(iterator.hasNext()); - BackendColumn second = iterator.next(); - Assert.assertArrayEquals(keyBytes(2), second.name); - - Assert.assertFalse(iterator.hasNext()); - Assert.assertArrayEquals(keyBytes(2), iterator.position()); - } - - @Test - public void testOrderedColumnIteratorPositionIgnoresPrefetchedKey() - throws Exception { - BackendColumnIterator iterator = newColumnIterator( - new TestIterator(1, 2, 3), true - ); - - Assert.assertTrue(iterator.hasNext()); - BackendColumn first = iterator.next(); - Assert.assertArrayEquals(keyBytes(1), first.name); - - Assert.assertTrue(iterator.hasNext()); - BackendColumn second = iterator.next(); - Assert.assertArrayEquals(keyBytes(2), second.name); - - Assert.assertTrue(iterator.hasNext()); - Assert.assertArrayEquals(keyBytes(2), iterator.position()); - } - - private static BackendColumnIterator newColumnIterator( - HgKvIterator iterator) throws Exception { - return newColumnIterator(iterator, false); - } - - private static BackendColumnIterator newColumnIterator( - HgKvIterator iterator, - boolean keepPositionAfterExhausted) throws Exception { - Class clazz = Class.forName(HstoreSessionsImpl.class.getName() + - "$ColumnIterator"); - Constructor constructor = clazz.getDeclaredConstructor( - String.class, HgKvIterator.class, byte[].class, byte[].class, - int.class, boolean.class - ); - constructor.setAccessible(true); - return (BackendColumnIterator) constructor.newInstance("test", iterator, - null, null, 0, - keepPositionAfterExhausted); - } - - private static byte[] keyBytes(int key) { - return new byte[]{(byte) key}; - } - - private static final class TestIterator implements HgKvIterator { - - private final List keys; - private int offset; - private HgKvEntry current; - - private TestIterator(Integer... keys) { - this.keys = Arrays.asList(keys); - this.offset = 0; - this.current = null; - } - - @Override - public boolean hasNext() { - return this.offset < this.keys.size(); - } - - @Override - public HgKvEntry next() { - if (!this.hasNext()) { - throw new NoSuchElementException(); - } - int key = this.keys.get(this.offset++); - this.current = new TestEntry(keyBytes(key)); - return this.current; - } - - @Override - public byte[] key() { - return this.current == null ? null : this.current.key(); - } - - @Override - public byte[] value() { - return this.current == null ? null : this.current.value(); - } - - @Override - public byte[] position() { - return this.key(); - } - } - - private static final class TestEntry implements HgKvEntry { - - private final byte[] key; - - private TestEntry(byte[] key) { - this.key = key; - } - - @Override - public byte[] key() { - return this.key; - } - - @Override - public byte[] value() { - return this.key; - } - - @Override - public int code() { - return 0; - } - } -} diff --git a/hugegraph-server/hugegraph-hstore/src/test/java/org/apache/hugegraph/backend/store/hstore/HstoreTableTest.java b/hugegraph-server/hugegraph-hstore/src/test/java/org/apache/hugegraph/backend/store/hstore/HstoreTableTest.java index e85c3ffe7c..c11cc37b9a 100644 --- a/hugegraph-server/hugegraph-hstore/src/test/java/org/apache/hugegraph/backend/store/hstore/HstoreTableTest.java +++ b/hugegraph-server/hugegraph-hstore/src/test/java/org/apache/hugegraph/backend/store/hstore/HstoreTableTest.java @@ -31,12 +31,9 @@ import org.apache.hugegraph.backend.store.BackendEntry.BackendColumn; import org.apache.hugegraph.backend.store.BackendEntry.BackendColumnIterator; import org.apache.hugegraph.backend.store.BackendEntryIterator; -import org.apache.hugegraph.store.client.util.HgStoreClientConfig; -import org.apache.hugegraph.store.client.util.HgStoreClientConst; import org.apache.hugegraph.type.HugeType; import org.junit.Assert; import org.junit.Test; -import org.mockito.ArgumentCaptor; import org.mockito.Mockito; public class HstoreTableTest { @@ -60,79 +57,7 @@ public void testRangeIndexPageStateIgnoresPrefetchedColumn() { } @Test - public void testRangeIndexPagingUsesPagePositionAsScanStart() { - byte[] originalStart = keyBytes(1); - byte[] pagePosition = keyBytes(2); - IdRangeQuery query = new IdRangeQuery(HugeType.RANGE_INT_INDEX, null, - IdGenerator.of(originalStart, - IdType.STRING), - true, - IdGenerator.of(keyBytes(9), - IdType.STRING), - false); - - query.page(""); - Assert.assertArrayEquals(originalStart, - HstoreTable.rangeIndexScanStart( - query, originalStart)); - - query.page(new PageState(pagePosition, 0, 1).toString()); - Assert.assertArrayEquals(pagePosition, - HstoreTable.rangeIndexScanStart( - query, originalStart)); - } - - @Test - public void testRangeIndexOrderedScanForOrderSensitiveQuery() { - IdRangeQuery query = rangeIndexQuery(); - Assert.assertFalse(HstoreTable.shouldUseOrderedRangeScan(query)); - - query.limit(10L); - Assert.assertTrue(HstoreTable.shouldUseOrderedRangeScan(query)); - - query = rangeIndexQuery(); - query.offset(1L); - Assert.assertTrue(HstoreTable.shouldUseOrderedRangeScan(query)); - - query = rangeIndexQuery(); - query.offset(1L); - query.limit(10L); - Assert.assertTrue(HstoreTable.shouldUseOrderedRangeScan(query)); - - query = rangeIndexQuery(); - query.page(""); - Assert.assertTrue(HstoreTable.shouldUseOrderedRangeScan(query)); - - query = rangeIndexQuery(); - query.page(""); - query.limit(10L); - Assert.assertTrue(HstoreTable.shouldUseOrderedRangeScan(query)); - - query = rangeIndexQuery(); - query.page(new PageState(keyBytes(2), 0, 1).toString()); - query.limit(10L); - Assert.assertTrue(HstoreTable.shouldUseOrderedRangeScan(query)); - - query = rangeIndexQuery(); - query.limit(HgStoreClientConfig.of().getNetKvScannerPageSize() + 1L); - Assert.assertTrue(HstoreTable.shouldUseOrderedRangeScan(query)); - - query = rangeIndexQuery(); - query.page(""); - query.limit(HgStoreClientConfig.of().getNetKvScannerPageSize() + 1L); - Assert.assertTrue(HstoreTable.shouldUseOrderedRangeScan(query)); - - query = new IdRangeQuery(HugeType.VERTEX, null, - IdGenerator.of(keyBytes(1), IdType.STRING), - true, - IdGenerator.of(keyBytes(9), IdType.STRING), - false); - query.limit(10L); - Assert.assertFalse(HstoreTable.shouldUseOrderedRangeScan(query)); - } - - @Test - public void testQueryByRangeUsesOrderedScanForOrderSensitiveRangeIndex() { + public void testQueryByRangeUsesLegacyScanForRangeIndexQueries() { HstoreTable table = new HstoreTable("graph", "index"); HstoreSessions.Session session = Mockito.mock( HstoreSessions.Session.class); @@ -145,16 +70,13 @@ public void testQueryByRangeUsesOrderedScanForOrderSensitiveRangeIndex() { query = rangeIndexQuery(); query.limit(10L); table.queryByRange(session, query); - OrderedScan orderedScan = verifyOrderedRangeScan(session); - Assert.assertEquals(10L, orderedScan.limit); + verifyLegacyRangeScan(session); Mockito.reset(session); query = rangeIndexQuery(); query.offset(1L); table.queryByRange(session, query); - orderedScan = verifyOrderedRangeScan(session); - Assert.assertEquals(HgStoreClientConst.NO_LIMIT, - orderedScan.limit); + verifyLegacyRangeScan(session); Mockito.reset(session); byte[] pagePosition = keyBytes(3); @@ -162,21 +84,7 @@ public void testQueryByRangeUsesOrderedScanForOrderSensitiveRangeIndex() { query.page(new PageState(pagePosition, 0, 1).toString()); query.limit(10L); table.queryByRange(session, query); - orderedScan = verifyOrderedRangeScan(session); - Assert.assertArrayEquals(pagePosition, orderedScan.keyFrom); - Assert.assertTrue(HstoreSessions.Session.matchScanType( - HstoreSessions.Session.SCAN_GT_BEGIN, - orderedScan.scanType)); - Assert.assertFalse(HstoreSessions.Session.matchScanType( - HstoreSessions.Session.SCAN_GTE_BEGIN, - orderedScan.scanType)); - - Mockito.reset(session); - query = rangeIndexQuery(); - query.limit(HgStoreClientConfig.of().getNetKvScannerPageSize() + 1L); - table.queryByRange(session, query); - orderedScan = verifyOrderedRangeScan(session); - Assert.assertEquals(query.total(), orderedScan.limit); + verifyLegacyRangeScan(session, pagePosition); } private static IdRangeQuery rangeIndexQuery() { @@ -234,6 +142,11 @@ public byte[] position() { } private static void verifyLegacyRangeScan(HstoreSessions.Session session) { + verifyLegacyRangeScan(session, null); + } + + private static void verifyLegacyRangeScan(HstoreSessions.Session session, + byte[] position) { Mockito.verify(session).scan(Mockito.anyString(), Mockito.any(byte[].class), Mockito.any(byte[].class), @@ -241,67 +154,13 @@ private static void verifyLegacyRangeScan(HstoreSessions.Session session) { Mockito.any(byte[].class), Mockito.anyInt(), Mockito.isNull(), - Mockito.isNull()); - verifyNoLegacyOrderedScan(session); - } - - private static OrderedScan verifyOrderedRangeScan( - HstoreSessions.Session session) { - ArgumentCaptor keyFrom = ArgumentCaptor.forClass(byte[].class); - ArgumentCaptor scanType = - ArgumentCaptor.forClass(Integer.class); - ArgumentCaptor limit = ArgumentCaptor.forClass(Long.class); - - Mockito.verify(session).scanOrdered(Mockito.anyString(), - Mockito.any(byte[].class), - Mockito.any(byte[].class), - keyFrom.capture(), - Mockito.any(byte[].class), - scanType.capture(), - Mockito.isNull(), - limit.capture()); - verifyNoLegacyRangeScan(session); - - return new OrderedScan(keyFrom.getValue(), scanType.getValue(), - limit.getValue()); + positionMatcher(position)); } - private static void verifyNoLegacyRangeScan( - HstoreSessions.Session session) { - Mockito.verify(session, Mockito.never()) - .scan(Mockito.anyString(), - Mockito.any(byte[].class), - Mockito.any(byte[].class), - Mockito.any(byte[].class), - Mockito.any(byte[].class), - Mockito.anyInt(), - Mockito.isNull(), - Mockito.isNull()); - } - - private static void verifyNoLegacyOrderedScan( - HstoreSessions.Session session) { - Mockito.verify(session, Mockito.never()) - .scanOrdered(Mockito.anyString(), - Mockito.any(byte[].class), - Mockito.any(byte[].class), - Mockito.any(byte[].class), - Mockito.any(byte[].class), - Mockito.anyInt(), - Mockito.isNull(), - Mockito.anyLong()); - } - - private static final class OrderedScan { - - private final byte[] keyFrom; - private final int scanType; - private final long limit; - - private OrderedScan(byte[] keyFrom, int scanType, long limit) { - this.keyFrom = keyFrom; - this.scanType = scanType; - this.limit = limit; + private static byte[] positionMatcher(byte[] position) { + if (position == null) { + return Mockito.isNull(); } + return Mockito.argThat(value -> Arrays.equals(position, value)); } } diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/backend/tx/GraphIndexTransactionTest.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/backend/tx/GraphIndexTransactionTest.java new file mode 100644 index 0000000000..a1b0d4e8c6 --- /dev/null +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/backend/tx/GraphIndexTransactionTest.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.backend.tx; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +import org.apache.hugegraph.backend.id.Id; +import org.apache.hugegraph.backend.id.IdGenerator; +import org.apache.hugegraph.backend.page.PageIds; +import org.apache.hugegraph.backend.query.ConditionQuery; +import org.apache.hugegraph.testutil.Assert; +import org.apache.hugegraph.type.HugeType; +import org.apache.hugegraph.util.InsertionOrderUtil; +import org.junit.Test; + +import com.google.common.collect.ImmutableList; + +public class GraphIndexTransactionTest { + + @Test + public void testSortedRangeBatchHolderKeepsPeekedBatch() { + ConditionQuery query = new ConditionQuery(HugeType.VERTEX); + Id id1 = IdGenerator.of(1); + Id id2 = IdGenerator.of(2); + Id id3 = IdGenerator.of(3); + Set ids = InsertionOrderUtil.newSet(); + ids.add(id1); + ids.add(id2); + ids.add(id3); + + GraphIndexTransaction.SortedRangeBatchIdHolder holder = + new GraphIndexTransaction.SortedRangeBatchIdHolder(query, ids); + + Assert.assertTrue(holder.keepOrder()); + + PageIds peeked = holder.peekNext(2); + Assert.assertEquals(ImmutableList.of(id1, id2), asList(peeked.ids())); + + PageIds firstBatch = holder.fetchNext(null, 2); + Assert.assertEquals(ImmutableList.of(id1, id2), + asList(firstBatch.ids())); + + PageIds secondBatch = holder.fetchNext(null, 2); + Assert.assertEquals(ImmutableList.of(id3), asList(secondBatch.ids())); + Assert.assertFalse(holder.hasNext()); + } + + @Test + public void testSortedRangeBatchHolderClosesOnZeroBatch() { + ConditionQuery query = new ConditionQuery(HugeType.VERTEX); + Set ids = InsertionOrderUtil.newSet(); + ids.add(IdGenerator.of(1)); + + GraphIndexTransaction.SortedRangeBatchIdHolder holder = + new GraphIndexTransaction.SortedRangeBatchIdHolder(query, ids); + + PageIds batch = holder.fetchNext(null, 0); + Assert.assertTrue(batch.empty()); + Assert.assertFalse(holder.hasNext()); + } + + private static List asList(Set ids) { + return new ArrayList<>(ids); + } +} diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/UnitTestSuite.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/UnitTestSuite.java index 11f8111949..9935909a02 100644 --- a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/UnitTestSuite.java +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/UnitTestSuite.java @@ -17,6 +17,7 @@ package org.apache.hugegraph.unit; +import org.apache.hugegraph.backend.tx.GraphIndexTransactionTest; import org.apache.hugegraph.core.RoleElectionStateMachineTest; import org.apache.hugegraph.meta.MetaManagerSchemaCacheClearEventTest; import org.apache.hugegraph.traversal.optimize.TraversalUtilOptimizeTest; @@ -38,7 +39,6 @@ import org.apache.hugegraph.unit.core.DataTypeTest; import org.apache.hugegraph.unit.core.DirectionsTest; import org.apache.hugegraph.unit.core.ExceptionTest; -import org.apache.hugegraph.unit.core.IdHolderTest; import org.apache.hugegraph.unit.core.LocksTableTest; import org.apache.hugegraph.unit.core.PageStateTest; import org.apache.hugegraph.unit.core.QueryResultsTest; @@ -123,6 +123,7 @@ BackendMutationTest.class, ConditionTest.class, ConditionQueryFlattenTest.class, + GraphIndexTransactionTest.class, QueryTest.class, QueryResultsTest.class, RangeTest.class, @@ -132,7 +133,6 @@ BackendStoreInfoTest.class, TraversalUtilTest.class, TraversalUtilOptimizeTest.class, - IdHolderTest.class, PageStateTest.class, SystemSchemaStoreTest.class, ServerInfoManagerTest.class, diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/core/IdHolderTest.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/core/IdHolderTest.java deleted file mode 100644 index f6bcd22b77..0000000000 --- a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/core/IdHolderTest.java +++ /dev/null @@ -1,254 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to You under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hugegraph.unit.core; - -import java.util.Collections; -import java.util.Iterator; -import java.util.NoSuchElementException; -import java.util.concurrent.atomic.AtomicInteger; - -import org.apache.hugegraph.backend.id.Id; -import org.apache.hugegraph.backend.id.IdGenerator; -import org.apache.hugegraph.backend.page.IdHolder.PagingIdHolder; -import org.apache.hugegraph.backend.page.IdHolderList; -import org.apache.hugegraph.backend.page.PageIds; -import org.apache.hugegraph.backend.page.PageState; -import org.apache.hugegraph.backend.page.QueryList; -import org.apache.hugegraph.backend.query.ConditionQuery; -import org.apache.hugegraph.backend.query.Query; -import org.apache.hugegraph.backend.query.QueryResults; -import org.apache.hugegraph.iterator.CIter; -import org.apache.hugegraph.testutil.Assert; -import org.apache.hugegraph.type.HugeType; -import org.junit.Test; - -public class IdHolderTest { - - @Test - public void testPagingHolderStopsShortPageByDefault() { - ConditionQuery query = new ConditionQuery(HugeType.SECONDARY_INDEX); - query.page(""); - - AtomicInteger calls = new AtomicInteger(); - PagingIdHolder holder = new PagingIdHolder(query, q -> { - calls.incrementAndGet(); - return new PageIds(Collections.singleton(IdGenerator.of(1L)), - new PageState(new byte[]{1}, 0, 1)); - }); - - PageIds first = holder.fetchNext("", 2L); - Assert.assertEquals(1, first.ids().size()); - - PageIds second = holder.fetchNext(first.page(), 2L); - Assert.assertTrue(second.empty()); - Assert.assertEquals(1, calls.get()); - } - - @Test - public void testOrderedPagingHolderStopsShortPageByDefault() { - ConditionQuery query = new ConditionQuery(HugeType.SECONDARY_INDEX); - query.page(""); - - AtomicInteger calls = new AtomicInteger(); - PagingIdHolder holder = new PagingIdHolder(query, q -> { - calls.incrementAndGet(); - return new PageIds(Collections.singleton(IdGenerator.of(1L)), - new PageState(new byte[]{1}, 0, 1)); - }, true); - - Assert.assertTrue(holder.keepOrder()); - - PageIds first = holder.fetchNext("", 2L); - Assert.assertEquals(1, first.ids().size()); - - PageIds second = holder.fetchNext(first.page(), 2L); - Assert.assertTrue(second.empty()); - Assert.assertEquals(1, calls.get()); - } - - @Test - public void testPagingHolderCanExplicitlyContinueShortPage() { - ConditionQuery query = new ConditionQuery(HugeType.SECONDARY_INDEX); - query.page(""); - - AtomicInteger calls = new AtomicInteger(); - PagingIdHolder holder = new PagingIdHolder(query, q -> { - int call = calls.incrementAndGet(); - if (call == 1) { - return new PageIds(Collections.singleton(IdGenerator.of(1L)), - new PageState(new byte[]{1}, 0, 1)); - } - if (call == 2) { - return new PageIds(Collections.singleton(IdGenerator.of(2L)), - PageState.EMPTY); - } - return PageIds.EMPTY; - }, true, false, true); - - PageIds first = holder.fetchNext("", 2L); - Assert.assertEquals(1, first.ids().size()); - - PageIds second = holder.fetchNext(first.page(), 2L); - Assert.assertEquals(1, second.ids().size()); - Assert.assertEquals(2, calls.get()); - } - - @Test - public void testPagingHolderStopsEmptyPageByDefault() { - ConditionQuery query = new ConditionQuery(HugeType.SECONDARY_INDEX); - query.page(""); - - AtomicInteger calls = new AtomicInteger(); - PagingIdHolder holder = new PagingIdHolder(query, q -> { - calls.incrementAndGet(); - return new PageIds(Collections.emptySet(), - new PageState(new byte[]{1}, 0, 0)); - }); - - PageIds first = holder.fetchNext("", 1L); - Assert.assertTrue(first.empty()); - Assert.assertNull(first.page()); - - PageIds second = holder.fetchNext(first.page(), 1L); - Assert.assertTrue(second.empty()); - Assert.assertEquals(1, calls.get()); - } - - @Test - public void testOrderedPagingIndexQueryStopsEmptyPageByDefault() { - ConditionQuery parent = new ConditionQuery(HugeType.VERTEX); - parent.page(""); - - ConditionQuery indexQuery = new ConditionQuery(HugeType.RANGE_INT_INDEX); - indexQuery.page(""); - - AtomicInteger calls = new AtomicInteger(); - PagingIdHolder holder = new PagingIdHolder(indexQuery, q -> { - int call = calls.incrementAndGet(); - if (call == 1) { - return new PageIds(Collections.emptySet(), - new PageState(new byte[]{1}, 0, 0)); - } - if (call == 2) { - return new PageIds(Collections.singleton(IdGenerator.of(2L)), - new PageState(PageState.EMPTY_BYTES, 0, 1)); - } - return PageIds.EMPTY; - }, true); - - IdHolderList holders = new IdHolderList(true); - holders.add(holder); - - QueryList queries = new QueryList<>(parent, q -> { - return new QueryResults<>(q.ids().iterator(), q); - }); - queries.add(holders, Query.QUERY_BATCH); - - QueryResults results = queries.fetch(1); - Iterator iterator = results.iterator(); - Assert.assertFalse(iterator.hasNext()); - Assert.assertEquals(1, calls.get()); - } - - @Test - public void testPagingIndexQueryCanExplicitlyContinueEmptyPage() { - ConditionQuery parent = new ConditionQuery(HugeType.VERTEX); - parent.page(""); - - ConditionQuery indexQuery = new ConditionQuery(HugeType.RANGE_INT_INDEX); - indexQuery.page(""); - - AtomicInteger calls = new AtomicInteger(); - PagingIdHolder holder = new PagingIdHolder(indexQuery, q -> { - int call = calls.incrementAndGet(); - if (call == 1) { - return new PageIds(Collections.emptySet(), - new PageState(new byte[]{1}, 0, 0)); - } - if (call == 2) { - return new PageIds(Collections.singleton(IdGenerator.of(2L)), - new PageState(PageState.EMPTY_BYTES, 0, 1)); - } - return PageIds.EMPTY; - }, true, true, true); - - IdHolderList holders = new IdHolderList(true); - holders.add(holder); - - QueryList queries = new QueryList<>(parent, q -> { - return new QueryResults<>(q.ids().iterator(), q); - }); - queries.add(holders, Query.QUERY_BATCH); - - QueryResults results = queries.fetch(1); - Iterator iterator = results.iterator(); - Assert.assertTrue(iterator.hasNext()); - Assert.assertEquals(IdGenerator.of(2L), iterator.next()); - Assert.assertFalse(iterator.hasNext()); - Assert.assertEquals(2, calls.get()); - } - - @Test - public void testOptimizedQueryStopsEmptyPageWithPageState() { - ConditionQuery parent = new ConditionQuery(HugeType.VERTEX); - parent.page(""); - - AtomicInteger calls = new AtomicInteger(); - QueryList queries = new QueryList<>(parent, q -> { - calls.incrementAndGet(); - return new QueryResults<>( - new EmptyPageIterator<>(new PageState(new byte[]{1}, 0, 0)), - q); - }); - queries.add(new ConditionQuery(HugeType.VERTEX)); - - QueryResults results = queries.fetch(1); - Iterator iterator = results.iterator(); - Assert.assertFalse(iterator.hasNext()); - Assert.assertEquals(1, calls.get()); - } - - private static class EmptyPageIterator implements CIter { - - private final PageState pageState; - - EmptyPageIterator(PageState pageState) { - this.pageState = pageState; - } - - @Override - public boolean hasNext() { - return false; - } - - @Override - public T next() { - throw new NoSuchElementException(); - } - - @Override - public Object metadata(String meta, Object... args) { - return this.pageState; - } - - @Override - public void close() throws Exception { - // pass - } - } -} diff --git a/hugegraph-store/hg-store-client/src/main/java/org/apache/hugegraph/store/client/NodeTkv.java b/hugegraph-store/hg-store-client/src/main/java/org/apache/hugegraph/store/client/NodeTkv.java index b9c28bcd9d..e78ced4c10 100644 --- a/hugegraph-store/hg-store-client/src/main/java/org/apache/hugegraph/store/client/NodeTkv.java +++ b/hugegraph-store/hg-store-client/src/main/java/org/apache/hugegraph/store/client/NodeTkv.java @@ -17,7 +17,6 @@ package org.apache.hugegraph.store.client; -import java.util.Arrays; import java.util.Objects; import javax.annotation.concurrent.ThreadSafe; @@ -40,16 +39,16 @@ class NodeTkv { NodeTkv(HgNodePartition nodePartition, String table, HgOwnerKey key) { this.nodePartition = nodePartition; this.table = table; - this.key = copyOf(key); - this.endKey = this.key; + this.key = key; + this.endKey = key; this.key.setKeyCode(this.nodePartition.getKeyCode()); } NodeTkv(HgNodePartition nodePartition, String table, HgOwnerKey key, int keyCode) { this.nodePartition = nodePartition; this.table = table; - this.key = copyOf(key); - this.endKey = this.key; + this.key = key; + this.endKey = key; this.key.setKeyCode(keyCode); } @@ -58,8 +57,8 @@ class NodeTkv { HgOwnerKey endKey) { this.nodePartition = nodePartition; this.table = table; - this.key = copyOf(startKey); - this.endKey = copyOf(endKey); + this.key = startKey; + this.endKey = endKey; this.key.setKeyCode(nodePartition.getStartKey()); this.endKey.setKeyCode(nodePartition.getEndKey()); } @@ -123,14 +122,4 @@ public HgStoreSession getSession() { public void setSession(HgStoreSession session) { this.session = session; } - - private static HgOwnerKey copyOf(HgOwnerKey key) { - HgOwnerKey copy = HgOwnerKey.of(Arrays.copyOf(key.getOwner(), - key.getOwner().length), - Arrays.copyOf(key.getKey(), - key.getKey().length)); - copy.setKeyCode(key.getKeyCode()); - copy.setSerialNo(key.getSerialNo()); - return copy; - } } diff --git a/hugegraph-store/hg-store-client/src/main/java/org/apache/hugegraph/store/client/NodeTxSessionProxy.java b/hugegraph-store/hg-store-client/src/main/java/org/apache/hugegraph/store/client/NodeTxSessionProxy.java index f33dae1778..65e3b76ec0 100644 --- a/hugegraph-store/hg-store-client/src/main/java/org/apache/hugegraph/store/client/NodeTxSessionProxy.java +++ b/hugegraph-store/hg-store-client/src/main/java/org/apache/hugegraph/store/client/NodeTxSessionProxy.java @@ -54,17 +54,12 @@ import org.apache.hugegraph.store.client.util.HgAssert; import org.apache.hugegraph.store.client.util.HgStoreClientConst; import org.apache.hugegraph.store.client.util.HgStoreClientUtil; -import org.apache.hugegraph.store.grpc.common.Header; -import org.apache.hugegraph.store.grpc.common.ScanMethod; -import org.apache.hugegraph.store.grpc.stream.ScanStreamReq; import org.apache.hugegraph.store.grpc.stream.ScanStreamReq.Builder; import org.apache.hugegraph.store.query.StoreQueryParam; import org.apache.hugegraph.store.term.HgPair; import org.apache.hugegraph.store.term.HgTriple; import org.apache.hugegraph.structure.BaseElement; -import com.google.protobuf.ByteString; - import lombok.extern.slf4j.Slf4j; /** @@ -490,26 +485,6 @@ public HgKvIterator scanIterator(String table, HgOwnerKey startKey, H } - public HgKvIterator scanIteratorOrdered(String table, HgOwnerKey startKey, - HgOwnerKey endKey, long limit, - int scanType, byte[] query) { - HgAssert.isFalse(HgAssert.isInvalid(table), "The argument is invalid: table"); - HgAssert.isFalse(startKey == null, "The argument is invalid: startKey"); - HgAssert.isFalse(endKey == null, "The argument is invalid: endKey"); - - List iterators = - this.toOrderedRangeNodeTkvList(table, startKey, endKey) - .parallelStream() - .map(e -> this.getStoreNode(e.getNodeId()) - .openSession(this.graphName) - .scanIterator(this.orderedRangeScanBuilder(e, - limit, - scanType, - query))) - .collect(Collectors.toList()); - return this.toOrderedHgKvIteratorProxy(iterators, limit); - } - @Override public HgKvIterator scanIterator(String table, int codeFrom, int codeTo, int scanType, byte[] query) { @@ -667,40 +642,18 @@ private BiFunction, HgScanQuery.ScanBuilder> toScanQueryFu } /*-- common --*/ - @SuppressWarnings("unchecked") - private HgKvIterator toOrderedHgKvIteratorProxy(List iteratorList, - long limit) { - return mergeOrderedRangeScanIterators(iteratorList, limit); - } - - @SuppressWarnings("unchecked") - static HgKvIterator mergeOrderedRangeScanIterators( - List iteratorList, long limit) { - List> iterators = - (List>) - (List) iteratorList; - return new OrderedKvIterator(iterators, limit); - } - private HgKvIterator toHgKvIteratorProxy(List iteratorList, long limit) { - return mergeRangeScanIterators(iteratorList, limit); - } - - @SuppressWarnings({"unchecked", "rawtypes"}) - static HgKvIterator mergeRangeScanIterators( - List iteratorList, long limit) { - List iterators = (List) (List) iteratorList; boolean isAllOrderedLimiter = iteratorList.stream() .allMatch( e -> e instanceof HgKvOrderedIterator); HgKvIterator iterator; if (isAllOrderedLimiter) { - iterator = new SequencedIterator(iterators.stream() - .map(e -> (HgKvOrderedIterator) e) - .collect(Collectors.toList()), limit); + iterator = new SequencedIterator(iteratorList.stream() + .map(e -> (HgKvOrderedIterator) e) + .collect(Collectors.toList()), limit); } else { - iterator = new TopWorkIteratorProxy(iterators, limit); + iterator = new TopWorkIteratorProxy(iteratorList, limit); } return iterator; @@ -795,35 +748,6 @@ private List toNodeTkvList(String table, HgOwnerKey startKey, HgOwnerKe return nodeTkvs; } - private List toOrderedRangeNodeTkvList(String table, - HgOwnerKey startKey, - HgOwnerKey endKey) { - return this.toNodeTkvList(table, startKey, endKey); - } - - private Builder orderedRangeScanBuilder(NodeTkv nodeTkv, long limit, - int scanType, byte[] query) { - long scanLimit = limit <= HgStoreClientConst.NO_LIMIT ? - Integer.MAX_VALUE : limit; - return ScanStreamReq.newBuilder() - .setHeader(Header.newBuilder() - .setGraph(this.graphName) - .build()) - .setMethod(ScanMethod.RANGE) - .setTable(nodeTkv.getTable()) - .setStart(toByteString(nodeTkv.getKey().getKey())) - .setEnd(toByteString(nodeTkv.getEndKey().getKey())) - .setLimit(scanLimit) - .setCode(nodeTkv.getKey().getKeyCode()) - .setScanType(scanType) - .setQuery(toByteString(query)); - } - - private static ByteString toByteString(byte[] bytes) { - return ByteString.copyFrom(bytes != null ? bytes : - HgStoreClientConst.EMPTY_BYTES); - } - private List toNodeTkvList(String table, int startCode, int endCode) { Collection partitions = this.doPartition(table, startCode, endCode); ArrayList nodeTkvs = new ArrayList<>(partitions.size()); diff --git a/hugegraph-store/hg-store-client/src/main/java/org/apache/hugegraph/store/client/OrderedKvIterator.java b/hugegraph-store/hg-store-client/src/main/java/org/apache/hugegraph/store/client/OrderedKvIterator.java deleted file mode 100644 index b537a15b82..0000000000 --- a/hugegraph-store/hg-store-client/src/main/java/org/apache/hugegraph/store/client/OrderedKvIterator.java +++ /dev/null @@ -1,167 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hugegraph.store.client; - -import java.util.Arrays; -import java.util.List; -import java.util.NoSuchElementException; -import java.util.PriorityQueue; - -import org.apache.hugegraph.store.HgKvEntry; -import org.apache.hugegraph.store.HgKvIterator; -import org.apache.hugegraph.store.client.util.HgStoreClientConst; - -final class OrderedKvIterator implements HgKvIterator { - - private final List> iterators; - private final PriorityQueue queue; - private final long limit; - - private boolean initialized; - private long count; - private HgKvEntry current; - private byte[] position; - private byte[] seekPosition; - private boolean closed; - - OrderedKvIterator(List> iterators, - long limit) { - this.iterators = iterators; - this.queue = new PriorityQueue<>((left, right) -> { - int result = Arrays.compareUnsigned(left.entry.key(), - right.entry.key()); - if (result != 0) { - return result; - } - return Integer.compare(left.source, right.source); - }); - this.limit = limit <= HgStoreClientConst.NO_LIMIT ? Long.MAX_VALUE : - limit; - this.initialized = false; - this.count = 0L; - this.current = null; - this.position = HgStoreClientConst.EMPTY_BYTES; - this.seekPosition = HgStoreClientConst.EMPTY_BYTES; - this.closed = false; - } - - @Override - public boolean hasNext() { - this.initialize(); - boolean hasNext = this.count < this.limit && !this.queue.isEmpty(); - if (!hasNext) { - this.close(); - } - return hasNext; - } - - @Override - public HgKvEntry next() { - if (!this.hasNext()) { - throw new NoSuchElementException(); - } - - SourceEntry entry = this.queue.poll(); - this.current = entry.entry; - this.position = entry.entry.key(); - this.count++; - - if (this.count < this.limit) { - HgKvIterator iterator = - this.iterators.get(entry.source); - this.addNext(entry.source, iterator); - } else { - this.close(); - } - return this.current; - } - - @Override - public byte[] key() { - return this.current == null ? null : this.current.key(); - } - - @Override - public byte[] value() { - return this.current == null ? null : this.current.value(); - } - - @Override - public byte[] position() { - return this.position; - } - - @Override - public void seek(byte[] position) { - if (this.initialized) { - throw new IllegalStateException("Can't seek after reading"); - } - this.seekPosition = position == null ? HgStoreClientConst.EMPTY_BYTES : - position; - } - - @Override - public void close() { - if (this.closed) { - return; - } - this.closed = true; - for (HgKvIterator iterator : this.iterators) { - iterator.close(); - } - this.queue.clear(); - } - - private void initialize() { - if (this.initialized) { - return; - } - for (int i = 0; i < this.iterators.size(); i++) { - HgKvIterator iterator = - this.iterators.get(i); - this.addNext(i, iterator); - } - this.initialized = true; - } - - private void addNext(int source, - HgKvIterator iterator) { - while (iterator.hasNext()) { - HgKvEntry entry = iterator.next(); - if (this.seekPosition.length == 0 || - Arrays.compareUnsigned(entry.key(), this.seekPosition) > 0) { - this.queue.add(new SourceEntry(source, entry)); - break; - } - } - if (this.queue.stream().noneMatch(entry -> entry.source == source)) { - iterator.close(); - } - } - - private static final class SourceEntry { - - private final int source; - private final HgKvEntry entry; - - private SourceEntry(int source, HgKvEntry entry) { - this.source = source; - this.entry = entry; - } - } -} diff --git a/hugegraph-store/hg-store-client/src/test/java/org/apache/hugegraph/store/client/NodeTxSessionProxyTest.java b/hugegraph-store/hg-store-client/src/test/java/org/apache/hugegraph/store/client/NodeTxSessionProxyTest.java deleted file mode 100644 index 188e6e7ef6..0000000000 --- a/hugegraph-store/hg-store-client/src/test/java/org/apache/hugegraph/store/client/NodeTxSessionProxyTest.java +++ /dev/null @@ -1,419 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hugegraph.store.client; - -import java.lang.reflect.Field; -import java.lang.reflect.InvocationHandler; -import java.lang.reflect.Method; -import java.lang.reflect.Proxy; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.NoSuchElementException; -import java.util.Set; - -import org.apache.hugegraph.store.HgKvEntry; -import org.apache.hugegraph.store.HgKvIterator; -import org.apache.hugegraph.store.HgOwnerKey; -import org.apache.hugegraph.store.HgStoreSession; -import org.apache.hugegraph.store.grpc.common.ScanMethod; -import org.apache.hugegraph.store.grpc.stream.ScanStreamReq.Builder; -import org.junit.Assert; -import org.junit.Test; - -public class NodeTxSessionProxyTest { - - @Test - public void testNodeTkvDoesNotMutateSharedOwnerKeys() { - HgOwnerKey start = HgOwnerKey.of(keyBytes(9), keyBytes(1)); - HgOwnerKey end = HgOwnerKey.of(keyBytes(9), keyBytes(5)); - - NodeTkv first = new NodeTkv(HgNodePartition.of(1L, 0, 10, 20), - "g+index", start, end); - NodeTkv second = new NodeTkv(HgNodePartition.of(2L, 0, 30, 40), - "g+index", start, end); - - Assert.assertEquals(0, start.getKeyCode()); - Assert.assertEquals(0, end.getKeyCode()); - Assert.assertEquals(10, first.getKey().getKeyCode()); - Assert.assertEquals(20, first.getEndKey().getKeyCode()); - Assert.assertEquals(30, second.getKey().getKeyCode()); - Assert.assertEquals(40, second.getEndKey().getKeyCode()); - } - - @Test - public void testMergeRangeScanIteratorsKeepsLegacyTopWorkOrder() { - HgKvIterator iterator = NodeTxSessionProxy.mergeRangeScanIterators( - Arrays.asList(new TestIterator(3, 4), new TestIterator(1, 2)), 0L); - - Assert.assertEquals(3, key(iterator.next())); - Assert.assertEquals(1, key(iterator.next())); - Assert.assertEquals(4, key(iterator.next())); - Assert.assertEquals(2, key(iterator.next())); - Assert.assertFalse(iterator.hasNext()); - } - - @Test - public void testMergeOrderedRangeScanIteratorsSortsByKey() { - HgKvIterator iterator = - NodeTxSessionProxy.mergeOrderedRangeScanIterators( - Arrays.asList(new TestIterator(3, 4), - new TestIterator(1, 2)), 0L); - - Assert.assertEquals(1, key(iterator.next())); - Assert.assertEquals(2, key(iterator.next())); - Assert.assertEquals(3, key(iterator.next())); - Assert.assertEquals(4, key(iterator.next())); - Assert.assertFalse(iterator.hasNext()); - } - - @Test - public void testMergeOrderedRangeScanIteratorsIsLazy() { - TestIterator first = new TestIterator(3, 4); - TestIterator second = new TestIterator(1, 2); - - HgKvIterator iterator = - NodeTxSessionProxy.mergeOrderedRangeScanIterators( - Arrays.asList(first, second), 0L); - - Assert.assertEquals(0, first.hasNextCalls); - Assert.assertEquals(0, second.hasNextCalls); - Assert.assertEquals(0, first.nextCalls); - Assert.assertEquals(0, second.nextCalls); - - Assert.assertEquals(1, key(iterator.next())); - Assert.assertEquals(2, key(iterator.next())); - Assert.assertEquals(3, key(iterator.next())); - Assert.assertEquals(4, key(iterator.next())); - Assert.assertFalse(iterator.hasNext()); - } - - @Test - public void testScanIteratorOrderedUsesStreamingBuildersLazily() - throws Exception { - HgStoreNodeManager manager = HgStoreNodeManager.getInstance(); - HgStoreNodePartitioner oldPartitioner = manager.getNodePartitioner(); - long firstNodeId = System.nanoTime(); - long secondNodeId = firstNodeId + 1L; - RecordingPartitioner partitioner = - new RecordingPartitioner(firstNodeId, secondNodeId); - TestIterator firstIterator = new TestIterator(3, 4); - TestIterator secondIterator = new TestIterator(1, 2); - RecordingSession firstSession = - new RecordingSession(firstIterator); - RecordingSession secondSession = - new RecordingSession(secondIterator); - String graph = "graph-" + firstNodeId; - manager.addNode(graph, new RecordingStoreNode(firstNodeId, - firstSession.proxy())); - manager.addNode(graph, new RecordingStoreNode(secondNodeId, - secondSession.proxy())); - manager.setNodePartitioner(partitioner); - try { - NodeTxSessionProxy proxy = new NodeTxSessionProxy(graph, manager); - HgKvIterator iterator = proxy.scanIteratorOrdered( - "table", HgOwnerKey.of(keyBytes(9), keyBytes(1)), - HgOwnerKey.of(keyBytes(9), keyBytes(5)), 5L, 123, - keyBytes(7)); - - Assert.assertEquals(1, firstSession.builders.size()); - Assert.assertEquals(1, secondSession.builders.size()); - Assert.assertEquals(0, firstSession.rangeScanCalls); - Assert.assertEquals(0, secondSession.rangeScanCalls); - Assert.assertEquals(0, firstIterator.hasNextCalls); - Assert.assertEquals(0, secondIterator.hasNextCalls); - assertOrderedRangeBuilder(firstSession.builders.get(0), 5L, 123, - keyBytes(7)); - assertOrderedRangeBuilder(secondSession.builders.get(0), 5L, 123, - keyBytes(7)); - - Assert.assertEquals(1, key(iterator.next())); - Assert.assertEquals(2, key(iterator.next())); - Assert.assertEquals(3, key(iterator.next())); - Assert.assertEquals(4, key(iterator.next())); - Assert.assertFalse(iterator.hasNext()); - } finally { - restoreNodePartitioner(manager, oldPartitioner); - } - } - - private static void assertOrderedRangeBuilder(Builder builder, long limit, - int scanType, - byte[] query) { - Assert.assertEquals(ScanMethod.RANGE, builder.getMethod()); - Assert.assertEquals("table", builder.getTable()); - Assert.assertEquals(limit, builder.getLimit()); - Assert.assertEquals(scanType, builder.getScanType()); - Assert.assertArrayEquals(query, builder.getQuery().toByteArray()); - } - - @Test - public void testMergeOrderedRangeScanIteratorsSortsPrefetchedSources() { - TestIterator first = new TestIterator(3, 4); - TestIterator second = new TestIterator(1, 2); - first.hasNext(); - second.hasNext(); - - HgKvIterator iterator = - NodeTxSessionProxy.mergeOrderedRangeScanIterators( - Arrays.asList(first, second), 0L); - Assert.assertEquals(1, key(iterator.next())); - Assert.assertEquals(2, key(iterator.next())); - Assert.assertEquals(3, key(iterator.next())); - Assert.assertEquals(4, key(iterator.next())); - Assert.assertFalse(iterator.hasNext()); - } - - @Test - public void testOrderedRangeNodeTkvListKeepsLegacyOwnerPartitioning() - throws Exception { - HgStoreNodeManager manager = HgStoreNodeManager.getInstance(); - HgStoreNodePartitioner oldPartitioner = manager.getNodePartitioner(); - RecordingPartitioner partitioner = new RecordingPartitioner(); - manager.setNodePartitioner(partitioner); - try { - NodeTxSessionProxy proxy = new NodeTxSessionProxy("graph", manager); - List nodeTkvs = toOrderedRangeNodeTkvList( - proxy, HgOwnerKey.of(keyBytes(9), keyBytes(1)), - HgOwnerKey.of(keyBytes(9), keyBytes(5))); - - Assert.assertEquals(1, partitioner.byteRangeCalls); - Assert.assertEquals(0, partitioner.codeRangeCalls); - Assert.assertArrayEquals(keyBytes(9), partitioner.lastStartKey); - Assert.assertArrayEquals(keyBytes(9), partitioner.lastEndKey); - Assert.assertEquals(2, nodeTkvs.size()); - Assert.assertEquals(10, nodeTkvs.get(0).getKey().getKeyCode()); - Assert.assertEquals(20, nodeTkvs.get(0).getEndKey().getKeyCode()); - Assert.assertEquals(30, nodeTkvs.get(1).getKey().getKeyCode()); - Assert.assertEquals(40, nodeTkvs.get(1).getEndKey().getKeyCode()); - } finally { - restoreNodePartitioner(manager, oldPartitioner); - } - } - - @SuppressWarnings("unchecked") - private static List toOrderedRangeNodeTkvList( - NodeTxSessionProxy proxy, HgOwnerKey startKey, - HgOwnerKey endKey) throws Exception { - Method method = NodeTxSessionProxy.class.getDeclaredMethod( - "toOrderedRangeNodeTkvList", String.class, HgOwnerKey.class, - HgOwnerKey.class); - method.setAccessible(true); - return (List) method.invoke(proxy, "table", startKey, endKey); - } - - private static void restoreNodePartitioner(HgStoreNodeManager manager, - HgStoreNodePartitioner old) - throws Exception { - Field field = HgStoreNodeManager.class.getDeclaredField( - "nodePartitioner"); - field.setAccessible(true); - field.set(manager, old); - } - - private static int key(HgKvEntry entry) { - return entry.key()[0] & 0xFF; - } - - private static byte[] keyBytes(int key) { - return new byte[]{(byte) key}; - } - - private static final class TestIterator implements HgKvIterator { - - private final List keys; - private int offset; - private HgKvEntry current; - private int hasNextCalls; - private int nextCalls; - - private TestIterator(Integer... keys) { - this.keys = Arrays.asList(keys); - this.offset = 0; - this.current = null; - this.hasNextCalls = 0; - this.nextCalls = 0; - } - - @Override - public boolean hasNext() { - this.hasNextCalls++; - return this.offset < this.keys.size(); - } - - @Override - public HgKvEntry next() { - if (!this.hasNext()) { - throw new NoSuchElementException(); - } - this.nextCalls++; - int key = this.keys.get(this.offset++); - this.current = new TestEntry(keyBytes(key)); - return this.current; - } - - @Override - public byte[] key() { - return this.current == null ? null : this.current.key(); - } - - @Override - public byte[] value() { - return this.current == null ? null : this.current.value(); - } - - @Override - public byte[] position() { - return this.key(); - } - } - - private static final class TestEntry implements HgKvEntry { - - private final byte[] key; - - private TestEntry(byte[] key) { - this.key = key; - } - - @Override - public byte[] key() { - return this.key; - } - - @Override - public byte[] value() { - return this.key; - } - } - - private static final class RecordingPartitioner - implements HgStoreNodePartitioner { - - private final long firstNodeId; - private final long secondNodeId; - private int byteRangeCalls; - private int codeRangeCalls; - private byte[] lastStartKey; - private byte[] lastEndKey; - - private RecordingPartitioner() { - this(1L, 2L); - } - - private RecordingPartitioner(long firstNodeId, long secondNodeId) { - this.firstNodeId = firstNodeId; - this.secondNodeId = secondNodeId; - } - - @Override - public int partition(HgNodePartitionerBuilder builder, - String graphName, byte[] startKey, - byte[] endKey) { - this.byteRangeCalls++; - this.lastStartKey = startKey; - this.lastEndKey = endKey; - builder.setPartitions(this.partitions()); - return 0; - } - - @Override - public int partition(HgNodePartitionerBuilder builder, - String graphName, int startCode, - int endCode) { - this.codeRangeCalls++; - builder.setPartitions(this.partitions()); - return 0; - } - - private Set partitions() { - Set partitions = new LinkedHashSet<>(); - partitions.add(HgNodePartition.of(this.firstNodeId, 10, 10, 20)); - partitions.add(HgNodePartition.of(this.secondNodeId, 30, 30, 40)); - return partitions; - } - } - - private static final class RecordingStoreNode implements HgStoreNode { - - private final Long nodeId; - private final HgStoreSession session; - - private RecordingStoreNode(Long nodeId, HgStoreSession session) { - this.nodeId = nodeId; - this.session = session; - } - - @Override - public Long getNodeId() { - return this.nodeId; - } - - @Override - public String getAddress() { - return "127.0.0.1:" + this.nodeId; - } - - @Override - public HgStoreSession openSession(String graphName) { - return this.session; - } - } - - private static final class RecordingSession - implements InvocationHandler { - - private final List builders; - private final TestIterator iterator; - private int rangeScanCalls; - - private RecordingSession(TestIterator iterator) { - this.builders = Collections.synchronizedList(new ArrayList<>()); - this.iterator = iterator; - this.rangeScanCalls = 0; - } - - private HgStoreSession proxy() { - return (HgStoreSession) Proxy.newProxyInstance( - HgStoreSession.class.getClassLoader(), - new Class[]{HgStoreSession.class}, this); - } - - @Override - public Object invoke(Object proxy, Method method, Object[] args) { - if ("scanIterator".equals(method.getName())) { - if (args != null && args.length == 1 && - args[0] instanceof Builder) { - this.builders.add(((Builder) args[0]).clone()); - return this.iterator; - } - this.rangeScanCalls++; - return this.iterator; - } - if ("isTx".equals(method.getName())) { - return false; - } - if ("toString".equals(method.getName())) { - return "RecordingSession"; - } - throw new UnsupportedOperationException(method.toString()); - } - } -} diff --git a/hugegraph-store/hg-store-client/src/test/java/org/apache/hugegraph/store/client/OrderedKvIteratorTest.java b/hugegraph-store/hg-store-client/src/test/java/org/apache/hugegraph/store/client/OrderedKvIteratorTest.java deleted file mode 100644 index 9a1aae631f..0000000000 --- a/hugegraph-store/hg-store-client/src/test/java/org/apache/hugegraph/store/client/OrderedKvIteratorTest.java +++ /dev/null @@ -1,208 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hugegraph.store.client; - -import java.util.Arrays; -import java.util.List; -import java.util.NoSuchElementException; - -import org.apache.hugegraph.store.HgKvEntry; -import org.apache.hugegraph.store.HgKvIterator; -import org.junit.Assert; -import org.junit.Test; - -public class OrderedKvIteratorTest { - - @Test - public void testMergeByKeyWithLimit() { - OrderedKvIterator iterator = new OrderedKvIterator(Arrays.asList( - new TestIterator(1, 4), - new TestIterator(2, 3) - ), 3L); - - Assert.assertTrue(iterator.hasNext()); - Assert.assertEquals(1, key(iterator.next())); - Assert.assertArrayEquals(keyBytes(1), iterator.position()); - - Assert.assertTrue(iterator.hasNext()); - Assert.assertEquals(2, key(iterator.next())); - Assert.assertArrayEquals(keyBytes(2), iterator.position()); - - Assert.assertTrue(iterator.hasNext()); - Assert.assertEquals(3, key(iterator.next())); - Assert.assertArrayEquals(keyBytes(3), iterator.position()); - - Assert.assertFalse(iterator.hasNext()); - } - - @Test - public void testNoLimitReturnsAllKeysInOrder() { - OrderedKvIterator iterator = new OrderedKvIterator(Arrays.asList( - new TestIterator(1, 5), - new TestIterator(2, 4) - ), 0L); - - Assert.assertEquals(1, key(iterator.next())); - Assert.assertEquals(2, key(iterator.next())); - Assert.assertEquals(4, key(iterator.next())); - Assert.assertEquals(5, key(iterator.next())); - Assert.assertFalse(iterator.hasNext()); - } - - @Test - public void testSeekBeforeReadSkipsBoundaryKey() { - OrderedKvIterator iterator = new OrderedKvIterator(Arrays.asList( - new TestIterator(1, 4), - new TestIterator(2, 3) - ), 0L); - - iterator.seek(keyBytes(3)); - - Assert.assertEquals(4, key(iterator.next())); - Assert.assertFalse(iterator.hasNext()); - } - - @Test - public void testSeekDoesNotSkipLowerSourceWithLargerKeys() { - OrderedKvIterator iterator = new OrderedKvIterator(Arrays.asList( - new TestIterator(1, 100), - new TestIterator(2, 3) - ), 0L); - - iterator.seek(keyBytes(2)); - - Assert.assertEquals(3, key(iterator.next())); - Assert.assertEquals(100, key(iterator.next())); - Assert.assertFalse(iterator.hasNext()); - } - - @Test - public void testCloseUnderlyingIteratorsWhenLimitReached() { - TestIterator first = new TestIterator(1, 4); - TestIterator second = new TestIterator(2, 3); - OrderedKvIterator iterator = new OrderedKvIterator(Arrays.asList(first, second), 1L); - - Assert.assertEquals(1, key(iterator.next())); - Assert.assertFalse(iterator.hasNext()); - Assert.assertTrue(first.closed()); - Assert.assertTrue(second.closed()); - } - - @Test - public void testCloseUnderlyingIteratorsWhenExhausted() { - TestIterator first = new TestIterator(1); - TestIterator second = new TestIterator(); - OrderedKvIterator iterator = new OrderedKvIterator(Arrays.asList(first, second), 0L); - - Assert.assertEquals(1, key(iterator.next())); - Assert.assertFalse(iterator.hasNext()); - Assert.assertTrue(first.closed()); - Assert.assertTrue(second.closed()); - } - - private static int key(HgKvEntry entry) { - return entry.key()[0] & 0xFF; - } - - private static byte[] keyBytes(int key) { - return new byte[]{(byte) key}; - } - - private static final class TestIterator implements HgKvIterator { - - private final List keys; - private int offset; - private HgKvEntry current; - private boolean closed; - - private TestIterator(Integer... keys) { - this.keys = Arrays.asList(keys); - this.offset = 0; - this.current = null; - this.closed = false; - } - - @Override - public boolean hasNext() { - return this.offset < this.keys.size(); - } - - @Override - public HgKvEntry next() { - if (!this.hasNext()) { - throw new NoSuchElementException(); - } - int key = this.keys.get(this.offset++); - this.current = new TestEntry(keyBytes(key)); - return this.current; - } - - @Override - public byte[] key() { - return this.current == null ? null : this.current.key(); - } - - @Override - public byte[] value() { - return this.current == null ? null : this.current.value(); - } - - @Override - public byte[] position() { - byte[] key = this.key(); - return key == null ? null : new byte[]{(byte) 0xFF, key[0]}; - } - - @Override - public void seek(byte[] position) { - int target = position[0] & 0xFF; - while (this.offset < this.keys.size() && - this.keys.get(this.offset) < target) { - this.offset++; - } - } - - @Override - public void close() { - this.closed = true; - } - - private boolean closed() { - return this.closed; - } - } - - private static final class TestEntry implements HgKvEntry { - - private final byte[] key; - - private TestEntry(byte[] key) { - this.key = key; - } - - @Override - public byte[] key() { - return this.key; - } - - @Override - public byte[] value() { - return this.key; - } - } -} From b16f5f13fcd627e6e27794000989289c486e5087 Mon Sep 17 00:00:00 2001 From: contrueCT Date: Tue, 16 Jun 2026 23:19:33 +0800 Subject: [PATCH 16/16] fix(hstore): defer ordered range paging follow-up --- .../backend/tx/GraphIndexTransaction.java | 263 ------------------ .../backend/store/hstore/HstoreTable.java | 5 + .../backend/tx/GraphIndexTransactionTest.java | 82 ------ .../apache/hugegraph/core/VertexCoreTest.java | 29 -- .../apache/hugegraph/unit/UnitTestSuite.java | 2 - 5 files changed, 5 insertions(+), 376 deletions(-) delete mode 100644 hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/backend/tx/GraphIndexTransactionTest.java diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/tx/GraphIndexTransaction.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/tx/GraphIndexTransaction.java index 048df9834a..2b8d02e581 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/tx/GraphIndexTransaction.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/tx/GraphIndexTransaction.java @@ -652,9 +652,6 @@ private void storeSelectedIndexField(IndexLabel indexLabel, @Watched(prefix = "index") private IdHolder doIndexQuery(IndexLabel indexLabel, ConditionQuery query) { - if (this.needMaterializedHstoreRangeOrder(indexLabel, query)) { - return this.doMaterializedHstoreRangeIndexQuery(indexLabel, query); - } if (!query.paging()) { return this.doIndexQueryBatch(indexLabel, query); } else { @@ -664,266 +661,6 @@ private IdHolder doIndexQuery(IndexLabel indexLabel, ConditionQuery query) { } } - private boolean needMaterializedHstoreRangeOrder(IndexLabel indexLabel, - ConditionQuery query) { - return this.store().provider().isHstore() && - indexLabel.indexType().isRange() && - (query.paging() || !query.noLimitAndOffset()); - } - - private IdHolder doMaterializedHstoreRangeIndexQuery(IndexLabel indexLabel, - ConditionQuery query) { - if (!query.paging()) { - Set ids = this.querySortedRangeIndexIds(indexLabel, query); - return this.newSortedRangeIndexBatchHolder(query, ids); - } - return new SortedRangePagingIdHolder(query, q -> { - return this.querySortedRangeIndexPage(indexLabel, q); - }); - } - - private BatchIdHolder newSortedRangeIndexBatchHolder(ConditionQuery query, - Set ids) { - return new SortedRangeBatchIdHolder(query, ids); - } - - private Set querySortedRangeIndexIds(IndexLabel indexLabel, - ConditionQuery query) { - List indexes = this.querySortedRangeIndexes(indexLabel, - query); - Set ids = InsertionOrderUtil.newSet(); - for (HugeIndex index : indexes) { - ids.addAll(index.elementIds()); - Query.checkForceCapacity(ids.size()); - } - return ids; - } - - private PageIds querySortedRangeIndexPage(IndexLabel indexLabel, - ConditionQuery query) { - List indexes = this.querySortedRangeIndexes(indexLabel, - query); - Set allIds = InsertionOrderUtil.newSet(); - for (HugeIndex index : indexes) { - allIds.addAll(index.elementIds()); - Query.checkForceCapacity(allIds.size()); - } - if (allIds.isEmpty()) { - return PageIds.EMPTY; - } - - int start = 0; - if (!query.page().isEmpty()) { - start = PageState.fromString(query.page()).offset(); - } - if (start >= allIds.size()) { - return PageIds.EMPTY; - } - - long total = allIds.size(); - long end = query.noLimit() ? total : - Math.min(total, (long) start + query.limit()); - Set pageIds = CollectionUtil.subSet(allIds, start, (int) end); - if (pageIds.isEmpty()) { - return PageIds.EMPTY; - } - - int next = (int) end; - PageState pageState; - if (next < total) { - pageState = new PageState(new byte[]{1}, next, pageIds.size()); - } else { - pageState = new PageState(PageState.EMPTY_BYTES, 0, - pageIds.size()); - } - return new PageIds(pageIds, pageState); - } - - private List querySortedRangeIndexes(IndexLabel indexLabel, - ConditionQuery query) { - List indexes = new ArrayList<>(); - Iterator entries = null; - String spaceGraph = this.params() - .graph().spaceGraphName(); - LockUtil.Locks locks = new LockUtil.Locks(spaceGraph); - ConditionQuery scanQuery = query.copy(); - scanQuery.page(null); - scanQuery.offset(0L); - scanQuery.limit(Query.NO_LIMIT); - try { - locks.lockReads(LockUtil.INDEX_LABEL_DELETE, indexLabel.id()); - locks.lockReads(LockUtil.INDEX_LABEL_REBUILD, indexLabel.id()); - if (!indexLabel.system()) { - graph().indexLabel(indexLabel.id()); - } - - entries = super.query(scanQuery).iterator(); - while (entries.hasNext()) { - HugeIndex index = this.readMatchedIndex(indexLabel, scanQuery, - entries.next()); - if (index == null) { - continue; - } - this.removeExpiredIndexIfNeeded(index, scanQuery.showExpired()); - this.recordIndexValue(scanQuery, index); - indexes.add(index); - Query.checkForceCapacity(indexes.size()); - } - } finally { - locks.unlock(); - CloseableIterator.closeIterator(entries); - } - - Collections.sort(indexes, this::compareRangeIndexValues); - return indexes; - } - - @SuppressWarnings({"rawtypes", "unchecked"}) - private int compareRangeIndexValues(HugeIndex left, HugeIndex right) { - Object leftValue = left.fieldValues(); - Object rightValue = right.fieldValues(); - E.checkArgument(leftValue instanceof Comparable, - "Invalid range index value '%s'", leftValue); - E.checkArgument(rightValue instanceof Comparable, - "Invalid range index value '%s'", rightValue); - - int result = ((Comparable) leftValue).compareTo(rightValue); - if (result != 0) { - return result; - } - return compareRangeIndexElementIds(left.elementIds(), - right.elementIds()); - } - - private static int compareRangeIndexElementIds(Set leftIds, - Set rightIds) { - Iterator left = leftIds.iterator(); - Iterator right = rightIds.iterator(); - while (left.hasNext() && right.hasNext()) { - int result = left.next().compareTo(right.next()); - if (result != 0) { - return result; - } - } - return Boolean.compare(left.hasNext(), right.hasNext()); - } - - static class SortedRangeBatchIdHolder extends BatchIdHolder { - - private final List idList; - private int offset; - private PageIds pendingBatch; - - SortedRangeBatchIdHolder(ConditionQuery query, Set ids) { - super(query, Collections.emptyIterator(), batch -> { - throw new IllegalStateException("Unexpected sorted index fetcher"); - }); - this.idList = new ArrayList<>(ids); - this.offset = 0; - this.pendingBatch = null; - } - - @Override - public boolean keepOrder() { - return true; - } - - @Override - public boolean hasNext() { - if (this.pendingBatch != null) { - return true; - } - if (this.exhausted) { - return false; - } - return this.offset < this.idList.size(); - } - - @Override - public IdHolder next() { - if (!this.hasNext()) { - throw new java.util.NoSuchElementException(); - } - return this; - } - - @Override - public PageIds fetchNext(String page, long batchSize) { - E.checkArgument(page == null, - "Not support page parameter by BatchIdHolder"); - E.checkArgument(batchSize >= 0L, - "Invalid batch size value: %s", batchSize); - if (this.pendingBatch != null) { - PageIds result = this.pendingBatch; - this.pendingBatch = null; - return result; - } - return this.fetchBatch(batchSize); - } - - @Override - public Set all() { - Set allIds = InsertionOrderUtil.newSet(); - if (this.pendingBatch != null) { - allIds.addAll(this.pendingBatch.ids()); - } - if (this.offset < this.idList.size()) { - allIds.addAll(this.idList.subList(this.offset, - this.idList.size())); - } - this.close(); - return allIds; - } - - @Override - public PageIds peekNext(long size) { - E.checkArgument(this.pendingBatch == null, - "Can't call peekNext() twice"); - this.pendingBatch = this.fetchBatch(size); - return this.pendingBatch; - } - - @Override - public void close() { - this.exhausted = true; - this.pendingBatch = null; - this.offset = this.idList.size(); - } - - private PageIds fetchBatch(long batchSize) { - if (this.offset >= this.idList.size() || batchSize == 0L) { - this.close(); - return PageIds.EMPTY; - } - - int end; - if (batchSize == Query.NO_LIMIT) { - end = this.idList.size(); - } else { - end = (int) Math.min((long) this.idList.size(), - this.offset + batchSize); - } - Set batchIds = InsertionOrderUtil.newSet(); - batchIds.addAll(this.idList.subList(this.offset, end)); - this.offset = end; - this.exhausted = this.offset >= this.idList.size(); - return new PageIds(batchIds, PageState.EMPTY); - } - } - - private static class SortedRangePagingIdHolder extends PagingIdHolder { - - SortedRangePagingIdHolder(ConditionQuery query, - Function fetcher) { - super(query, fetcher); - } - - @Override - public boolean keepOrder() { - return true; - } - } - @Watched(prefix = "index") private IdHolder doIndexQueryBatch(IndexLabel indexLabel, ConditionQuery query) { diff --git a/hugegraph-server/hugegraph-hstore/src/main/java/org/apache/hugegraph/backend/store/hstore/HstoreTable.java b/hugegraph-server/hugegraph-hstore/src/main/java/org/apache/hugegraph/backend/store/hstore/HstoreTable.java index 415a2577da..20cb66bb82 100755 --- a/hugegraph-server/hugegraph-hstore/src/main/java/org/apache/hugegraph/backend/store/hstore/HstoreTable.java +++ b/hugegraph-server/hugegraph-hstore/src/main/java/org/apache/hugegraph/backend/store/hstore/HstoreTable.java @@ -648,6 +648,11 @@ private boolean onlyOwnerVertex(Condition condition) { protected BackendColumnIterator queryByRange(Session session, IdRangeQuery query) { + /* + * FIXME: multi-partition HStore range-index paging still needs a + * bounded ordered merge in the store-client layer before it can + * guarantee globally ordered pages. + */ byte[] start = query.start().asBytes(); byte[] end = query.end() == null ? null : query.end().asBytes(); int type = query.inclusiveStart() ? diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/backend/tx/GraphIndexTransactionTest.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/backend/tx/GraphIndexTransactionTest.java deleted file mode 100644 index a1b0d4e8c6..0000000000 --- a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/backend/tx/GraphIndexTransactionTest.java +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hugegraph.backend.tx; - -import java.util.ArrayList; -import java.util.List; -import java.util.Set; - -import org.apache.hugegraph.backend.id.Id; -import org.apache.hugegraph.backend.id.IdGenerator; -import org.apache.hugegraph.backend.page.PageIds; -import org.apache.hugegraph.backend.query.ConditionQuery; -import org.apache.hugegraph.testutil.Assert; -import org.apache.hugegraph.type.HugeType; -import org.apache.hugegraph.util.InsertionOrderUtil; -import org.junit.Test; - -import com.google.common.collect.ImmutableList; - -public class GraphIndexTransactionTest { - - @Test - public void testSortedRangeBatchHolderKeepsPeekedBatch() { - ConditionQuery query = new ConditionQuery(HugeType.VERTEX); - Id id1 = IdGenerator.of(1); - Id id2 = IdGenerator.of(2); - Id id3 = IdGenerator.of(3); - Set ids = InsertionOrderUtil.newSet(); - ids.add(id1); - ids.add(id2); - ids.add(id3); - - GraphIndexTransaction.SortedRangeBatchIdHolder holder = - new GraphIndexTransaction.SortedRangeBatchIdHolder(query, ids); - - Assert.assertTrue(holder.keepOrder()); - - PageIds peeked = holder.peekNext(2); - Assert.assertEquals(ImmutableList.of(id1, id2), asList(peeked.ids())); - - PageIds firstBatch = holder.fetchNext(null, 2); - Assert.assertEquals(ImmutableList.of(id1, id2), - asList(firstBatch.ids())); - - PageIds secondBatch = holder.fetchNext(null, 2); - Assert.assertEquals(ImmutableList.of(id3), asList(secondBatch.ids())); - Assert.assertFalse(holder.hasNext()); - } - - @Test - public void testSortedRangeBatchHolderClosesOnZeroBatch() { - ConditionQuery query = new ConditionQuery(HugeType.VERTEX); - Set ids = InsertionOrderUtil.newSet(); - ids.add(IdGenerator.of(1)); - - GraphIndexTransaction.SortedRangeBatchIdHolder holder = - new GraphIndexTransaction.SortedRangeBatchIdHolder(query, ids); - - PageIds batch = holder.fetchNext(null, 0); - Assert.assertTrue(batch.empty()); - Assert.assertFalse(holder.hasNext()); - } - - private static List asList(Set ids) { - return new ArrayList<>(ids); - } -} diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/VertexCoreTest.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/VertexCoreTest.java index 967ca0049a..4746826ee8 100644 --- a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/VertexCoreTest.java +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/VertexCoreTest.java @@ -8776,26 +8776,6 @@ public void testQueryByRangeIndexInPage() { Assert.assertTrue(CollectionUtil.intersect(vertices1, vertices2) .isEmpty()); - // There are 12 vertices matched, ordered by range-index key - iter = g.V().hasLabel("software") - .has("price", P.between(100, 400)) - .has("~page", "") - .limit(5); - - vertices1 = IteratorUtils.list(iter); - Assert.assertEquals(5, vertices1.size()); - assertSortedByPrice(vertices1); - page = TraversalUtil.page(iter); - vertices2 = g.V().hasLabel("software") - .has("price", P.between(100, 400)) - .has("~page", page).limit(5).toList(); - Assert.assertEquals(5, vertices2.size()); - assertSortedByPrice(vertices2); - Assert.assertTrue((int) vertices1.get(vertices1.size() - 1) - .value("price") <= - (int) vertices2.get(0).value("price")); - Assert.assertTrue(CollectionUtil.intersect(vertices1, vertices2) - .isEmpty()); } @Test @@ -9624,15 +9604,6 @@ private Vertex vertex(String label, String pkName, Object pkValue) { return vertices.size() == 1 ? vertices.get(0) : null; } - private static void assertSortedByPrice(List vertices) { - int previous = Integer.MIN_VALUE; - for (Vertex vertex : vertices) { - int current = vertex.value("price"); - Assert.assertTrue(previous <= current); - previous = current; - } - } - private static void assertContains(List vertices, Object... keyValues) { Assert.assertTrue(Utils.contains(vertices, new FakeObjects.FakeVertex(keyValues))); diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/UnitTestSuite.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/UnitTestSuite.java index 9935909a02..3c856937bd 100644 --- a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/UnitTestSuite.java +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/UnitTestSuite.java @@ -17,7 +17,6 @@ package org.apache.hugegraph.unit; -import org.apache.hugegraph.backend.tx.GraphIndexTransactionTest; import org.apache.hugegraph.core.RoleElectionStateMachineTest; import org.apache.hugegraph.meta.MetaManagerSchemaCacheClearEventTest; import org.apache.hugegraph.traversal.optimize.TraversalUtilOptimizeTest; @@ -123,7 +122,6 @@ BackendMutationTest.class, ConditionTest.class, ConditionQueryFlattenTest.class, - GraphIndexTransactionTest.class, QueryTest.class, QueryResultsTest.class, RangeTest.class,