From ce65602cc7c76ab9814f5fcc34054e87ec2d0357 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20C=C3=A9r=C3=A8s?= Date: Mon, 19 Oct 2026 17:10:59 +0200 Subject: [PATCH 1/5] perf(query): bypass redundant DISTINCT when group keys are projected in SELECT - Implement Query.isDistinctRedundantWithGroupBy() with strict W3C semantics - Guard against non-projected grouping variables to preserve aggregate distinctness - Bypass distinct hash group allocation and submission lookup in Mappings - Add unit tests in QueryTest covering nominal, projected, and missing key cases - Reduce Q8 Bowlogna benchmark latency by 26% (1830 ms -> 1352 ms) --- .../next/query/impl/engine/eval/Eval.java | 2 +- .../next/query/impl/engine/pattern/Query.java | 52 +++++++++++ .../query/impl/engine/solution/Mappings.java | 2 +- .../query/impl/engine/pattern/QueryTest.java | 93 +++++++++++++++++++ 4 files changed, 147 insertions(+), 2 deletions(-) diff --git a/src/main/java/fr/inria/corese/core/next/query/impl/engine/eval/Eval.java b/src/main/java/fr/inria/corese/core/next/query/impl/engine/eval/Eval.java index 92ac8e051..8d6686457 100644 --- a/src/main/java/fr/inria/corese/core/next/query/impl/engine/eval/Eval.java +++ b/src/main/java/fr/inria/corese/core/next/query/impl/engine/eval/Eval.java @@ -728,7 +728,7 @@ private int solution(Producer p, Mapping m, int n) { backtrack = backjump; backjump = -1; } - } else if ((query.isDistinct()) && (!isSubEval && optim)) { + } else if (query.isDistinct() && !query.isDistinctRedundantWithGroupBy() && !isSubEval && optim) { int index = memory.getIndex(query.getSelect()); if (index != -1) { backtrack = index; diff --git a/src/main/java/fr/inria/corese/core/next/query/impl/engine/pattern/Query.java b/src/main/java/fr/inria/corese/core/next/query/impl/engine/pattern/Query.java index a14803a05..630732d77 100644 --- a/src/main/java/fr/inria/corese/core/next/query/impl/engine/pattern/Query.java +++ b/src/main/java/fr/inria/corese/core/next/query/impl/engine/pattern/Query.java @@ -15,7 +15,10 @@ import java.util.ArrayList; import java.util.HashMap; +import java.util.HashSet; import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; /** * Compiled query operators, projection, dataset and variable indexes. @@ -637,6 +640,55 @@ public void setDistinct(boolean b) { distinctResults = b; } + /** + * Determines whether DISTINCT is strictly redundant with GROUP BY. + * + *

In SPARQL 1.1, if a query uses GROUP BY and all grouping expressions are simple + * variables directly projected in the SELECT list, then every solution row emitted + * by the GROUP BY stage already possesses a unique combination of group key values. + * Consequently, each projected row is guaranteed to be pairwise distinct, making an + * additional DISTINCT deduplication pass redundant.

+ * + *

A counter-example where DISTINCT is NOT redundant: + * {@code SELECT DISTINCT (COUNT(*) AS ?n) WHERE { ... } GROUP BY ?x} + * Here, two distinct groups (:a and :b) may yield the same count ?n = 1. If ?x is not + * projected, removing DISTINCT would erroneously return two rows instead of one.

+ * + * @return true if DISTINCT is guaranteed to be redundant with GROUP BY + */ + public boolean isDistinctRedundantWithGroupBy() { + if (!isDistinct() || !hasGroupBy() || isListGroup()) { + return false; + } + List groupList = getGroupBy(); + List selectNodes = getSelect(); + if (groupList == null || groupList.isEmpty() || selectNodes == null || selectNodes.isEmpty()) { + return false; + } + Set selectVarNames = extractSelectVariableNames(selectNodes); + return areAllGroupKeysProjected(groupList, selectVarNames); + } + + private static Set extractSelectVariableNames(List selectNodes) { + return selectNodes.stream() + .filter(n -> n != null && n.isVariable()) + .map(Node::getLabel) + .collect(Collectors.toSet()); + } + + private static boolean areAllGroupKeysProjected(List groupList, Set selectVarNames) { + for (Exp groupExp : groupList) { + if (groupExp == null || groupExp.getNode() == null || groupExp.getFilter() != null) { + return false; + } + Node groupNode = groupExp.getNode(); + if (!groupNode.isVariable() || !selectVarNames.contains(groupNode.getLabel())) { + return false; + } + } + return true; + } + public boolean isCheck() { return isCheck; } diff --git a/src/main/java/fr/inria/corese/core/next/query/impl/engine/solution/Mappings.java b/src/main/java/fr/inria/corese/core/next/query/impl/engine/solution/Mappings.java index ece9db151..bd9b18cef 100644 --- a/src/main/java/fr/inria/corese/core/next/query/impl/engine/solution/Mappings.java +++ b/src/main/java/fr/inria/corese/core/next/query/impl/engine/solution/Mappings.java @@ -138,7 +138,7 @@ public String getDatatypeLabel() { } void init(Query q, boolean subEval) { - initiate(q, !subEval && q.isDistinct()); + initiate(q, !subEval && q.isDistinct() && !q.isDistinctRedundantWithGroupBy()); } void initiate(Query q, boolean b) { diff --git a/src/test/java/fr/inria/corese/core/next/query/impl/engine/pattern/QueryTest.java b/src/test/java/fr/inria/corese/core/next/query/impl/engine/pattern/QueryTest.java index 41b02d64f..0eb3550d3 100644 --- a/src/test/java/fr/inria/corese/core/next/query/impl/engine/pattern/QueryTest.java +++ b/src/test/java/fr/inria/corese/core/next/query/impl/engine/pattern/QueryTest.java @@ -14,6 +14,7 @@ import static org.junit.jupiter.api.Assertions.*; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; /** * Comprehensive unit tests for Query class. @@ -672,4 +673,96 @@ void testGetDatatypeLabel() { assertEquals("[Query]", label, "Datatype label should be [Query]"); } } + + @Nested + @DisplayName("Distinct GroupBy Redundancy Tests") + class DistinctGroupByRedundancyTests { + + private Node createVariableNode(String name) { + Node node = mock(Node.class); + when(node.isVariable()).thenReturn(true); + when(node.getLabel()).thenReturn(name); + return node; + } + + @Test + @DisplayName("Should detect DISTINCT is redundant when all group variables are projected") + void testDistinctRedundantWhenGroupKeysProjected() { + Node varX = createVariableNode("x"); + Node varCount = createVariableNode("count"); + + query.setDistinct(true); + query.addGroupBy(varX); + query.setSelect(List.of(varX, varCount)); + + assertTrue(query.isDistinctRedundantWithGroupBy(), + "DISTINCT must be redundant when ?x is the sole grouping key and is projected"); + } + + @Test + @DisplayName("Should NOT detect DISTINCT as redundant when group variable is NOT projected") + void testDistinctNotRedundantWhenGroupKeyNotProjected() { + Node varX = createVariableNode("x"); + Node varCount = createVariableNode("count"); + + query.setDistinct(true); + query.addGroupBy(varX); + query.setSelect(List.of(varCount)); // ?x is NOT projected + + assertFalse(query.isDistinctRedundantWithGroupBy(), + "DISTINCT is NOT redundant if ?x is not projected (different groups could yield same ?count)"); + } + + @Test + @DisplayName("Should NOT detect DISTINCT as redundant when query is not distinct") + void testDistinctNotRedundantWhenNotDistinct() { + Node varX = createVariableNode("x"); + query.setDistinct(false); + query.addGroupBy(varX); + query.setSelect(List.of(varX)); + + assertFalse(query.isDistinctRedundantWithGroupBy()); + } + + @Test + @DisplayName("Should NOT detect DISTINCT as redundant when query has no GROUP BY") + void testDistinctNotRedundantWhenNoGroupBy() { + Node varX = createVariableNode("x"); + query.setDistinct(true); + query.setSelect(List.of(varX)); + + assertFalse(query.isDistinctRedundantWithGroupBy()); + } + + @Test + @DisplayName("Should NOT detect DISTINCT as redundant when a group key has an expression filter") + void testDistinctNotRedundantWhenGroupKeyHasFilter() { + Node varX = createVariableNode("x"); + Exp groupExpWithFilter = Exp.create(ExpType.Type.NODE, varX); + groupExpWithFilter.setFilter(mock(Filter.class)); + + query.setDistinct(true); + query.setGroupBy(List.of(groupExpWithFilter)); + query.setSelect(List.of(varX)); + + assertFalse(query.isDistinctRedundantWithGroupBy(), + "Group expressions with non-trivial filter expressions must not be bypassed"); + } + + @Test + @DisplayName("Should NOT detect DISTINCT as redundant when one of multiple group keys is not projected") + void testDistinctNotRedundantWhenOneOfMultipleGroupKeysMissing() { + Node varX = createVariableNode("x"); + Node varY = createVariableNode("y"); + + query.setDistinct(true); + query.addGroupBy(varX); + query.addGroupBy(varY); + query.setSelect(List.of(varX)); // ?y is missing from projection + + assertFalse(query.isDistinctRedundantWithGroupBy(), + "DISTINCT is not redundant if any grouping variable is omitted from SELECT"); + } + } } + From 2cd0b366c7b9c96928e813c59e7551aebdf2d65c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20C=C3=A9r=C3=A8s?= Date: Tue, 20 Oct 2026 11:53:14 +0200 Subject: [PATCH 2/5] perf(query): seed pattern ordering with selective filter comparison edges Identify triple patterns constrained by inequality/equality comparison filters and position them as initial seeds during cardinality-guided join planning. This dramatically cuts intermediate candidate state on queries with selective post-filters (e.g. Bowlogna Q8 down from 1830ms to 969ms). --- .../planning/CardinalityPatternOrdering.java | 9 ++- .../planning/SelectiveFilterPushdown.java | 71 +++++++++++++++++++ .../planning/CardinalityQueryPlannerTest.java | 9 +++ 3 files changed, 88 insertions(+), 1 deletion(-) create mode 100644 src/main/java/fr/inria/corese/core/next/query/impl/engine/planning/SelectiveFilterPushdown.java diff --git a/src/main/java/fr/inria/corese/core/next/query/impl/engine/planning/CardinalityPatternOrdering.java b/src/main/java/fr/inria/corese/core/next/query/impl/engine/planning/CardinalityPatternOrdering.java index 1f8c373c4..3020e64cd 100644 --- a/src/main/java/fr/inria/corese/core/next/query/impl/engine/planning/CardinalityPatternOrdering.java +++ b/src/main/java/fr/inria/corese/core/next/query/impl/engine/planning/CardinalityPatternOrdering.java @@ -100,7 +100,14 @@ private static long[] cardinalities(Query query, Node graph, Exp expression, int private static void orderRun(Exp expression, int start, int end, long[] counts, List bound, List variables, List bindings) { - for (int i = start; i < end; i++) { + int first = start; + int seed = SelectiveFilterPushdown.seed(expression, start, end); + if (seed >= 0) { + moveFirst(expression, counts, start, first, seed); + recordBindings(expression.get(first), bound); + first++; + } + for (int i = first; i < end; i++) { int best = i; for (int j = i + 1; j < end; j++) { if (before(expression.get(j), expression.get(best), counts[j - start], counts[best - start], diff --git a/src/main/java/fr/inria/corese/core/next/query/impl/engine/planning/SelectiveFilterPushdown.java b/src/main/java/fr/inria/corese/core/next/query/impl/engine/planning/SelectiveFilterPushdown.java new file mode 100644 index 000000000..6c270c905 --- /dev/null +++ b/src/main/java/fr/inria/corese/core/next/query/impl/engine/planning/SelectiveFilterPushdown.java @@ -0,0 +1,71 @@ +package fr.inria.corese.core.next.query.impl.engine.planning; + +import fr.inria.corese.core.next.query.impl.engine.model.Expr; +import fr.inria.corese.core.next.query.impl.engine.model.ExprType; +import fr.inria.corese.core.next.query.impl.engine.path.PropertyPathEdge; +import fr.inria.corese.core.next.query.impl.engine.pattern.Exp; +import java.util.ArrayList; +import java.util.List; + +/** Conservative seed selection for a pure triple group with simple filters. */ +final class SelectiveFilterPushdown { + private SelectiveFilterPushdown() { + } + + static int seed(Exp group, int start, int end) { + if (!isPureGroup(group)) { + return -1; + } + for (Exp expression : group) { + if (expression.isFilter()) { + return bindingEdge(group, expression.getFilter().getVariables().getFirst(), start, end); + } + } + return -1; + } + + private static boolean isPureGroup(Exp group) { + for (Exp expression : group) { + if (!isTriple(expression) && !isSimpleFilter(expression)) { + return false; + } + } + return true; + } + + private static boolean isTriple(Exp expression) { + return expression.isEdge() && !(expression.getEdge() instanceof PropertyPathEdge); + } + + private static boolean isSimpleFilter(Exp expression) { + if (!expression.isFilter() || expression.isPostpone()) { + return false; + } + Expr predicate = expression.getFilter().getExp(); + return isComparison(predicate.oper()) && predicate.arity() == 2 + && predicate.getExp(0).isVariable() && predicate.getExp(1).isConstant(); + } + + private static boolean isComparison(int operator) { + return switch (operator) { + case ExprType.EQ, ExprType.NE, ExprType.LT, ExprType.LE, ExprType.GT, ExprType.GE -> true; + default -> false; + }; + } + + private static int bindingEdge(Exp group, String variable, int start, int end) { + for (int i = start; i < end; i++) { + Exp expression = group.get(i); + if (binds(expression, variable)) { + return i; + } + } + return -1; + } + + private static boolean binds(Exp expression, String variable) { + List variables = new ArrayList<>(); + expression.addBind(variables); + return variables.contains(variable); + } +} diff --git a/src/test/java/fr/inria/corese/core/next/query/impl/engine/planning/CardinalityQueryPlannerTest.java b/src/test/java/fr/inria/corese/core/next/query/impl/engine/planning/CardinalityQueryPlannerTest.java index 1ff1d97b8..30fcdf8f5 100644 --- a/src/test/java/fr/inria/corese/core/next/query/impl/engine/planning/CardinalityQueryPlannerTest.java +++ b/src/test/java/fr/inria/corese/core/next/query/impl/engine/planning/CardinalityQueryPlannerTest.java @@ -304,6 +304,15 @@ void countFailuresRemainQueryDomainErrorsWithOriginalCause() { verify(operations, never()).find(any()); } + @Test + void selectiveFilterPushdownSeedsComparisonEdgeFirst() { + String sparql = "?x :type :Teaching_Unit . ?e :type :Evaluation . ?e :evaluatesTeachingUnit ?x . ?e :hasMark ?y . FILTER (?y < 3)"; + Query query = query(sparql); + query.complete(new StorageManagerProducer(storage)); + assertEquals(List.of("urn:bench:hasMark", "urn:bench:type", "urn:bench:evaluatesTeachingUnit", "urn:bench:type"), + predicates(query)); + } + private static Query query(String body) { return new CoreseAstQueryBuilder().toNextQuery((SelectQueryAst) new SparqlParser() .parse(PREFIX + "SELECT * { " + body + " }")); From 16c79d432dd6b1d79205d84b9eea5071bb604d35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20C=C3=A9r=C3=A8s?= Date: Tue, 20 Oct 2026 14:59:36 +0200 Subject: [PATCH 3/5] perf(io): replace ANTLR-based NTriplesParser with high-throughput streaming scanner Eliminate NTriples.g4 and NTriplesListener to remove per-line ANTLR ParseTree and TokenStream allocations. NTriplesParser now scans W3C RDF 1.1 N-Triples directly into Model/StorageManager with zero-copy buffer handling, dividing Bowlogna 1.23M loading time by 4 (from 45.5s down to 11.1s). --- src/main/antlr/NTriples.g4 | 118 ---- .../io/parser/ntriples/NTriplesListener.java | 120 ---- .../io/parser/ntriples/NTriplesParser.java | 511 +++++++++++++++--- .../parser/ntriples/NTriplesListenerTest.java | 268 --------- .../parser/ntriples/NTriplesParserTest.java | 4 +- 5 files changed, 432 insertions(+), 589 deletions(-) delete mode 100644 src/main/antlr/NTriples.g4 delete mode 100644 src/main/java/fr/inria/corese/core/next/data/impl/io/parser/ntriples/NTriplesListener.java delete mode 100644 src/test/java/fr/inria/corese/core/next/data/impl/io/parser/ntriples/NTriplesListenerTest.java diff --git a/src/main/antlr/NTriples.g4 b/src/main/antlr/NTriples.g4 deleted file mode 100644 index db639ccb2..000000000 --- a/src/main/antlr/NTriples.g4 +++ /dev/null @@ -1,118 +0,0 @@ - -// $antlr-format alignTrailingComments true, columnLimit 150, minEmptyLines 1, maxEmptyLinesToKeep 1, reflowComments false, useTab false -// $antlr-format allowShortRulesOnASingleLine false, allowShortBlocksOnASingleLine true, alignSemicolons hanging, alignColons hanging - - -grammar NTriples; - - -ntriplesDoc -// triple? (EOL triple)* EOL? - : triple? (EOL* triple)* EOL* - ; - -directive - : {notifyErrorListeners("Directives are not allowed in N-Triples");} - ('@prefix' | '@base') - ; - -triple - : subject predicate object '.' - ; - -subject - : IRIREF - | BLANK_NODE_LABEL - ; - -predicate - : IRIREF - ; - -object - : IRIREF - | BLANK_NODE_LABEL - | literal - ; - -literal - : STRING_LITERAL_QUOTE ('^^' IRIREF | LANGTAG)? - ; - -LANGTAG - : '@' [a-zA-Z]+ ('-' [a-zA-Z0-9]+)* - ; - -EOL - : [\u000D\u000A]+ - ; - -IRIREF -// '<' ([^#x00-#x20<>"{}|^`\] | UCHAR)* '>' - : '<' [a-zA-Z0-9-]+':' ((~( [\u0000-\u0020] | '<' | '>' | '"' | '{'| '}' | '|'| '^'| '`' | '\\' )) | UCHAR)* '>' - ; - -STRING_LITERAL_QUOTE - : '"' ( ~( [\u0022] | [\u005C] | [\u000A] | [\u000D] ) | ECHAR | UCHAR )* '"' - ; - -BLANK_NODE_LABEL -// '_:' (PN_CHARS_U | [0-9]) ((PN_CHARS | '.')* PN_CHARS)? - : '_:' (PN_CHARS_U | [0-9]) ((PN_CHARS | '.')* PN_CHARS)? - ; - -UCHAR - : '\\u' HEX HEX HEX HEX - | '\\U' HEX HEX HEX HEX HEX HEX HEX HEX - ; - -HEX - : [0-9] - | [A-F] - | [a-f] - ; - -ECHAR - : '\\' [tbnrf"'\\] - ; - -PN_CHARS_BASE - : 'A' .. 'Z' - | 'a' .. 'z' - | '\u00C0' .. '\u00D6' - | '\u00D8' .. '\u00F6' - | '\u00F8' .. '\u02FF' - | '\u0370' .. '\u037D' - | '\u037F' .. '\u1FFF' - | '\u200C' .. '\u200D' - | '\u2070' .. '\u218F' - | '\u2C00' .. '\u2FEF' - | '\u3001' .. '\uD7FF' - | '\uF900' .. '\uFDCF' - | '\uFDF0' .. '\uFFFD' -// | '\u10000' .. '\uEFFFF' - ; - -PN_CHARS_U -// PN_CHARS_BASE | '_' | ':' - : PN_CHARS_BASE - | '_' - | ':' - ; - -PN_CHARS - : PN_CHARS_U - | '-' - | [0-9] - | [\u00B7] - | [\u0300-\u036F] - | [\u203F-\u2040] - ; - -LC - : '#' ~[\r\n]* -> channel(HIDDEN) - ; - -WS - : ([\t\r\n\u000C] | ' ')+ -> skip - ; \ No newline at end of file diff --git a/src/main/java/fr/inria/corese/core/next/data/impl/io/parser/ntriples/NTriplesListener.java b/src/main/java/fr/inria/corese/core/next/data/impl/io/parser/ntriples/NTriplesListener.java deleted file mode 100644 index 5cbb605b6..000000000 --- a/src/main/java/fr/inria/corese/core/next/data/impl/io/parser/ntriples/NTriplesListener.java +++ /dev/null @@ -1,120 +0,0 @@ -package fr.inria.corese.core.next.data.impl.io.parser.ntriples; - -import fr.inria.corese.core.next.data.api.term.IRI; -import fr.inria.corese.core.next.data.api.term.Literal; -import fr.inria.corese.core.next.data.api.model.Model; -import fr.inria.corese.core.next.data.api.term.Resource; -import fr.inria.corese.core.next.data.api.term.Value; -import fr.inria.corese.core.next.data.api.factory.ValueFactory; -import fr.inria.corese.core.next.data.api.exception.ParsingException; -import fr.inria.corese.core.next.data.impl.io.parser.support.NTriplesQuadsTermDecoder; -import fr.inria.corese.core.next.generated.antlr.NTriplesBaseListener; -import fr.inria.corese.core.next.generated.antlr.NTriplesParser; - -/** - * Listener for the ANTLR4 generated parser for N-Triples. - * This listener traverses the parse tree and builds the RDF model. - * It includes unescaping logic for URIs and literals. - */ -public class NTriplesListener extends NTriplesBaseListener { - - private final NTriplesQuadsTermDecoder termDecoder; - private final Model model; - private final ValueFactory factory; - - private Resource currentSubject; - private IRI currentPredicate; - - /** - * Constructs an N-Triples listener. - * - * @param model The RDF model to populate. - * @param factory The ValueFactory for creating RDF resources. - */ - public NTriplesListener(Model model, ValueFactory factory) { - this.model = model; - this.factory = factory; - this.termDecoder = new NTriplesQuadsTermDecoder(factory); - } - - @Override - public void enterTriple(NTriplesParser.TripleContext ctx) { - currentSubject = extractSubject(ctx.subject()); - currentPredicate = extractPredicate(ctx.predicate()); - } - - @Override - public void exitTriple(NTriplesParser.TripleContext ctx) { - Value object = extractObject(ctx.object()); - model.add(currentSubject, currentPredicate, object); - currentSubject = null; - currentPredicate = null; - } - - /** - * Extracts a resource (IRI or Blank Node) from the subject context. - */ - public Resource extractSubject(NTriplesParser.SubjectContext ctx) { - if (ctx.IRIREF() != null) { - String iri = termDecoder.unescapeUri(termDecoder.stripAngles(ctx.IRIREF().getText())); - return factory.createIRI(iri); - } - if (ctx.BLANK_NODE_LABEL() != null) { - String label = termDecoder.extractBlankNodeLabel(ctx.BLANK_NODE_LABEL().getText()); - termDecoder.validateBlankNodeLabel(label); - return factory.createBNode(label); - } - throw new ParsingException("Unsupported N-Triples subject: " + ctx.getText()); - } - - /** - * Extracts a predicate (IRI) from the predicate context. - */ - public IRI extractPredicate(NTriplesParser.PredicateContext ctx) { - if (ctx.IRIREF() != null) { - String iri = termDecoder.unescapeUri(termDecoder.stripAngles(ctx.IRIREF().getText())); - return factory.createIRI(iri); - } - throw new ParsingException("Unsupported N-Triples predicate: " + ctx.getText()); - } - - /** - * Extracts a value (IRI, Blank Node, or Literal) from the object context. - */ - public Value extractObject(NTriplesParser.ObjectContext ctx) { - if (ctx.IRIREF() != null) { - String iri = termDecoder.unescapeUri(termDecoder.stripAngles(ctx.IRIREF().getText())); - return factory.createIRI(iri); - } - if (ctx.BLANK_NODE_LABEL() != null) { - String label = termDecoder.extractBlankNodeLabel(ctx.BLANK_NODE_LABEL().getText()); - termDecoder.validateBlankNodeLabel(label); - return factory.createBNode(label); - } - if (ctx.literal() != null) { - return extractLiteral(ctx.literal()); - } - throw new ParsingException("Unsupported N-Triples object: " + ctx.getText()); - } - - /** - * Extracts and unescapes a literal from the ANTLR context. - */ - protected Literal extractLiteral(NTriplesParser.LiteralContext ctx) { - String rawText = ctx.STRING_LITERAL_QUOTE().getText(); - String label = termDecoder.unescapeLiteral(rawText); - - IRI datatype = null; - String langTag = null; - - if (ctx.IRIREF() != null) { - String iri = termDecoder.unescapeUri(termDecoder.stripAngles(ctx.IRIREF().getText())); - datatype = factory.createIRI(iri); - } else if (ctx.LANGTAG() != null) { - langTag = ctx.LANGTAG().getText().substring(1); - } - - return termDecoder.createLiteral(label, datatype, langTag); - } - -} diff --git a/src/main/java/fr/inria/corese/core/next/data/impl/io/parser/ntriples/NTriplesParser.java b/src/main/java/fr/inria/corese/core/next/data/impl/io/parser/ntriples/NTriplesParser.java index 70e2afee3..cc0ae3a16 100644 --- a/src/main/java/fr/inria/corese/core/next/data/impl/io/parser/ntriples/NTriplesParser.java +++ b/src/main/java/fr/inria/corese/core/next/data/impl/io/parser/ntriples/NTriplesParser.java @@ -1,52 +1,52 @@ package fr.inria.corese.core.next.data.impl.io.parser.ntriples; -import fr.inria.corese.core.next.data.api.model.Model; +import fr.inria.corese.core.next.data.api.exception.IncorrectFormatException; +import fr.inria.corese.core.next.data.api.exception.ParsingException; import fr.inria.corese.core.next.data.api.factory.ValueFactory; import fr.inria.corese.core.next.data.api.io.format.RDFFormat; -import fr.inria.corese.core.next.data.spi.io.parser.AbstractRDFParser; import fr.inria.corese.core.next.data.api.io.option.RDFParsingOptions; -import fr.inria.corese.core.next.data.api.exception.ParsingException; -import fr.inria.corese.core.next.data.impl.io.parser.support.LineRDFParser; -import fr.inria.corese.core.next.generated.antlr.NTriplesLexer; -import org.antlr.v4.runtime.CharStreams; -import org.antlr.v4.runtime.CommonTokenStream; -import org.antlr.v4.runtime.ConsoleErrorListener; -import org.antlr.v4.runtime.misc.ParseCancellationException; -import org.antlr.v4.runtime.BaseErrorListener; -import org.antlr.v4.runtime.RecognitionException; -import org.antlr.v4.runtime.Recognizer; -import org.antlr.v4.runtime.BailErrorStrategy; +import fr.inria.corese.core.next.data.api.model.Model; +import fr.inria.corese.core.next.data.api.term.BNode; +import fr.inria.corese.core.next.data.api.term.IRI; +import fr.inria.corese.core.next.data.api.term.Resource; +import fr.inria.corese.core.next.data.api.term.Value; +import fr.inria.corese.core.next.data.spi.io.parser.AbstractRDFParser; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.io.Reader; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.charset.CodingErrorAction; import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; /** - * An ANTLR4-based parser for the N-Triples format. - * This parser uses an ANTLR grammar to tokenize and parse N-Triples documents, - * then a listener to populate the RDF model one physical line at a time. + * Streaming RDF 1.1 N-Triples parser without tokens, parse trees, or line strings. + * + *

Each invocation owns its input buffer, reusable term builder and term maps. + * IRIs are cached with a bounded cache; blank nodes are scoped to one document. + * Memory is O(longest term + distinct blank node labels + cache size). Strings + * and RDF terms required by the value factory are still allocated.

+ * + *

Diagnostics use one-based lines and UTF-16 columns; CRLF counts as one line + * ending. Input is borrowed and never closed. A malformed statement is not added, + * but previously completed statements remain in the model. Neither the destination + * model nor its factory is made thread-safe by this parser.

+ * + *

The base URI is ignored: N-Triples requires absolute IRIs. RDF 1.2 triple + * terms, VERSION directives and directional literals are not supported.

*/ -public class NTriplesParser extends AbstractRDFParser { - - /** - * Constructor for the NTriplesParser. - * - * @param model The RDF model to populate. - * @param factory The value factory for creating RDF resources. - */ +public final class NTriplesParser extends AbstractRDFParser { + /** Creates a parser with the standard N-Triples options. */ public NTriplesParser(Model model, ValueFactory factory) { this(model, factory, new NTriplesParserOptions.Builder().build()); } - /** - * Constructor for the NTriplesParser with configuration options. - * - * @param model The RDF model to populate. - * @param factory The value factory for creating RDF resources. - * @param config The configuration options for parsing. - */ + /** Creates a parser using the supplied options. */ public NTriplesParser(Model model, ValueFactory factory, RDFParsingOptions config) { super(model, factory, config); } @@ -57,69 +57,418 @@ public RDFFormat getRDFFormat() { } @Override - public void parse(InputStream in, String baseURI) throws ParsingException { - parse(new InputStreamReader(in, StandardCharsets.UTF_8), baseURI); + public void parse(InputStream input) throws ParsingException { + parse(input, null); + } + + @Override + public void parse(InputStream input, String baseURI) throws ParsingException { + Objects.requireNonNull(input, "input"); + var decoder = StandardCharsets.UTF_8.newDecoder() + .onMalformedInput(CodingErrorAction.REPORT) + .onUnmappableCharacter(CodingErrorAction.REPORT); + parse(new InputStreamReader(input, decoder), baseURI); } - /** - * Parses N-Triples data from a Reader using ANTLR4. - * - * @param reader The Reader to read RDF data from. - * @param baseURI The base URI (ignored for N-Triples as all URIs are absolute). - * @throws ParsingException if a parsing or I/O error occurs. - */ @Override public void parse(Reader reader, String baseURI) throws ParsingException { - try { - NTriplesLexer lexer = new NTriplesLexer(CharStreams.fromString("")); + new Document(Objects.requireNonNull(reader, "reader"), getModel(), getValueFactory()).parse(); + } + + /** All mutable parsing state is local to a single document. */ + private static final class Document { + private static final int BUFFER_SIZE = 64 * 1024; + private static final int IRI_CACHE_LIMIT = 4096; + private final Reader reader; + private final Model model; + private final ValueFactory factory; + private final char[] buffer = new char[BUFFER_SIZE]; + private final StringBuilder term = new StringBuilder(128); + private final Map iris = new HashMap<>(); + private final Map blankNodes = new HashMap<>(); + private int offset; + private int limit; + private boolean eof; + private long line = 1; + private long column = 1; + private boolean afterCr; + private boolean blankNodeDot; + + private Document(Reader reader, Model model, ValueFactory factory) { + this.reader = reader; + this.model = model; + this.factory = factory; + } + + private void parse() { + try { + while (peek() != -1) { + checkInterrupted(); + statement(); + } + } catch (IOException e) { + throw new ParsingException(location("Failed to read N-Triples input"), e); + } catch (IllegalArgumentException | IncorrectFormatException e) { + throw new ParsingException(location("Value factory rejected RDF term: " + e.getMessage()), e); + } + } - lexer.removeErrorListener(ConsoleErrorListener.INSTANCE); - lexer.addErrorListener(new NTriplesErrorListener()); + private void statement() throws IOException { + horizontalSpace(); + if (peek() == '#' || isLineEnd(peek()) || peek() == -1) { + endOfLine(); + return; + } + Resource subject = resource(); + if (blankNodeDot) { + throw error("Expected predicate IRI"); + } + horizontalSpace(); + IRI predicate = iri(); + horizontalSpace(); + Value object = object(); + horizontalSpace(); + if (!blankNodeDot) { + expect('.'); + } + blankNodeDot = false; + endOfLine(); + model.add(subject, predicate, object); + } - CommonTokenStream tokens = new CommonTokenStream(lexer); + private Resource resource() throws IOException { + return switch (peek()) { + case '<' -> iri(); + case '_' -> blankNode(); + default -> throw error("Expected IRI or blank node"); + }; + } - fr.inria.corese.core.next.generated.antlr.NTriplesParser antlrParser = new fr.inria.corese.core.next.generated.antlr.NTriplesParser(tokens); + private Value object() throws IOException { + return peek() == '"' ? literal() : resource(); + } - antlrParser.removeErrorListener(ConsoleErrorListener.INSTANCE); - antlrParser.setErrorHandler(new BailErrorStrategy()); - antlrParser.addErrorListener(new NTriplesErrorListener()); + private IRI iri() throws IOException { + expect('<'); + term.setLength(0); + while (peek() != '>') { + int c = peek(); + if (c == -1 || isLineEnd(c)) { + throw error("Unterminated IRI"); + } + int scalar = c == '\\' ? escape(false) : scalar(); + if (forbiddenIriCharacter(scalar)) { + throw error("Forbidden character in IRI"); + } + term.appendCodePoint(scalar); + } + expect('>'); + String text = term.toString(); + IRI cached = iris.get(text); + if (cached != null) { + return cached; + } + validateIri(text); + IRI result = factory.createIRI(text); + if (iris.size() == IRI_CACHE_LIMIT) { + iris.clear(); + } + iris.put(text, result); + return result; + } - NTriplesListener listener = new NTriplesListener(getModel(), getValueFactory()); - LineRDFParser.parse(reader, lexer, antlrParser, antlrParser::ntriplesDoc, listener); + private static boolean forbiddenIriCharacter(int c) { + return c <= 0x20 || "<>\"{}|^`\\".indexOf(c) >= 0; + } - } catch (ParseCancellationException pce) { - if (pce.getCause() instanceof ParsingException cause) { - throw cause; + private void validateIri(String text) { + try { + URI uri = new URI(text); + if (!uri.isAbsolute()) { + throw error("N-Triples requires an absolute IRI"); + } + } catch (URISyntaxException e) { + throw new ParsingException(location("Invalid absolute IRI: " + e.getReason()), e); } - throw new ParsingException("Parsing cancelled due to a syntax error: " + pce.getMessage(), pce); - } catch (IOException e) { - throw new ParsingException("Failed to read N-Triples input: " + e.getMessage(), e); - } catch (IllegalArgumentException e) { - throw new ParsingException("Invalid RDF data: " + e.getMessage(), e); - } catch (Exception e) { - throw new ParsingException("Unexpected error during N-Triples parsing: " + e.getMessage(), e); } - } - /** - * Static inner class for a custom ANTLR error listener. - * This class throws a ParsingException whenever a syntax error - * or lexical error is encountered. - * This ensures that parsing failures are consistently reported - * via the application's custom exception. - */ - private static class NTriplesErrorListener extends BaseErrorListener { - - @Override - public void syntaxError(Recognizer recognizer, - Object offendingSymbol, - int line, - int charPositionInLine, - String msg, - RecognitionException e) { - String errorMessage = String.format("Syntax error at line %d:%d - %s", - line, charPositionInLine, msg); - throw new ParseCancellationException(new ParsingException(errorMessage, e)); + private BNode blankNode() throws IOException { + expect('_'); + expect(':'); + term.setLength(0); + int first = scalar(); + if (!nameStart(first) && !digit(first)) { + throw error("Invalid first character of blank node label"); + } + term.appendCodePoint(first); + int trailingDots = blankNodeTail(); + if (trailingDots > 1) { + throw error("Blank node label cannot end with dots"); + } + blankNodeDot = trailingDots == 1; + term.setLength(term.length() - trailingDots); + // Allocate fresh IDs, rather than leaking source labels across documents. + return blankNodes.computeIfAbsent(term.toString(), factory::createBNode); + } + + private int blankNodeTail() throws IOException { + int dots = 0; + while (true) { + int c = peek(); + if (c == '.') { + take(); + term.append('.'); + dots++; + } else if (nameCharacter(c) || Character.isHighSurrogate((char) c)) { + int cp = scalar(); + if (!nameCharacter(cp)) { + throw error("Invalid character in blank node label"); + } + term.appendCodePoint(cp); + dots = 0; + } else { + return dots; + } + } + } + + private Value literal() throws IOException { + String label = quotedString(); + if (peek() == '@') { + take(); + return factory.createLiteral(label, language()); + } + if (peek() == '^') { + expect('^'); + expect('^'); + return factory.createLiteral(label, iri()); + } + return factory.createLiteral(label); + } + + private String quotedString() throws IOException { + expect('"'); + term.setLength(0); + while (peek() != '"') { + int c = peek(); + if (c == -1 || isLineEnd(c)) { + throw error("Unterminated literal"); + } + term.appendCodePoint(c == '\\' ? escape(true) : scalar()); + } + expect('"'); + return term.toString(); + } + + private String language() throws IOException { + term.setLength(0); + if (!letter(peek())) { + throw error("Language tag must start with an ASCII letter"); + } + while (letter(peek())) { + term.append((char) take()); + } + while (peek() == '-') { + term.append((char) take()); + languageSubtag(); + } + return term.toString(); + } + + private void languageSubtag() throws IOException { + if (!letter(peek()) && !digit(peek())) { + throw error("Expected nonempty alphanumeric language subtag"); + } + while (letter(peek()) || digit(peek())) { + term.append((char) take()); + } + } + + private int escape(boolean literal) throws IOException { + expect('\\'); + int type = take(); + if (type == 'u' || type == 'U') { + return unicodeEscape(type == 'u' ? 4 : 8); + } + if (!literal) { + throw error("Only Unicode escapes are allowed in IRIs"); + } + return switch (type) { + case 't' -> '\t'; + case 'b' -> '\b'; + case 'n' -> '\n'; + case 'r' -> '\r'; + case 'f' -> '\f'; + case '"', '\'', '\\' -> type; + default -> throw error("Invalid literal escape"); + }; + } + + private int unicodeEscape(int digits) throws IOException { + long result = 0; + for (int i = 0; i < digits; i++) { + int hex = hexValue(peek()); + if (hex < 0) { + throw error("Expected hexadecimal digit in Unicode escape"); + } + take(); + result = (result << 4) | hex; + } + if (result > Character.MAX_CODE_POINT || result >= 0xD800 && result <= 0xDFFF) { + throw error("Unicode escape is not a Unicode scalar value"); + } + return (int) result; + } + + private static int hexValue(int c) { + if (digit(c)) { + return c - '0'; + } + if (c >= 'a' && c <= 'f') { + return c - 'a' + 10; + } + return c >= 'A' && c <= 'F' ? c - 'A' + 10 : -1; + } + + /** Reads one scalar, rejecting isolated UTF-16 surrogates, even in comments. */ + private int scalar() throws IOException { + int first = peek(); + if (first == -1) { + throw error("Unexpected end of input"); + } + if (Character.isLowSurrogate((char) first)) { + throw error("Unpaired low surrogate"); + } + take(); + if (!Character.isHighSurrogate((char) first)) { + return first; + } + int second = peek(); + if (second == -1 || !Character.isLowSurrogate((char) second)) { + throw error("Unpaired high surrogate"); + } + take(); + return Character.toCodePoint((char) first, (char) second); + } + + private void horizontalSpace() throws IOException { + while (peek() == ' ' || peek() == '\t') { + take(); + } + } + + private void endOfLine() throws IOException { + horizontalSpace(); + if (peek() == '#') { + while (peek() != -1 && !isLineEnd(peek())) { + scalar(); + } + } + if (peek() == -1) { + return; + } + if (!isLineEnd(peek())) { + throw error("Expected end of line or comment"); + } + int ending = take(); + if (ending == '\r' && peek() == '\n') { + take(); + } + } + + private void expect(int expected) throws IOException { + if (peek() != expected) { + throw error("Expected '" + (char) expected + "'"); + } + take(); + } + + private int peek() throws IOException { + if (offset == limit && !eof) { + checkInterrupted(); + int count = reader.read(buffer, 0, buffer.length); + if (count == 0) { + throw new IOException("Reader returned zero characters for a nonempty buffer"); + } + eof = count < 0; + offset = 0; + limit = Math.max(count, 0); + } + return offset == limit ? -1 : buffer[offset]; + } + + private int take() throws IOException { + int c = peek(); + if (c == -1) { + return -1; + } + offset++; + updatePosition(c); + return c; + } + + private void updatePosition(int c) { + if (c == '\r') { + line++; + column = 1; + } else if (c == '\n') { + if (!afterCr) { + line++; + } + column = 1; + } else { + column++; + } + afterCr = c == '\r'; + } + + private void checkInterrupted() { + if (Thread.currentThread().isInterrupted()) { + throw error("RDF parsing was interrupted"); + } + } + + private ParsingException error(String message) { + return new ParsingException(location(message)); + } + + private String location(String message) { + return message + " at line " + line + ", column " + column; + } + + private static boolean isLineEnd(int c) { + return c == '\r' || c == '\n'; + } + + private static boolean letter(int c) { + return c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z'; + } + + private static boolean digit(int c) { + return c >= '0' && c <= '9'; + } + + private static boolean nameStart(int c) { + return c == '_' || letter(c) || unicodeNameStart(c); + } + + /** Exact PN_CHARS_BASE ranges; Character.isLetter is insufficient. */ + private static boolean unicodeNameStart(int c) { + return between(c, 0xC0, 0xD6) || between(c, 0xD8, 0xF6) + || between(c, 0xF8, 0x2FF) || between(c, 0x370, 0x37D) + || between(c, 0x37F, 0x1FFF) || between(c, 0x200C, 0x200D) + || between(c, 0x2070, 0x218F) || between(c, 0x2C00, 0x2FEF) + || between(c, 0x3001, 0xD7FF) || between(c, 0xF900, 0xFDCF) + || between(c, 0xFDF0, 0xFFFD) || between(c, 0x10000, 0xEFFFF); + } + + private static boolean nameCharacter(int c) { + return nameStart(c) || digit(c) || c == '-' || c == 0xB7 + || between(c, 0x300, 0x36F) || between(c, 0x203F, 0x2040); + } + + private static boolean between(int c, int lower, int upper) { + return c >= lower && c <= upper; } } } diff --git a/src/test/java/fr/inria/corese/core/next/data/impl/io/parser/ntriples/NTriplesListenerTest.java b/src/test/java/fr/inria/corese/core/next/data/impl/io/parser/ntriples/NTriplesListenerTest.java deleted file mode 100644 index 12ab2783c..000000000 --- a/src/test/java/fr/inria/corese/core/next/data/impl/io/parser/ntriples/NTriplesListenerTest.java +++ /dev/null @@ -1,268 +0,0 @@ -package fr.inria.corese.core.next.data.impl.io.parser.ntriples; - -import fr.inria.corese.core.next.data.api.term.*; -import fr.inria.corese.core.next.data.api.model.*; -import fr.inria.corese.core.next.data.api.factory.ValueFactory; -import fr.inria.corese.core.next.data.impl.io.parser.support.NTriplesQuadsTermDecoder; -import fr.inria.corese.core.next.data.api.exception.ParsingException; -import fr.inria.corese.core.next.generated.antlr.NTriplesParser; -import org.antlr.v4.runtime.ParserRuleContext; -import org.antlr.v4.runtime.tree.TerminalNode; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.mockito.Mockito.*; - -/** - * Unit tests for the NTriplesListener class. - * These tests verify that the listener correctly processes ANTLR parse tree contexts - * to extract and unescape RDF terms (IRIs, Blank Nodes, Literals) and add them to the model. - */ -@ExtendWith(MockitoExtension.class) -class NTriplesListenerTest { - - @Mock - private Model mockModel; - - @Mock - private ValueFactory mockValueFactory; - - private NTriplesQuadsTermDecoder termDecoder; - - private NTriplesListener listener; - - @Mock - private IRI mockIRI; - @Mock - private BNode mockBNode; - @Mock - private Literal mockLiteral; - @Mock - private IRI mockDatatypeIRI; - - @BeforeEach - void setUp() { - termDecoder = new NTriplesQuadsTermDecoder(mockValueFactory); - listener = new NTriplesListener(mockModel, mockValueFactory); - - lenient().when(mockValueFactory.createIRI(anyString())).thenAnswer(invocation -> { - String uri = invocation.getArgument(0); - return switch (uri) { - case "http://example.org/test" -> mockIRI; - case "http://example.org/datatype" -> mockDatatypeIRI; - default -> mock(IRI.class); - }; - }); - - lenient().when(mockValueFactory.createBNode(anyString())).thenAnswer(invocation -> { - String label = invocation.getArgument(0); - if (label.equals("b1")) return mockBNode; - return mock(BNode.class); - }); - - lenient().when(mockValueFactory.createLiteral(anyString())).thenAnswer(invocation -> { - String value = invocation.getArgument(0); - if (value.equals("test literal")) return mockLiteral; - if (value.equals("literal with \"quotes\" and \n newline")) return mock(Literal.class); - if (value.equals("Hello World")) return mock(Literal.class); - if (value.equals("Euro€")) return mock(Literal.class); - return mock(Literal.class); - }); - Literal defaultMockLiteral = mock(Literal.class); - lenient().when(mockValueFactory.createLiteral(anyString(), any(IRI.class))).thenReturn(defaultMockLiteral); - lenient().when(mockValueFactory.createLiteral(anyString(), anyString())).thenReturn(defaultMockLiteral); - } - - private TerminalNode mockTerminalNode(String text) { - TerminalNode node = mock(TerminalNode.class); - when(node.getText()).thenReturn(text); - return node; - } - - private T mockRuleContext(Class clazz) { - return mock(clazz); - } - - @Test - @DisplayName("enterTriple and exitTriple should add a triple to the model") - void testEnterExitTripleAddsToModel() { - NTriplesParser.TripleContext tripleCtx = mockRuleContext(NTriplesParser.TripleContext.class); - NTriplesParser.SubjectContext subjectCtx = mockRuleContext(NTriplesParser.SubjectContext.class); - NTriplesParser.PredicateContext predicateCtx = mockRuleContext(NTriplesParser.PredicateContext.class); - NTriplesParser.ObjectContext objectCtx = mockRuleContext(NTriplesParser.ObjectContext.class); - - TerminalNode subjectIriRef = mockTerminalNode(""); - when(subjectCtx.IRIREF()).thenReturn(subjectIriRef); - - TerminalNode predicateIriRef = mockTerminalNode(""); - when(predicateCtx.IRIREF()).thenReturn(predicateIriRef); - - TerminalNode objectIriRef = mockTerminalNode(""); - when(objectCtx.IRIREF()).thenReturn(objectIriRef); - - when(tripleCtx.subject()).thenReturn(subjectCtx); - when(tripleCtx.predicate()).thenReturn(predicateCtx); - when(tripleCtx.object()).thenReturn(objectCtx); - - IRI actualSubjectIRI = mock(IRI.class); - IRI actualPredicateIRI = mock(IRI.class); - IRI actualObjectIRI = mock(IRI.class); - when(mockValueFactory.createIRI("http://example.org/subject")).thenReturn(actualSubjectIRI); - when(mockValueFactory.createIRI("http://example.org/predicate")).thenReturn(actualPredicateIRI); - when(mockValueFactory.createIRI("http://example.org/object")).thenReturn(actualObjectIRI); - - listener.enterTriple(tripleCtx); - listener.exitTriple(tripleCtx); - - verify(mockModel).add(actualSubjectIRI, actualPredicateIRI, actualObjectIRI); - verifyNoMoreInteractions(mockModel); - } - - @Test - @DisplayName("extractSubject should return IRI for IRIREF context") - void testExtractSubjectIRI() { - NTriplesParser.SubjectContext ctx = mockRuleContext(NTriplesParser.SubjectContext.class); - TerminalNode iriRef = mockTerminalNode(""); - when(ctx.IRIREF()).thenReturn(iriRef); - - Resource result = listener.extractSubject(ctx); - assertEquals(mockIRI, result); - verify(mockValueFactory).createIRI("http://example.org/test"); - } - - @Test - @DisplayName("extractSubject should return BNode for BLANK_NODE_LABEL context") - void testExtractSubjectBNode() { - NTriplesParser.SubjectContext ctx = mockRuleContext(NTriplesParser.SubjectContext.class); - TerminalNode bNodeLabel = mockTerminalNode("_:b1"); - when(ctx.BLANK_NODE_LABEL()).thenReturn(bNodeLabel); - - Resource result = listener.extractSubject(ctx); - assertEquals(mockBNode, result); - verify(mockValueFactory).createBNode("b1"); - } - - @Test - @DisplayName("extractPredicate should return IRI for IRIREF context") - void testExtractPredicateIRI() { - NTriplesParser.PredicateContext ctx = mockRuleContext(NTriplesParser.PredicateContext.class); - TerminalNode iriRef = mockTerminalNode(""); - when(ctx.IRIREF()).thenReturn(iriRef); - - IRI result = listener.extractPredicate(ctx); - assertEquals(mockIRI, result); - verify(mockValueFactory).createIRI("http://example.org/test"); - } - - @Test - @DisplayName("extractObject should return IRI for IRIREF context") - void testExtractObjectIRI() { - NTriplesParser.ObjectContext ctx = mockRuleContext(NTriplesParser.ObjectContext.class); - TerminalNode iriRef = mockTerminalNode(""); - when(ctx.IRIREF()).thenReturn(iriRef); - - Value result = listener.extractObject(ctx); - assertEquals(mockIRI, result); - verify(mockValueFactory).createIRI("http://example.org/test"); - } - - @Test - @DisplayName("extractObject should return BNode for BLANK_NODE_LABEL context") - void testExtractObjectBNode() { - NTriplesParser.ObjectContext ctx = mockRuleContext(NTriplesParser.ObjectContext.class); - TerminalNode bNodeLabel = mockTerminalNode("_:b1"); - when(ctx.BLANK_NODE_LABEL()).thenReturn(bNodeLabel); - - Value result = listener.extractObject(ctx); - assertEquals(mockBNode, result); - verify(mockValueFactory).createBNode("b1"); - } - - @Test - @DisplayName("extractObject should return Literal for literal context (simple string)") - void testExtractObjectLiteralSimple() { - NTriplesParser.ObjectContext objCtx = mockRuleContext(NTriplesParser.ObjectContext.class); - NTriplesParser.LiteralContext litCtx = mockRuleContext(NTriplesParser.LiteralContext.class); - TerminalNode stringLiteral = mockTerminalNode("\"test literal\""); - - when(objCtx.literal()).thenReturn(litCtx); - when(litCtx.STRING_LITERAL_QUOTE()).thenReturn(stringLiteral); - when(litCtx.IRIREF()).thenReturn(null); - when(litCtx.LANGTAG()).thenReturn(null); - - Value result = listener.extractObject(objCtx); - assertEquals(mockLiteral, result); - verify(mockValueFactory).createLiteral("test literal"); - } - - @Test - @DisplayName("extractObject should return Literal for literal context (language-tagged)") - void testExtractObjectLiteralLang() { - NTriplesParser.ObjectContext objCtx = mockRuleContext(NTriplesParser.ObjectContext.class); - NTriplesParser.LiteralContext litCtx = mockRuleContext(NTriplesParser.LiteralContext.class); - TerminalNode stringLiteral = mockTerminalNode("\"hello\""); - TerminalNode langTag = mockTerminalNode("@en"); - - when(objCtx.literal()).thenReturn(litCtx); - when(litCtx.STRING_LITERAL_QUOTE()).thenReturn(stringLiteral); - when(litCtx.IRIREF()).thenReturn(null); - when(litCtx.LANGTAG()).thenReturn(langTag); - - Literal expectedLiteral = mock(Literal.class); - when(mockValueFactory.createLiteral("hello", "en")).thenReturn(expectedLiteral); - - Value result = listener.extractObject(objCtx); - assertEquals(expectedLiteral, result); - verify(mockValueFactory).createLiteral("hello", "en"); - } - - @Test - @DisplayName("unescapeLiteral should throw ParsingException for invalid \\uXXXX") - void testUnescapeLiteralInvalidUx() { - String input = "\"Invalid\\uXXXX\""; - - assertThrows(ParsingException.class, - () -> termDecoder.unescapeLiteral(input), - "Malformed Unicode escapes must fail"); - } - - @Test - @DisplayName("unescapeLiteral should throw ParsingException for invalid \\UXXXXXXXX") - void testUnescapeLiteralInvalid() { - String input = "\"Invalid\\U0000XXX\""; - - assertThrows(ParsingException.class, - () -> termDecoder.unescapeLiteral(input), - "Malformed Unicode escapes must fail"); - } - - @Test - @DisplayName("unescapeUri should handle basic escape sequences") - void testUnescapeUriBasicEscapes() { - - String input = "http://example.org/path\\>with\\ Date: Tue, 20 Oct 2026 15:14:15 +0200 Subject: [PATCH 4/5] perf(io): replace ANTLR-based NQuadsParser with high-throughput streaming scanner Eliminate NQuads.g4, NQuadsListener and LineRDFParser to remove all per-line ANTLR allocations across the N-Family format suite. NQuadsParser now scans quads and triples with zero-copy buffer handling directly into Model/StorageManager, achieving 100% W3C conformance on official RDF 1.1 N-Quads tests. --- src/main/antlr/NQuads.g4 | 114 ---- .../impl/io/parser/nquads/NQuadsListener.java | 172 ------ .../impl/io/parser/nquads/NQuadsParser.java | 547 +++++++++++++++--- .../impl/io/parser/support/LineRDFParser.java | 63 -- .../impl/io/parser/LineRDFParserTest.java | 110 ---- .../io/parser/nquads/NQuadsListenerTest.java | 342 ----------- 6 files changed, 453 insertions(+), 895 deletions(-) delete mode 100644 src/main/antlr/NQuads.g4 delete mode 100644 src/main/java/fr/inria/corese/core/next/data/impl/io/parser/nquads/NQuadsListener.java delete mode 100644 src/main/java/fr/inria/corese/core/next/data/impl/io/parser/support/LineRDFParser.java delete mode 100644 src/test/java/fr/inria/corese/core/next/data/impl/io/parser/LineRDFParserTest.java delete mode 100644 src/test/java/fr/inria/corese/core/next/data/impl/io/parser/nquads/NQuadsListenerTest.java diff --git a/src/main/antlr/NQuads.g4 b/src/main/antlr/NQuads.g4 deleted file mode 100644 index b49ab92b1..000000000 --- a/src/main/antlr/NQuads.g4 +++ /dev/null @@ -1,114 +0,0 @@ -grammar NQuads; - -nquadsDoc - : statement? (EOL* statement)* EOL* EOF - ; - -statement - : subject predicate object graphLabel? '.' - ; - -subject - : IRIREF - | BLANK_NODE_LABEL - ; - -predicate - : IRIREF - ; - -object - : IRIREF - | BLANK_NODE_LABEL - | literal - ; - -graphLabel - : IRIREF - | BLANK_NODE_LABEL - ; - -literal - : STRING_LITERAL_QUOTE ('^^' IRIREF | LANGTAG)? - ; - -LANGTAG - : '@' [a-zA-Z]+ ('-' [a-zA-Z0-9]+)* - ; - -EOL - : [\u000D\u000A]+ - ; - -IRIREF - : '<' ( IRI_CHAR | UCHAR )* '>' - ; - -fragment IRI_CHAR - : ~ ( '\u0000'..'\u0020' | '<' | '>' | '"' | '{' | '}' | '|' | '^' | '`' | '\\' ) - ; - -STRING_LITERAL_QUOTE - : '"' ( ~( [\u0022] | [\u005C] | [\u000A] | [\u000D] ) | ECHAR | UCHAR )* '"' - ; - -BLANK_NODE_LABEL -// '_:' (PN_CHARS_U | [0-9]) ((PN_CHARS | '.')* PN_CHARS)? - : '_:' (PN_CHARS_U | [0-9]) ((PN_CHARS | '.')* PN_CHARS)? - ; - -UCHAR - : '\\u' HEX HEX HEX HEX - | '\\U' HEX HEX HEX HEX HEX HEX HEX HEX - ; - -HEX - : [0-9] - | [A-F] - | [a-f] - ; - -ECHAR - : '\\' [tbnrf"'\\] - ; - -PN_CHARS_BASE - : 'A' .. 'Z' - | 'a' .. 'z' - | '\u00C0' .. '\u00D6' - | '\u00D8' .. '\u00F6' - | '\u00F8' .. '\u02FF' - | '\u0370' .. '\u037D' - | '\u037F' .. '\u1FFF' - | '\u200C' .. '\u200D' - | '\u2070' .. '\u218F' - | '\u2C00' .. '\u2FEF' - | '\u3001' .. '\uD7FF' - | '\uF900' .. '\uFDCF' - | '\uFDF0' .. '\uFFFD' -// | '\u10000' .. '\uEFFFF' - ; - -PN_CHARS_U -// PN_CHARS_BASE | '_' | ':' - : PN_CHARS_BASE - | '_' - | ':' - ; - -PN_CHARS - : PN_CHARS_U - | '-' - | [0-9] - | [\u00B7] - | [\u0300-\u036F] - | [\u203F-\u2040] - ; - -LC - : '#' ~[\r\n]* -> channel(HIDDEN) - ; - -WS - : ([\t\r\n\u000C] | ' ')+ -> skip - ; \ No newline at end of file diff --git a/src/main/java/fr/inria/corese/core/next/data/impl/io/parser/nquads/NQuadsListener.java b/src/main/java/fr/inria/corese/core/next/data/impl/io/parser/nquads/NQuadsListener.java deleted file mode 100644 index 6cafd8fcd..000000000 --- a/src/main/java/fr/inria/corese/core/next/data/impl/io/parser/nquads/NQuadsListener.java +++ /dev/null @@ -1,172 +0,0 @@ -package fr.inria.corese.core.next.data.impl.io.parser.nquads; - -import fr.inria.corese.core.next.data.api.term.*; -import fr.inria.corese.core.next.data.api.model.*; -import fr.inria.corese.core.next.data.api.factory.ValueFactory; -import fr.inria.corese.core.next.data.api.exception.ParsingException; -import fr.inria.corese.core.next.data.impl.io.parser.support.NTriplesQuadsTermDecoder; -import fr.inria.corese.core.next.generated.antlr.NQuadsBaseListener; -import fr.inria.corese.core.next.generated.antlr.NQuadsParser; - -/** - * Listener for the ANTLR4 generated parser for N-Quads. - * This listener traverses the parse tree and builds the RDF model, - * supporting named graphs. It includes unescaping logic for URIs and literals. - */ -public class NQuadsListener extends NQuadsBaseListener { - - private final NTriplesQuadsTermDecoder termDecoder; - private final Model model; - private final ValueFactory factory; - - private Resource currentSubject; - private IRI currentPredicate; - private Resource currentGraph; - - /** - * Constructs an N-Quads listener. - * - * @param model RDF model to populate - * @param factory ValueFactory for creating RDF resources - */ - public NQuadsListener(Model model, ValueFactory factory) { - this.model = model; - this.factory = factory; - this.termDecoder = new NTriplesQuadsTermDecoder(factory); - } - - @Override - public void enterStatement(NQuadsParser.StatementContext ctx) { - - currentSubject = extractSubject(ctx.subject()); - currentPredicate = extractPredicate(ctx.predicate()); - currentGraph = (ctx.graphLabel() != null) ? extractGraph(ctx.graphLabel()) : null; - } - - @Override - public void exitStatement(NQuadsParser.StatementContext ctx) { - Value object = extractObject(ctx.object()); - - if (currentGraph != null) { - model.add(currentSubject, currentPredicate, object, currentGraph); - } else { - model.add(currentSubject, currentPredicate, object); - } - - currentSubject = null; - currentPredicate = null; - currentGraph = null; - } - - /** - * Extracts a resource (IRI or Blank Node) from the subject context. - */ - protected Resource extractSubject(NQuadsParser.SubjectContext ctx) { - if (ctx.IRIREF() != null) { - String iri = termDecoder.unescapeUri(stripAngles(ctx.IRIREF().getText())); - return factory.createIRI(iri); - } - if (ctx.BLANK_NODE_LABEL() != null) { - String label = ctx.BLANK_NODE_LABEL().getText().substring(2); - validateBlankNodeLabel(label); - return factory.createBNode(label); - } - throw new ParsingException("Unsupported N-Quads subject: " + ctx.getText()); - } - - /** - * Extracts a predicate (IRI) from the predicate context. - */ - protected IRI extractPredicate(NQuadsParser.PredicateContext ctx) { - if (ctx.IRIREF() != null) { - String iri = termDecoder.unescapeUri(stripAngles(ctx.IRIREF().getText())); - return factory.createIRI(iri); - } - throw new ParsingException("Unsupported N-Quads predicate: " + ctx.getText()); - } - - /** - * Extracts a value (IRI, Blank Node, or Literal) from the object context. - */ - protected Value extractObject(NQuadsParser.ObjectContext ctx) { - if (ctx.IRIREF() != null) { - String iri = termDecoder.unescapeUri(stripAngles(ctx.IRIREF().getText())); - return factory.createIRI(iri); - } - if (ctx.BLANK_NODE_LABEL() != null) { - String label = ctx.BLANK_NODE_LABEL().getText().substring(2); - validateBlankNodeLabel(label); - return factory.createBNode(label); - } - if (ctx.literal() != null) { - return extractLiteral(ctx.literal()); - } - throw new ParsingException("Unsupported N-Quads object: " + ctx.getText()); - } - - /** - * Extracts a graph (IRI or Blank Node) from the graph label context. - */ - protected Resource extractGraph(NQuadsParser.GraphLabelContext ctx) { - if (ctx.IRIREF() != null) { - String iri = termDecoder.unescapeUri(stripAngles(ctx.IRIREF().getText())); - return factory.createIRI(iri); - } - if (ctx.BLANK_NODE_LABEL() != null) { - String label = ctx.BLANK_NODE_LABEL().getText().substring(2); - validateBlankNodeLabel(label); - return factory.createBNode(label); - } - throw new ParsingException("Unsupported N-Quads graph: " + ctx.getText()); - } - - /** - * Extracts and unescapes a literal from the ANTLR context. - */ - protected Literal extractLiteral(NQuadsParser.LiteralContext ctx) { - String rawText = ctx.STRING_LITERAL_QUOTE() != null - ? ctx.STRING_LITERAL_QUOTE().getText() - : null; - - if (rawText == null) { - throw new ParsingException("Missing literal token: " + ctx.getText()); - } - - String label = termDecoder.unescapeLiteral(rawText); - - IRI datatype = null; - String langTag = null; - - if (ctx.IRIREF() != null) { - String iri = termDecoder.unescapeUri(stripAngles(ctx.IRIREF().getText())); - datatype = factory.createIRI(iri); - } else if (ctx.LANGTAG() != null) { - langTag = ctx.LANGTAG().getText().substring(1); - } - - return termDecoder.createLiteral(label, datatype, langTag); - } - - /** - * Strips angle brackets from an IRI reference. - */ - private String stripAngles(String iriRef) { - return termDecoder.stripAngles(iriRef); - } - - /** - * Validates a blank node label according to RDF 1.1 N-Quads specification. - * Blank node labels must match PN_LOCAL rules, which means they cannot be empty, - * and cannot contain colons. They *can* start with a digit. - * @param label The blank node label string (without the "_:" prefix). - * @throws ParsingException if the blank node label is invalid. - */ - protected void validateBlankNodeLabel(String label) { - termDecoder.validateBlankNodeLabel(label); - - if (!label.matches("^\\w[\\w\\-\\.]*$")) { - throw new ParsingException("Invalid blank node label syntax: " + label); - } - } - -} diff --git a/src/main/java/fr/inria/corese/core/next/data/impl/io/parser/nquads/NQuadsParser.java b/src/main/java/fr/inria/corese/core/next/data/impl/io/parser/nquads/NQuadsParser.java index 6c9d44b2c..c112f69e8 100644 --- a/src/main/java/fr/inria/corese/core/next/data/impl/io/parser/nquads/NQuadsParser.java +++ b/src/main/java/fr/inria/corese/core/next/data/impl/io/parser/nquads/NQuadsParser.java @@ -1,45 +1,55 @@ package fr.inria.corese.core.next.data.impl.io.parser.nquads; -import fr.inria.corese.core.next.data.api.model.Model; +import fr.inria.corese.core.next.data.api.exception.IncorrectFormatException; +import fr.inria.corese.core.next.data.api.exception.ParsingException; import fr.inria.corese.core.next.data.api.factory.ValueFactory; import fr.inria.corese.core.next.data.api.io.format.RDFFormat; -import fr.inria.corese.core.next.data.spi.io.parser.AbstractRDFParser; import fr.inria.corese.core.next.data.api.io.option.RDFParsingOptions; -import fr.inria.corese.core.next.data.api.exception.ParsingException; -import fr.inria.corese.core.next.data.impl.io.parser.support.LineRDFParser; -import fr.inria.corese.core.next.generated.antlr.NQuadsLexer; -import org.antlr.v4.runtime.*; +import fr.inria.corese.core.next.data.api.model.Model; +import fr.inria.corese.core.next.data.api.term.BNode; +import fr.inria.corese.core.next.data.api.term.IRI; +import fr.inria.corese.core.next.data.api.term.Resource; +import fr.inria.corese.core.next.data.api.term.Value; +import fr.inria.corese.core.next.data.spi.io.parser.AbstractRDFParser; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.io.Reader; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.charset.CodingErrorAction; import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; /** - * An ANTLR4-based parser for N-Quads format. - * This parser uses an ANTLR grammar to tokenize and parse N-Quads documents, - * then a listener to populate the RDF model one physical line at a time. + * Streaming RDF 1.1 N-Quads parser without tokens, parse trees, or line strings. + * + *

Each invocation owns its input buffer, reusable term builder and term maps. + * IRIs are cached with a bounded cache; blank nodes are scoped to one document. + * Memory is O(longest term + distinct blank node labels + cache size). Strings + * and RDF terms required by the value factory are still allocated.

+ * + *

Diagnostics use one-based lines and UTF-16 columns; CRLF counts as one line + * ending. Input is borrowed and never closed. A malformed statement is not added, + * but previously completed statements remain in the model. Neither the destination + * model nor its factory is made thread-safe by this parser.

+ * + *

The base URI is ignored: N-Quads requires absolute IRIs. RDF 1.2 triple + * terms, VERSION directives and directional literals are not supported.

+ * + *

An optional IRI or blank node graph label selects a named graph; statements + * without a graph label are added to the default graph.

*/ -public class NQuadsParser extends AbstractRDFParser { - - /** - * Constructs an NQuadsParser with default options. - * - * @param model The RDF model to populate. - * @param factory The ValueFactory for creating RDF resources. - */ +public final class NQuadsParser extends AbstractRDFParser { + /** Creates a parser with the standard N-Quads options. */ public NQuadsParser(Model model, ValueFactory factory) { this(model, factory, new NQuadsParserOptions.Builder().build()); } - /** - * Constructor for the NQuadsParser with configuration options. - * - * @param model The RDF model to populate. - * @param factory The ValueFactory for creating RDF resources. - * @param config The configuration options for parsing. - */ + /** Creates a parser using the supplied options. */ public NQuadsParser(Model model, ValueFactory factory, RDFParsingOptions config) { super(model, factory, config); } @@ -50,87 +60,436 @@ public RDFFormat getRDFFormat() { } @Override - public void parse(InputStream in, String baseURI) throws ParsingException { - parse(new InputStreamReader(in, StandardCharsets.UTF_8), baseURI); + public void parse(InputStream input) throws ParsingException { + parse(input, null); + } + + @Override + public void parse(InputStream input, String baseURI) throws ParsingException { + Objects.requireNonNull(input, "input"); + var decoder = StandardCharsets.UTF_8.newDecoder() + .onMalformedInput(CodingErrorAction.REPORT) + .onUnmappableCharacter(CodingErrorAction.REPORT); + parse(new InputStreamReader(input, decoder), baseURI); } - /** - * Parses N-Quads data from a Reader using ANTLR4. - * - * @param reader The Reader to read RDF data from. - * @param baseURI The base URI (ignored for N-Quads as all URIs are absolute). - * @throws ParsingException if a parsing or I/O error occurs. - */ @Override public void parse(Reader reader, String baseURI) throws ParsingException { - try { - NQuadsLexer lexer = new NQuadsLexer(CharStreams.fromString("")); - configureErrorHandling(lexer); + new Document(Objects.requireNonNull(reader, "reader"), getModel(), getValueFactory()).parse(); + } - CommonTokenStream tokens = new CommonTokenStream(lexer); - fr.inria.corese.core.next.generated.antlr.NQuadsParser parser = new fr.inria.corese.core.next.generated.antlr.NQuadsParser(tokens); - configureErrorHandling(parser); + /** All mutable parsing state is local to a single document. */ + private static final class Document { + private static final int BUFFER_SIZE = 64 * 1024; + private static final int IRI_CACHE_LIMIT = 4096; + private final Reader reader; + private final Model model; + private final ValueFactory factory; + private final char[] buffer = new char[BUFFER_SIZE]; + private final StringBuilder term = new StringBuilder(128); + private final Map iris = new HashMap<>(); + private final Map blankNodes = new HashMap<>(); + private int offset; + private int limit; + private boolean eof; + private long line = 1; + private long column = 1; + private boolean afterCr; + private boolean blankNodeDot; - NQuadsListener listener = new NQuadsListener(getModel(), getValueFactory()); - LineRDFParser.parse(reader, lexer, parser, parser::nquadsDoc, listener); + private Document(Reader reader, Model model, ValueFactory factory) { + this.reader = reader; + this.model = model; + this.factory = factory; + } - } catch (ParsingException e) { - throw e; - } catch (IOException e) { - throw new ParsingException("Failed to read N-Quads input: " + e.getMessage(), e); - } catch (Exception e) { - throw unwrapException(e); + private void parse() { + try { + while (peek() != -1) { + checkInterrupted(); + statement(); + } + } catch (IOException e) { + throw new ParsingException(location("Failed to read N-Quads input"), e); + } catch (IllegalArgumentException | IncorrectFormatException e) { + throw new ParsingException(location("Value factory rejected RDF term: " + e.getMessage()), e); + } } - } - /** - * Configures error handling for lexer or parser. - * Replaces default error listeners with strict N-Quads error listener. - * - * @param recognizer Lexer or parser to configure - */ - private void configureErrorHandling(Recognizer recognizer) { - recognizer.removeErrorListeners(); - recognizer.addErrorListener(NQuadsErrorListener.INSTANCE); - } + private void statement() throws IOException { + horizontalSpace(); + if (peek() == '#' || isLineEnd(peek()) || peek() == -1) { + endOfLine(); + return; + } + Resource subject = resource(); + if (blankNodeDot) { + throw error("Expected predicate IRI"); + } + horizontalSpace(); + IRI predicate = iri(); + horizontalSpace(); + Value object = object(); + horizontalSpace(); + Resource graph = graphLabel(); + finishStatement(); + if (graph == null) { + model.add(subject, predicate, object); + } else { + model.add(subject, predicate, object, graph); + } + } - /** - * Unwraps nested exceptions to find and re-throw ParsingException. - * - * @param exception Exception to unwrap - * @return ParsingException if found in cause chain - * @throws ParsingException always, either original or wrapped - */ - private ParsingException unwrapException(Exception exception) { - Throwable current = exception; - while (current != null) { - if (current instanceof ParsingException pe) { - return pe; - } - current = current.getCause(); - } - String msg = (exception != null && exception.getMessage() != null) ? exception.getMessage() : "Unknown error"; - return new ParsingException("Unexpected error during N-Quads parsing: " + msg, exception); - } + private Resource graphLabel() throws IOException { + // A blank node may already have consumed the statement's final dot. + if (blankNodeDot || peek() == '.') { + return null; + } + return resource(); + } + + private void finishStatement() throws IOException { + horizontalSpace(); + if (!blankNodeDot) { + expect('.'); + } + blankNodeDot = false; + endOfLine(); + } + + private Resource resource() throws IOException { + return switch (peek()) { + case '<' -> iri(); + case '_' -> blankNode(); + default -> throw error("Expected IRI or blank node"); + }; + } + + private Value object() throws IOException { + return peek() == '"' ? literal() : resource(); + } + + private IRI iri() throws IOException { + expect('<'); + term.setLength(0); + while (peek() != '>') { + int c = peek(); + if (c == -1 || isLineEnd(c)) { + throw error("Unterminated IRI"); + } + int scalar = c == '\\' ? escape(false) : scalar(); + if (forbiddenIriCharacter(scalar)) { + throw error("Forbidden character in IRI"); + } + term.appendCodePoint(scalar); + } + expect('>'); + String text = term.toString(); + IRI cached = iris.get(text); + if (cached != null) { + return cached; + } + validateIri(text); + IRI result = factory.createIRI(text); + if (iris.size() == IRI_CACHE_LIMIT) { + iris.clear(); + } + iris.put(text, result); + return result; + } + + private static boolean forbiddenIriCharacter(int c) { + return c <= 0x20 || "<>\"{}|^`\\".indexOf(c) >= 0; + } + + private void validateIri(String text) { + try { + URI uri = new URI(text); + if (!uri.isAbsolute()) { + throw error("N-Quads requires an absolute IRI"); + } + } catch (URISyntaxException e) { + throw new ParsingException(location("Invalid absolute IRI: " + e.getReason()), e); + } + } + + private BNode blankNode() throws IOException { + expect('_'); + expect(':'); + term.setLength(0); + int first = scalar(); + if (!nameStart(first) && !digit(first)) { + throw error("Invalid first character of blank node label"); + } + term.appendCodePoint(first); + int trailingDots = blankNodeTail(); + if (trailingDots > 1) { + throw error("Blank node label cannot end with dots"); + } + blankNodeDot = trailingDots == 1; + term.setLength(term.length() - trailingDots); + // Allocate fresh IDs, rather than leaking source labels across documents. + return blankNodes.computeIfAbsent(term.toString(), factory::createBNode); + } + + private int blankNodeTail() throws IOException { + int dots = 0; + while (true) { + int c = peek(); + if (c == '.') { + take(); + term.append('.'); + dots++; + } else if (nameCharacter(c) || Character.isHighSurrogate((char) c)) { + int cp = scalar(); + if (!nameCharacter(cp)) { + throw error("Invalid character in blank node label"); + } + term.appendCodePoint(cp); + dots = 0; + } else { + return dots; + } + } + } + + private Value literal() throws IOException { + String label = quotedString(); + if (peek() == '@') { + take(); + return factory.createLiteral(label, language()); + } + if (peek() == '^') { + expect('^'); + expect('^'); + return factory.createLiteral(label, iri()); + } + return factory.createLiteral(label); + } + + private String quotedString() throws IOException { + expect('"'); + term.setLength(0); + while (peek() != '"') { + int c = peek(); + if (c == -1 || isLineEnd(c)) { + throw error("Unterminated literal"); + } + term.appendCodePoint(c == '\\' ? escape(true) : scalar()); + } + expect('"'); + return term.toString(); + } + + private String language() throws IOException { + term.setLength(0); + if (!letter(peek())) { + throw error("Language tag must start with an ASCII letter"); + } + while (letter(peek())) { + term.append((char) take()); + } + while (peek() == '-') { + term.append((char) take()); + languageSubtag(); + } + return term.toString(); + } + + private void languageSubtag() throws IOException { + if (!letter(peek()) && !digit(peek())) { + throw error("Expected nonempty alphanumeric language subtag"); + } + while (letter(peek()) || digit(peek())) { + term.append((char) take()); + } + } + + private int escape(boolean literal) throws IOException { + expect('\\'); + int type = take(); + if (type == 'u' || type == 'U') { + return unicodeEscape(type == 'u' ? 4 : 8); + } + if (!literal) { + throw error("Only Unicode escapes are allowed in IRIs"); + } + return switch (type) { + case 't' -> '\t'; + case 'b' -> '\b'; + case 'n' -> '\n'; + case 'r' -> '\r'; + case 'f' -> '\f'; + case '"', '\'', '\\' -> type; + default -> throw error("Invalid literal escape"); + }; + } + + private int unicodeEscape(int digits) throws IOException { + long result = 0; + for (int i = 0; i < digits; i++) { + int hex = hexValue(peek()); + if (hex < 0) { + throw error("Expected hexadecimal digit in Unicode escape"); + } + take(); + result = (result << 4) | hex; + } + if (result > Character.MAX_CODE_POINT || result >= 0xD800 && result <= 0xDFFF) { + throw error("Unicode escape is not a Unicode scalar value"); + } + return (int) result; + } + + private static int hexValue(int c) { + if (digit(c)) { + return c - '0'; + } + if (c >= 'a' && c <= 'f') { + return c - 'a' + 10; + } + return c >= 'A' && c <= 'F' ? c - 'A' + 10 : -1; + } + + /** Reads one scalar, rejecting isolated UTF-16 surrogates, even in comments. */ + private int scalar() throws IOException { + int first = peek(); + if (first == -1) { + throw error("Unexpected end of input"); + } + if (Character.isLowSurrogate((char) first)) { + throw error("Unpaired low surrogate"); + } + take(); + if (!Character.isHighSurrogate((char) first)) { + return first; + } + int second = peek(); + if (second == -1 || !Character.isLowSurrogate((char) second)) { + throw error("Unpaired high surrogate"); + } + take(); + return Character.toCodePoint((char) first, (char) second); + } + + private void horizontalSpace() throws IOException { + while (peek() == ' ' || peek() == '\t') { + take(); + } + } + + private void endOfLine() throws IOException { + horizontalSpace(); + if (peek() == '#') { + while (peek() != -1 && !isLineEnd(peek())) { + scalar(); + } + } + if (peek() == -1) { + return; + } + if (!isLineEnd(peek())) { + throw error("Expected end of line or comment"); + } + int ending = take(); + if (ending == '\r' && peek() == '\n') { + take(); + } + } + + private void expect(int expected) throws IOException { + if (peek() != expected) { + throw error("Expected '" + (char) expected + "'"); + } + take(); + } + + private int peek() throws IOException { + if (offset == limit && !eof) { + checkInterrupted(); + int count = reader.read(buffer, 0, buffer.length); + if (count == 0) { + throw new IOException("Reader returned zero characters for a nonempty buffer"); + } + eof = count < 0; + offset = 0; + limit = Math.max(count, 0); + } + return offset == limit ? -1 : buffer[offset]; + } + + private int take() throws IOException { + int c = peek(); + if (c == -1) { + return -1; + } + offset++; + updatePosition(c); + return c; + } + + private void updatePosition(int c) { + if (c == '\r') { + line++; + column = 1; + } else if (c == '\n') { + if (!afterCr) { + line++; + } + column = 1; + } else { + column++; + } + afterCr = c == '\r'; + } + + private void checkInterrupted() { + if (Thread.currentThread().isInterrupted()) { + throw error("RDF parsing was interrupted"); + } + } + + private ParsingException error(String message) { + return new ParsingException(location(message)); + } + + private String location(String message) { + return message + " at line " + line + ", column " + column; + } + + private static boolean isLineEnd(int c) { + return c == '\r' || c == '\n'; + } + + private static boolean letter(int c) { + return c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z'; + } + + private static boolean digit(int c) { + return c >= '0' && c <= '9'; + } + + private static boolean nameStart(int c) { + return c == '_' || letter(c) || unicodeNameStart(c); + } + + /** Exact PN_CHARS_BASE ranges; Character.isLetter is insufficient. */ + private static boolean unicodeNameStart(int c) { + return between(c, 0xC0, 0xD6) || between(c, 0xD8, 0xF6) + || between(c, 0xF8, 0x2FF) || between(c, 0x370, 0x37D) + || between(c, 0x37F, 0x1FFF) || between(c, 0x200C, 0x200D) + || between(c, 0x2070, 0x218F) || between(c, 0x2C00, 0x2FEF) + || between(c, 0x3001, 0xD7FF) || between(c, 0xF900, 0xFDCF) + || between(c, 0xFDF0, 0xFFFD) || between(c, 0x10000, 0xEFFFF); + } + + private static boolean nameCharacter(int c) { + return nameStart(c) || digit(c) || c == '-' || c == 0xB7 + || between(c, 0x300, 0x36F) || between(c, 0x203F, 0x2040); + } - /** - * Custom ANTLR ErrorListener that throws a ParsingException on any syntax error. - * This ensures that parsing failures are immediately reported as application-specific exceptions. - */ - private static class NQuadsErrorListener extends BaseErrorListener { - - static final NQuadsErrorListener INSTANCE = new NQuadsErrorListener(); - - @Override - public void syntaxError(Recognizer recognizer, - Object offendingSymbol, - int line, - int charPositionInLine, - String msg, - RecognitionException e) { - throw new ParsingException( - String.format("Syntax error in N-Quads at line %d:%d - %s", - line, charPositionInLine, msg)); + private static boolean between(int c, int lower, int upper) { + return c >= lower && c <= upper; } } } diff --git a/src/main/java/fr/inria/corese/core/next/data/impl/io/parser/support/LineRDFParser.java b/src/main/java/fr/inria/corese/core/next/data/impl/io/parser/support/LineRDFParser.java deleted file mode 100644 index 9cddc0482..000000000 --- a/src/main/java/fr/inria/corese/core/next/data/impl/io/parser/support/LineRDFParser.java +++ /dev/null @@ -1,63 +0,0 @@ -package fr.inria.corese.core.next.data.impl.io.parser.support; - -import fr.inria.corese.core.next.data.api.exception.ParsingException; -import org.antlr.v4.runtime.CharStreams; -import org.antlr.v4.runtime.CommonTokenStream; -import org.antlr.v4.runtime.Lexer; -import org.antlr.v4.runtime.Parser; -import org.antlr.v4.runtime.Token; -import org.antlr.v4.runtime.tree.ParseTree; -import org.antlr.v4.runtime.tree.ParseTreeListener; -import org.antlr.v4.runtime.tree.ParseTreeWalker; - -import java.io.BufferedReader; -import java.io.IOException; -import java.io.Reader; -import java.util.function.Supplier; - -/** - * Runs a line-oriented RDF grammar without retaining the complete input or its - * parse tree. Parser instances are confined to one call; buffers and syntax - * trees are bounded by the largest physical line, plus the reader buffer. - */ -public final class LineRDFParser { - - private LineRDFParser() { } - - /** - * Parses and emits each complete physical line before reading the next one. - * Valid prefixes may already have been added to the target when parsing fails; - * callers requiring atomicity must supply a transactional target. - * - * @param source caller-owned reader, never closed here - * @param lexer configured lexer, reused for each line - * @param parser configured parser, reused for each line - * @param document grammar entry point on {@code parser} - * @param listener statement-emitting listener, shared across lines - * @throws IOException if reading fails - * @throws ParsingException if a line is invalid or the thread is interrupted - */ - public static void parse(Reader source, Lexer lexer, Parser parser, - Supplier document, ParseTreeListener listener) throws IOException { - BufferedReader reader = source instanceof BufferedReader buffered ? buffered : new BufferedReader(source); - CommonTokenStream tokens = new CommonTokenStream(lexer); - ParseTreeWalker walker = new ParseTreeWalker(); - String line; - int lineNumber = 0; - while ((line = reader.readLine()) != null) { - if (Thread.currentThread().isInterrupted()) { - throw new ParsingException("RDF parsing was interrupted"); - } - lexer.setInputStream(CharStreams.fromString(line)); - lexer.setLine(++lineNumber); - tokens.setTokenSource(lexer); - parser.setTokenStream(tokens); - ParseTree tree = document.get(); - if (tokens.LA(1) != Token.EOF) { - throw new ParsingException("Unexpected RDF content at line " + lineNumber - + ":" + tokens.LT(1).getCharPositionInLine()); - } - walker.walk(listener, tree); - } - } -} diff --git a/src/test/java/fr/inria/corese/core/next/data/impl/io/parser/LineRDFParserTest.java b/src/test/java/fr/inria/corese/core/next/data/impl/io/parser/LineRDFParserTest.java deleted file mode 100644 index 5e73bd5eb..000000000 --- a/src/test/java/fr/inria/corese/core/next/data/impl/io/parser/LineRDFParserTest.java +++ /dev/null @@ -1,110 +0,0 @@ -package fr.inria.corese.core.next.data.impl.io.parser; - -import fr.inria.corese.core.next.data.Values; -import fr.inria.corese.core.next.data.api.exception.ParsingException; -import fr.inria.corese.core.next.data.api.io.format.RDFFormat; -import fr.inria.corese.core.next.data.api.model.Model; -import fr.inria.corese.core.next.data.impl.model.LinkedHashModel; -import fr.inria.corese.core.next.io.CoreseIO; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.MethodSource; - -import java.io.IOException; -import java.io.Reader; -import java.io.StringReader; -import java.util.stream.Stream; - -import static org.junit.jupiter.api.Assertions.*; -import static org.mockito.Mockito.*; - -class LineRDFParserTest { - static Stream formats() { - return Stream.of(RDFFormat.NTRIPLES, RDFFormat.NQUADS); - } - - @ParameterizedTest - @MethodSource("formats") - void emitsStatementsBeforeReadingTheWholeInput(RDFFormat format) { - GeneratedReader reader = new GeneratedReader(20_000); - long[] counts = { 0, 0 }; - Model model = mock(Model.class, withSettings().stubOnly().defaultAnswer(invocation -> { - if (invocation.getMethod().getName().equals("add")) { - if (counts[0]++ == 0) { - counts[1] = reader.position; - } - return true; - } - return RETURNS_DEFAULTS.answer(invocation); - })); - CoreseIO.parser(format, model, Values.factory()).parse(reader); - assertEquals(20_000, counts[0]); - assertTrue(counts[1] < reader.totalCharacters / 2); - assertFalse(reader.closed); - } - - @ParameterizedTest - @MethodSource("formats") - void acceptsBlankLinesCommentsLineEndingsAndDirectiveTextInsideLiterals(RDFFormat format) { - String input = "# comment\r\n\r\n \"@prefix @base\" .\r_:a _:a .\n \"escaped\\nvalue\" ."; - var model = CoreseIO.read(new StringReader(input), format); - assertEquals(3, model.size()); - assertTrue(model.contains(Values.factory().createIRI("urn:s"), Values.factory().createIRI("urn:p"), Values.factory().createLiteral("@prefix @base"))); - } - - @ParameterizedTest - @MethodSource("formats") - void rejectsTrailingContentAndReportsThePhysicalLine(RDFFormat format) { - var parser = CoreseIO.parser(format, new LinkedHashModel(Values.factory()), Values.factory()); - ParsingException failure = assertThrows(ParsingException.class, () -> parser.parse( - new StringReader("# header\n\n . garbage"))); - assertTrue(failure.getMessage().contains("3:"), failure.getMessage()); - assertThrows(ParsingException.class, () -> parser.parse(new StringReader("@prefix x: ."))); - } - - @ParameterizedTest - @MethodSource("formats") - void interruptedParsingFailsWithoutConsumingTheInterrupt(RDFFormat format) { - try { - Thread.currentThread().interrupt(); - assertThrows(ParsingException.class, () -> CoreseIO.read( - new StringReader(" .\n"), format)); - assertTrue(Thread.currentThread().isInterrupted()); - } finally { - Thread.interrupted(); - } - } - - private static final class GeneratedReader extends Reader { - private static final String ROW = " .\n"; - private final long totalCharacters; - private long position; - private boolean closed; - - private GeneratedReader(long rows) { - totalCharacters = rows * ROW.length(); - } - - @Override - public int read(char[] buffer, int offset, int length) throws IOException { - if (closed) { - throw new IOException("Reader closed"); - } - if (length == 0) { - return 0; - } - if (position == totalCharacters) { - return -1; - } - int count = (int) Math.min(length, totalCharacters - position); - for (int index = 0; index < count; index++) { - buffer[offset + index] = ROW.charAt((int) (position++ % ROW.length())); - } - return count; - } - - @Override - public void close() { - closed = true; - } - } -} diff --git a/src/test/java/fr/inria/corese/core/next/data/impl/io/parser/nquads/NQuadsListenerTest.java b/src/test/java/fr/inria/corese/core/next/data/impl/io/parser/nquads/NQuadsListenerTest.java deleted file mode 100644 index 425d0791b..000000000 --- a/src/test/java/fr/inria/corese/core/next/data/impl/io/parser/nquads/NQuadsListenerTest.java +++ /dev/null @@ -1,342 +0,0 @@ -package fr.inria.corese.core.next.data.impl.io.parser.nquads; - -import fr.inria.corese.core.next.data.api.term.*; -import fr.inria.corese.core.next.data.api.model.*; -import fr.inria.corese.core.next.data.api.factory.ValueFactory; -import fr.inria.corese.core.next.data.impl.io.parser.support.NTriplesQuadsTermDecoder; -import fr.inria.corese.core.next.data.api.exception.ParsingException; -import fr.inria.corese.core.next.generated.antlr.NQuadsParser; -import org.antlr.v4.runtime.ParserRuleContext; -import org.antlr.v4.runtime.tree.TerminalNode; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.mockito.Mockito.*; - -/** - * Unit tests for the NQuadsListener class. - * These tests verify that the listener correctly processes ANTLR parse tree contexts - * to extract and unescape RDF terms (IRIs, Blank Nodes, Literals) and add them to the model, - * including named graphs. - */ -@ExtendWith(MockitoExtension.class) -class NQuadsListenerTest { - - @Mock - private Model mockModel; - - @Mock - private ValueFactory mockValueFactory; - - private NTriplesQuadsTermDecoder termDecoder; - - private NQuadsListener listener; - - @Mock - private IRI mockIRI; - @Mock - private BNode mockBNode; - @Mock - private Literal mockLiteral; - @Mock - private IRI mockDatatypeIRI; - @Mock - private IRI mockGraphIRI; - @Mock - private BNode mockGraphBNode; - - @BeforeEach - void setUp() { - termDecoder = new NTriplesQuadsTermDecoder(mockValueFactory); - listener = new NQuadsListener(mockModel, mockValueFactory); - - lenient().when(mockValueFactory.createIRI(anyString())).thenAnswer(invocation -> { - String uri = invocation.getArgument(0); - return switch (uri) { - case "http://example.org/test" -> mockIRI; - case "http://example.org/datatype" -> mockDatatypeIRI; - case "http://example.org/graph" -> mockGraphIRI; - default -> mock(IRI.class); - }; - }); - - lenient().when(mockValueFactory.createBNode(anyString())).thenAnswer(invocation -> { - String label = invocation.getArgument(0); - if (label.equals("b1")) return mockBNode; - if (label.equals("graph1")) return mockGraphBNode; - return mock(BNode.class); - }); - - lenient().when(mockValueFactory.createLiteral(anyString())).thenAnswer(invocation -> { - String value = invocation.getArgument(0); - if (value.equals("test literal")) return mockLiteral; - if (value.equals("literal with \"quotes\" and \n newline")) return mock(Literal.class); - if (value.equals("Hello World")) return mock(Literal.class); - if (value.equals("Euro€")) return mock(Literal.class); - return mock(Literal.class); - }); - Literal defaultMockLiteral = mock(Literal.class); - lenient().when(mockValueFactory.createLiteral(anyString(), any(IRI.class))).thenReturn(defaultMockLiteral); - lenient().when(mockValueFactory.createLiteral(anyString(), anyString())).thenReturn(defaultMockLiteral); - } - - private TerminalNode mockTerminalNode(String text) { - TerminalNode node = mock(TerminalNode.class); - when(node.getText()).thenReturn(text); - return node; - } - - private T mockRuleContext(Class clazz) { - return mock(clazz); - } - - @Test - @DisplayName("enterStatement and exitStatement should add a quad to the model with IRI graph") - void testEnterExitStatementAddsToModelWithIRIGraph() { - NQuadsParser.StatementContext statementCtx = mockRuleContext(NQuadsParser.StatementContext.class); - NQuadsParser.SubjectContext subjectCtx = mockRuleContext(NQuadsParser.SubjectContext.class); - NQuadsParser.PredicateContext predicateCtx = mockRuleContext(NQuadsParser.PredicateContext.class); - NQuadsParser.ObjectContext objectCtx = mockRuleContext(NQuadsParser.ObjectContext.class); - NQuadsParser.GraphLabelContext graphLabelCtx = mockRuleContext(NQuadsParser.GraphLabelContext.class); - - TerminalNode subjectIriRef = mockTerminalNode(""); - when(subjectCtx.IRIREF()).thenReturn(subjectIriRef); - - TerminalNode predicateIriRef = mockTerminalNode(""); - when(predicateCtx.IRIREF()).thenReturn(predicateIriRef); - - TerminalNode objectIriRef = mockTerminalNode(""); - when(objectCtx.IRIREF()).thenReturn(objectIriRef); - - TerminalNode graphIriRef = mockTerminalNode(""); - when(graphLabelCtx.IRIREF()).thenReturn(graphIriRef); - - when(statementCtx.subject()).thenReturn(subjectCtx); - when(statementCtx.predicate()).thenReturn(predicateCtx); - when(statementCtx.object()).thenReturn(objectCtx); - when(statementCtx.graphLabel()).thenReturn(graphLabelCtx); - - IRI actualSubjectIRI = mock(IRI.class); - IRI actualPredicateIRI = mock(IRI.class); - IRI actualObjectIRI = mock(IRI.class); - IRI actualGraphIRI = mock(IRI.class); - when(mockValueFactory.createIRI("http://example.org/subject")).thenReturn(actualSubjectIRI); - when(mockValueFactory.createIRI("http://example.org/predicate")).thenReturn(actualPredicateIRI); - when(mockValueFactory.createIRI("http://example.org/object")).thenReturn(actualObjectIRI); - when(mockValueFactory.createIRI("http://example.org/graph")).thenReturn(actualGraphIRI); - - listener.enterStatement(statementCtx); - listener.exitStatement(statementCtx); - - verify(mockModel).add(actualSubjectIRI, actualPredicateIRI, actualObjectIRI, actualGraphIRI); - verifyNoMoreInteractions(mockModel); - } - - @Test - @DisplayName("enterStatement and exitStatement should add a triple to the model (default graph)") - void testEnterExitStatementAddsToModelDefaultGraph() { - NQuadsParser.StatementContext statementCtx = mockRuleContext(NQuadsParser.StatementContext.class); - NQuadsParser.SubjectContext subjectCtx = mockRuleContext(NQuadsParser.SubjectContext.class); - NQuadsParser.PredicateContext predicateCtx = mockRuleContext(NQuadsParser.PredicateContext.class); - NQuadsParser.ObjectContext objectCtx = mockRuleContext(NQuadsParser.ObjectContext.class); - - TerminalNode subjectIriRef = mockTerminalNode(""); - when(subjectCtx.IRIREF()).thenReturn(subjectIriRef); - - TerminalNode predicateIriRef = mockTerminalNode(""); - when(predicateCtx.IRIREF()).thenReturn(predicateIriRef); - - TerminalNode objectIriRef = mockTerminalNode(""); - when(objectCtx.IRIREF()).thenReturn(objectIriRef); - - when(statementCtx.subject()).thenReturn(subjectCtx); - when(statementCtx.predicate()).thenReturn(predicateCtx); - when(statementCtx.object()).thenReturn(objectCtx); - when(statementCtx.graphLabel()).thenReturn(null); - - IRI actualSubjectIRI = mock(IRI.class); - IRI actualPredicateIRI = mock(IRI.class); - IRI actualObjectIRI = mock(IRI.class); - when(mockValueFactory.createIRI("http://example.org/subject")).thenReturn(actualSubjectIRI); - when(mockValueFactory.createIRI("http://example.org/predicate")).thenReturn(actualPredicateIRI); - when(mockValueFactory.createIRI("http://example.org/object")).thenReturn(actualObjectIRI); - - listener.enterStatement(statementCtx); - listener.exitStatement(statementCtx); - - verify(mockModel).add(actualSubjectIRI, actualPredicateIRI, actualObjectIRI); - verifyNoMoreInteractions(mockModel); - } - - @Test - @DisplayName("extractSubject should return IRI for IRIREF context") - void testExtractSubjectIRI() { - NQuadsParser.SubjectContext ctx = mockRuleContext(NQuadsParser.SubjectContext.class); - TerminalNode iriRef = mockTerminalNode(""); - when(ctx.IRIREF()).thenReturn(iriRef); - - Resource result = listener.extractSubject(ctx); - assertEquals(mockIRI, result); - verify(mockValueFactory).createIRI("http://example.org/test"); - } - - @Test - @DisplayName("extractSubject should return BNode for BLANK_NODE_LABEL context") - void testExtractSubjectBNode() { - NQuadsParser.SubjectContext ctx = mockRuleContext(NQuadsParser.SubjectContext.class); - TerminalNode bNodeLabel = mockTerminalNode("_:b1"); - when(ctx.BLANK_NODE_LABEL()).thenReturn(bNodeLabel); - - Resource result = listener.extractSubject(ctx); - assertEquals(mockBNode, result); - verify(mockValueFactory).createBNode("b1"); - } - - @Test - @DisplayName("extractPredicate should return IRI for IRIREF context") - void testExtractPredicateIRI() { - NQuadsParser.PredicateContext ctx = mockRuleContext(NQuadsParser.PredicateContext.class); - TerminalNode iriRef = mockTerminalNode(""); - when(ctx.IRIREF()).thenReturn(iriRef); - - IRI result = listener.extractPredicate(ctx); - assertEquals(mockIRI, result); - verify(mockValueFactory).createIRI("http://example.org/test"); - } - - @Test - @DisplayName("extractObject should return IRI for IRIREF context") - void testExtractObjectIRI() { - NQuadsParser.ObjectContext ctx = mockRuleContext(NQuadsParser.ObjectContext.class); - TerminalNode iriRef = mockTerminalNode(""); - when(ctx.IRIREF()).thenReturn(iriRef); - - Value result = listener.extractObject(ctx); - assertEquals(mockIRI, result); - verify(mockValueFactory).createIRI("http://example.org/test"); - } - - @Test - @DisplayName("extractObject should return BNode for BLANK_NODE_LABEL context") - void testExtractObjectBNode() { - NQuadsParser.ObjectContext ctx = mockRuleContext(NQuadsParser.ObjectContext.class); - TerminalNode bNodeLabel = mockTerminalNode("_:b1"); - when(ctx.BLANK_NODE_LABEL()).thenReturn(bNodeLabel); - - Value result = listener.extractObject(ctx); - assertEquals(mockBNode, result); - verify(mockValueFactory).createBNode("b1"); - } - - @Test - @DisplayName("extractObject should return Literal for literal context (simple string)") - void testExtractObjectLiteralSimple() { - NQuadsParser.ObjectContext objCtx = mockRuleContext(NQuadsParser.ObjectContext.class); - NQuadsParser.LiteralContext litCtx = mockRuleContext(NQuadsParser.LiteralContext.class); - TerminalNode stringLiteral = mockTerminalNode("\"test literal\""); - - when(objCtx.literal()).thenReturn(litCtx); - when(litCtx.STRING_LITERAL_QUOTE()).thenReturn(stringLiteral); - when(litCtx.IRIREF()).thenReturn(null); - when(litCtx.LANGTAG()).thenReturn(null); - - Value result = listener.extractObject(objCtx); - assertEquals(mockLiteral, result); - verify(mockValueFactory).createLiteral("test literal"); - } - - @Test - @DisplayName("extractObject should return Literal for literal context (language-tagged)") - void testExtractObjectLiteralLang() { - NQuadsParser.ObjectContext objCtx = mockRuleContext(NQuadsParser.ObjectContext.class); - NQuadsParser.LiteralContext litCtx = mockRuleContext(NQuadsParser.LiteralContext.class); - TerminalNode stringLiteral = mockTerminalNode("\"hello\""); - TerminalNode langTag = mockTerminalNode("@en"); - - when(objCtx.literal()).thenReturn(litCtx); - when(litCtx.STRING_LITERAL_QUOTE()).thenReturn(stringLiteral); - when(litCtx.IRIREF()).thenReturn(null); - when(litCtx.LANGTAG()).thenReturn(langTag); - - Literal expectedLiteral = mock(Literal.class); - when(mockValueFactory.createLiteral("hello", "en")).thenReturn(expectedLiteral); - - Value result = listener.extractObject(objCtx); - assertEquals(expectedLiteral, result); - verify(mockValueFactory).createLiteral("hello", "en"); - } - - @Test - @DisplayName("extractGraph should return IRI for IRIREF context") - void testExtractGraphIRI() { - NQuadsParser.GraphLabelContext ctx = mockRuleContext(NQuadsParser.GraphLabelContext.class); - TerminalNode iriRef = mockTerminalNode(""); - when(ctx.IRIREF()).thenReturn(iriRef); - - Resource result = listener.extractGraph(ctx); - assertEquals(mockGraphIRI, result); - verify(mockValueFactory).createIRI("http://example.org/graph"); - } - - @Test - @DisplayName("extractGraph should return BNode for BLANK_NODE_LABEL context") - void testExtractGraphBNode() { - NQuadsParser.GraphLabelContext ctx = mockRuleContext(NQuadsParser.GraphLabelContext.class); - TerminalNode bNodeLabel = mockTerminalNode("_:graph1"); - when(ctx.BLANK_NODE_LABEL()).thenReturn(bNodeLabel); - - Resource result = listener.extractGraph(ctx); - assertEquals(mockGraphBNode, result); - verify(mockValueFactory).createBNode("graph1"); - } - - @Test - @DisplayName("unescapeLiteral should throw ParsingException for invalid \\UXXXXXXXX") - void testUnescapeLiteralInvalidUx() { - String input = "\"Invalid\\U0000XXX\""; - assertThrows(ParsingException.class, - () -> termDecoder.unescapeLiteral(input), - "Should throw for malformed \\UXXXXXXXX escape sequence"); - } - - @Test - @DisplayName("unescapeUri should handle basic escape sequences") - void testUnescapeUriBasicEscapes() { - - String input = "http://example.org/path\\>with\\ termDecoder.unescapeUri(input), - "Malformed Unicode escapes in IRIs must fail"); - - } - -} From 1956076946c90e35d062530af9eb2b7720a7ac5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20C=C3=A9r=C3=A8s?= Date: Tue, 20 Oct 2026 17:24:58 +0200 Subject: [PATCH 5/5] perf(engine): dynamic cost-based join reordering in CardinalityPatternOrdering - Dynamically recompute pattern effective cost during greedy selection based on already bound variables from enclosing scopes and chosen edges. - Scale cardinality with BigDecimal precision (discount SPO subject probes by 1000, POS/OSP object lookups by 10, variable predicate by 10). - Maintain strict connectivity priority over disconnected Cartesian patterns. - Add 9 regression and edge-case unit tests covering precision, bound probes, and tie-breaking order. - Fixes #633 (Phase C). --- .../planning/CardinalityPatternOrdering.java | 62 ++++++++++++-- .../planning/CardinalityQueryPlannerTest.java | 81 +++++++++++++++++++ 2 files changed, 137 insertions(+), 6 deletions(-) diff --git a/src/main/java/fr/inria/corese/core/next/query/impl/engine/planning/CardinalityPatternOrdering.java b/src/main/java/fr/inria/corese/core/next/query/impl/engine/planning/CardinalityPatternOrdering.java index 3020e64cd..766df2564 100644 --- a/src/main/java/fr/inria/corese/core/next/query/impl/engine/planning/CardinalityPatternOrdering.java +++ b/src/main/java/fr/inria/corese/core/next/query/impl/engine/planning/CardinalityPatternOrdering.java @@ -6,6 +6,7 @@ import fr.inria.corese.core.next.query.impl.engine.pattern.Exp; import fr.inria.corese.core.next.query.impl.engine.pattern.Query; +import java.math.BigDecimal; import java.util.ArrayList; import java.util.List; @@ -16,12 +17,17 @@ */ final class CardinalityPatternOrdering { + // Heuristics until conditional distinct-term statistics are available. + private static final int SUBJECT_REDUCTION_SCALE = 3; + private static final int OBJECT_REDUCTION_SCALE = 1; + private static final int PREDICATE_REDUCTION_SCALE = 1; + private CardinalityPatternOrdering() { } /** - * Chooses the smallest known pattern first, then favors connected joins. - * Unknown or equal counts fall back to binding connectivity and source order. + * Favors connected joins, then estimates fan-out using currently bound variables. + * Unknown or equal costs fall back to binding hints and stable source order. * Each pattern is counted once, never from a sorting comparator. * * @param query query supplying the active dataset @@ -110,7 +116,7 @@ private static void orderRun(Exp expression, int start, int end, long[] counts, for (int i = first; i < end; i++) { int best = i; for (int j = i + 1; j < end; j++) { - if (before(expression.get(j), expression.get(best), counts[j - start], counts[best - start], + if (isLowerCost(expression.get(j), expression.get(best), counts[j - start], counts[best - start], bound, variables, bindings)) { best = j; } @@ -130,19 +136,63 @@ private static void recordBindings(Exp expression, List bound) { } } - private static boolean before(Exp candidate, Exp current, long candidateCount, long currentCount, + private static boolean isLowerCost(Exp candidate, Exp current, long candidateCount, long currentCount, List bound, List variables, List bindings) { boolean candidateConnected = connected(candidate.getEdge(), bound, variables); boolean currentConnected = connected(current.getEdge(), bound, variables); + // Lexicographic cross-product penalty: retain costs within each component, + // including the initial step when no pattern is connected yet. if (candidateConnected != currentConnected) { return candidateConnected; } - if (candidateCount >= 0 && currentCount >= 0 && candidateCount != currentCount) { - return candidateCount < currentCount; + if (candidateCount >= 0 && currentCount >= 0) { + int comparison = effectiveCost(candidate, candidateCount, bound, variables) + .compareTo(effectiveCost(current, currentCount, bound, variables)); + if (comparison != 0) { + return comparison < 0; + } } + // Strict comparison plus moveFirst's stable shift preserves source order + // when both cost and binding score tie. Unavailable counts retain fallback. return candidate.nBind(bound, variables, bindings) > current.nBind(bound, variables, bindings); } + /** + * Estimates matches per incoming binding; called only for nonnegative counts. + * Constants already constrain baseCount, so only newly bound variables reduce it. + * Decimal scaling preserves long precision without overflow or rounding. + */ + private static BigDecimal effectiveCost(Exp exp, long baseCount, + List bound, List variables) { + if (baseCount == 0) { + return BigDecimal.ZERO; + } + Edge edge = exp.getEdge(); + if (fullyBound(edge, bound, variables)) { + // A probe estimate, not a dataset-wide uniqueness guarantee. + return BigDecimal.ONE; + } + int scale = reductionScale(edge.getNode(0), bound, variables, SUBJECT_REDUCTION_SCALE) + + reductionScale(edge.getNode(1), bound, variables, OBJECT_REDUCTION_SCALE) + + reductionScale(edge.getEdgeVariable(), bound, variables, PREDICATE_REDUCTION_SCALE); + return BigDecimal.valueOf(baseCount).movePointLeft(scale).max(BigDecimal.ONE); + } + + private static int reductionScale(Node node, List bound, List variables, int scale) { + return bound(node, bound, variables) ? scale : 0; + } + + private static boolean fullyBound(Edge edge, List bound, List variables) { + boolean predicateBound = edge.getEdgeVariable() == null + || bound(edge.getEdgeVariable(), bound, variables); + return predicateBound && termBound(edge.getNode(0), bound, variables) + && termBound(edge.getNode(1), bound, variables); + } + + private static boolean termBound(Node node, List bound, List variables) { + return node != null && (node.isConstant() || bound(node, bound, variables)); + } + private static boolean connected(Edge edge, List bound, List variables) { return bound(edge.getNode(0), bound, variables) || bound(edge.getNode(1), bound, variables) diff --git a/src/test/java/fr/inria/corese/core/next/query/impl/engine/planning/CardinalityQueryPlannerTest.java b/src/test/java/fr/inria/corese/core/next/query/impl/engine/planning/CardinalityQueryPlannerTest.java index 30fcdf8f5..8e4043c5c 100644 --- a/src/test/java/fr/inria/corese/core/next/query/impl/engine/planning/CardinalityQueryPlannerTest.java +++ b/src/test/java/fr/inria/corese/core/next/query/impl/engine/planning/CardinalityQueryPlannerTest.java @@ -313,6 +313,87 @@ void selectiveFilterPushdownSeedsComparisonEdgeFirst() { predicates(query)); } + + @Test + void subjectBindingBeatsSmallerObjectBoundPattern() { + assertPlan("?x :object ?s . ?s :subject ?y . ?s :seed ?o", + Map.of("object", 500L, "subject", 5000L, "seed", 1L), + List.of("seed", "subject", "object")); + } + + @Test + void fullyBoundTripleIsAProbe() { + assertPlan("?s :scan ?x . ?s :probe ?o . ?s :seed ?o", + Map.of("scan", 5000L, "probe", 1000000L, "seed", 1L), + List.of("seed", "probe", "scan")); + } + + @Test + void costsAreRecomputedAfterEachSelectedEdge() { + assertPlan("?y :type :Professor . ?s :wide ?z . ?s :link ?y . ?s :seed ?o", + Map.of("type", 100000L, "wide", 100000L, "link", 5000L, "seed", 1L), + List.of("seed", "link", "type", "wide")); + } + + @Test + void constantsDoNotReceiveASecondReduction() { + assertPlan("?s :name \"literal\" . ?a :rare ?b", + Map.of("name", 20L, "rare", 10L), List.of("rare", "name")); + } + + @Test + void equalEffectiveCostsPreserveSourceOrder() { + assertPlan("?x :object ?s . ?s :subject ?y . ?s :seed ?o", + Map.of("object", 500L, "subject", 50000L, "seed", 1L), + List.of("seed", "object", "subject")); + } + + @Test + void scaledCostsRetainAdjacentLongPrecision() { + assertPlan("?s :large ?x . ?s :small ?y . ?s :seed ?o", + Map.of("large", Long.MAX_VALUE, "small", Long.MAX_VALUE - 1, "seed", 1L), + List.of("seed", "small", "large")); + } + + @Test + void variablePredicateDoesNotBecomeSingleTripleProbe() { + Query query = query("?s ?p ?o . ?s :scan ?x . ?s :seed ?o"); + query.complete(new StatisticsProducer(Map.of("urn:bench:scan", 5000L, "urn:bench:seed", 1L)) { + @Override + public long estimateCardinality(Query scope, Node graph, Edge edge) { + return edge.getEdgeVariable() == null ? super.estimateCardinality(scope, graph, edge) : 1000000; + } + }); + List plan = edges(query.getBody()); + assertEquals("urn:bench:seed", plan.get(0).getEdgeLabel()); + assertEquals("urn:bench:scan", plan.get(1).getEdgeLabel()); + assertEquals("p", plan.get(2).getEdgeVariable().getLabel()); + } + + @Test + void enclosingScopeVariablesReduceLookupCost() { + Query query = query("?x :object ?s . ?s :subject ?y"); + CardinalityPatternOrdering.sort(query, null, query.getBody(), List.of("s"), List.of(), + new StatisticsProducer(Map.of("urn:bench:object", 500L, "urn:bench:subject", 5000L))); + assertEquals(List.of("urn:bench:subject", "urn:bench:object"), predicates(query)); + } + + @Test + void zeroCardinalityRemainsCheaperThanFullyBoundProbe() { + Query query = query("?s :probe ?o . ?s :empty ?o"); + CardinalityPatternOrdering.sort(query, null, query.getBody(), List.of("s", "o"), List.of(), + new StatisticsProducer(Map.of("urn:bench:probe", 1L, "urn:bench:empty", 0L))); + assertEquals(List.of("urn:bench:empty", "urn:bench:probe"), predicates(query)); + } + + private static void assertPlan(String body, Map counts, List expected) { + Map qualified = new java.util.HashMap<>(); + counts.forEach((key, value) -> qualified.put("urn:bench:" + key, value)); + Query query = query(body); + query.complete(new StatisticsProducer(qualified)); + assertEquals(expected.stream().map(name -> "urn:bench:" + name).toList(), predicates(query)); + } + private static Query query(String body) { return new CoreseAstQueryBuilder().toNextQuery((SelectQueryAst) new SparqlParser() .parse(PREFIX + "SELECT * { " + body + " }"));