From 26ee158291d61dbe12b2ca578cda08b0d00b4425 Mon Sep 17 00:00:00 2001 From: RomeCore <62770895+RomeCore@users.noreply.github.com> Date: Sat, 15 Nov 2025 01:28:27 +0500 Subject: [PATCH 1/3] Change one of FailIf fail predicate --- src/RCParsing/Building/TokenBuilder.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/RCParsing/Building/TokenBuilder.cs b/src/RCParsing/Building/TokenBuilder.cs index 5a818f1..0719fbc 100644 --- a/src/RCParsing/Building/TokenBuilder.cs +++ b/src/RCParsing/Building/TokenBuilder.cs @@ -867,7 +867,7 @@ public TokenBuilder FailIf(Action builderAction, Func return Token(new BuildableFailIfTokenPattern { Child = builder.BuildingPattern.Value, - Condition = v => condition((T)v), + Condition = v => v is T t && condition(t), ErrorMessage = errorMessage }); } From 4987382abfd075c5a50ebc0ab7577ddda54ea11d Mon Sep 17 00:00:00 2001 From: RomeCore <62770895+RomeCore@users.noreply.github.com> Date: Sun, 25 Jan 2026 23:05:03 +0500 Subject: [PATCH 2/3] Provided debug symbols, enhanced error display for BuildableSimpleSkipStrategy --- .../BuildableSimpleSkipStrategy.cs | 14 + src/RCParsing/Parser.cs | 4 +- src/RCParsing/RCParsing.csproj | 14 +- .../LLT/ModernLLTGrammarTests.cs | 128 ++++++ tests/RCParsing.Tests/LLT/ModernLLTParser.cs | 421 ++++++++++++++++++ ...TGrammarTests.cs => OldLLTGrammarTests.cs} | 4 +- .../RCParsing.Tests/Rules/ParserRuleTests.cs | 2 +- 7 files changed, 581 insertions(+), 6 deletions(-) create mode 100644 tests/RCParsing.Tests/LLT/ModernLLTGrammarTests.cs create mode 100644 tests/RCParsing.Tests/LLT/ModernLLTParser.cs rename tests/RCParsing.Tests/LLT/{LLTGrammarTests.cs => OldLLTGrammarTests.cs} (99%) diff --git a/src/RCParsing/Building/SkipStrategies/BuildableSimpleSkipStrategy.cs b/src/RCParsing/Building/SkipStrategies/BuildableSimpleSkipStrategy.cs index 4fd6b70..4e6cc3f 100644 --- a/src/RCParsing/Building/SkipStrategies/BuildableSimpleSkipStrategy.cs +++ b/src/RCParsing/Building/SkipStrategies/BuildableSimpleSkipStrategy.cs @@ -31,6 +31,20 @@ public class BuildableSimpleSkipStrategy : BuildableSkipStrategy public override SkipStrategy BuildTyped(List? ruleChildren, List? tokenChildren, List? elementChildren) { + if (ruleChildren[0] == -1) + switch (Strategy) + { + case ParserSkippingStrategy.SkipBeforeParsing: + case ParserSkippingStrategy.SkipBeforeParsingGreedy: + case ParserSkippingStrategy.SkipBeforeParsingLazy: + case ParserSkippingStrategy.TryParseThenSkip: + case ParserSkippingStrategy.TryParseThenSkipGreedy: + case ParserSkippingStrategy.TryParseThenSkipLazy: + case ParserSkippingStrategy.TryParseNonEmptyThenSkip: + case ParserSkippingStrategy.TryParseNonEmptyThenSkipGreedy: + case ParserSkippingStrategy.TryParseNonEmptyThenSkipLazy: + throw new ParserBuildingException($"Skipping sule is not set for this strategy type: {Strategy}."); + } switch (Strategy) { case ParserSkippingStrategy.Default: diff --git a/src/RCParsing/Parser.cs b/src/RCParsing/Parser.cs index f60e0d9..77d710c 100644 --- a/src/RCParsing/Parser.cs +++ b/src/RCParsing/Parser.cs @@ -187,7 +187,7 @@ public TokenPattern GetTokenPattern(int id) { if (id >= 0 && id < _tokenPatterns.Length) return _tokenPatterns[id]; - throw new ArgumentOutOfRangeException(nameof(id), "Invalid token pattern ID."); + throw new ArgumentOutOfRangeException(nameof(id), $"Invalid token pattern ID: {id}."); } /// @@ -213,7 +213,7 @@ public ParserRule GetRule(int id) { if (id >= 0 && id < _rules.Length) return _rules[id]; - throw new ArgumentOutOfRangeException(nameof(id), "Invalid rule ID."); + throw new ArgumentOutOfRangeException(nameof(id), $"Invalid rule ID: {id}."); } /// diff --git a/src/RCParsing/RCParsing.csproj b/src/RCParsing/RCParsing.csproj index 2b3e0e9..adc0c55 100644 --- a/src/RCParsing/RCParsing.csproj +++ b/src/RCParsing/RCParsing.csproj @@ -8,7 +8,7 @@ RCParsing - 5.0.0 + 5.1.0 Roman K. RomeCore RCParsing @@ -26,6 +26,18 @@ + + true + embedded + true + true + true + + + + + + diff --git a/tests/RCParsing.Tests/LLT/ModernLLTGrammarTests.cs b/tests/RCParsing.Tests/LLT/ModernLLTGrammarTests.cs new file mode 100644 index 0000000..d9e472a --- /dev/null +++ b/tests/RCParsing.Tests/LLT/ModernLLTGrammarTests.cs @@ -0,0 +1,128 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace RCParsing.Tests.LLT +{ + public class ModernLLTGrammarTests + { + [Fact] + public void SimpleTemplateParsing() + { + string templateStr = + """ + @// The main template. + @template MainTemplate { + @* + Use the template to display a greeting message. + *@ + Hello, @user.name! + + @// Display a number. + Value: @number + + @// Method call with parameters. + Method test: @data.getItem(1, "arg").fieldName + + @// Simple arithmetic and logical operations. + Result: @(a * (b + c) - 42 / value % 3 > 0 && !flag || isAdmin) + + @if user.age >= 18 { + Adult content + @if user.isAdmin { + Welcome, mighty @user.role! + } else { + Regular user detected + } + } else @* AAA *@ { + @@You are too {{young}}. + } + + @foreach item in user.items { + Item: @item.id - @item.name + } + + @// String literals, booleans, and null values. + String literal: @'hello ''world''' + Boolean true: @true + Boolean false: @false + Null test: @null + } + + @// Second template with no expressions. + @template Secondary { + Static text only + No expressions here + } + + @// Third template with mixed content. + @template Mixed { + @// Nested if-else statements. + @if flag { + YES + } else { + @if otherFlag { + NESTED YES + } else { + DEEP ELSE + } + } + + @// Foreach loop with nested foreach. + @foreach group in groups { + Group: @group.name + @foreach member in group.members { + Member: @member.id - @member.name + } + } + } + """; + + var parser = ModernLLTParser.CreateParser(); + parser.Parse(templateStr); + } + + [Fact] + public void SimpleMessagesTemplateParsing() + { + string templateStr = + """ + @messages template ChatBot { + @metadata { + language: 'ru', + version: 1 + } + + @system message { + You are a helpful assistant. + } + + @user message { + Hello! + } + + @if user.isAdmin { + @assistant message { + Welcome back, admin! + } + } else { + @assistant message { + Welcome, user! + } + } + + @foreach item in items { + @assistant message { + Processing item: @item + } + } + } + """; + + var parser = ModernLLTParser.CreateParser(); + parser.Parse(templateStr); + } + } +} \ No newline at end of file diff --git a/tests/RCParsing.Tests/LLT/ModernLLTParser.cs b/tests/RCParsing.Tests/LLT/ModernLLTParser.cs new file mode 100644 index 0000000..2ebc3ac --- /dev/null +++ b/tests/RCParsing.Tests/LLT/ModernLLTParser.cs @@ -0,0 +1,421 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace RCParsing.Tests.LLT +{ + public static class ModernLLTParser + { + private static void DeclareValues(ParserBuilder builder) + { + builder.CreateToken("identifier") + .Identifier() + .Transform(v => v.Text); + + builder.CreateToken("method_name") + .Identifier() + .Transform(v => v.Text); + + builder.CreateToken("field_name") + .Identifier() + .Transform(v => v.Text); + + builder.CreateToken("number") + .Number(); + + builder.CreateToken("string") + .Literal('\'') + .EscapedTextDoubleChars('\'') + .Literal('\'') + .Pass(1); + + builder.CreateToken("raw_string") + .Literal('\'') + .EscapedTextDoubleChars('\'') + .Literal('\'') + .Pass(1) + .Transform(v => v.GetIntermediateValue()); + + builder.CreateToken("boolean") + .LiteralChoice("true", "false"); + + builder.CreateToken("null") + .Literal("null"); + + // Constants // + + builder.CreateRule("constant") + .Choice( + c => c.Token("number"), + c => c.Token("string"), + c => c.Token("boolean"), + c => c.Token("null"), + c => c.Rule("constant_array"), + c => c.Rule("constant_object")); + + builder.CreateRule("constant_pair") + .Token("identifier") + .Literal(":") + .Rule("constant"); + + builder.CreateRule("constant_array") + .Literal("[") + .ZeroOrMoreSeparated(b => b.Rule("constant"), b => b.Literal(","), allowTrailingSeparator: true) + .ConfigureLast(c => c.Skip(b => b.Rule("skip"), ParserSkippingStrategy.SkipBeforeParsingGreedy)) + .Literal("]"); + + builder.CreateRule("constant_object") + .Literal("{") + .ZeroOrMoreSeparated(b => b.Rule("constant_pair"), b => b.Literal(","), allowTrailingSeparator: true) + .ConfigureLast(c => c.Skip(b => b.Rule("skip"), ParserSkippingStrategy.SkipBeforeParsingGreedy)) + .Literal("}"); + + // Expression values // + + builder.CreateRule("function_access") + .Identifier() + .Literal('(') + .ZeroOrMoreSeparated(b => b + .Rule("expression"), b => b.Literal(',')) + .Literal(')'); + + builder.CreateRule("context_access") + .Choice( + b => b + .Literal("ctx"), + b => b + .Identifier()); + + builder.CreateRule("primary") + .Choice( + c => c.Rule("constant"), + c => c.Rule("function_access"), + c => c.Rule("context_access"), + c => c.Literal('(').Rule("expression").Literal(')')); + + builder.CreateRule("value") + .Rule("primary"); // Add alias for 'primary' rule + } + + private static void DeclareExpressions(ParserBuilder builder) + { + builder.CreateRule("postfix_member") + .Rule("primary") + .ZeroOrMore(b => b.Choice( + b => b.Literal('.') // Method call + .Token("method_name") + .Literal('(') + .ZeroOrMoreSeparated(a => a.Rule("expression"), s => s.Literal(',')) + .Literal(')'), + + b => b.Literal('.') // Field access + .Token("field_name"), + + b => b.Literal('[') // Index access + .Rule("expression") + .Literal(']') + )); + + builder.CreateRule("prefix_operator") + .ZeroOrMore(b => b.LiteralChoice("+", "-", "!")) + .Rule("postfix_member"); + + builder.CreateRule("simple_expression") + .Rule("prefix_operator"); + + // Operators // + + builder.CreateRule("multiplicative_operator") // multiplicative + .OneOrMoreSeparated(b => b.Rule("prefix_operator"), + o => o.LiteralChoice("*", "/", "%"), + includeSeparatorsInResult: true); + + builder.CreateRule("additive_operator") // additive + .OneOrMoreSeparated(b => b.Rule("multiplicative_operator"), + o => o.LiteralChoice("+", "-"), + includeSeparatorsInResult: true); + + builder.CreateRule("relational_operator") // relational (<, <=, >, >=) + .OneOrMoreSeparated(b => b.Rule("additive_operator"), + o => o.LiteralChoice("<", "<=", ">", ">="), + includeSeparatorsInResult: true); + + builder.CreateRule("equality_operator") // equality (==, !=) + .OneOrMoreSeparated(b => b.Rule("relational_operator"), + o => o.LiteralChoice("==", "!="), + includeSeparatorsInResult: true); + + builder.CreateRule("logical_and_operator") // logical AND + .OneOrMoreSeparated(b => b.Rule("equality_operator"), + o => o.Literal("&&"), + includeSeparatorsInResult: true); + + builder.CreateRule("logical_or_operator") // logical OR + .OneOrMoreSeparated(b => b.Rule("logical_and_operator"), + o => o.Literal("||"), + includeSeparatorsInResult: true); + + builder.CreateRule("ternary_operator") // ternary (? :) + .Rule("logical_or_operator") + .Optional(b => b + .Literal('?') + .Rule("expression") + .Literal(':') + .Rule("expression")); + + // Final expression = lowest precedence + + builder.CreateRule("expression") + .Rule("ternary_operator"); + } + + private static void DeclareMessagesTemplates(ParserBuilder builder) + { + builder.CreateRule("messages_template") + .Literal('@') + .Keyword("messages") + .Keyword("template") + .Optional(b => b.Token("identifier")) // 3 + .Literal('{') + .Optional(b => // 5 + b.Rule("metadata_block")) + .Rule("message_statements") // 6 + .Literal('}'); + + builder.CreateRule("messages_template_block") + .Literal('{') + .Rule("message_statements") + .Literal('}') + .Transform(v => v.GetValue(1)); + + builder.CreateRule("message_statements") + .ZeroOrMore(b => b + .Literal('@') + .Choice( + c => c.Rule("message_block"), + c => c.Rule("messages_if"), + c => c.Rule("messages_foreach"), + c => c.Rule("messages_while"), + c => c.Rule("messages_render"), + c => c.Rule("messages_var_assignment"))); + + builder.CreateRule("message_block") + .Choice( + b => b.Rule("message_block_explicit_role"), + b => b.Rule("message_block_variable_role")); + + builder.CreateRule("message_block_explicit_role") + .KeywordChoice("system", "user", "assistant", "tool") + .Keyword("message") + .Rule("text_template_block"); + + builder.CreateRule("message_block_variable_role") + .Keyword("message") + .Literal('{') + .Literal('@').Keyword("role").Rule("simple_expression") // 4 + .Rule("text_statements") // 5 + .Literal('}'); + + builder.CreateRule("messages_if") + .Keyword("if") + .Rule("expression") + .Rule("messages_template_block") + .Optional( + b => b + .Keyword("else") + .Choice( + b => b.Rule("messages_if"), + b => b.Rule("messages_template_block"))); + + builder.CreateRule("messages_foreach") + .Keyword("foreach") + .Token("identifier") + .Keyword("in") + .Rule("expression") + .Rule("messages_template_block"); + + builder.CreateRule("messages_while") + .Keyword("while") + .Rule("expression") + .Rule("messages_template_block"); + + builder.CreateRule("messages_render") + .Keyword("render") + .Rule("simple_expression") + .Optional(b => b + .Keyword("with") + .Rule("simple_expression")); + + builder.CreateRule("messages_var_assignment") + .Choice(b => b + .Keyword("let") + .Token("identifier") + .Literal("=") + .Rule("expression"), + b => b + .Token("identifier") + .Literal("=") + .Rule("expression")); + } + + private static void DeclareTextTemplates(ParserBuilder builder) + { + builder.CreateRule("text_template") + .Literal('@') + .Keyword("template") + .Optional(b => b.Token("identifier")) // 2 + .Literal('{') + .Optional(b => b.Rule("metadata_block")) // 4 + .Rule("text_statements") // 5 + .Literal('}'); + + builder.CreateRule("text_content") + .EscapedTextDoubleChars("@{}", allowsEmpty: false); + + builder.CreateRule("text_statements") + .ZeroOrMore(b => b.Choice( + c => c.Rule("text_content"), + c => c.Rule("text_statement"))); + + builder.CreateRule("text_template_block") + .Literal('{') + .Rule("text_statements") + .Literal('}') + .Transform(v => v.GetValue(1)); + + builder.CreateRule("text_statement") + .Literal('@') + .Choice( + b => b.Rule("text_if"), + b => b.Rule("text_foreach"), + b => b.Rule("text_render"), + b => b.Rule("text_while"), + b => b.Rule("text_var_assignment"), + b => b.Rule("text_expression") + ) + .Transform(v => v.GetValue(1)); + + builder.CreateRule("text_expression_inner") + .Rule("simple_expression") // We don't want to use binary expressions in text statements + .Optional(b => b + .Literal(':') + .Token("raw_string")); + + builder.CreateRule("text_expression") + .Custom( + (self, ctx, sett, childSett, children, childrenIds) => + { + var result = self.ParseRule(childrenIds[0], ctx, childSett); + if (!result.success) + return result; + var modifierChild = result.children![1]; + if (modifierChild.length == 0) + result.length = result.children[0].length; + return result; + }, + b => b.Rule("text_expression_inner") + ); + + builder.CreateRule("text_if") + .Keyword("if") + .Rule("expression") + .Rule("text_template_block") + .Optional( + b => b + .Keyword("else") + .Choice( + b => b.Rule("text_if"), + b => b.Rule("text_template_block"))); + + builder.CreateRule("text_foreach") + .Keyword("foreach") + .Token("identifier") + .Keyword("in") + .Rule("expression") + .Rule("text_template_block"); + + builder.CreateRule("text_while") + .Keyword("while") + .Rule("expression") + .Rule("text_template_block"); + + builder.CreateRule("text_render") + .Keyword("render") + .Rule("prefix_operator") + .Optional(b => b + .Literal("with") + .Rule("prefix_operator")); + + builder.CreateRule("text_var_assignment") + .Choice(b => b + .Keyword("let") + .Token("identifier") + .Literal("=") + .Rule("simple_expression"), + b => b + .Token("identifier") + .Literal("=") + .Rule("simple_expression")); + } + + private static void DeclareMainRules(ParserBuilder builder) + { + builder.CreateRule("template") + .Choice( + b => b.Rule("text_template"), + b => b.Rule("messages_template")) + + //.Recovery(r => r.SkipUntil(s => s.Literal('@').Optional(o => o.Keyword("messages")).Keyword("template"))) + + ; + + builder.CreateRule("metadata_block") + .Literal('@') + .Literal("metadata") + .Rule("constant_object"); + + builder.CreateMainRule("file_content") + .ZeroOrMore(b => b.Rule("template")) + .EOF(); + } + + public static void FillWithRules(ParserBuilder builder) + { + builder.CreateRule("skip") + .Choice( + c => c.Whitespaces(), + c => c.Literal("@/").TextUntil('\n', '\r'), // @/ C#-like comments + c => c.Literal("@*").TextUntil("*@").Literal("*@")) // @*...*@ comments + .ConfigureForSkip(); // Ignore all errors when parsing comments and unnecessary whitespace + + // Settings // + builder.Settings + .Skip(b => b.Rule("skip"), ParserSkippingStrategy.TryParseThenSkipLazy) // Allows rules to capture skip-rules contents if can, such as whitespaces + .UseCaching(); // If caching is disabled, prepare to wait for a long time (seconds) when encountering an error :P (you will also get a million of errors, seriously) + + // ---- Values ---- // + DeclareValues(builder); + + // ---- Expressions ---- // + DeclareExpressions(builder); + + // ---- Messages templates ---- // + DeclareMessagesTemplates(builder); + + // ---- Text templates ---- // + DeclareTextTemplates(builder); + + // ---- Main rules ---- // + DeclareMainRules(builder); + } + + public static Parser CreateParser() + { + var builder = new ParserBuilder(); + FillWithRules(builder); + return builder.Build(); + } + } +} \ No newline at end of file diff --git a/tests/RCParsing.Tests/LLT/LLTGrammarTests.cs b/tests/RCParsing.Tests/LLT/OldLLTGrammarTests.cs similarity index 99% rename from tests/RCParsing.Tests/LLT/LLTGrammarTests.cs rename to tests/RCParsing.Tests/LLT/OldLLTGrammarTests.cs index 6e6e575..0026f88 100644 --- a/tests/RCParsing.Tests/LLT/LLTGrammarTests.cs +++ b/tests/RCParsing.Tests/LLT/OldLLTGrammarTests.cs @@ -11,7 +11,7 @@ namespace RCParsing.Tests.LLT /// /// The tests for parsing grammar for LLT (Large Language Template) language. /// - public class LLTGrammarTests + public class OldLLTGrammarTests { private static readonly Parser _parser, _optParser; @@ -392,7 +392,7 @@ private static void FillWithRules(ParserBuilder builder) DeclareMainRules(builder); } - static LLTGrammarTests() + static OldLLTGrammarTests() { var builder = new ParserBuilder(); FillWithRules(builder); diff --git a/tests/RCParsing.Tests/Rules/ParserRuleTests.cs b/tests/RCParsing.Tests/Rules/ParserRuleTests.cs index 697e62a..9762374 100644 --- a/tests/RCParsing.Tests/Rules/ParserRuleTests.cs +++ b/tests/RCParsing.Tests/Rules/ParserRuleTests.cs @@ -186,7 +186,7 @@ public void SimpleJSONParsing() .Number(TokenPatterns.NumberFlags.StrictScientific); builder.CreateToken("boolean") - .LiteralChoice(["true", "false"]).Transform(v => v.Text == "true"); // LiteralChoice uses Trie + .LiteralChoice("true", "false").Transform(v => v.Text == "true"); // LiteralChoice uses Trie builder.CreateToken("null") .Literal("null").Transform(v => null); From 82239fb6eb6b0b9027005a79ae057659df75e95c Mon Sep 17 00:00:00 2001 From: RomeCore <62770895+RomeCore@users.noreply.github.com> Date: Sun, 25 Jan 2026 23:06:08 +0500 Subject: [PATCH 3/3] Update BuildableSimpleSkipStrategy.cs --- .../Building/SkipStrategies/BuildableSimpleSkipStrategy.cs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/RCParsing/Building/SkipStrategies/BuildableSimpleSkipStrategy.cs b/src/RCParsing/Building/SkipStrategies/BuildableSimpleSkipStrategy.cs index 4e6cc3f..350ddd3 100644 --- a/src/RCParsing/Building/SkipStrategies/BuildableSimpleSkipStrategy.cs +++ b/src/RCParsing/Building/SkipStrategies/BuildableSimpleSkipStrategy.cs @@ -25,9 +25,7 @@ public class BuildableSimpleSkipStrategy : BuildableSkipStrategy public override IEnumerable>? RuleChildren => - Strategy == ParserSkippingStrategy.Default || Strategy == ParserSkippingStrategy.Whitespaces - ? null - : new[] { SkipRule ?? default }; + new[] { SkipRule ?? default }; public override SkipStrategy BuildTyped(List? ruleChildren, List? tokenChildren, List? elementChildren) {