diff --git a/src/RCParsing/Building/ParserElementBuilder.cs b/src/RCParsing/Building/ParserElementBuilder.cs index fa15f3e..f44c5c1 100644 --- a/src/RCParsing/Building/ParserElementBuilder.cs +++ b/src/RCParsing/Building/ParserElementBuilder.cs @@ -1389,7 +1389,16 @@ public T Empty() { return Token(new EmptyTokenPattern()); } - + + /// + /// Adds an any character token to the current sequence. It matches any single character, and returns it as intermediate value. + /// + /// Current instance for method chaining. + public T AnyChar() + { + return Token(new AnyCharTokenPattern()); + } + /// /// Adds a fail token to the current sequence. It always fails. /// @@ -2005,5 +2014,20 @@ public T TextUntil(params string[] forbidden) { return Token(EscapedTextTokenPattern.CreateUntil(forbidden)); } + + // The pizdec + + /// + /// Adds a token pattern that uses an entire parser for matching. + /// The parser will be invoked as a token in the current sequence. + /// + /// The parser to use for matching. + /// The alias for the rule to parse. If null, uses the main rule of the parser. + /// Optional token alias to match instead of a rule. If both this and ruleAlias are null, uses main rule. + /// Current instance for method chaining. + public T Parser(Parser matchParser, string? ruleAlias = null, string? tokenAlias = null) + { + return Token(new ParserTokenPattern(matchParser, ruleAlias, tokenAlias)); + } } } \ No newline at end of file diff --git a/src/RCParsing/ParsedRuleResultBase.cs b/src/RCParsing/ParsedRuleResultBase.cs index 451c7c5..94aced6 100644 --- a/src/RCParsing/ParsedRuleResultBase.cs +++ b/src/RCParsing/ParsedRuleResultBase.cs @@ -198,14 +198,16 @@ IEnumerator IEnumerable.GetEnumerator() /// /// The type of value to retrieve. /// The intermediate value associated with this AST node. - public T GetIntermediateValue() => (T)IntermediateValue; + public T GetIntermediateValue() => IntermediateValue is T res ? res : + throw new SemanticException(this, $"Expected an intermediate value of type {typeof(T).Name} but got {IntermediateValue?.GetType().Name ?? "null"}."); /// /// Gets the intermediate value associated with child AST node at the specific index as an instance of type . /// /// The type of value to retrieve. /// The intermediate value associated with child AST node. - public T GetIntermediateValue(int index) => (T)this[index].IntermediateValue; + public T GetIntermediateValue(int index) => this[index].IntermediateValue is T res ? res : + throw new SemanticException(this[index], $"Expected an intermediate value of type {typeof(T).Name} but got {IntermediateValue?.GetType().Name ?? "null"}."); /// /// Tries to get the intermediate value associated with this AST node as an instance of type . @@ -227,40 +229,72 @@ IEnumerator IEnumerable.GetEnumerator() /// /// The type of value to retrieve. /// The intermediate value associated with this AST node. - public T ConvertIntermediateValue() => (T)Convert.ChangeType(IntermediateValue, typeof(T)); + public T ConvertIntermediateValue() + { + try + { + return (T)Convert.ChangeType(IntermediateValue, typeof(T)); + } + catch (Exception ex) + { + throw new SemanticException(this, $"Failed to convert intermediate value to {typeof(T).Name}: {ex.Message}", ex); + } + } /// /// Gets the intermediate value associated with child AST node at the specific index converted to type . /// /// The type of value to retrieve. /// The intermediate value associated with child AST node. - public T ConvertIntermediateValue(int index) => (T)Convert.ChangeType(this[index].IntermediateValue, typeof(T)); + public T ConvertIntermediateValue(int index) + { + try + { + return (T)Convert.ChangeType(this[index].IntermediateValue, typeof(T)); + } + catch (Exception ex) + { + throw new SemanticException(this[index], $"Failed to convert intermediate value to {typeof(T).Name}: {ex.Message}", ex); + } + } /// /// Gets the value associated with this AST node as not-null object. If the value is null, throws an exception. /// /// The value associated with this AST node. - public object GetValue() => Value ?? throw new InvalidOperationException("ParsedRuleResult.Value is null"); + public object GetValue() => Value ?? throw new SemanticException(this, "ParsedRuleResult.Value is null"); /// /// Gets the value associated with child AST node at the specific index as not-null object. If the value is null, throws an exception. /// /// The value associated with child AST node. - public object GetValue(int index) => this[index].Value ?? throw new InvalidOperationException("ParsedRuleResult.Value is null"); + public object GetValue(int index) => this[index].Value ?? throw new SemanticException(this[index], "ParsedRuleResult[index].Value is null"); /// /// Gets the value associated with this AST node as an instance of type . /// /// The type of value to retrieve. /// The value associated with this AST node. - public T GetValue() => (T)Value; + public T GetValue() + { + var value = Value; + return value is T res ? res : + throw new SemanticException(this, + $"Expected a value of type {typeof(T).Name} but got {value?.GetType().Name ?? "null"}."); + } /// /// Gets the value associated with child AST node at the specific index as an instance of type . /// /// The type of value to retrieve. /// The value associated with child AST node. - public T GetValue(int index) => (T)this[index].Value; + public T GetValue(int index) + { + var value = this[index].Value; + return value is T res ? res : + throw new SemanticException(this[index], + $"Expected a value of type {typeof(T).Name} but got {value?.GetType().Name ?? "null"}."); + } /// /// Tries to get the value associated with this AST node as an instance of type or value. @@ -302,7 +336,17 @@ IEnumerator IEnumerable.GetEnumerator() /// /// The type of value to retrieve. /// The value associated with this AST node. - public T ConvertValue() => (T)Convert.ChangeType(Value, typeof(T)); + public T ConvertValue() + { + try + { + return (T)Convert.ChangeType(Value, typeof(T)); + } + catch (Exception ex) + { + throw new SemanticException(this, $"Failed to convert value to {typeof(T).Name}: {ex.Message}", ex); + } + } /// /// Gets the value associated with child AST node at the specific index converted to type . @@ -312,14 +356,25 @@ IEnumerator IEnumerable.GetEnumerator() /// /// The type of value to retrieve. /// The value associated with child AST node. - public T ConvertValue(int index) => (T)Convert.ChangeType(this[index].Value, typeof(T)); + public T ConvertValue(int index) + { + try + { + return (T)Convert.ChangeType(this[index].Value, typeof(T)); + } + catch (Exception ex) + { + throw new SemanticException(this[index], $"Failed to convert value to {typeof(T).Name}: {ex.Message}", ex); + } + } /// /// Gets the parsing parameter associated with parser context as an instance of type . /// /// The type of parsing parameter to retrieve. /// The parsing parameter associated with the parser context. - public T GetParsingParameter() => (T)ParsingParameter; + public T GetParsingParameter() => ParsingParameter is T res ? res : + throw new SemanticException(this, $"Expected a parsing parameter of type {typeof(T).Name} but got {ParsingParameter?.GetType().Name ?? "null"}."); /// /// Tries to get the parsing parameter associated with parser context as an instance of type or value. diff --git a/src/RCParsing/Parser.findallmatches.cs b/src/RCParsing/Parser.findallmatches.cs index 978ad1e..19052fb 100644 --- a/src/RCParsing/Parser.findallmatches.cs +++ b/src/RCParsing/Parser.findallmatches.cs @@ -3,6 +3,7 @@ using System.Linq; using System.Text; using System.Text.RegularExpressions; +using RCParsing.Utils; namespace RCParsing { @@ -313,7 +314,7 @@ public string ReplaceAllMatches(string input, object? parameter, FuncThe input string with all matches replaced. public string ReplaceAllMatches(string input, Func replacementSelector) { - return ReplaceAllMatches(input, null, replacementSelector); + return ReplaceAllMatches(input, parameter: null, replacementSelector); } /// @@ -357,7 +358,7 @@ public string ReplaceAllMatches(string ruleAlias, string input, object? paramete /// The input string with all matches replaced. public string ReplaceAllMatches(string ruleAlias, string input, Func replacementSelector) { - return ReplaceAllMatches(ruleAlias, input, fallbackReplacement: null, replacementSelector); + return ReplaceAllMatches(ruleAlias, input, parameter: null, replacementSelector); } /// @@ -400,7 +401,7 @@ public string ReplaceAllMatches(string input, object? parameter, FuncThe input string with all matches replaced. public string ReplaceAllMatches(string input, Func replacementSelector) { - return ReplaceAllMatches(input, null, replacementSelector); + return ReplaceAllMatches(input, parameter: null, replacementSelector); } /// @@ -465,7 +466,7 @@ public string ReplaceAllMatches(string ruleAlias, ParserContext context, Func /// /// Splits the input string into substrings by all non-overlapping matches of the specified rule. - /// Works similarly to . + /// Works similarly to . /// /// The rule ID to use as a delimiter. /// Parser context for the input. @@ -562,5 +563,401 @@ public IEnumerable Split(string ruleAlias, ParserContext context) return Split(ruleId, context); } + + + + /// + /// Splits the input string into segments by all non-overlapping matches of the specified rule. + /// Works similarly to . + /// + /// The rule ID to use as a delimiter. + /// Parser context for the input. + /// An enumerable sequence of segments between matches. + internal IEnumerable SplitSegments(int ruleId, ParserContext context) + { + if (context.parser != this) + throw new InvalidOperationException("Parser context is not associated with this parser."); + + EmitBarriers(ref context); + + var input = context.input; + + // Non-overlapping: the same as ReplaceAllMatches (overlap = false). + var matches = FindAllMatches(ruleId, context, GlobalSettings, overlap: false); + + var currentIndex = context.position; + + foreach (var match in matches) + { + var start = match.startIndex; + var length = match.length; + + if (start < currentIndex) + continue; + + // Segment before match. + yield return new StringSegment(input, currentIndex, start - currentIndex); + + currentIndex = start + length; + } + + // Tail after last match. + if (currentIndex < context.maxPosition) + yield return new StringSegment(input, currentIndex, context.maxPosition - currentIndex); + } + + /// + /// Splits the input string into segments by all non-overlapping matches of the main rule. + /// Works similarly to . + /// + /// The input text to split. + /// Optional parameter to pass to the parser. + /// An enumerable sequence of segments between matches. + public IEnumerable SplitSegments(string input, object? parameter = null) + { + if (_mainRuleId == -1) + throw new InvalidOperationException("Main rule is not set."); + + var context = new ParserContext(this, input, parameter); + return SplitSegments(_mainRuleId, context); + } + + /// + /// Splits the input represented by the specified context into segments by + /// all non-overlapping matches of the main rule. + /// + /// The parser context to use for parsing. + /// An enumerable sequence of segments between matches. + public IEnumerable SplitSegments(ParserContext context) + { + if (_mainRuleId == -1) + throw new InvalidOperationException("Main rule is not set."); + + return SplitSegments(_mainRuleId, context); + } + + /// + /// Splits the input string into segments by all non-overlapping matches of the specified rule. + /// + /// The alias for the parser rule to use as a delimiter. + /// The input text to split. + /// Optional parameter to pass to the parser. + /// An enumerable sequence of segments between matches. + public IEnumerable SplitSegments(string ruleAlias, string input, object? parameter = null) + { + if (!_rulesAliases.TryGetValue(ruleAlias, out var ruleId)) + throw new ArgumentException("Invalid rule alias", nameof(ruleAlias)); + + var context = new ParserContext(this, input, parameter); + return SplitSegments(ruleId, context); + } + + /// + /// Splits the input represented by the specified context into segments by + /// all non-overlapping matches of the specified rule. + /// + /// The alias for the parser rule to use as a delimiter. + /// The parser context to use for parsing. + /// An enumerable sequence of segments between matches. + public IEnumerable SplitSegments(string ruleAlias, ParserContext context) + { + if (!_rulesAliases.TryGetValue(ruleAlias, out var ruleId)) + throw new ArgumentException("Invalid rule alias", nameof(ruleAlias)); + + return SplitSegments(ruleId, context); + } + + + + /// + /// Finds all matches of target rule, then returns sequence of segments between matches and matches themselves. + /// + /// The rule ID to use as a delimiter. + /// Parser context for the input. + /// An enumerable sequence of segments between matches and matches. + internal IEnumerable> Scan(int ruleId, ParserContext context) + { + if (context.parser != this) + throw new InvalidOperationException("Parser context is not associated with this parser."); + + EmitBarriers(ref context); + + var input = context.input; + + // Non-overlapping: the same as ReplaceAllMatches (overlap = false). + var matches = FindAllMatches(ruleId, context, GlobalSettings, overlap: false); + + var currentIndex = context.position; + + foreach (var match in matches) + { + var start = match.startIndex; + var length = match.length; + + if (start < currentIndex) + { + var capturedMatch1 = match; + yield return CreateResult(ref context, ref capturedMatch1); + continue; + } + + // Segment before match. + yield return new StringSegment(input, currentIndex, start - currentIndex); + currentIndex = start + length; + + var capturedMatch2 = match; + yield return CreateResult(ref context, ref capturedMatch2); + } + + // Tail after last match. + if (currentIndex < context.maxPosition) + yield return new StringSegment(input, currentIndex, context.maxPosition - currentIndex); + } + + /// + /// Scans the input using the main rule, returning segments between matches and the matches themselves. + /// + /// The parser context to use for scanning. + /// An enumerable sequence of segments between matches and matches. + public IEnumerable> Scan(ParserContext context) + { + if (context.parser != this) + throw new InvalidOperationException("Parser context is not associated with this parser."); + if (_mainRuleId == -1) + throw new InvalidOperationException("Main rule is not set."); + + return Scan(_mainRuleId, context); + } + + /// + /// Scans the input using the main rule, returning segments between matches and the matches themselves. + /// + /// The input text to scan. + /// Optional parameter to pass to the parser. + /// An enumerable sequence of segments between matches and matches. + public IEnumerable> Scan(string input, object? parameter = null) + { + if (_mainRuleId == -1) + throw new InvalidOperationException("Main rule is not set."); + + var context = new ParserContext(this, input, parameter); + return Scan(_mainRuleId, context); + } + + /// + /// Scans the input using the specified rule, returning segments between matches and the matches themselves. + /// + /// The alias for the parser rule to use. + /// The parser context to use for scanning. + /// An enumerable sequence of segments between matches and matches. + public IEnumerable> Scan(string ruleAlias, ParserContext context) + { + if (context.parser != this) + throw new InvalidOperationException("Parser context is not associated with this parser."); + if (!_rulesAliases.TryGetValue(ruleAlias, out var ruleId)) + throw new ArgumentException("Invalid rule alias", nameof(ruleAlias)); + + return Scan(ruleId, context); + } + + /// + /// Scans the input using the specified rule, returning segments between matches and the matches themselves. + /// + /// The alias for the parser rule to use. + /// The input text to scan. + /// Optional parameter to pass to the parser. + /// An enumerable sequence of segments between matches and matches. + public IEnumerable> Scan(string ruleAlias, string input, object? parameter = null) + { + if (!_rulesAliases.TryGetValue(ruleAlias, out var ruleId)) + throw new ArgumentException("Invalid rule alias", nameof(ruleAlias)); + + var context = new ParserContext(this, input, parameter); + return Scan(ruleId, context); + } + + /// + /// Scans the input using the main rule, applying the specified factories to convert each segment into a uniform result type. + /// + /// The type of the result. + /// The parser context to use for scanning. + /// Factory to convert a into . + /// Factory to convert a (parsed match) into . + /// An enumerable sequence of converted results. + public IEnumerable Scan(ParserContext context, + Func rawFactory, + Func matchFactory) + { + foreach (var segment in Scan(context)) + { + if (segment.VariantIndex == 0) + yield return rawFactory(segment.AsT1()); + else + yield return matchFactory(segment.AsT2()); + } + } + + /// + /// Scans the input using the main rule, applying the specified factories to convert each segment into a uniform result type. + /// + /// The type of the result. + /// The input text to scan. + /// Factory to convert a into . + /// Factory to convert a (parsed match) into . + /// Optional parameter to pass to the parser. + /// An enumerable sequence of converted results. + public IEnumerable Scan(string input, + Func rawFactory, + Func matchFactory, + object? parameter = null) + { + foreach (var segment in Scan(input, parameter)) + { + if (segment.VariantIndex == 0) + yield return rawFactory(segment.AsT1()); + else + yield return matchFactory(segment.AsT2()); + } + } + + /// + /// Scans the input using the specified rule, applying the specified factories to convert each segment into a uniform result type. + /// + /// The type of the result. + /// The alias for the parser rule to use. + /// The parser context to use for scanning. + /// Factory to convert a into . + /// Factory to convert a (parsed match) into . + /// An enumerable sequence of converted results. + public IEnumerable Scan(string ruleAlias, ParserContext context, + Func rawFactory, + Func matchFactory) + { + foreach (var segment in Scan(ruleAlias, context)) + { + if (segment.VariantIndex == 0) + yield return rawFactory(segment.AsT1()); + else + yield return matchFactory(segment.AsT2()); + } + } + + /// + /// Scans the input using the specified rule, applying the specified factories to convert each segment into a uniform result type. + /// + /// The type of the result. + /// The alias for the parser rule to use. + /// The input text to scan. + /// Factory to convert a into . + /// Factory to convert a (parsed match) into . + /// Optional parameter to pass to the parser. + /// An enumerable sequence of converted results. + public IEnumerable Scan(string ruleAlias, string input, + Func rawFactory, + Func matchFactory, + object? parameter = null) + { + foreach (var segment in Scan(ruleAlias, input, parameter)) + { + if (segment.VariantIndex == 0) + yield return rawFactory(segment.AsT1()); + else + yield return matchFactory(segment.AsT2()); + } + } + + /// + /// Scans the input using the main rule, applying the specified factories to convert each segment into a discriminated union result. + /// + /// The type for raw text segments. + /// The type for parsed match results. + /// The parser context to use for scanning. + /// Factory to convert a into . + /// Factory to convert a (parsed match) into . + /// An enumerable sequence of values representing either raw text or a parsed match. + public IEnumerable> Scan(ParserContext context, + Func rawFactory, + Func matchFactory) + { + foreach (var segment in Scan(context)) + { + if (segment.VariantIndex == 0) + yield return rawFactory(segment.AsT1()); + else + yield return matchFactory(segment.AsT2()); + } + } + + /// + /// Scans the input using the main rule, applying the specified factories to convert each segment into a discriminated union result. + /// + /// The type for raw text segments. + /// The type for parsed match results. + /// The input text to scan. + /// Factory to convert a into . + /// Factory to convert a (parsed match) into . + /// Optional parameter to pass to the parser. + /// An enumerable sequence of values representing either raw text or a parsed match. + public IEnumerable> Scan(string input, + Func rawFactory, + Func matchFactory, + object? parameter = null) + { + foreach (var segment in Scan(input, parameter)) + { + if (segment.VariantIndex == 0) + yield return rawFactory(segment.AsT1()); + else + yield return matchFactory(segment.AsT2()); + } + } + + /// + /// Scans the input using the specified rule, applying the specified factories to convert each segment into a discriminated union result. + /// + /// The type for raw text segments. + /// The type for parsed match results. + /// The alias for the parser rule to use. + /// The parser context to use for scanning. + /// Factory to convert a into . + /// Factory to convert a (parsed match) into . + /// An enumerable sequence of values representing either raw text or a parsed match. + public IEnumerable> Scan(string ruleAlias, ParserContext context, + Func rawFactory, + Func matchFactory) + { + foreach (var segment in Scan(ruleAlias, context)) + { + if (segment.VariantIndex == 0) + yield return rawFactory(segment.AsT1()); + else + yield return matchFactory(segment.AsT2()); + } + } + + /// + /// Scans the input using the specified rule, applying the specified factories to convert each segment into a discriminated union result. + /// + /// The type for raw text segments. + /// The type for parsed match results. + /// The alias for the parser rule to use. + /// The input text to scan. + /// Factory to convert a into . + /// Factory to convert a (parsed match) into . + /// Optional parameter to pass to the parser. + /// An enumerable sequence of values representing either raw text or a parsed match. + public IEnumerable> Scan(string ruleAlias, string input, + Func rawFactory, + Func matchFactory, + object? parameter = null) + { + foreach (var segment in Scan(ruleAlias, input, parameter)) + { + if (segment.VariantIndex == 0) + yield return rawFactory(segment.AsT1()); + else + yield return matchFactory(segment.AsT2()); + } + } } } \ No newline at end of file diff --git a/src/RCParsing/Parser.incremental.cs b/src/RCParsing/Parser.incremental.cs index 38d8add..c32894f 100644 --- a/src/RCParsing/Parser.incremental.cs +++ b/src/RCParsing/Parser.incremental.cs @@ -112,8 +112,14 @@ internal ParsedRule ParseIncrementally(ParserContext context, ParserSettings set var recovery = rule.ErrorRecovery ?? ErrorRecoveryStrategy.NoRecovery; result = recovery.TryRecover(context, settings, rule, ruleContext, ruleSettings, ruleChildSettings); - result = result.ChangeVersion(newVersion); - return result; + + if (result.success) + { + result = result.ChangeVersion(newVersion); + return result; + } + + return ParsedRule.Fail; } } } \ No newline at end of file diff --git a/src/RCParsing/Parser.tokens.cs b/src/RCParsing/Parser.tokens.cs index f4665f6..c0806f0 100644 --- a/src/RCParsing/Parser.tokens.cs +++ b/src/RCParsing/Parser.tokens.cs @@ -490,6 +490,27 @@ public bool MatchesToken(string tokenPatternAlias, string input, out int matched return parsedToken.success; } + /// + /// Checks if the given input matches the specified token pattern by its alias. + /// + /// + /// Does not calculates intermediate value. + /// + /// The alias for the token pattern to use. + /// The input text to parse. + /// The parsed token result. Does not include intermediate value. + /// Optional parameter to pass to the parser. Can be used to pass additional information to the custom token patterns. + /// if token matches the input string; otherwise, . + public bool MatchesToken(string tokenPatternAlias, string input, out ParsedElement parsedToken, object? parameter = null) + { + if (!_tokenPatternsAliases.TryGetValue(tokenPatternAlias, out var tokenPatternId)) + throw new ArgumentException("Invalid token pattern alias", nameof(tokenPatternAlias)); + + parsedToken = MatchToken(tokenPatternId, input, 0, + input.Length, parameter, false, out _); + return parsedToken.success; + } + /// /// Checks if the given input matches the specified token pattern by its alias. /// @@ -534,6 +555,28 @@ public bool MatchesToken(string tokenPatternAlias, string input, int startIndex, return parsedToken.success; } + /// + /// Checks if the given input matches the specified token pattern by its alias. + /// + /// + /// Does not calculates intermediate value. + /// + /// The alias for the token pattern to use. + /// The input text to parse. + /// Starting index in the input text to parse. + /// The parsed token result. Does not include intermediate value. + /// Optional parameter to pass to the parser. Can be used to pass additional information to the custom token patterns. + /// if token matches the input string; otherwise, . + public bool MatchesToken(string tokenPatternAlias, string input, int startIndex, out ParsedElement parsedToken, object? parameter = null) + { + if (!_tokenPatternsAliases.TryGetValue(tokenPatternAlias, out var tokenPatternId)) + throw new ArgumentException("Invalid token pattern alias", nameof(tokenPatternAlias)); + + parsedToken = MatchToken(tokenPatternId, input, startIndex, + input.Length, parameter, false, out _); + return parsedToken.success; + } + /// /// Checks if the given input matches the specified token pattern by its alias. /// @@ -579,5 +622,28 @@ public bool MatchesToken(string tokenPatternAlias, string input, int startIndex, matchedLength = parsedToken.startIndex + parsedToken.length - startIndex; return parsedToken.success; } + + /// + /// Checks if the given input matches the specified token pattern by its alias. + /// + /// + /// Does not calculates intermediate value. + /// + /// The alias for the token pattern to use. + /// The input text to parse. + /// Starting index in the input text to parse. + /// Number of characters to parse from the input text. + /// The parsed token result. Does not include intermediate value. + /// Optional parameter to pass to the parser. Can be used to pass additional information to the custom token patterns. + /// if token matches the input string; otherwise, . + public bool MatchesToken(string tokenPatternAlias, string input, int startIndex, int length, out ParsedElement parsedToken, object? parameter = null) + { + if (!_tokenPatternsAliases.TryGetValue(tokenPatternAlias, out var tokenPatternId)) + throw new ArgumentException("Invalid token pattern alias", nameof(tokenPatternAlias)); + + parsedToken = MatchToken(tokenPatternId, input, startIndex, + startIndex + length, parameter, false, out _); + return parsedToken.success; + } } } \ No newline at end of file diff --git a/src/RCParsing/ParserRules/RepeatParserRule.cs b/src/RCParsing/ParserRules/RepeatParserRule.cs index ce758e3..595baac 100644 --- a/src/RCParsing/ParserRules/RepeatParserRule.cs +++ b/src/RCParsing/ParserRules/RepeatParserRule.cs @@ -65,7 +65,7 @@ protected override void Initialize(ParserInitFlags initFlags) ParsedRule Parse(ref ParserContext context, ref ParserSettings settings, ref ParserSettings childSettings) { var rules = new List(); - var initialPosition = context.position; + var initialPosition = -1; ParsedRule parsedRule = default; for (int i = 0; i < this.MaxCount || this.MaxCount == -1; i++) @@ -78,6 +78,9 @@ ParsedRule Parse(ref ParserContext context, ref ParserSettings settings, ref Par if (!parsedRule.success) break; + if (initialPosition == -1) + initialPosition = parsedRule.startIndex; + context.position = parsedRule.startIndex + parsedRule.length; context.passedBarriers = parsedRule.passedBarriers; parsedRule.occurency = i; @@ -90,6 +93,9 @@ ParsedRule Parse(ref ParserContext context, ref ParserSettings settings, ref Par return ParsedRule.Fail; } + if (initialPosition == -1) + initialPosition = context.position; + return new ParsedRule(Id, initialPosition, context.position - initialPosition, context.passedBarriers, null, rules); }; diff --git a/src/RCParsing/ParserRules/SeparatedRepeatParserRule.cs b/src/RCParsing/ParserRules/SeparatedRepeatParserRule.cs index c077376..9241f2f 100644 --- a/src/RCParsing/ParserRules/SeparatedRepeatParserRule.cs +++ b/src/RCParsing/ParserRules/SeparatedRepeatParserRule.cs @@ -93,7 +93,7 @@ protected override void Initialize(ParserInitFlags initFlags) ParsedRule Parse(ref ParserContext context, ref ParserSettings settings, ref ParserSettings childSettings) { - var initialPosition = context.position; + int initialPosition = -1; // Put some parameters from heap to stack for max performance int minCount = MinCount, maxCount = MaxCount; @@ -112,7 +112,7 @@ ParsedRule Parse(ref ParserContext context, ref ParserSettings settings, ref Par // If minCount == 0 — OK: empty sequence if (minCount == 0) { - return new ParsedRule(Id, initialPosition, 0, context.passedBarriers, Array.Empty()); + return new ParsedRule(Id, context.position, 0, context.passedBarriers, Array.Empty()); } else { @@ -129,9 +129,10 @@ ParsedRule Parse(ref ParserContext context, ref ParserSettings settings, ref Par return ParsedRule.Fail; } - var elements = new List(); + initialPosition = firstElement.startIndex; int count = 1; - firstElement.occurency = elements.Count; + firstElement.occurency = 0; + var elements = new List(); elements.Add(firstElement); context.position = firstElement.startIndex + firstElement.length; context.passedBarriers = firstElement.passedBarriers; diff --git a/src/RCParsing/ParserRules/SequenceParserRule.cs b/src/RCParsing/ParserRules/SequenceParserRule.cs index cdf010e..ee31438 100644 --- a/src/RCParsing/ParserRules/SequenceParserRule.cs +++ b/src/RCParsing/ParserRules/SequenceParserRule.cs @@ -88,7 +88,7 @@ protected override void Initialize(ParserInitFlags initFlags) ParsedRule Parse(ref ParserContext context, ref ParserSettings settings, ref ParserSettings childSettings) { - var startIndex = context.position; + var initialPosition = -1; ParsedRule[]? rules = null; for (int i = 0; i < parseFunctions.Length; i++) @@ -100,6 +100,9 @@ ParsedRule Parse(ref ParserContext context, ref ParserSettings settings, ref Par return ParsedRule.Fail; } + if (initialPosition == -1) + initialPosition = parsedRule.startIndex; + rules ??= new ParsedRule[_rules.Length]; parsedRule.occurency = i; rules[i] = parsedRule; @@ -108,7 +111,10 @@ ParsedRule Parse(ref ParserContext context, ref ParserSettings settings, ref Par context.passedBarriers = parsedRule.passedBarriers; } - return new ParsedRule(Id, startIndex, context.position - startIndex, + if (initialPosition == -1) + initialPosition = context.position; + + return new ParsedRule(Id, initialPosition, context.position - initialPosition, context.passedBarriers, rules); }; diff --git a/src/RCParsing/PositionalFormatter.cs b/src/RCParsing/PositionalFormatter.cs index 1528ef4..d1ad837 100644 --- a/src/RCParsing/PositionalFormatter.cs +++ b/src/RCParsing/PositionalFormatter.cs @@ -107,6 +107,28 @@ public static string Format(string str, int position) { Decompose(str, position, out int lineStart, out int lineLength, out int lineNumber, out int columnNumber, out int visualColumnNumber); + return Format(str, lineStart, lineLength, lineNumber, columnNumber, visualColumnNumber); + } + + /// + /// Extracts a line containing a specified position in a text and formats it for display. + /// + /// + /// Useful for debugging and displaying errors in a user-friendly manner. + /// + /// The input text. + /// The zero-based index of the start of the line. + /// The length of the line. + /// The one-based index of the line number. + /// The one-based index of the column number. + /// The one-based index of the visual column number. + /// + /// A formatted string containing the line at the specified position + /// along with line number and column information for the specified position. + /// + /// Thrown if the specified position is out of range for the input text. + public static string Format(string str, int lineStart, int lineLength, int lineNumber, int columnNumber, int visualColumnNumber) + { string lineAndColumn = $"line {lineNumber}, column {columnNumber}"; string pointerLine; diff --git a/src/RCParsing/RCParsing.csproj b/src/RCParsing/RCParsing.csproj index adc0c55..4a55ad3 100644 --- a/src/RCParsing/RCParsing.csproj +++ b/src/RCParsing/RCParsing.csproj @@ -8,7 +8,7 @@ RCParsing - 5.1.0 + 5.2.0 Roman K. RomeCore RCParsing diff --git a/src/RCParsing/SemanticException.cs b/src/RCParsing/SemanticException.cs new file mode 100644 index 0000000..6883f82 --- /dev/null +++ b/src/RCParsing/SemanticException.cs @@ -0,0 +1,156 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using RCParsing.Utils; + +namespace RCParsing +{ + /// + /// Represents an exception that occurs during semantic analysis (e.g. transforming into a value). + /// + public class SemanticException : Exception + { + /// + /// Gets the parsed rule result associated with this semantic exception. + /// + public ParsedRuleResultBase AssociatedResult { get; } + + /// + /// Gets the original exception message that have been passed to constructor. + /// + public string OriginalMessage { get; } + + /// + /// Gets the children exceptions associated with this semantic exception. + /// + public IReadOnlyList Children { get; } + + public SemanticException(ParsedRuleResultBase result, string message, Exception? inner = null) : + base(FormatMessage(result, message, Enumerable.Empty()), inner) + { + AssociatedResult = result; + OriginalMessage = message; + Children = Array.Empty(); + } + + public SemanticException(ParsedRuleResultBase result, string message, params SemanticException[] children) : + base(FormatMessage(result, message, children)) + { + AssociatedResult = result; + OriginalMessage = message; + Children = children.AsReadOnlyList(); + } + + public SemanticException(ParsedRuleResultBase result, string message, Exception? inner, params SemanticException[] children) : + base(FormatMessage(result, message, children), inner) + { + AssociatedResult = result; + OriginalMessage = message; + Children = children.AsReadOnlyList(); + } + + public SemanticException(ParsedRuleResultBase result, string message, IEnumerable children) : + base(FormatMessage(result, message, children)) + { + AssociatedResult = result; + OriginalMessage = message; + Children = children.AsReadOnlyList(); + } + + public SemanticException(ParsedRuleResultBase result, string message, Exception? inner, IEnumerable children) : + base(FormatMessage(result, message, children), inner) + { + AssociatedResult = result; + OriginalMessage = message; + Children = children.AsReadOnlyList(); + } + + private static string FormatMessage(ParsedRuleResultBase result, string message, IEnumerable children) + { + var sb = new StringBuilder(); + + sb.Append("A semantic error occurred: "); + sb.AppendLine(message); + sb.AppendLine(); + + var input = result.Context.input; + if (result.Length is 0 or 1) + { + PositionalFormatter.Decompose(input, result.StartIndex, + out int lineStart, out int lineLength, out int lineNumber, out int columnNumber, out int visualColumnNumber, + result.Context.parser.MainSettings.tabSize); + + sb.AppendLine("Location:"); + + string lineAndColumn = $"line {lineNumber}, column {columnNumber}"; + string pointerLine; + if (visualColumnNumber <= lineAndColumn.Length + 2) + pointerLine = new string(' ', visualColumnNumber - 1) + '^' + ' ' + lineAndColumn; + else + pointerLine = new string(' ', visualColumnNumber - 2 - lineAndColumn.Length) + lineAndColumn + ' ' + '^'; + + sb.AppendLine(input.Substring(lineStart, lineLength)); + sb.AppendLine(pointerLine); + } + else + { + PositionalFormatter.Decompose(input, result.StartIndex, + out int lineStart1, out int lineLength1, out int lineNumber1, out int columnNumber1, out int visualColumnNumber1, + result.Context.parser.MainSettings.tabSize); + + PositionalFormatter.Decompose(input, result.EndIndex - 1, + out int lineStart2, out int lineLength2, out int lineNumber2, out int columnNumber2, out int visualColumnNumber2, + result.Context.parser.MainSettings.tabSize); + + sb.AppendLine("Location:"); + + if (lineNumber1 == lineNumber2) + { + // At the same line + string lineAndColumn = $"line {lineNumber1}, column {columnNumber1}, length {result.Length}"; + string pointerLine; + if (visualColumnNumber1 <= lineAndColumn.Length + 2) + pointerLine = new string(' ', visualColumnNumber1 - 1) + new string('^', result.Length) + ' ' + lineAndColumn; + else + pointerLine = new string(' ', visualColumnNumber1 - 2 - lineAndColumn.Length) + lineAndColumn + ' ' + new string('^', result.Length); + + sb.AppendLine(input.Substring(lineStart1, lineLength1)); + sb.AppendLine(pointerLine); + } + else + { + // At different lines + sb.Append(lineNumber1.ToString().PadLeft(6) + ": "); + sb.AppendLine(input.Substring(lineStart1, lineLength1)); + sb.AppendLine(new string('^', lineLength1 - visualColumnNumber1 - 2).PadLeft(lineLength1)); + + if (lineNumber1 + 1 != lineNumber2) + sb.AppendLine("..."); + + sb.Append(lineNumber2.ToString().PadLeft(6) + ": "); + sb.AppendLine(input.Substring(lineStart2, lineLength2)); + sb.AppendLine(new string('^', lineLength2 - visualColumnNumber2 - 2).PadRight(lineLength2)); + } + } + + sb.AppendLine().AppendLine("The rule that failed:"); + sb.Append(result.Rule.ToString()); + + var childList = children.ToList(); + if (childList.Count > 0) + { + sb.AppendLine().AppendLine("Children errors:"); + for (int i = 0; i < childList.Count; i++) + { + if (i < childList.Count - 1) + sb.AppendLine(childList[i].OriginalMessage); + else + sb.Append(childList[i].OriginalMessage); + } + } + + return sb.ToString(); + } + } +} \ No newline at end of file diff --git a/src/RCParsing/StringSegment.cs b/src/RCParsing/StringSegment.cs new file mode 100644 index 0000000..036195a --- /dev/null +++ b/src/RCParsing/StringSegment.cs @@ -0,0 +1,62 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace RCParsing +{ + /// + /// Represents a segment of a string. + /// + public struct StringSegment + { + /// + /// Gets the source string that contains the segment. + /// + public readonly string Source { get; } + + /// + /// Gets the starting index of the segment within the source string. The index is zero-based. + /// + public readonly int StartIndex { get; } + + /// + /// Gets the length of the segment. The length is the number of characters from StartIndex to the end of the string. + /// + public readonly int Length { get; } + + /// + /// Gets the end index of the segment. This is calculated as StartIndex + Length. + /// + public readonly int EndIndex => StartIndex + Length; + + /// + /// Gets a span representing the segment. + /// + public readonly ReadOnlySpan Span => Source.AsSpan(StartIndex, Length); + + private string? _slice; + /// + /// Gets a copy of the segment as a new string. This is cached for future use. + /// + public string Slice => _slice ??= Span.ToString(); + + /// + /// Initializes a new instance of the struct. + /// + /// The source string that this segment belongs to. + /// The starting index of the segment within the source string. + /// The length of the segment. + public StringSegment(string source, int startIndex, int length) + { + Source = source; + StartIndex = startIndex; + Length = length; + _slice = null; + } + + public override string ToString() + { + return Slice; + } + } +} \ No newline at end of file diff --git a/src/RCParsing/TokenPatterns/AnyCharTokenPattern.cs b/src/RCParsing/TokenPatterns/AnyCharTokenPattern.cs new file mode 100644 index 0000000..8ea41ed --- /dev/null +++ b/src/RCParsing/TokenPatterns/AnyCharTokenPattern.cs @@ -0,0 +1,54 @@ +using System.Collections.Generic; + +namespace RCParsing.TokenPatterns +{ + /// + /// Represents an any character token pattern that matches any single character and returns it as intermediate value. + /// + public class AnyCharTokenPattern : TokenPattern + { + /// + /// Initializes a new instance of class. + /// + public AnyCharTokenPattern() + { + } + + protected override HashSet FirstCharsCore => new(); + protected override bool IsFirstCharDeterministicCore => true; + protected override bool IsOptionalCore => false; + + + + public override ParsedElement Match(string input, int position, int barrierPosition, + object? parserParameter, bool calculateIntermediateValue, ref ParsingError furthestError) + { + if (position >= barrierPosition) + { + if (position >= furthestError.position) + furthestError = new ParsingError(position, 0, "Cannot match any char token, position exceeds the barrier or end of input.", Id, true); + return ParsedElement.Fail; + } + + return new ParsedElement(position, 1, calculateIntermediateValue ? input[position] : null); + } + + + + public override bool Equals(object obj) + { + return base.Equals(obj) && + obj is AnyCharTokenPattern; + } + + public override int GetHashCode() + { + return base.GetHashCode(); + } + + public override string ToStringOverride(int remainingDepth) + { + return "anychar"; + } + } +} \ No newline at end of file diff --git a/src/RCParsing/TokenPatterns/Combinators/BetweenTokenPattern.cs b/src/RCParsing/TokenPatterns/Combinators/BetweenTokenPattern.cs index 9687132..7272b8d 100644 --- a/src/RCParsing/TokenPatterns/Combinators/BetweenTokenPattern.cs +++ b/src/RCParsing/TokenPatterns/Combinators/BetweenTokenPattern.cs @@ -91,12 +91,11 @@ protected override void PreInitialize(ParserInitFlags initFlags) public override ParsedElement Match(string input, int position, int barrierPosition, object? parserParameter, bool calculateIntermediateValue, ref ParsingError furthestError) { - var initialPosition = position; - var first = _first.Match(input, position, barrierPosition, parserParameter, calculateIntermediateValue: false, ref furthestError); if (!first.success) return ParsedElement.Fail; + var initialPosition = first.startIndex; position = first.startIndex + first.length; var middle = _middle.Match(input, position, barrierPosition, parserParameter, diff --git a/src/RCParsing/TokenPatterns/Combinators/CaptureTextTokenPattern.cs b/src/RCParsing/TokenPatterns/Combinators/CaptureTextTokenPattern.cs index 7ac523c..8f08ae8 100644 --- a/src/RCParsing/TokenPatterns/Combinators/CaptureTextTokenPattern.cs +++ b/src/RCParsing/TokenPatterns/Combinators/CaptureTextTokenPattern.cs @@ -65,23 +65,19 @@ public override ParsedElement Match(string input, int position, int barrierPosit if (!child.success) return ParsedElement.Fail; - var initialPosition = position; - position = child.startIndex + child.length; - - object? value = null; int trimStart = Math.Min(TrimStart, child.length); int trimEnd = Math.Min(TrimEnd, child.length - trimStart); if (child.length - trimStart - trimEnd > 0) { - value = input.Substring(child.startIndex + trimStart, child.length - trimStart - trimEnd); + child.intermediateValue = input.Substring(child.startIndex + trimStart, child.length - trimStart - trimEnd); } else { - value = string.Empty; + child.intermediateValue = string.Empty; } - return new ParsedElement(initialPosition, position - initialPosition, value); + return child; } diff --git a/src/RCParsing/TokenPatterns/Combinators/FirstTokenPattern.cs b/src/RCParsing/TokenPatterns/Combinators/FirstTokenPattern.cs index 2449a43..b034250 100644 --- a/src/RCParsing/TokenPatterns/Combinators/FirstTokenPattern.cs +++ b/src/RCParsing/TokenPatterns/Combinators/FirstTokenPattern.cs @@ -76,12 +76,11 @@ protected override void PreInitialize(ParserInitFlags initFlags) public override ParsedElement Match(string input, int position, int barrierPosition, object? parserParameter, bool calculateIntermediateValue, ref ParsingError furthestError) { - var initialPosition = position; - var first = _first.Match(input, position, barrierPosition, parserParameter, calculateIntermediateValue, ref furthestError); if (!first.success) return ParsedElement.Fail; + var initialPosition = first.startIndex; position = first.startIndex + first.length; var second = _second.Match(input, position, barrierPosition, parserParameter, false, ref furthestError); diff --git a/src/RCParsing/TokenPatterns/Combinators/RepeatTokenPattern.cs b/src/RCParsing/TokenPatterns/Combinators/RepeatTokenPattern.cs index feefa07..b69d477 100644 --- a/src/RCParsing/TokenPatterns/Combinators/RepeatTokenPattern.cs +++ b/src/RCParsing/TokenPatterns/Combinators/RepeatTokenPattern.cs @@ -71,7 +71,7 @@ public override ParsedElement Match(string input, int position, int barrierPosit { if (PassageFunction == null || !calculateIntermediateValue) { - var initialPosition = position; + var initialPosition = -1; int count = 0; for (int i = 0; i < MaxCount || MaxCount == -1; i++) @@ -83,6 +83,9 @@ public override ParsedElement Match(string input, int position, int barrierPosit break; } + if (initialPosition == -1) + initialPosition = matchedToken.startIndex; + position = matchedToken.startIndex + matchedToken.length; count++; } @@ -90,6 +93,9 @@ public override ParsedElement Match(string input, int position, int barrierPosit if (count < MinCount) return ParsedElement.Fail; + if (initialPosition == -1) + initialPosition = position; + return new ParsedElement(initialPosition, position - initialPosition); } else diff --git a/src/RCParsing/TokenPatterns/Combinators/ReturnTokenPattern.cs b/src/RCParsing/TokenPatterns/Combinators/ReturnTokenPattern.cs index 2510e84..d204552 100644 --- a/src/RCParsing/TokenPatterns/Combinators/ReturnTokenPattern.cs +++ b/src/RCParsing/TokenPatterns/Combinators/ReturnTokenPattern.cs @@ -49,13 +49,11 @@ public override ParsedElement Match(string input, int position, int barrierPosit return _child.Match(input, position, barrierPosition, parserParameter, false, ref furthestError); - var initialPosition = position; var child = _child.Match(input, position, barrierPosition, parserParameter, false, ref furthestError); if (!child.success) return ParsedElement.Fail; - position = child.startIndex + child.length; - return new ParsedElement(initialPosition, position - initialPosition, Value); + return new ParsedElement(child.startIndex, child.length, Value); } diff --git a/src/RCParsing/TokenPatterns/Combinators/SecondTokenPattern.cs b/src/RCParsing/TokenPatterns/Combinators/SecondTokenPattern.cs index 8f46057..d48b18a 100644 --- a/src/RCParsing/TokenPatterns/Combinators/SecondTokenPattern.cs +++ b/src/RCParsing/TokenPatterns/Combinators/SecondTokenPattern.cs @@ -75,12 +75,11 @@ protected override void PreInitialize(ParserInitFlags initFlags) public override ParsedElement Match(string input, int position, int barrierPosition, object? parserParameter, bool calculateIntermediateValue, ref ParsingError furthestError) { - var initialPosition = position; - var first = _first.Match(input, position, barrierPosition, parserParameter, false, ref furthestError); if (!first.success) return ParsedElement.Fail; + var initialPosition = first.startIndex; position = first.startIndex + first.length; var second = _second.Match(input, position, barrierPosition, parserParameter, diff --git a/src/RCParsing/TokenPatterns/Combinators/SeparatedRepeatTokenPattern.cs b/src/RCParsing/TokenPatterns/Combinators/SeparatedRepeatTokenPattern.cs index 4d75bf8..03e3580 100644 --- a/src/RCParsing/TokenPatterns/Combinators/SeparatedRepeatTokenPattern.cs +++ b/src/RCParsing/TokenPatterns/Combinators/SeparatedRepeatTokenPattern.cs @@ -108,13 +108,14 @@ private ParsedElement MatchWithoutCalculation(string input, int position, int ba if (maxCount == 0) return new ParsedElement(position, 0, null); - var initialPosition = position; + var initialPosition = -1; var firstElement = _token.Match(input, position, barrierPosition, parserParameter, false, ref furthestError); + if (!firstElement.success) { if (minCount == 0) - return new ParsedElement(initialPosition, position - initialPosition); + return new ParsedElement(position, 0); else return ParsedElement.Fail; } @@ -123,6 +124,7 @@ private ParsedElement MatchWithoutCalculation(string input, int position, int ba return ParsedElement.Fail; int count = 1; + initialPosition = firstElement.startIndex; position = firstElement.startIndex + firstElement.length; while (maxCount == -1 || count < maxCount || (allowTrailing && count == maxCount)) diff --git a/src/RCParsing/TokenPatterns/Combinators/SequenceTokenPattern.cs b/src/RCParsing/TokenPatterns/Combinators/SequenceTokenPattern.cs index 06c47f1..2cc1365 100644 --- a/src/RCParsing/TokenPatterns/Combinators/SequenceTokenPattern.cs +++ b/src/RCParsing/TokenPatterns/Combinators/SequenceTokenPattern.cs @@ -86,7 +86,7 @@ public override ParsedElement Match(string input, int position, int barrierPosit { if (calculateIntermediateValue && PassageFunction != null) { - var initialPosition = position; + var initialPosition = -1; object?[]? intermediateValues = null; for (int i = 0; i < _patterns.Length; i++) @@ -96,18 +96,23 @@ public override ParsedElement Match(string input, int position, int barrierPosit if (!token.success) return ParsedElement.Fail; + if (initialPosition == -1) + initialPosition = token.startIndex; + intermediateValues ??= new object?[_patterns.Length]; intermediateValues[i] = token.intermediateValue; position = token.startIndex + token.length; } + if (initialPosition == -1) + initialPosition = position; var intermediateValue = PassageFunction(intermediateValues); return new ParsedElement(initialPosition, position - initialPosition, intermediateValue); } else { - var initialPosition = position; + var initialPosition = -1; for (int i = 0; i < _patterns.Length; i++) { @@ -116,9 +121,15 @@ public override ParsedElement Match(string input, int position, int barrierPosit if (!token.success) return ParsedElement.Fail; + if (initialPosition == -1) + initialPosition = token.startIndex; + position = token.startIndex + token.length; } + if (initialPosition == -1) + initialPosition = position; + return new ParsedElement(initialPosition, position - initialPosition); } } diff --git a/src/RCParsing/TokenPatterns/Combinators/SkipWhitespacesTokenPattern.cs b/src/RCParsing/TokenPatterns/Combinators/SkipWhitespacesTokenPattern.cs index 0e4ea40..80be893 100644 --- a/src/RCParsing/TokenPatterns/Combinators/SkipWhitespacesTokenPattern.cs +++ b/src/RCParsing/TokenPatterns/Combinators/SkipWhitespacesTokenPattern.cs @@ -42,23 +42,13 @@ protected override void PreInitialize(ParserInitFlags initFlags) public override ParsedElement Match(string input, int position, int barrierPosition, object? parserParameter, bool calculateIntermediateValue, ref ParsingError furthestError) { - var initialPosition = position; - // Skip any whitespace characters while (position < barrierPosition && char.IsWhiteSpace(input[position])) position++; // Match the child pattern at the new position - var result = _pattern.Match(input, position, barrierPosition, parserParameter, + return _pattern.Match(input, position, barrierPosition, parserParameter, calculateIntermediateValue, ref furthestError); - - if (!result.success) - return ParsedElement.Fail; - - // Calculate the total length including skipped whitespace - var totalLength = (result.startIndex + result.length) - initialPosition; - - return new ParsedElement(initialPosition, totalLength, result.intermediateValue); } public override string ToStringOverride(int remainingDepth) diff --git a/src/RCParsing/TokenPatterns/LiteralCharTokenPattern.cs b/src/RCParsing/TokenPatterns/LiteralCharTokenPattern.cs index 8987193..5dc476a 100644 --- a/src/RCParsing/TokenPatterns/LiteralCharTokenPattern.cs +++ b/src/RCParsing/TokenPatterns/LiteralCharTokenPattern.cs @@ -52,7 +52,7 @@ public LiteralCharTokenPattern(char literal, StringComparison comparison = Strin public override ParsedElement Match(string input, int position, int barrierPosition, object? parserParameter, bool calculateIntermediateValue, ref ParsingError furthestError) { - if (position + 1 > barrierPosition) + if (position >= barrierPosition) { return ParsedElement.Fail; } diff --git a/src/RCParsing/TokenPatterns/ParserTokenPattern.cs b/src/RCParsing/TokenPatterns/ParserTokenPattern.cs new file mode 100644 index 0000000..60814be --- /dev/null +++ b/src/RCParsing/TokenPatterns/ParserTokenPattern.cs @@ -0,0 +1,104 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace RCParsing.TokenPatterns +{ + /// + /// Represents a token pattern that uses an entire for matching. + /// + public class ParserTokenPattern : TokenPattern + { + /// + /// The parser to use for matching. + /// + public Parser MatchParser { get; } + + /// + /// The alias for the rule to parse. + /// + public string? RuleAlias { get; } + + /// + /// The alias for the token to parse. + /// + public string? TokenAlias { get; } + + /// + /// Initializes a new instance of the class. + /// + /// The parser to use for matching. + /// The alias for the rule to parse. + /// The alias for the token to parse. + public ParserTokenPattern(Parser matchParser, string? ruleAlias = null, string? tokenAlias = null) + { + MatchParser = matchParser ?? throw new ArgumentNullException(nameof(matchParser)); + RuleAlias = ruleAlias; + TokenAlias = tokenAlias; + } + + protected override HashSet FirstCharsCore => new HashSet(); + protected override bool IsFirstCharDeterministicCore => false; + protected override bool IsOptionalCore => true; + + public override ParsedElement Match(string input, int position, int barrierPosition, object? parserParameter, + bool calculateIntermediateValue, ref ParsingError furthestError) + { + if (position > barrierPosition) + { + return ParsedElement.Fail; + } + + try + { + if (RuleAlias != null || TokenAlias == null) + { + var context = new ParserContext(MatchParser, input, parserParameter) + { + position = position, + maxPosition = barrierPosition + }; + var result = MatchParser.ParseRule(RuleAlias, context); + if (calculateIntermediateValue) + return new ParsedElement(result.StartIndex, result.Length, result.Value); + else + return new ParsedElement(result.StartIndex, result.Length, null); + } + else + { + if (calculateIntermediateValue) + { + var result = MatchParser.TryMatchToken(TokenAlias, input, position, barrierPosition - position); + if (!result.Success) + return ParsedElement.Fail; + if (calculateIntermediateValue) + return new ParsedElement(result.StartIndex, result.Length, result.IntermediateValue); + else + return new ParsedElement(result.StartIndex, result.Length, null); + } + else + { + if (MatchParser.MatchesToken(TokenAlias, input, position, barrierPosition - position, out ParsedElement matchedToken, parserParameter)) + return matchedToken; + return ParsedElement.Fail; + } + } + } + catch (Exception ex) + { + if (position >= furthestError.position) + furthestError = new ParsingError(position, 0, ex.Message, Id, true); + return ParsedElement.Fail; + } + } + + public override string ToStringOverride(int remainingDepth) + { + if (RuleAlias != null) + return $"parser(rule:{RuleAlias})"; + if (TokenAlias != null) + return $"parser(token:{TokenAlias})"; + return "parser(main)"; + } + } +} diff --git a/src/RCParsing/TokenPatterns/RegexTokenPattern.cs b/src/RCParsing/TokenPatterns/RegexTokenPattern.cs index b6c2d48..25b3726 100644 --- a/src/RCParsing/TokenPatterns/RegexTokenPattern.cs +++ b/src/RCParsing/TokenPatterns/RegexTokenPattern.cs @@ -84,7 +84,7 @@ public override ParsedElement Match(string input, int position, int barrierPosit public override string ToStringOverride(int remainingDepth) { - return $"regex '{RegexPattern}'"; + return $"regex '{RegexPattern ?? "unknown"}'"; } public override bool Equals(object? obj) diff --git a/tests/RCParsing.Tests/ReplaceAllMatchesTests.cs b/tests/RCParsing.Tests/ReplaceAllMatchesTests.cs new file mode 100644 index 0000000..cef1a7f --- /dev/null +++ b/tests/RCParsing.Tests/ReplaceAllMatchesTests.cs @@ -0,0 +1,170 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using RCParsing.Building; +using RCParsing.ParserRules; +using RCParsing.TokenPatterns; + +namespace RCParsing.Tests +{ + public class ReplaceAllMatchesTests + { + [Fact] + public void Replace_WithCustomSelector() + { + var builder = new ParserBuilder(); + builder.CreateToken("hello").Literal("hello"); + builder.CreateMainRule().Token("hello"); + + var result = builder.Build().ReplaceAllMatches("hello world hello", + replacementSelector: r => "X"); + + Assert.Equal("X world X", result); + } + + [Fact] + public void Replace_NoMatches_ReturnsOriginal() + { + var builder = new ParserBuilder(); + builder.CreateToken("number").Number(); + builder.CreateMainRule().Token("number"); + + var result = builder.Build().ReplaceAllMatches("hello world"); + Assert.Equal("hello world", result); + } + + [Fact] + public void Replace_ByRuleAlias() + { + var builder = new ParserBuilder(); + builder.CreateToken("num").Number(); + builder.CreateRule("value").Token("num"); + builder.CreateMainRule().Rule("value"); + + var result = builder.Build().ReplaceAllMatches("value", "a 1 b 2 c", + replacementSelector: r => (r.GetValue() * 12).ToString(CultureInfo.InvariantCulture)); + + Assert.Equal("a 12 b 24 c", result); + } + + [Fact] + public void Replace_SequenceWithTransform_RemovesSpaces() + { + var builder = new ParserBuilder(); + builder.Settings.SkipWhitespaces(); + + builder.CreateMainRule() + .Literal("(") + .Number() + .Literal(")") + .Transform(v => + { + var num = v.GetValue(1); + return (num * 10).ToString(CultureInfo.InvariantCulture); + }); + + // SkipWhitespaces removes spaces between tokens + var result = builder.Build().ReplaceAllMatches("( 1 ) text ( 2 ) more ( 3 )", + replacementSelector: r => r.GetValue()); + + Assert.Equal("10 text 20 more 30", result); + } + + [Fact] + public void Replace_WithContext() + { + var builder = new ParserBuilder(); + builder.CreateToken("num").Number(); + builder.CreateMainRule().Token("num"); + + var parser = builder.Build(); + var context = parser.CreateContext("x 5 y"); + + var result = parser.ReplaceAllMatches(context, + r => r.GetValue().ToString(CultureInfo.InvariantCulture)); + + Assert.Equal("x 5 y", result); + } + + [Fact] + public void Replace_WithParameter() + { + var builder = new ParserBuilder(); + + builder.CreateMainRule() + .Literal("x") + .Transform(v => + { + var param = v.GetParsingParameter(); + return $"x{param}"; + }); + + var result = builder.Build().ReplaceAllMatches("x x", parameter: 42, + replacementSelector: r => r.GetValue()); + + Assert.Equal("x42 x42", result); + } + + [Fact] + public void Replace_NoOverlap() + { + var builder = new ParserBuilder(); + builder.CreateMainRule().Literal("aa"); + + var parser = builder.Build(); + + var result = parser.ReplaceAllMatches("aaa", + replacementSelector: r => "X"); + + Assert.Equal("Xa", result); + } + + [Fact] + public void Replace_MainRuleNotSet_Throws() + { + var parser = new ParserBuilder().Build(); + Assert.Throws(() => parser.ReplaceAllMatches("test")); + } + + [Fact] + public void Replace_InvalidAlias_Throws() + { + var builder = new ParserBuilder(); + builder.CreateMainRule().Literal("a"); + var parser = builder.Build(); + + Assert.Throws(() => + parser.ReplaceAllMatches("nonexistent", "test", replacementSelector: r => "X")); + } + + [Fact] + public void Replace_EmptyInput_ReturnsEmpty() + { + var builder = new ParserBuilder(); + builder.CreateToken("number").Number(); + builder.CreateMainRule().Token("number"); + + var result = builder.Build().ReplaceAllMatches("", replacementSelector: r => "X"); + Assert.Empty(result); + } + + [Fact] + public void Replace_WithFallbackReplacement_NullValue() + { + var builder = new ParserBuilder(); + + builder.CreateMainRule() + .Literal("x") + .Transform(v => null); // explicitly return null + + var result = builder.Build().ReplaceAllMatches("x y x", + fallbackReplacement: "NULL"); + + // Value is null -> fallbackReplacement kicks in + Assert.Equal("NULL y NULL", result); + } + } +} diff --git a/tests/RCParsing.Tests/ScanTests.cs b/tests/RCParsing.Tests/ScanTests.cs new file mode 100644 index 0000000..ffcad06 --- /dev/null +++ b/tests/RCParsing.Tests/ScanTests.cs @@ -0,0 +1,262 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using RCParsing.Building; +using RCParsing.ParserRules; +using RCParsing.TokenPatterns; + +namespace RCParsing.Tests +{ + /// + /// Tests for method. + /// + public class ScanTests + { + [Fact] + public void Scan_BasicInterleaving() + { + var builder = new ParserBuilder(); + + builder.CreateToken("number").Number(); + builder.CreateMainRule().Token("number"); + + var input = "abc 123 def 456 ghi"; + + var results = builder.Build().Scan(input).ToList(); + + // Expected: [raw("abc "), match(123), raw(" def "), match(456), raw(" ghi")] + Assert.Equal(5, results.Count); + + Assert.Equal(0, results[0].VariantIndex); // raw + Assert.Equal("abc ", results[0].AsT1().Slice); + + Assert.Equal(1, results[1].VariantIndex); // match + Assert.Equal(123, results[1].AsT2().GetValue()); + + Assert.Equal(0, results[2].VariantIndex); // raw + Assert.Equal(" def ", results[2].AsT1().Slice); + + Assert.Equal(1, results[3].VariantIndex); // match + Assert.Equal(456, results[3].AsT2().GetValue()); + + Assert.Equal(0, results[4].VariantIndex); // raw + Assert.Equal(" ghi", results[4].AsT1().Slice); + } + + [Fact] + public void Scan_NoMatches_ReturnsSingleRawSegment() + { + var builder = new ParserBuilder(); + + builder.CreateToken("number").Number(); + builder.CreateMainRule().Token("number"); + + var input = "hello world"; + + var results = builder.Build().Scan(input).ToList(); + + Assert.Single(results); + Assert.Equal(0, results[0].VariantIndex); // raw + Assert.Equal("hello world", results[0].AsT1().Slice); + } + + [Fact] + public void Scan_EntireInputMatches_ReturnsEmptyRawBeforeMatch() + { + var builder = new ParserBuilder(); + + builder.CreateToken("number").Number(); + builder.CreateMainRule().Token("number"); + + var input = "12345"; + + var results = builder.Build().Scan(input).ToList(); + + // Even if whole input matches, there's an empty raw segment before the match + Assert.Equal(2, results.Count); + Assert.Equal(0, results[0].VariantIndex); // raw (empty) + Assert.Empty(results[0].AsT1().Slice); + Assert.Equal(1, results[1].VariantIndex); // match + Assert.Equal(12345, results[1].AsT2().GetValue()); + } + + [Fact] + public void Scan_WithTransformation_ReturnsTransformedValues() + { + var builder = new ParserBuilder(); + builder.Settings.SkipWhitespaces(); + + builder.CreateMainRule() + .Literal("Price:") + .Number() + .LiteralChoice("USD", "EUR") + .Transform(v => + { + var number = v.GetValue(1); + var currency = v.GetValue(2); + return $"{number.ToString(CultureInfo.InvariantCulture)} {currency}"; + }); + + var input = "Text Price: 42.99 USD more Price: 99.50 EUR end"; + + var results = builder.Build().Scan(input).ToList(); + + // Expected: raw("Text "), match("42.99 USD"), raw(" more "), match("99.50 EUR"), raw(" end") + Assert.Equal(5, results.Count); + + Assert.Equal("Text ", results[0].AsT1().Slice); + Assert.Equal("42.99 USD", results[1].AsT2().GetValue()); + Assert.Equal(" more ", results[2].AsT1().Slice); + Assert.Equal("99.5 EUR", results[3].AsT2().GetValue()); + Assert.Equal(" end", results[4].AsT1().Slice); + } + + [Fact] + public void Scan_TypedOr_WithFactories_ReturnsOrUnion() + { + var builder = new ParserBuilder(); + + builder.CreateToken("number").Number(); + builder.CreateMainRule().Token("number"); + + var input = "a 5 b"; + + // Use different types for T1 and T2 (Or disallows same types) + var results = builder.Build().Scan( + input, + rawFactory: seg => seg, + matchFactory: match => match.GetValue() + ).ToList(); + + Assert.Equal(3, results.Count); + + Assert.Equal(0, results[0].VariantIndex); // raw + Assert.Equal("a ", results[0].AsT1().Slice); + + Assert.Equal(1, results[1].VariantIndex); // match + Assert.Equal(5, results[1].AsT2()); + + Assert.Equal(0, results[2].VariantIndex); // raw + Assert.Equal(" b", results[2].AsT1().Slice); + } + + [Fact] + public void Scan_ByRuleAlias() + { + var builder = new ParserBuilder(); + + builder.CreateToken("num").Number(); + builder.CreateRule("value").Token("num"); + builder.CreateMainRule().Rule("value"); + + var input = "abc 42 def"; + + var results = builder.Build().Scan("value", input).ToList(); + + Assert.Equal(3, results.Count); + Assert.Equal("abc ", results[0].AsT1().Slice); + Assert.Equal(1, results[1].VariantIndex); + Assert.Equal(42, results[1].AsT2().GetValue()); + Assert.Equal(" def", results[2].AsT1().Slice); + } + + [Fact] + public void Scan_WithContext() + { + var builder = new ParserBuilder(); + + builder.CreateToken("number").Number(); + builder.CreateMainRule().Token("number"); + + var parser = builder.Build(); + var context = parser.CreateContext("x 7 y"); + + var results = parser.Scan(context).ToList(); + + Assert.Equal(3, results.Count); + Assert.Equal("x ", results[0].AsT1().Slice); + Assert.Equal(7, results[1].AsT2().GetValue()); + Assert.Equal(" y", results[2].AsT1().Slice); + } + + [Fact] + public void Scan_MainRuleNotSet_Throws() + { + var parser = new ParserBuilder().Build(); + + Assert.Throws(() => parser.Scan("test")); + } + + [Fact] + public void Scan_WrongContext_Throws() + { + var builder1 = new ParserBuilder(); + builder1.CreateMainRule().Literal("a"); + var parser1 = builder1.Build(); + + var builder2 = new ParserBuilder(); + builder2.CreateMainRule().Literal("b"); + var parser2 = builder2.Build(); + + var ctx = parser2.CreateContext("test"); + + Assert.Throws(() => parser1.Scan(ctx)); + } + + [Fact] + public void Scan_InvalidRuleAlias_Throws() + { + var builder = new ParserBuilder(); + builder.CreateMainRule().Literal("a"); + var parser = builder.Build(); + + Assert.Throws(() => parser.Scan("nonexistent", "test")); + } + + [Fact] + public void Scan_EmptyInput_ReturnsEmpty() + { + var builder = new ParserBuilder(); + + builder.CreateToken("number").Number(); + builder.CreateMainRule().Token("number"); + + var results = builder.Build().Scan("").ToList(); + + // Empty input yields no segments at all (nothing to scan) + Assert.Empty(results); + } + + [Fact] + public void Scan_TypedOr_FromUntypedScan_ProducesCorrectSequence() + { + var builder = new ParserBuilder(); + + builder.CreateToken("number").Number(); + builder.CreateMainRule().Token("number"); + + var input = "x 10 y 20 z"; + + var rawResults = builder.Build().Scan(input).ToList(); + + // Convert manually to check the pattern + var typed = rawResults.Select(seg => + { + if (seg.VariantIndex == 0) + return $"RAW:{seg.AsT1().Slice}"; + else + return $"NUM:{seg.AsT2().GetValue()}"; + }).ToList(); + + Assert.Equal(5, typed.Count); + Assert.Equal("RAW:x ", typed[0]); + Assert.Equal("NUM:10", typed[1]); + Assert.Equal("RAW: y ", typed[2]); + Assert.Equal("NUM:20", typed[3]); + Assert.Equal("RAW: z", typed[4]); + } + } +} diff --git a/tests/RCParsing.Tests/SplitSegmentsTests.cs b/tests/RCParsing.Tests/SplitSegmentsTests.cs new file mode 100644 index 0000000..8090a30 --- /dev/null +++ b/tests/RCParsing.Tests/SplitSegmentsTests.cs @@ -0,0 +1,107 @@ +namespace RCParsing.Tests +{ + /// + /// Tests for method. + /// + public class SplitSegmentsTests + { + [Fact] + public void SplitSegments_Basic() + { + var builder = new ParserBuilder(); + + builder.CreateToken("number").Number(); + builder.CreateMainRule().Token("number"); + + var input = "a 1 b 2 c"; + + var segments = builder.Build().SplitSegments(input).ToList(); + + Assert.Equal(3, segments.Count); + Assert.Equal("a ", segments[0].Slice); + Assert.Equal(" b ", segments[1].Slice); + Assert.Equal(" c", segments[2].Slice); + } + + [Fact] + public void SplitSegments_NoMatches_ReturnsWholeInput() + { + var builder = new ParserBuilder(); + + builder.CreateToken("number").Number(); + builder.CreateMainRule().Token("number"); + + var segments = builder.Build().SplitSegments("hello").ToList(); + + Assert.Single(segments); + Assert.Equal("hello", segments[0].Slice); + } + + [Fact] + public void SplitSegments_ByRuleAlias() + { + var builder = new ParserBuilder(); + + builder.CreateToken("n").Number(); + builder.CreateRule("val").Token("n"); + builder.CreateMainRule().Rule("val"); + + var segments = builder.Build().SplitSegments("val", "x 1 y").ToList(); + + // One match => two segments: before and after + Assert.Equal(2, segments.Count); + Assert.Equal("x ", segments[0].Slice); + Assert.Equal(" y", segments[1].Slice); + } + + [Fact] + public void SplitSegments_WithContext() + { + var builder = new ParserBuilder(); + + builder.CreateToken("number").Number(); + builder.CreateMainRule().Token("number"); + + var parser = builder.Build(); + var context = parser.CreateContext("q 8 r"); + + var segments = parser.SplitSegments(context).ToList(); + + // One match => two segments: before and after + Assert.Equal(2, segments.Count); + Assert.Equal("q ", segments[0].Slice); + Assert.Equal(" r", segments[1].Slice); + } + + [Fact] + public void SplitSegments_MainRuleNotSet_Throws() + { + var parser = new ParserBuilder().Build(); + + Assert.Throws(() => parser.SplitSegments("test")); + } + + [Fact] + public void SplitSegments_InvalidAlias_Throws() + { + var builder = new ParserBuilder(); + builder.CreateMainRule().Literal("a"); + var parser = builder.Build(); + + Assert.Throws(() => parser.SplitSegments("nonexistent", "test")); + } + + [Fact] + public void SplitSegments_EmptyInput_ReturnsEmpty() + { + var builder = new ParserBuilder(); + + builder.CreateToken("number").Number(); + builder.CreateMainRule().Token("number"); + + var segments = builder.Build().SplitSegments("").ToList(); + + Assert.Empty(segments); + } + } +} diff --git a/tests/RCParsing.Tests/Tokens/CombinatorTokensTests.cs b/tests/RCParsing.Tests/Tokens/CombinatorTokensTests.cs index 3f82f60..aaa34e1 100644 --- a/tests/RCParsing.Tests/Tokens/CombinatorTokensTests.cs +++ b/tests/RCParsing.Tests/Tokens/CombinatorTokensTests.cs @@ -85,7 +85,7 @@ public void Token_Value_String() Assert.True(result.Success); Assert.Equal((double)999, result.IntermediateValue); - Assert.True(parser.MatchesToken("value", " \"hello\" ", out var matchedLength)); + Assert.True(parser.MatchesToken("value", " \"hello\" ", out int matchedLength)); Assert.Equal(8, matchedLength); Assert.True(parser.MatchesToken("value", " 999 ", out matchedLength)); diff --git a/tests/RCParsing.Tests/Tokens/KeywordTokenTests.cs b/tests/RCParsing.Tests/Tokens/KeywordTokenTests.cs index 77e10ac..827925e 100644 --- a/tests/RCParsing.Tests/Tokens/KeywordTokenTests.cs +++ b/tests/RCParsing.Tests/Tokens/KeywordTokenTests.cs @@ -18,7 +18,7 @@ public void CaseInsensitiveMatching() var parser = builder.Build(); - Assert.True(parser.MatchesToken("1", "keyword", out var matchedLen)); + Assert.True(parser.MatchesToken("1", "keyword", out int matchedLen)); Assert.Equal(7, matchedLen); Assert.True(parser.MatchesToken("1", "KEYWORD", out matchedLen)); @@ -42,7 +42,7 @@ public void Choice_CaseInsensitiveMatching() var parser = builder.Build(); - Assert.True(parser.MatchesToken("1", "keyword", out var matchedLen)); + Assert.True(parser.MatchesToken("1", "keyword", out int matchedLen)); Assert.Equal(7, matchedLen); Assert.True(parser.MatchesToken("1", "KEYWORD", out matchedLen)); @@ -83,7 +83,7 @@ public void CaseInsensitiveMatching_UnicodeLanguages() var parser = builder.Build(); // Russian - Cyrillic - Assert.True(parser.MatchesToken("1", "привет", out var matchedLen)); + Assert.True(parser.MatchesToken("1", "привет", out int matchedLen)); Assert.Equal(6, matchedLen); // 6 characters in "Привет" Assert.True(parser.MatchesToken("1", "ПРИВЕТ", out matchedLen)); Assert.Equal(6, matchedLen); @@ -134,7 +134,7 @@ public void Choice_CaseInsensitiveMatching_UnicodeLanguages() var parser = builder.Build(); // Mixed languages - token 1 - Assert.True(parser.MatchesToken("1", "привет", out var matchedLen)); + Assert.True(parser.MatchesToken("1", "привет", out int matchedLen)); Assert.Equal(6, matchedLen); Assert.True(parser.MatchesToken("1", "ПРИВЕТ", out matchedLen)); Assert.Equal(6, matchedLen);