From 849c676b0314b4a956f7a253e2e58867b3bf169a Mon Sep 17 00:00:00 2001 From: Marcos Passos Date: Sat, 15 Aug 2026 16:32:20 -0300 Subject: [PATCH 1/2] Fix hash and equals contract and align the simulator with Java Values that compare equal now hash equally for ATN configs, prediction contexts and bit sets, so sets and maps stop degrading into linear scans. MurmurHash matches Java bit for bit, using PHP's native murmur3a when available with a pure PHP fallback. ATNState::hashCode() returns the state number as in Java and C#. The anonymous Equivalence class in ATNConfigSet is now a named class so the set can be serialized. setTrace(false) clears the tracer so isTrace() reports the right value. The lexer decodes its input once instead of on every read, and the hot loops in both simulators avoid repeated method calls. --- src/Atn/ATNConfig.php | 30 ++- src/Atn/ATNConfigSet.php | 94 ++++---- src/Atn/ATNDeserializationOptions.php | 12 +- src/Atn/ATNSimulator.php | 9 +- src/Atn/Actions/LexerAction.php | 2 +- src/Atn/Actions/LexerChannelAction.php | 4 +- src/Atn/Actions/LexerCustomAction.php | 9 +- src/Atn/Actions/LexerIndexedCustomAction.php | 9 +- src/Atn/Actions/LexerModeAction.php | 4 +- src/Atn/Actions/LexerMoreAction.php | 10 +- src/Atn/Actions/LexerPopModeAction.php | 10 +- src/Atn/Actions/LexerPushModeAction.php | 4 +- src/Atn/Actions/LexerSkipAction.php | 10 +- src/Atn/Actions/LexerTypeAction.php | 4 +- src/Atn/AltAndContextEquivalence.php | 57 +++++ src/Atn/ConfigEquivalence.php | 61 +++++ src/Atn/LexerATNConfig.php | 17 +- src/Atn/LexerATNSimulator.php | 41 +++- src/Atn/LexerActionExecutor.php | 22 +- src/Atn/ParserATNSimulator.php | 77 +++++-- src/Atn/PredictionMode.php | 35 +-- src/Atn/SemanticContexts/AndOperator.php | 15 +- src/Atn/SemanticContexts/OrOperator.php | 15 +- .../SemanticContexts/PrecedencePredicate.php | 3 +- src/Atn/SemanticContexts/Predicate.php | 11 +- src/Atn/SemanticContexts/SemanticContext.php | 15 +- src/Atn/States/ATNState.php | 11 +- src/Atn/Transitions/AtomTransition.php | 2 +- src/Atn/Transitions/RangeTransition.php | 2 +- src/Atn/Transitions/Transition.php | 11 + src/BufferedTokenStream.php | 8 +- src/CommonToken.php | 6 +- src/CommonTokenFactory.php | 6 +- src/Comparison/Equality.php | 5 +- src/Comparison/MurmurHash.php | 208 ++++++++++++++++++ src/Dfa/DFAState.php | 8 +- src/InputStream.php | 125 +++++++++-- src/Interval.php | 19 +- src/IntervalSet.php | 45 +++- src/LL1Analyzer.php | 11 +- src/Lexer.php | 6 +- src/Parser.php | 54 +++-- src/ParserTraceListener.php | 12 +- .../ArrayPredictionContext.php | 21 +- .../EmptyPredictionContext.php | 7 +- src/PredictionContexts/PredictionContext.php | 83 +++++-- .../SingletonPredictionContext.php | 35 ++- src/Recognizer.php | 17 +- src/RuleContext.php | 6 +- src/StdoutMessageLogger.php | 2 +- src/Tree/ParseTreeWalker.php | 6 +- src/Utils/BitSet.php | 43 +++- src/Utils/Map.php | 2 + src/Utils/Pair.php | 7 +- src/Utils/Set.php | 35 +-- src/VocabularyImpl.php | 6 +- 56 files changed, 1053 insertions(+), 336 deletions(-) create mode 100644 src/Atn/AltAndContextEquivalence.php create mode 100644 src/Atn/ConfigEquivalence.php create mode 100644 src/Comparison/MurmurHash.php diff --git a/src/Atn/ATNConfig.php b/src/Atn/ATNConfig.php index 7e9c9b9..a0eba41 100644 --- a/src/Atn/ATNConfig.php +++ b/src/Atn/ATNConfig.php @@ -8,7 +8,7 @@ use Antlr\Antlr4\Runtime\Atn\States\ATNState; use Antlr\Antlr4\Runtime\Comparison\Equality; use Antlr\Antlr4\Runtime\Comparison\Hashable; -use Antlr\Antlr4\Runtime\Comparison\Hasher; +use Antlr\Antlr4\Runtime\Comparison\MurmurHash; use Antlr\Antlr4\Runtime\PredictionContexts\PredictionContext; /** @@ -127,22 +127,25 @@ public function equals(object $other): bool return true; } + // Field order follows Java's, cheapest discriminator first. Comparing the + // state by number rather than through `Equality::equals()` also avoids a + // full object comparison on the hottest path in the simulator. return $other instanceof self + && $this->state->stateNumber === $other->state->stateNumber && $this->alt === $other->alt - && $this->isPrecedenceFilterSuppressed() === $other->isPrecedenceFilterSuppressed() + && Equality::equals($this->context, $other->context) && $this->semanticContext->equals($other->semanticContext) - && Equality::equals($this->state, $other->state) - && Equality::equals($this->context, $other->context); + && $this->isPrecedenceFilterSuppressed() === $other->isPrecedenceFilterSuppressed(); } public function hashCode(): int { - return Hasher::hash( + return MurmurHash::hash([ $this->state->stateNumber, $this->alt, $this->context, $this->semanticContext, - ); + ], 7); } public function toString(bool $showAlt): string @@ -172,15 +175,10 @@ public function toString(bool $showAlt): string public function __toString(): string { - return \sprintf( - '(%s,%d%s%s%s)', - $this->state, - $this->alt, - $this->context !== null ? ',[' . $this->context . ']' : '', - $this->semanticContext->equals(SemanticContext::none()) - ? '' - : ',' . $this->semanticContext, - $this->reachesIntoOuterContext > 0 ? ',up=' . $this->reachesIntoOuterContext : '', - ); + // Java's no-arg `toString()` is `toString(null, true)`. Duplicating the + // formatting here is what let the two drift: this copy printed the raw + // `reachesIntoOuterContext` field, which carries the + // SUPPRESS_PRECEDENCE_FILTER bit, instead of `getOuterContextDepth()`. + return $this->toString(true); } } diff --git a/src/Atn/ATNConfigSet.php b/src/Atn/ATNConfigSet.php index 319cd6b..805e61e 100644 --- a/src/Atn/ATNConfigSet.php +++ b/src/Atn/ATNConfigSet.php @@ -5,10 +5,9 @@ namespace Antlr\Antlr4\Runtime\Atn; use Antlr\Antlr4\Runtime\Atn\SemanticContexts\SemanticContext; +use Antlr\Antlr4\Runtime\Atn\States\ATNState; use Antlr\Antlr4\Runtime\Comparison\Equality; -use Antlr\Antlr4\Runtime\Comparison\Equivalence; use Antlr\Antlr4\Runtime\Comparison\Hashable; -use Antlr\Antlr4\Runtime\Comparison\Hasher; use Antlr\Antlr4\Runtime\PredictionContexts\PredictionContext; use Antlr\Antlr4\Runtime\Utils\BitSet; use Antlr\Antlr4\Runtime\Utils\DoubleKeyMap; @@ -33,6 +32,8 @@ class ATNConfigSet implements Hashable /** * All configs but hashed by (s, i, _, pi) not including context. Wiped out * when we go readonly as this set becomes a DFA state. + * + * @var Set|null */ public ?Set $configLookup = null; @@ -82,32 +83,7 @@ public function __construct(bool $fullCtx = true) * not including context. Wiped out when we go readonly as this se * becomes a DFA state. */ - $this->configLookup = new Set(new class implements Equivalence { - public function equivalent(Hashable $left, Hashable $right): bool - { - if ($left === $right) { - return true; - } - - if (!$left instanceof ATNConfig || !$right instanceof ATNConfig) { - return false; - } - - return $left->alt === $right->alt - && $left->semanticContext->equals($right->semanticContext) - && Equality::equals($left->state, $right->state); - } - - public function hash(Hashable $value): int - { - return $value->hashCode(); - } - - public function equals(object $other): bool - { - return $other instanceof self; - } - }); + $this->configLookup = new Set(new ConfigEquivalence()); $this->fullCtx = $fullCtx; } @@ -121,6 +97,8 @@ public function equals(object $other): bool * This method updates {@see ATNConfigSet::$dipsIntoOuterContext} and * {@see ATNConfigSet::$hasSemanticContext} when necessary. * + * @param DoubleKeyMap|null $mergeCache + * * @throws \InvalidArgumentException */ public function add(ATNConfig $config, ?DoubleKeyMap $mergeCache = null): bool @@ -140,7 +118,10 @@ public function add(ATNConfig $config, ?DoubleKeyMap $mergeCache = null): bool /** @var ATNConfig $existing */ $existing = $this->configLookup->getOrAdd($config); - if ($existing->equals($config)) { + // Identity, not equality: `getOrAdd` returns the argument only when it was + // genuinely new. Comparing with `equals()` would also take this branch for + // a distinct-but-equal configuration and append a duplicate to `$configs`. + if ($existing === $config) { $this->cachedHashCode = null; $this->configs[] = $config; // track order here @@ -186,8 +167,12 @@ public function elements(): array return $this->configs; } + /** + * @return Set + */ public function getStates(): Set { + /** @var Set $states */ $states = new Set(); foreach ($this->configs as $config) { $states->add($config->state); @@ -268,25 +253,43 @@ public function equals(object $other): bool return false; } - return $this->fullCtx === $other->fullCtx + // Field order and comparison kinds follow Java. In particular + // `conflictingAlts` is compared by **reference** there + // (`this.conflictingAlts == other.conflictingAlts`), not by value: + // comparing it by value merged config sets that Java keeps distinct. + return Equality::equals($this->configs, $other->configs) + && $this->fullCtx === $other->fullCtx && $this->uniqueAlt === $other->uniqueAlt + && $this->conflictingAlts === $other->conflictingAlts && $this->hasSemanticContext === $other->hasSemanticContext - && $this->dipsIntoOuterContext === $other->dipsIntoOuterContext - && Equality::equals($this->configs, $other->configs) - && Equality::equals($this->conflictingAlts, $other->conflictingAlts); + && $this->dipsIntoOuterContext === $other->dipsIntoOuterContext; } public function hashCode(): int { + // Only a read-only set may cache: while the set is still mutable its + // configurations keep having their contexts merged underneath it. if (!$this->isReadOnly()) { - return Hasher::hash($this->configs); + return $this->computeHashCode(); } - if ($this->cachedHashCode === null) { - $this->cachedHashCode = Hasher::hash($this->configs); + return $this->cachedHashCode ??= $this->computeHashCode(); + } + + /** + * Java's `ATNConfigSet.hashCode()` is `configs.hashCode()` — that is, + * `AbstractList.hashCode()`: a 31-based accumulation over the elements, + * wrapping at 32 bits. + */ + private function computeHashCode(): int + { + $hash = 1; + + foreach ($this->configs as $config) { + $hash = (31 * $hash + $config->hashCode()) & 0xFFFFFFFF; } - return $this->cachedHashCode; + return $hash >= 0x80000000 ? $hash - 0x100000000 : $hash; } public function getLength(): int @@ -313,6 +316,9 @@ public function containsFast(ATNConfig $item): bool return $this->contains($item); } + /** + * @return \Iterator + */ public function getIterator(): \Iterator { return new \ArrayIterator($this->configs); @@ -325,8 +331,14 @@ public function clear(): void } $this->configs = []; - $this->cachedHashCode = -1; - $this->configLookup = new Set(); + // `null` is the "not computed" sentinel; `-1` was a valid cached hash and + // pinned every cleared set to the same value. (Java's sentinel *is* -1, + // which is why the port picked it up.) + $this->cachedHashCode = null; + // Clear in place rather than replacing the set: a fresh `Set` would fall + // back to the default equivalence and silently drop `ConfigEquivalence` + // (or, in `OrderedATNConfigSet`, its own), disabling context merging. + $this->configLookup?->clear(); } public function isReadOnly(): bool @@ -358,7 +370,9 @@ public function __toString(): string return \sprintf( '[%s]%s%s%s%s', \implode(', ', $this->configs), - $this->hasSemanticContext ? ',hasSemanticContext=' . $this->hasSemanticContext : '', + // Java appends the boolean itself, which prints `true`; interpolating + // a PHP bool printed `1`. + $this->hasSemanticContext ? ',hasSemanticContext=true' : '', $this->uniqueAlt !== ATN::INVALID_ALT_NUMBER ? ',uniqueAlt=' . $this->uniqueAlt : '', $this->conflictingAlts !== null ? ',conflictingAlts=' . $this->conflictingAlts : '', $this->dipsIntoOuterContext ? ',dipsIntoOuterContext' : '', diff --git a/src/Atn/ATNDeserializationOptions.php b/src/Atn/ATNDeserializationOptions.php index d04724b..ac5e3fd 100644 --- a/src/Atn/ATNDeserializationOptions.php +++ b/src/Atn/ATNDeserializationOptions.php @@ -12,16 +12,16 @@ final class ATNDeserializationOptions private bool $generateRuleBypassTransitions; + private static ?self $defaultOptions = null; + public static function defaultOptions(): ATNDeserializationOptions { - static $defaultOptions; - - if ($defaultOptions === null) { - $defaultOptions = new ATNDeserializationOptions(); - $defaultOptions->readOnly = true; + if (self::$defaultOptions === null) { + self::$defaultOptions = new ATNDeserializationOptions(); + self::$defaultOptions->readOnly = true; } - return $defaultOptions; + return self::$defaultOptions; } public function __construct(?ATNDeserializationOptions $options = null) diff --git a/src/Atn/ATNSimulator.php b/src/Atn/ATNSimulator.php index 5ecce88..c23e359 100644 --- a/src/Atn/ATNSimulator.php +++ b/src/Atn/ATNSimulator.php @@ -62,11 +62,11 @@ public function __construct(ATN $atn, PredictionContextCache $sharedContextCache $this->sharedContextCache = $sharedContextCache; } + private static ?DFAState $error = null; + public static function error(): DFAState { - static $error; - - return $error ?? ($error = new DFAState(new ATNConfigSet(), 0x7FFFFFFF)); + return self::$error ??= new DFAState(new ATNConfigSet(), 0x7FFFFFFF); } abstract public function reset(): void; @@ -92,7 +92,8 @@ public function getSharedContextCache(): PredictionContextCache public function getCachedContext(PredictionContext $context): PredictionContext { - $visited = []; + /** @var \SplObjectStorage $visited */ + $visited = new \SplObjectStorage(); return PredictionContext::getCachedPredictionContext( $context, diff --git a/src/Atn/Actions/LexerAction.php b/src/Atn/Actions/LexerAction.php index a8ab492..0b4da8e 100644 --- a/src/Atn/Actions/LexerAction.php +++ b/src/Atn/Actions/LexerAction.php @@ -14,7 +14,7 @@ * * @author Sam Harwell */ -interface LexerAction extends Hashable +interface LexerAction extends Hashable, \Stringable { /** * Gets the serialization type of the lexer action. diff --git a/src/Atn/Actions/LexerChannelAction.php b/src/Atn/Actions/LexerChannelAction.php index fe05d3c..57f8b9a 100644 --- a/src/Atn/Actions/LexerChannelAction.php +++ b/src/Atn/Actions/LexerChannelAction.php @@ -4,7 +4,7 @@ namespace Antlr\Antlr4\Runtime\Atn\Actions; -use Antlr\Antlr4\Runtime\Comparison\Hasher; +use Antlr\Antlr4\Runtime\Comparison\MurmurHash; use Antlr\Antlr4\Runtime\Lexer; /** @@ -70,7 +70,7 @@ public function execute(Lexer $lexer): void public function hashCode(): int { - return Hasher::hash($this->getActionType(), $this->channel); + return MurmurHash::hash([$this->getActionType(), $this->channel]); } public function equals(object $other): bool diff --git a/src/Atn/Actions/LexerCustomAction.php b/src/Atn/Actions/LexerCustomAction.php index 93a2f7b..c364cd1 100644 --- a/src/Atn/Actions/LexerCustomAction.php +++ b/src/Atn/Actions/LexerCustomAction.php @@ -4,7 +4,7 @@ namespace Antlr\Antlr4\Runtime\Atn\Actions; -use Antlr\Antlr4\Runtime\Comparison\Hasher; +use Antlr\Antlr4\Runtime\Comparison\MurmurHash; use Antlr\Antlr4\Runtime\Lexer; /** @@ -97,9 +97,14 @@ public function execute(Lexer $lexer): void $lexer->action(null, $this->ruleIndex, $this->actionIndex); } + public function __toString(): string + { + return \sprintf('custom(%d:%d)', $this->ruleIndex, $this->actionIndex); + } + public function hashCode(): int { - return Hasher::hash($this->getActionType(), $this->ruleIndex, $this->actionIndex); + return MurmurHash::hash([$this->getActionType(), $this->ruleIndex, $this->actionIndex]); } public function equals(object $other): bool diff --git a/src/Atn/Actions/LexerIndexedCustomAction.php b/src/Atn/Actions/LexerIndexedCustomAction.php index 36c8f97..249306e 100644 --- a/src/Atn/Actions/LexerIndexedCustomAction.php +++ b/src/Atn/Actions/LexerIndexedCustomAction.php @@ -4,7 +4,7 @@ namespace Antlr\Antlr4\Runtime\Atn\Actions; -use Antlr\Antlr4\Runtime\Comparison\Hasher; +use Antlr\Antlr4\Runtime\Comparison\MurmurHash; use Antlr\Antlr4\Runtime\Lexer; /** @@ -102,9 +102,14 @@ public function execute(Lexer $lexer): void $this->action->execute($lexer); } + public function __toString(): string + { + return \sprintf('%s@%d', $this->action, $this->offset); + } + public function hashCode(): int { - return Hasher::hash($this->getActionType(), $this->offset, $this->action); + return MurmurHash::hash([$this->offset, $this->action]); } public function equals(object $other): bool diff --git a/src/Atn/Actions/LexerModeAction.php b/src/Atn/Actions/LexerModeAction.php index 7377ebd..2f1da9b 100644 --- a/src/Atn/Actions/LexerModeAction.php +++ b/src/Atn/Actions/LexerModeAction.php @@ -4,7 +4,7 @@ namespace Antlr\Antlr4\Runtime\Atn\Actions; -use Antlr\Antlr4\Runtime\Comparison\Hasher; +use Antlr\Antlr4\Runtime\Comparison\MurmurHash; use Antlr\Antlr4\Runtime\Lexer; final class LexerModeAction implements LexerAction @@ -64,7 +64,7 @@ public function execute(Lexer $lexer): void public function hashCode(): int { - return Hasher::hash($this->getActionType(), $this->mode); + return MurmurHash::hash([$this->getActionType(), $this->mode]); } public function equals(object $other): bool diff --git a/src/Atn/Actions/LexerMoreAction.php b/src/Atn/Actions/LexerMoreAction.php index e1c5a03..86c623c 100644 --- a/src/Atn/Actions/LexerMoreAction.php +++ b/src/Atn/Actions/LexerMoreAction.php @@ -4,7 +4,7 @@ namespace Antlr\Antlr4\Runtime\Atn\Actions; -use Antlr\Antlr4\Runtime\Comparison\Hasher; +use Antlr\Antlr4\Runtime\Comparison\MurmurHash; use Antlr\Antlr4\Runtime\Lexer; /** @@ -20,11 +20,11 @@ final class LexerMoreAction implements LexerAction /** * Provides a singleton instance of this parameterless lexer action. */ + private static ?self $instance = null; + public static function instance(): self { - static $instance; - - return $instance ??= new self(); + return self::$instance ??= new self(); } /** @@ -59,7 +59,7 @@ public function execute(Lexer $lexer): void public function hashCode(): int { - return Hasher::hash($this->getActionType()); + return MurmurHash::hash([$this->getActionType()]); } public function equals(object $other): bool diff --git a/src/Atn/Actions/LexerPopModeAction.php b/src/Atn/Actions/LexerPopModeAction.php index b5de30b..45d1b03 100644 --- a/src/Atn/Actions/LexerPopModeAction.php +++ b/src/Atn/Actions/LexerPopModeAction.php @@ -4,7 +4,7 @@ namespace Antlr\Antlr4\Runtime\Atn\Actions; -use Antlr\Antlr4\Runtime\Comparison\Hasher; +use Antlr\Antlr4\Runtime\Comparison\MurmurHash; use Antlr\Antlr4\Runtime\Lexer; /** @@ -20,11 +20,11 @@ final class LexerPopModeAction implements LexerAction /** * Provides a singleton instance of this parameterless lexer action. */ + private static ?self $instance = null; + public static function instance(): self { - static $instance; - - return $instance ??= new self(); + return self::$instance ??= new self(); } /** @@ -59,7 +59,7 @@ public function execute(Lexer $lexer): void public function hashCode(): int { - return Hasher::hash($this->getActionType()); + return MurmurHash::hash([$this->getActionType()]); } public function equals(object $other): bool diff --git a/src/Atn/Actions/LexerPushModeAction.php b/src/Atn/Actions/LexerPushModeAction.php index 5b45318..1eb1104 100644 --- a/src/Atn/Actions/LexerPushModeAction.php +++ b/src/Atn/Actions/LexerPushModeAction.php @@ -4,7 +4,7 @@ namespace Antlr\Antlr4\Runtime\Atn\Actions; -use Antlr\Antlr4\Runtime\Comparison\Hasher; +use Antlr\Antlr4\Runtime\Comparison\MurmurHash; use Antlr\Antlr4\Runtime\Lexer; /** @@ -65,7 +65,7 @@ public function execute(Lexer $lexer): void public function hashCode(): int { - return Hasher::hash($this->getActionType(), $this->mode); + return MurmurHash::hash([$this->getActionType(), $this->mode]); } public function equals(object $other): bool diff --git a/src/Atn/Actions/LexerSkipAction.php b/src/Atn/Actions/LexerSkipAction.php index fbb9a8a..7e276b5 100644 --- a/src/Atn/Actions/LexerSkipAction.php +++ b/src/Atn/Actions/LexerSkipAction.php @@ -4,7 +4,7 @@ namespace Antlr\Antlr4\Runtime\Atn\Actions; -use Antlr\Antlr4\Runtime\Comparison\Hasher; +use Antlr\Antlr4\Runtime\Comparison\MurmurHash; use Antlr\Antlr4\Runtime\Lexer; /** @@ -20,11 +20,11 @@ final class LexerSkipAction implements LexerAction /** * Provides a singleton instance of this parameterless lexer action. */ + private static ?self $instance = null; + public static function instance(): self { - static $instance; - - return $instance ??= new self(); + return self::$instance ??= new self(); } /** @@ -59,7 +59,7 @@ public function execute(Lexer $lexer): void public function hashCode(): int { - return Hasher::hash($this->getActionType()); + return MurmurHash::hash([$this->getActionType()]); } public function equals(object $other): bool diff --git a/src/Atn/Actions/LexerTypeAction.php b/src/Atn/Actions/LexerTypeAction.php index ab046d9..aaa6ba4 100644 --- a/src/Atn/Actions/LexerTypeAction.php +++ b/src/Atn/Actions/LexerTypeAction.php @@ -4,7 +4,7 @@ namespace Antlr\Antlr4\Runtime\Atn\Actions; -use Antlr\Antlr4\Runtime\Comparison\Hasher; +use Antlr\Antlr4\Runtime\Comparison\MurmurHash; use Antlr\Antlr4\Runtime\Lexer; /** @@ -70,7 +70,7 @@ public function execute(Lexer $lexer): void public function hashCode(): int { - return Hasher::hash($this->getActionType(), $this->type); + return MurmurHash::hash([$this->getActionType(), $this->type]); } public function equals(object $other): bool diff --git a/src/Atn/AltAndContextEquivalence.php b/src/Atn/AltAndContextEquivalence.php new file mode 100644 index 0000000..31b98f0 --- /dev/null +++ b/src/Atn/AltAndContextEquivalence.php @@ -0,0 +1,57 @@ +state->stateNumber === $right->state->stateNumber + && Equality::equals($left->context, $right->context); + } + + public function hash(Hashable $value): int + { + if (!$value instanceof ATNConfig) { + throw new \InvalidArgumentException('Unsupported value.'); + } + + // The hash is a function of the state number and the context only. + return MurmurHash::hash([$value->state->stateNumber, $value->context], 7); + } + + public function equals(object $other): bool + { + return $other instanceof self; + } +} diff --git a/src/Atn/ConfigEquivalence.php b/src/Atn/ConfigEquivalence.php new file mode 100644 index 0000000..c023ffa --- /dev/null +++ b/src/Atn/ConfigEquivalence.php @@ -0,0 +1,61 @@ +state->stateNumber === $right->state->stateNumber + && $left->alt === $right->alt + && $left->semanticContext->equals($right->semanticContext); + } + + public function hash(Hashable $value): int + { + if (!$value instanceof ATNConfig) { + return 0; + } + + $hash = 7; + $hash = 31 * $hash + $value->state->stateNumber; + $hash = 31 * $hash + $value->alt; + + return 31 * $hash + $value->semanticContext->hashCode(); + } + + public function equals(object $other): bool + { + return $other instanceof self; + } +} diff --git a/src/Atn/LexerATNConfig.php b/src/Atn/LexerATNConfig.php index 0b353c1..39a0bac 100644 --- a/src/Atn/LexerATNConfig.php +++ b/src/Atn/LexerATNConfig.php @@ -7,7 +7,7 @@ use Antlr\Antlr4\Runtime\Atn\States\ATNState; use Antlr\Antlr4\Runtime\Atn\States\DecisionState; use Antlr\Antlr4\Runtime\Comparison\Equality; -use Antlr\Antlr4\Runtime\Comparison\Hasher; +use Antlr\Antlr4\Runtime\Comparison\MurmurHash; use Antlr\Antlr4\Runtime\PredictionContexts\PredictionContext; final class LexerATNConfig extends ATNConfig @@ -43,14 +43,14 @@ public function isPassedThroughNonGreedyDecision(): bool public function hashCode(): int { - return Hasher::hash( + return MurmurHash::hash([ $this->state->stateNumber, $this->alt, $this->context, $this->semanticContext, - $this->passedThroughNonGreedyDecision, + $this->passedThroughNonGreedyDecision ? 1 : 0, $this->lexerActionExecutor, - ); + ], 7); } public function equals(object $other): bool @@ -63,15 +63,18 @@ public function equals(object $other): bool return false; } - if (!parent::equals($other)) { + // Java checks the two cheap lexer-specific fields *before* delegating to + // the parent, which is the expensive comparison (context and semantic + // context). Doing it the other way round paid that cost on every miss. + if ($this->passedThroughNonGreedyDecision !== $other->passedThroughNonGreedyDecision) { return false; } - if ($this->passedThroughNonGreedyDecision !== $other->passedThroughNonGreedyDecision) { + if (!Equality::equals($this->lexerActionExecutor, $other->lexerActionExecutor)) { return false; } - return Equality::equals($this->lexerActionExecutor, $other->lexerActionExecutor); + return parent::equals($other); } private static function checkNonGreedyDecision(LexerATNConfig $source, ATNState $target): bool diff --git a/src/Atn/LexerATNSimulator.php b/src/Atn/LexerATNSimulator.php index 0a64035..38857cc 100644 --- a/src/Atn/LexerATNSimulator.php +++ b/src/Atn/LexerATNSimulator.php @@ -51,6 +51,12 @@ class LexerATNSimulator extends ATNSimulator /** @var array */ public array $decisionToDFA = []; + /** + * Counts `match()` calls. Java keeps the same counter for debugging; it is + * not read by the algorithm. + */ + private static int $matchCalls = 0; + protected int $mode = Lexer::DEFAULT_MODE; /** @@ -107,13 +113,7 @@ public function setCharPositionInLine(int $charPositionInLine): void */ public function match(CharStream $input, int $mode): int { - static $match_calls; - - if ($match_calls === null) { - $match_calls = 0; - } - - $match_calls++; + self::$matchCalls++; $this->mode = $mode; $mark = $input->mark(); @@ -175,6 +175,7 @@ protected function execATN(CharStream $input, DFAState $ds0): int $t = $input->LA(1); $s = $ds0; // s is current/from DFA state + $error = ATNSimulator::error(); while (true) { // As we move src->trg, src->trg, we keep track of the previous trg to @@ -195,13 +196,21 @@ protected function execATN(CharStream $input, DFAState $ds0): int // A character will take us back to an existing DFA state // that already has lots of edges out of it. e.g., .* in comments. // print("Target for:" + str(s) + " and:" + str(t)) - $target = $this->getExistingTargetState($s, $t); + // `getExistingTargetState()` and `consume()` are inlined here, and + // only here. This loop runs once per input character — it was the + // hottest path in the runtime — and each call PHP does not make is + // worth more than the readability. Both methods remain for callers + // and subclasses; keep them and this copy in step. + $edges = $s->edges; + $target = $edges !== null && $t >= self::MIN_DFA_EDGE && $t <= self::MAX_DFA_EDGE + ? $edges[$t - self::MIN_DFA_EDGE] ?? null + : null; if ($target === null) { $target = $this->computeTargetState($input, $s, $t); } - if ($target === ATNSimulator::error()) { + if ($target === $error) { break; } @@ -210,11 +219,21 @@ protected function execATN(CharStream $input, DFAState $ds0): int // position accurately reflect the state of the interpreter at the // end of the token. if ($t !== Token::EOF) { - $this->consume($input); + // Inline of `consume()`: the character it would re-read with + // `LA(1)` is already in `$t`. + if ($t === self::NEW_LINE_CODE) { + $this->line++; + $this->charPositionInLine = 0; + } else { + $this->charPositionInLine++; + } + + $input->consume(); } if ($target->isAcceptState) { $this->captureSimState($this->prevAccept, $input, $target); + if ($t === Token::EOF) { break; } @@ -506,7 +525,7 @@ protected function getEpsilonTarget( ): ?LexerATNConfig { $cfg = null; - switch ($t->getSerializationType()) { + switch ($t->serializationType) { case Transition::RULE: if (!$t instanceof RuleTransition) { throw new \LogicException('Unexpected transition type.'); diff --git a/src/Atn/LexerActionExecutor.php b/src/Atn/LexerActionExecutor.php index e6e4e57..86b799f 100644 --- a/src/Atn/LexerActionExecutor.php +++ b/src/Atn/LexerActionExecutor.php @@ -8,8 +8,8 @@ use Antlr\Antlr4\Runtime\Atn\Actions\LexerIndexedCustomAction; use Antlr\Antlr4\Runtime\CharStream; use Antlr\Antlr4\Runtime\Comparison\Equality; -use Antlr\Antlr4\Runtime\Comparison\Equatable; -use Antlr\Antlr4\Runtime\Comparison\Hasher; +use Antlr\Antlr4\Runtime\Comparison\Hashable; +use Antlr\Antlr4\Runtime\Comparison\MurmurHash; use Antlr\Antlr4\Runtime\Lexer; /** @@ -22,7 +22,7 @@ * * @author Sam Harwell */ -final class LexerActionExecutor implements Equatable +final class LexerActionExecutor implements Hashable { /** @var array */ private array $lexerActions; @@ -187,11 +187,9 @@ public function execute(Lexer $lexer, CharStream $input, int $startIndex): void public function hashCode(): int { - if ($this->cachedHashCode === null) { - $this->cachedHashCode = Hasher::hash($this->lexerActions); - } - - return $this->cachedHashCode; + // Java computes this once in the constructor: + // `initialize()`, one `update()` per action, `finish(hash, actions.length)`. + return $this->cachedHashCode ??= MurmurHash::hash($this->lexerActions); } public function equals(object $other): bool @@ -207,9 +205,11 @@ public function equals(object $other): bool public function __toString(): string { - return \sprintf( - 'LexerActionExecutor[%s]', - \implode(', ', \array_map('\strval', $this->lexerActions)), + $actions = \array_map( + static fn (LexerAction $action): string => (string) $action, + $this->lexerActions, ); + + return \sprintf('LexerActionExecutor[%s]', \implode(', ', $actions)); } } diff --git a/src/Atn/ParserATNSimulator.php b/src/Atn/ParserATNSimulator.php index 57fb55a..10d6a76 100644 --- a/src/Atn/ParserATNSimulator.php +++ b/src/Atn/ParserATNSimulator.php @@ -250,6 +250,8 @@ final class ParserATNSimulator extends ATNSimulator * This maps graphs a and b to merged result c. (a,b)→c. We can avoid * the merge if we ever see a and b again. Note that (b,a)→c should * also be examined during cache lookup. + * + * @var DoubleKeyMap|null */ protected ?DoubleKeyMap $mergeCache = null; @@ -372,9 +374,7 @@ public function adaptivePredict(TokenStream $input, int $decision, ParserRuleCon } } - $alt = $this->execATN($dfa, $s0, $input, $index, $outerContext); - - return $alt ?? 0; + return $this->execATN($dfa, $s0, $input, $index, $outerContext); } finally { $this->mergeCache = null; // wack cache after each prediction $this->dfa = null; @@ -422,7 +422,7 @@ public function execATN( TokenStream $input, int $startIndex, ParserRuleContext $outerContext, - ): ?int { + ): int { if (self::$traceAtnSimulation) { $this->logger->debug( 'execATN decision {decision}, DFA state {state}, LA(1)=={token} line {line}:{pos}', @@ -889,6 +889,7 @@ protected function computeReachSet(ATNConfigSet $closure, int $t, bool $fullCtx) // operation on the intermediate set to compute its initial value. if ($reach === null) { $reach = new ATNConfigSet($fullCtx); + /** @var Set $closureBusy */ $closureBusy = new Set(); $treatEofAsEpsilon = $t === Token::EOF; @@ -1014,6 +1015,7 @@ protected function computeStartState(ATNState $p, RuleContext $ctx, bool $fullCt foreach ($p->getTransitions() as $i => $t) { $c = new ATNConfig(null, $t->target, $initialContext, null, $i + 1); + /** @var Set $closureBusy */ $closureBusy = new Set(); $this->closure($c, $configs, $closureBusy, true, $fullCtx, false); @@ -1478,6 +1480,9 @@ protected function evalSemanticContextOne( * waste to pursue the closure. Might have to advance when we do * ambig detection thought :( */ + /** + * @param Set $closureBusy + */ protected function closure( ATNConfig $config, ATNConfigSet $configs, @@ -1503,6 +1508,9 @@ protected function closure( } } + /** + * @param Set $closureBusy + */ protected function closureCheckingStopState( ATNConfig $config, ATNConfigSet $configs, @@ -1585,6 +1593,9 @@ protected function closureCheckingStopState( /** * Do the actual work of walking epsilon edges. */ + /** + * @param Set $closureBusy + */ protected function closure_( ATNConfig $config, ATNConfigSet $configs, @@ -1597,25 +1608,30 @@ protected function closure_( $p = $config->state; // optimization - if (!$p->onlyHasEpsilonTransitions()) { + // The accessors are read directly here: this runs hundreds of thousands + // of times per parse and each getter is a call PHP cannot inline. + if (!$p->epsilonOnlyTransitions) { // make sure to not return here, because EOF transitions can act as // both epsilon transitions and non-epsilon transitions. $configs->add($config, $this->mergeCache); } - foreach ($p->getTransitions() as $i => $t) { + $isRuleStop = $p instanceof RuleStopState; + $atDepthZero = $depth === 0; + + foreach ($p->transitions as $i => $t) { if ($i === 0 && $this->canDropLoopEntryEdgeInLeftRecursiveRule($config)) { continue; } $continueCollecting = $collectPredicates && !$t instanceof ActionTransition; - $c = $this->getEpsilonTarget($config, $t, $continueCollecting, $depth === 0, $fullCtx, $treatEofAsEpsilon); + $c = $this->getEpsilonTarget($config, $t, $continueCollecting, $atDepthZero, $fullCtx, $treatEofAsEpsilon); if ($c !== null) { $newDepth = $depth; - if ($config->state instanceof RuleStopState) { + if ($isRuleStop) { if ($fullCtx) { throw new \LogicException('Unexpected error.'); } @@ -1660,15 +1676,32 @@ protected function closure_( } } - $this->closureCheckingStopState( - $c, - $configs, - $closureBusy, - $continueCollecting, - $fullCtx, - $newDepth, - $treatEofAsEpsilon, - ); + // `closureCheckingStopState()` forwards straight to `closure_()` + // unless the state is a rule stop, so for the common case the + // frame is skipped. It also emits the ATN trace line, hence the + // guard: with tracing on, the original path is always taken and + // the trace stays byte-identical. + if (self::$traceAtnSimulation || $c->state instanceof RuleStopState) { + $this->closureCheckingStopState( + $c, + $configs, + $closureBusy, + $continueCollecting, + $fullCtx, + $newDepth, + $treatEofAsEpsilon, + ); + } else { + $this->closure_( + $c, + $configs, + $closureBusy, + $continueCollecting, + $fullCtx, + $newDepth, + $treatEofAsEpsilon, + ); + } } } } @@ -1773,12 +1806,18 @@ protected function canDropLoopEntryEdgeInLeftRecursiveRule(ATNConfig $config): b * Are we the special loop entry/exit state? or SLL wildcard */ + // Cheapest discriminator first: this is called for the first transition + // out of every state, and almost none of them are loop entries. + if (!$p instanceof StarLoopEntryState) { + return false; + } + if ($config->context === null) { throw new \LogicException('Prediction context cannot be null.'); } if ($p->getStateType() !== ATNState::STAR_LOOP_ENTRY - || ($p instanceof StarLoopEntryState && !$p->isPrecedenceDecision) + || !$p->isPrecedenceDecision || $config->context->isEmpty() || $config->context->hasEmptyPath()) { return false; @@ -1875,7 +1914,7 @@ protected function getEpsilonTarget( bool $fullCtx, bool $treatEofAsEpsilon, ): ?ATNConfig { - switch ($t->getSerializationType()) { + switch ($t->serializationType) { case Transition::RULE: if (!$t instanceof RuleTransition) { throw new \LogicException('Unexpected transition type.'); diff --git a/src/Atn/PredictionMode.php b/src/Atn/PredictionMode.php index b08a3de..a69c364 100644 --- a/src/Atn/PredictionMode.php +++ b/src/Atn/PredictionMode.php @@ -7,10 +7,6 @@ use Antlr\Antlr4\Runtime\Atn\SemanticContexts\SemanticContext; use Antlr\Antlr4\Runtime\Atn\States\ATNState; use Antlr\Antlr4\Runtime\Atn\States\RuleStopState; -use Antlr\Antlr4\Runtime\Comparison\Equality; -use Antlr\Antlr4\Runtime\Comparison\Equivalence; -use Antlr\Antlr4\Runtime\Comparison\Hashable; -use Antlr\Antlr4\Runtime\Comparison\Hasher; use Antlr\Antlr4\Runtime\Utils\BitSet; use Antlr\Antlr4\Runtime\Utils\Map; @@ -495,31 +491,7 @@ public static function getAlts(array $altSets): BitSet public static function getConflictingAltSubsets(ATNConfigSet $configs): array { /** @var Map $configToAlts */ - $configToAlts = new Map( - new class implements Equivalence { - public function equals(object $other): bool - { - return $other instanceof self; - } - - public function equivalent(Hashable $left, Hashable $right): bool - { - return $left instanceof ATNConfig - && $right instanceof ATNConfig - && $left->state->stateNumber === $right->state->stateNumber - && Equality::equals($left->context, $right->context); - } - - public function hash(Hashable $value): int - { - if (!$value instanceof ATNConfig) { - throw new \InvalidArgumentException('Unsupported value.'); - } - - return Hasher::hash($value->state->stateNumber, $value->context); - } - }, - ); + $configToAlts = new Map(AltAndContextEquivalence::instance()); foreach ($configs->elements() as $config) { $alts = $configToAlts->get($config); @@ -541,6 +513,9 @@ public function hash(Hashable $value): int * * map[c.{@see ATNConfig::$state}] U= c.{@see ATNConfig::$alt} */ + /** + * @return Map<\Antlr\Antlr4\Runtime\Atn\States\ATNState, \Antlr\Antlr4\Runtime\Utils\BitSet> + */ public static function getStateToAltMap(ATNConfigSet $configs): Map { /** @var Map $map */ @@ -563,7 +538,7 @@ public static function getStateToAltMap(ATNConfigSet $configs): Map public static function hasStateAssociatedWithOneAlt(ATNConfigSet $configs): bool { foreach (self::getStateToAltMap($configs)->getValues() as $value) { - if ($value instanceof BitSet && $value->length() === 1) { + if ($value->length() === 1) { return true; } } diff --git a/src/Atn/SemanticContexts/AndOperator.php b/src/Atn/SemanticContexts/AndOperator.php index 31efb17..7a58d78 100644 --- a/src/Atn/SemanticContexts/AndOperator.php +++ b/src/Atn/SemanticContexts/AndOperator.php @@ -5,7 +5,7 @@ namespace Antlr\Antlr4\Runtime\Atn\SemanticContexts; use Antlr\Antlr4\Runtime\Comparison\Equality; -use Antlr\Antlr4\Runtime\Comparison\Hasher; +use Antlr\Antlr4\Runtime\Comparison\MurmurHash; use Antlr\Antlr4\Runtime\Recognizer; use Antlr\Antlr4\Runtime\RuleContext; use Antlr\Antlr4\Runtime\Utils\Set; @@ -16,6 +16,12 @@ */ final class AndOperator extends Operator { + /** + * Java seeds this with `AND.class.hashCode()`, a JVM identity hash that + * differs between runs, so it cannot be mirrored exactly. A fixed seed + * keeps the value deterministic, which is what the collections need. + */ + private const HASH_SEED = 41; /** @var array */ public array $operands; @@ -127,9 +133,14 @@ public function equals(object $other): bool return Equality::equals($this->operands, $other->operands); } + /** + * Memoised: every field feeding it is set once in the constructor. + */ + private ?int $cachedHashCode = null; + public function hashCode(): int { - return Hasher::hash(41, $this->operands); + return $this->cachedHashCode ??= MurmurHash::hash($this->operands, self::HASH_SEED); } public function __toString(): string diff --git a/src/Atn/SemanticContexts/OrOperator.php b/src/Atn/SemanticContexts/OrOperator.php index ff6f2bd..bf43fbc 100644 --- a/src/Atn/SemanticContexts/OrOperator.php +++ b/src/Atn/SemanticContexts/OrOperator.php @@ -5,7 +5,7 @@ namespace Antlr\Antlr4\Runtime\Atn\SemanticContexts; use Antlr\Antlr4\Runtime\Comparison\Equality; -use Antlr\Antlr4\Runtime\Comparison\Hasher; +use Antlr\Antlr4\Runtime\Comparison\MurmurHash; use Antlr\Antlr4\Runtime\Recognizer; use Antlr\Antlr4\Runtime\RuleContext; use Antlr\Antlr4\Runtime\Utils\Set; @@ -16,6 +16,12 @@ */ final class OrOperator extends Operator { + /** + * Java seeds this with `OR.class.hashCode()`, a JVM identity hash that + * differs between runs, so it cannot be mirrored exactly. A fixed seed + * keeps the value deterministic, which is what the collections need. + */ + private const HASH_SEED = 37; /** @var array */ public array $operand; @@ -129,9 +135,14 @@ public function equals(object $other): bool return Equality::equals($this->operand, $other->operand); } + /** + * Memoised: every field feeding it is set once in the constructor. + */ + private ?int $cachedHashCode = null; + public function hashCode(): int { - return Hasher::hash(37, $this->operand); + return $this->cachedHashCode ??= MurmurHash::hash($this->operand, self::HASH_SEED); } public function __toString(): string diff --git a/src/Atn/SemanticContexts/PrecedencePredicate.php b/src/Atn/SemanticContexts/PrecedencePredicate.php index 122d305..2af4f11 100644 --- a/src/Atn/SemanticContexts/PrecedencePredicate.php +++ b/src/Atn/SemanticContexts/PrecedencePredicate.php @@ -4,7 +4,6 @@ namespace Antlr\Antlr4\Runtime\Atn\SemanticContexts; -use Antlr\Antlr4\Runtime\Comparison\Hasher; use Antlr\Antlr4\Runtime\Recognizer; use Antlr\Antlr4\Runtime\RuleContext; @@ -33,7 +32,7 @@ public function evalPrecedence(Recognizer $parser, RuleContext $parserCallStack) public function hashCode(): int { - return Hasher::hash(31, $this->precedence); + return 31 + $this->precedence; } public function compareTo(PrecedencePredicate $other): int diff --git a/src/Atn/SemanticContexts/Predicate.php b/src/Atn/SemanticContexts/Predicate.php index 0574386..c68d227 100644 --- a/src/Atn/SemanticContexts/Predicate.php +++ b/src/Atn/SemanticContexts/Predicate.php @@ -4,7 +4,7 @@ namespace Antlr\Antlr4\Runtime\Atn\SemanticContexts; -use Antlr\Antlr4\Runtime\Comparison\Hasher; +use Antlr\Antlr4\Runtime\Comparison\MurmurHash; use Antlr\Antlr4\Runtime\Recognizer; use Antlr\Antlr4\Runtime\RuleContext; @@ -30,9 +30,16 @@ public function eval(Recognizer $parser, RuleContext $parserCallStack): bool return $parser->sempred($localctx, $this->ruleIndex, $this->predIndex); } + /** + * Memoised: every field feeding it is set once in the constructor. + */ + private ?int $cachedHashCode = null; + public function hashCode(): int { - return Hasher::hash($this->ruleIndex, $this->predIndex, $this->isCtxDependent); + return $this->cachedHashCode ??= MurmurHash::hash( + [$this->ruleIndex, $this->predIndex, $this->isCtxDependent ? 1 : 0], + ); } public function equals(object $other): bool diff --git a/src/Atn/SemanticContexts/SemanticContext.php b/src/Atn/SemanticContexts/SemanticContext.php index 3431f6f..0511ae4 100644 --- a/src/Atn/SemanticContexts/SemanticContext.php +++ b/src/Atn/SemanticContexts/SemanticContext.php @@ -24,11 +24,11 @@ abstract class SemanticContext implements Hashable * The default {@see SemanticContext}, which is semantically equivalent to * a predicate of the form `{true}?`. */ + private static ?Predicate $none = null; + public static function none(): Predicate { - static $none; - - return $none ??= new Predicate(); + return self::$none ??= new Predicate(); } public static function andContext(?self $a, ?self $b): ?self @@ -103,14 +103,23 @@ public function evalPrecedence(Recognizer $parser, RuleContext $parserCallStack) } /** + * @param Set $set + * * @return array */ public static function filterPrecedencePredicates(Set $set): array { $result = []; + foreach ($set->getValues() as $context) { if ($context instanceof PrecedencePredicate) { $result[] = $context; + + // Java's version is a *mutating* filter — `iterator.remove()`. + // Only collecting them left every precedence predicate in the + // operand set, so `AND`/`OR` never reduced them to the single + // strictest/loosest bound and both survived into evaluation. + $set->remove($context); } } diff --git a/src/Atn/States/ATNState.php b/src/Atn/States/ATNState.php index 8a9d51e..2bcf6c1 100644 --- a/src/Atn/States/ATNState.php +++ b/src/Atn/States/ATNState.php @@ -60,9 +60,12 @@ abstract class ATNState implements Hashable /** * Track the transitions emanating from this ATN state. * + * Public, as in the reference runtime, so the simulator's inner loops can + * read it without a getter call. + * * @var array */ - protected array $transitions = []; + public array $transitions = []; /** * Used to cache lookahead during parsing, not used during construction. @@ -75,7 +78,9 @@ public function equals(object $other): bool return true; } - return $other instanceof static + // Java compares against `ATNState`, not the runtime class, so two states + // of different concrete types are still comparable. + return $other instanceof self && $this->stateNumber === $other->stateNumber; } @@ -152,7 +157,7 @@ public function __toString(): string public function hashCode(): int { - return $this->getStateType(); + return $this->stateNumber; } abstract public function getStateType(): int; diff --git a/src/Atn/Transitions/AtomTransition.php b/src/Atn/Transitions/AtomTransition.php index ef67b7d..b5035f9 100644 --- a/src/Atn/Transitions/AtomTransition.php +++ b/src/Atn/Transitions/AtomTransition.php @@ -18,7 +18,7 @@ public function __construct(ATNState $target, int $label) $this->label = $label; } - public function label(): ?IntervalSet + public function label(): IntervalSet { return IntervalSet::fromInt($this->label); } diff --git a/src/Atn/Transitions/RangeTransition.php b/src/Atn/Transitions/RangeTransition.php index d4470c4..b67fc12 100644 --- a/src/Atn/Transitions/RangeTransition.php +++ b/src/Atn/Transitions/RangeTransition.php @@ -22,7 +22,7 @@ public function __construct(ATNState $target, int $from, int $to) $this->to = $to; } - public function label(): ?IntervalSet + public function label(): IntervalSet { return IntervalSet::fromRange($this->from, $this->to); } diff --git a/src/Atn/Transitions/Transition.php b/src/Atn/Transitions/Transition.php index b6d5d3a..38c9a73 100644 --- a/src/Atn/Transitions/Transition.php +++ b/src/Atn/Transitions/Transition.php @@ -39,9 +39,20 @@ abstract class Transition implements Equatable */ public ATNState $target; + /** + * The serialization type, resolved once at construction. + * + * `getSerializationType()` is called for every transition the closure walks, + * hundreds of thousands of times per parse, and each call returns the same + * constant. Reading a property instead removes that call from the + * simulator's inner loop. + */ + public int $serializationType; + public function __construct(ATNState $target) { $this->target = $target; + $this->serializationType = $this->getSerializationType(); } /** diff --git a/src/BufferedTokenStream.php b/src/BufferedTokenStream.php index f213170..7ea23a1 100644 --- a/src/BufferedTokenStream.php +++ b/src/BufferedTokenStream.php @@ -4,8 +4,6 @@ namespace Antlr\Antlr4\Runtime; -use Antlr\Antlr4\Runtime\Utils\Set; - /** * This implementation of {@see TokenStream} loads tokens from a * {@see TokenSource} on-demand, and places the tokens in a buffer to provide @@ -286,9 +284,11 @@ public function getAllTokens(): array /** * Get all tokens from start..stop inclusively * + * @param array|null $types token types to keep, or null for all + * * @return array|null */ - public function getTokens(int $start, int $stop, ?Set $types = null): ?array + public function getTokens(int $start, int $stop, ?array $types = null): ?array { if ($start < 0 || $stop < 0) { return null; @@ -308,7 +308,7 @@ public function getTokens(int $start, int $stop, ?Set $types = null): ?array break; } - if ($types === null || $types->contains($t->getType())) { + if ($types === null || \in_array($t->getType(), $types, true)) { $subset[] = $t; } } diff --git a/src/CommonToken.php b/src/CommonToken.php index 4571bc0..9b6fd5c 100644 --- a/src/CommonToken.php +++ b/src/CommonToken.php @@ -104,11 +104,11 @@ public function __construct( * An empty {@see Pair}, which is used as the default value of * {@see CommonToken::source()} for tokens that do not have a source. */ + private static ?Pair $emptySource = null; + public static function emptySource(): Pair { - static $source; - - return $source ??= new Pair(null, null); + return self::$emptySource ??= new Pair(null, null); } /** diff --git a/src/CommonTokenFactory.php b/src/CommonTokenFactory.php index ee3df1b..28705f8 100644 --- a/src/CommonTokenFactory.php +++ b/src/CommonTokenFactory.php @@ -48,11 +48,11 @@ public function __construct(bool $copyText = false) * This token factory does not explicitly copy token text when constructing * tokens. */ + private static ?self $default = null; + public static function default(): self { - static $default; - - return $default ??= new CommonTokenFactory(); + return self::$default ??= new CommonTokenFactory(); } public function createEx( diff --git a/src/Comparison/Equality.php b/src/Comparison/Equality.php index a83de26..9dc0ea1 100644 --- a/src/Comparison/Equality.php +++ b/src/Comparison/Equality.php @@ -39,7 +39,10 @@ private static function deeplyEquals(array $left, array $right): bool } foreach ($left as $key => $value) { - if (!isset($right[$key])) { + // `array_key_exists`, not `isset`: a legitimately `null` element — + // prediction-context parent arrays are full of them — would otherwise + // read as a missing key and make two identical arrays compare unequal. + if (!\array_key_exists($key, $right)) { return false; } diff --git a/src/Comparison/MurmurHash.php b/src/Comparison/MurmurHash.php new file mode 100644 index 0000000..1df5673 --- /dev/null +++ b/src/Comparison/MurmurHash.php @@ -0,0 +1,208 @@ + update... -> finish` for the same list. + * + * `null` contributes 0 and a {@see Hashable} contributes its own + * `hashCode()`, exactly as Java's `update(int, Object)` overload does. + * + * @param array $values + */ + public static function hash(array $values, int $seed = self::DEFAULT_SEED): int + { + $words = []; + + foreach ($values as $value) { + // `hashCode()` is the hottest call in the simulator, and the vast + // majority of words are already integers, so the common cases are + // inlined rather than dispatched through `valueOf()`. `pack('V')` + // truncates to 32 bits on its own, so no masking is needed here. + if (\is_int($value)) { + $words[] = $value; + } elseif ($value instanceof Hashable) { + $words[] = $value->hashCode(); + } else { + $words[] = self::valueOf($value); + } + } + + self::$native ??= \in_array('murmur3a', \hash_algos(), true); + + return self::$native + ? self::nativeHash($words, $seed) + : self::fallbackHash($words, $seed); + } + + /** + * Java's `String.hashCode()`: `s[0]*31^(n-1) + ... + s[n-1]`, over UTF-16 + * code units, wrapping at 32 bits. + */ + public static function hashString(string $value): int + { + $hash = 0; + + // Java hashes UTF-16 code units, so anything outside the BMP has to + // contribute its surrogate pair rather than its code point. + $utf16 = \mb_convert_encoding($value, 'UTF-16BE', 'UTF-8'); + $length = \strlen($utf16); + + for ($i = 0; $i < $length; $i += 2) { + $unit = (\ord($utf16[$i]) << 8) | \ord($utf16[$i + 1]); + $hash = self::multiply($hash, 31) + $unit & self::MASK; + } + + return self::toSigned($hash); + } + + /** + * @param array $words + */ + private static function nativeHash(array $words, int $seed): int + { + $digest = \hash( + 'murmur3a', + \pack('V*', ...$words), + true, + ['seed' => $seed & self::MASK], + ); + + /** @var array{1: int} $unpacked */ + $unpacked = \unpack('N', $digest); + + return self::toSigned($unpacked[1]); + } + + /** + * The reference fold, kept verbatim so the native path above has something + * to be checked against. + * + * @param array $words + */ + private static function fallbackHash(array $words, int $seed): int + { + $hash = $seed & self::MASK; + + foreach ($words as $word) { + $k = self::multiply($word, 0xCC9E2D51); + $k = self::rotateLeft($k, 15); + $k = self::multiply($k, 0x1B873593); + + $hash = self::rotateLeft(($hash ^ $k) & self::MASK, 13); + $hash = self::multiply($hash, 5) + 0xE6546B64 & self::MASK; + } + + $hash = ($hash ^ \count($words) * 4) & self::MASK; + $hash ^= $hash >> 16; + $hash = self::multiply($hash, 0x85EBCA6B); + $hash ^= $hash >> 13; + $hash = self::multiply($hash, 0xC2B2AE35); + $hash ^= $hash >> 16; + + return self::toSigned($hash & self::MASK); + } + + private static function valueOf(mixed $value): int + { + if (\is_int($value)) { + return $value; + } + + if ($value === null) { + return 0; + } + + if ($value instanceof Hashable) { + return $value->hashCode(); + } + + if (\is_bool($value)) { + return $value ? 1 : 0; + } + + if (\is_string($value)) { + return self::hashString($value); + } + + if (\is_object($value)) { + // Java folds in `Object.hashCode()`, which is identity-based for a + // type that does not override it. `Pair` holds exactly such values. + return \spl_object_id($value); + } + + throw new \InvalidArgumentException(\sprintf( + 'Cannot hash a value of type "%s".', + \get_debug_type($value), + )); + } + + /** + * Multiplies two 32-bit values with Java's wrap-around semantics. + * + * The operands are split into 16-bit halves because a full 32x32 product + * overflows PHP's signed 64-bit integer and would silently become a float. + */ + private static function multiply(int $a, int $b): int + { + $a &= self::MASK; + $b &= self::MASK; + + $high = ($a >> 16) & 0xFFFF; + $low = $a & 0xFFFF; + + return (($high * $b & 0xFFFF) << 16) + ($low * $b) & self::MASK; + } + + private static function rotateLeft(int $value, int $bits): int + { + $value &= self::MASK; + + return (($value << $bits) | ($value >> 32 - $bits)) & self::MASK; + } + + private static function toSigned(int $value): int + { + return $value >= 0x80000000 ? $value - 0x100000000 : $value; + } +} diff --git a/src/Dfa/DFAState.php b/src/Dfa/DFAState.php index 0ba4686..bd4a524 100644 --- a/src/Dfa/DFAState.php +++ b/src/Dfa/DFAState.php @@ -8,7 +8,7 @@ use Antlr\Antlr4\Runtime\Atn\LexerActionExecutor; use Antlr\Antlr4\Runtime\Comparison\Equality; use Antlr\Antlr4\Runtime\Comparison\Hashable; -use Antlr\Antlr4\Runtime\Comparison\Hasher; +use Antlr\Antlr4\Runtime\Comparison\MurmurHash; /** * A DFA state represents a set of possible ATN configurations. @@ -137,6 +137,10 @@ public function __toString(): string public function hashCode(): int { - return Hasher::hash($this->configs); + // Hashing through `ATNConfigSet::hashCode()` matters: that value is cached + // once the set goes read-only, and a DFA state is looked up on every + // transition. Hashing the configurations directly here re-walked the whole + // set each time. + return MurmurHash::hash([$this->configs], 7); } } diff --git a/src/InputStream.php b/src/InputStream.php index d645ca3..d7fedb0 100644 --- a/src/InputStream.php +++ b/src/InputStream.php @@ -4,10 +4,19 @@ namespace Antlr\Antlr4\Runtime; -use Antlr\Antlr4\Runtime\Utils\StringUtils; - /** * Vacuum all input from a string and then treat it like a buffer. + * + * Indexing is by Unicode code point, matching Java's `CodePointCharStream`, so + * an astral-plane character occupies one position rather than the two a UTF-16 + * stream would use. + * + * The decoding is done **once**, up front, into an array of code points. The + * lexer reads `LA()` several times per input character — it was the single + * hottest call in the runtime — and decoding on each read made every one of + * those a `mb_ord()` call. Text extraction needs the original bytes, so a byte + * offset per code point is kept alongside, letting `getText()` be a plain + * `substr()` instead of a scan. */ final class InputStream implements CharStream { @@ -19,24 +28,81 @@ final class InputStream implements CharStream public string $input; - /** @var array */ - public array $characters = []; + /** + * The decoded input, one Unicode code point per element. + * + * @var array + */ + private array $codePoints = []; /** - * @param array $characters + * Byte offset of each code point within {@see InputStream::$input}, with one + * extra entry holding the total length so a slice's end is always known. + * Empty when the input is pure ASCII, where the offset is the index itself. + * + * @var array */ - private function __construct(string $input, array $characters) + private array $byteOffsets = []; + + /** + * @param array $codePoints + * @param array $byteOffsets + */ + private function __construct(string $input, array $codePoints, array $byteOffsets) { $this->input = $input; - $this->characters = $characters; - $this->size = \count($this->characters); + $this->codePoints = $codePoints; + $this->byteOffsets = $byteOffsets; + $this->size = \count($codePoints); } public static function fromString(string $input): InputStream { - $chars = \preg_split('//u', $input, -1, \PREG_SPLIT_NO_EMPTY); + if ($input === '') { + return new self($input, [], []); + } + + $length = \strlen($input); + + // Pure ASCII is the overwhelmingly common case and needs no decoding at + // all: byte value is code point, and byte offset is index. + if (\preg_match('/[\x80-\xFF]/', $input) === 0) { + /** @var array $bytes */ + $bytes = \unpack('C*', $input); + + return new self($input, \array_values($bytes), []); + } + + // Converting the whole string at once keeps the decoding inside mbstring + // rather than paying a PHP call per character. + $utf32 = @\mb_convert_encoding($input, 'UTF-32BE', 'UTF-8'); + + if ($utf32 === '') { + return new self($input, [], []); + } + + /** @var array $unpacked */ + $unpacked = \unpack('N*', $utf32); + $codePoints = \array_values($unpacked); + + // A code point's UTF-8 width is a function of its value, so the offsets + // follow from the decoded points without re-walking the bytes. + $byteOffsets = []; + $offset = 0; + + foreach ($codePoints as $codePoint) { + $byteOffsets[] = $offset; + $offset += match (true) { + $codePoint < 0x80 => 1, + $codePoint < 0x800 => 2, + $codePoint < 0x10000 => 3, + default => 4, + }; + } + + $byteOffsets[] = $length; - return new self($input, $chars === false ? [] : $chars); + return new self($input, $codePoints, $byteOffsets); } public static function fromPath(string $path): InputStream @@ -47,7 +113,25 @@ public static function fromPath(string $path): InputStream throw new \InvalidArgumentException(\sprintf('File not found at %s.', $path)); } - return self::fromString($content); + // `CharStreams.fromPath()` records the path as the stream's source name; + // it surfaces through `Lexer::getSourceName()` and `Parser::getSourceName()`. + $stream = self::fromString($content); + $stream->name = $path; + + return $stream; + } + + /** + * The decoded code points backing this stream. + * + * Exposed so the lexer's inner loop can read characters without a method + * call per access; treat it as read-only. + * + * @return array + */ + public function getCodePoints(): array + { + return $this->codePoints; } public function getIndex(): int @@ -88,7 +172,7 @@ public function LA(int $offset): int return Token::EOF; } - return StringUtils::codePoint($this->characters[$pos]); + return $this->codePoints[$pos]; } public function LT(int $offset): int @@ -132,16 +216,27 @@ public function getText(int $start, int $stop): string $stop = $this->size - 1; } - if ($start >= $this->size) { + if ($start >= $this->size || $start > $stop) { return ''; } - return \implode(\array_slice($this->characters, $start, $stop - $start + 1)); + if ($this->byteOffsets === []) { + // ASCII: index and byte offset coincide. + return \substr($this->input, $start, $stop - $start + 1); + } + + $from = $this->byteOffsets[$start]; + + return \substr($this->input, $from, $this->byteOffsets[$stop + 1] - $from); } public function getSourceName(): string { - return ''; + // Java falls back to `IntStream.UNKNOWN_SOURCE_NAME` rather than an empty + // string, and the constant already existed here unused. + return $this->name === '' || $this->name === '' + ? IntStream::UNKNOWN_SOURCE_NAME + : $this->name; } public function __toString(): string diff --git a/src/Interval.php b/src/Interval.php index 1cbdea9..2a42ee0 100644 --- a/src/Interval.php +++ b/src/Interval.php @@ -21,11 +21,11 @@ public function __construct(int $start, int $stop) $this->stop = $stop; } + private static ?self $invalid = null; + public static function invalid(): self { - static $invalid; - - return $invalid ?? $invalid = new Interval(-1, -2); + return self::$invalid ??= new Interval(-1, -2); } public function contains(int $item): bool @@ -35,6 +35,13 @@ public function contains(int $item): bool public function getLength(): int { + // An inverted interval is empty, not negatively long. `IntervalSet` + // sums these to report its cardinality, so a negative would silently + // corrupt the total. + if ($this->stop < $this->start) { + return 0; + } + return $this->stop - $this->start + 1; } @@ -124,10 +131,8 @@ public function intersection(Interval $other): self public function __toString(): string { - if ($this->start === $this->stop) { - return (string) $this->start; - } - + // Java renders `a..b` unconditionally; collapsing a single-element + // interval to just `a` was a PHP-only shorthand. return $this->start . '..' . $this->stop; } } diff --git a/src/IntervalSet.php b/src/IntervalSet.php index 357d73f..9198a73 100644 --- a/src/IntervalSet.php +++ b/src/IntervalSet.php @@ -5,7 +5,8 @@ namespace Antlr\Antlr4\Runtime; use Antlr\Antlr4\Runtime\Comparison\Equality; -use Antlr\Antlr4\Runtime\Comparison\Equatable; +use Antlr\Antlr4\Runtime\Comparison\Hashable; +use Antlr\Antlr4\Runtime\Comparison\MurmurHash; use Antlr\Antlr4\Runtime\Utils\StringUtils; /** @@ -19,7 +20,7 @@ * the range {@see Integer::MIN_VALUE} to {@see Integer::MAX_VALUE} * (inclusive). */ -final class IntervalSet implements Equatable +final class IntervalSet implements Hashable { /** @var array */ protected array $intervals = []; @@ -323,19 +324,21 @@ public function removeOne(int $v): void } // check for upper boundary - if ($v === $i->stop - 1) { + // Java compares against `b`, not `b - 1`; testing `stop - 1` meant + // the last element of an interval could never be removed. + if ($v === $i->stop) { $this->intervals[$k] = new Interval($i->start, $i->stop - 1); return; } // split existing range - if ($v < $i->stop - 1) { - $x = new Interval($i->start, $v); - - $i->start = $v + 1; - - \array_splice($this->intervals, $k, 0, [$x]); + if ($v > $i->start && $v < $i->stop) { + // The left half ends *before* the removed element. Using `$v` as + // the stop left the element in the set. + $stop = $i->stop; + $this->intervals[$k] = new Interval($i->start, $v - 1); + $this->addRange($v + 1, $stop); return; } @@ -487,6 +490,30 @@ protected function elementName(Vocabulary $vocabulary, int $a): string return $vocabulary->getDisplayName($a); } + /** + * Java's `IntervalSet` defines `hashCode()`; the port implemented only + * `equals()`, leaving the two out of contract for any hash-based lookup. + */ + public function hashCode(): int + { + $words = []; + + foreach ($this->intervals as $interval) { + $words[] = $interval->start; + $words[] = $interval->stop; + } + + return MurmurHash::hash($words); + } + + /** + * @return array + */ + public function getIntervals(): array + { + return $this->intervals; + } + public function equals(object $other): bool { if ($this === $other) { diff --git a/src/LL1Analyzer.php b/src/LL1Analyzer.php index 57b4bbf..02a6347 100644 --- a/src/LL1Analyzer.php +++ b/src/LL1Analyzer.php @@ -54,6 +54,7 @@ public function getDecisionLookahead(?ATNState $s): ?array $look = []; for ($alt = 0; $alt < $s->getNumberOfTransitions(); $alt++) { $interval = new IntervalSet(); + /** @var Set $lookBusy */ $lookBusy = new Set(); $seeThruPreds = false; // fail to get lookahead upon pred @@ -108,12 +109,15 @@ public function look(ATNState $s, ?ATNState $stopState, ?RuleContext $context): PredictionContext::fromRuleContext($s->atn, $context) : null; + /** @var Set $lookBusy */ + $lookBusy = new Set(); + $this->lookRecursively( $s, $stopState, $lookContext, $r, - new Set(), + $lookBusy, new BitSet(), $seeThruPreds, true, @@ -142,7 +146,7 @@ public function look(ATNState $s, ?ATNState $stopState, ?RuleContext $context): * if the outer context should * not be used. * @param IntervalSet $look The result lookahead set. - * @param Set $lookBusy A set used for preventing + * @param Set $lookBusy A set used for preventing * epsilon closures in the ATN * from causing a stack overflow. * Outside code should pass @@ -170,6 +174,9 @@ public function look(ATNState $s, ?ATNState $stopState, ?RuleContext $context): * has no effect if `context` * is `null`. */ + /** + * @param Set $lookBusy + */ protected function lookRecursively( ATNState $s, ?ATNState $stopState, diff --git a/src/Lexer.php b/src/Lexer.php index f596fe5..86f196b 100644 --- a/src/Lexer.php +++ b/src/Lexer.php @@ -247,7 +247,11 @@ public function popMode(): int public function getSourceName(): string { - return $this->input === null ? '' : $this->input->getSourceName(); + // With no input Java would NPE; reporting the documented unknown-source + // sentinel is the closest honest answer. + return $this->input === null + ? IntStream::UNKNOWN_SOURCE_NAME + : $this->input->getSourceName(); } public function getInputStream(): ?IntStream diff --git a/src/Parser.php b/src/Parser.php index 2be0e4b..9157169 100644 --- a/src/Parser.php +++ b/src/Parser.php @@ -499,7 +499,7 @@ public function consume(): Token $this->tokenStream()->consume(); } - if ($this->buildParseTree || \count($this->parseListeners) > 0) { + if ($this->buildParseTree || $this->parseListeners !== []) { if ($this->errorHandler->inErrorRecoveryMode($this)) { $node = $this->context()->addErrorNode($this->createErrorNode($this->context(), $o)); @@ -564,30 +564,35 @@ public function enterRule(ParserRuleContext $localctx, int $state, int $ruleInde { $this->setState($state); $this->ctx = $localctx; - $this->context()->start = $this->tokenStream()->LT(1); + $localctx->start = $this->tokenStream()->LT(1); if ($this->buildParseTree) { $this->addContextToParseTree(); } - $this->triggerEnterRuleEvent(); + // Skipped rather than called into when there are no listeners: this runs + // once per rule invocation. + if ($this->parseListeners !== []) { + $this->triggerEnterRuleEvent(); + } } public function exitRule(): void { - if ($this->matchedEOF) { - // if we have matched EOF, it cannot consume past EOF so we use LT(1) here - $this->context()->stop = $this->tokenStream()->LT(1); // LT(1) will be end of file - } else { - $this->context()->stop = $this->tokenStream()->LT(-1); // stop node is what we just matched - } + // Resolved once: this runs on every rule exit and each accessor is a call. + $context = $this->context(); + + // if we have matched EOF, it cannot consume past EOF so we use LT(1) here + $context->stop = $this->tokenStream()->LT($this->matchedEOF ? 1 : -1); // trigger event on _ctx, before it reverts to parent - $this->triggerExitRuleEvent(); + if ($this->parseListeners !== []) { + $this->triggerExitRuleEvent(); + } - $this->setState($this->context()->invokingState); + $this->setState($context->invokingState); - $parent = $this->context()->getParent(); + $parent = $context->getParent(); if ($parent === null || $parent instanceof ParserRuleContext) { $this->ctx = $parent; @@ -905,14 +910,29 @@ public function getSourceName(): string */ public function setTrace(bool $trace): void { - if ($this->tracer !== null) { - $this->removeParseListener($this->tracer); + if (!$trace) { + if ($this->tracer !== null) { + $this->removeParseListener($this->tracer); + + // Discarded, not just detached: `isTrace()` reports on this + // reference, so keeping it would leave tracing switched off yet + // reported as on. + $this->tracer = null; + } + + return; } - if ($trace) { - $this->tracer = new ParserTraceListener($this); - $this->addParseListener($this->tracer); + $tracer = $this->tracer; + + if ($tracer === null) { + $tracer = new ParserTraceListener($this); + $this->tracer = $tracer; + } else { + $this->removeParseListener($tracer); } + + $this->addParseListener($tracer); } /** diff --git a/src/ParserTraceListener.php b/src/ParserTraceListener.php index d194b11..9caaff9 100644 --- a/src/ParserTraceListener.php +++ b/src/ParserTraceListener.php @@ -22,17 +22,19 @@ public function enterEveryRule(ParserRuleContext $context): void $stream = $this->parser->getTokenStream(); $token = $stream?->LT(1); + // Java's TraceListener uses `println`. Without the newline every trace + // line ran together on one unreadable line — issue #33. echo \sprintf( - 'enter %s, LT(1)=%s', + 'enter %s, LT(1)=%s' . \PHP_EOL, $this->parser->getRuleNames()[$context->getRuleIndex()], - $token === null? '' : $token->getText() ?? '', + $token === null ? '' : $token->getText() ?? '', ); } public function visitTerminal(TerminalNode $node): void { echo \sprintf( - 'consume %s rule %s', + 'consume %s rule %s' . \PHP_EOL, $node->getSymbol(), $this->parser->getCurrentRuleName(), ); @@ -44,9 +46,9 @@ public function exitEveryRule(ParserRuleContext $context): void $token = $stream?->LT(1); echo \sprintf( - 'exit %s, LT(1)=%s', + 'exit %s, LT(1)=%s' . \PHP_EOL, $this->parser->getRuleNames()[$context->getRuleIndex()], - $token === null? '' : $token->getText() ?? '', + $token === null ? '' : $token->getText() ?? '', ); } diff --git a/src/PredictionContexts/ArrayPredictionContext.php b/src/PredictionContexts/ArrayPredictionContext.php index a174752..7501854 100644 --- a/src/PredictionContexts/ArrayPredictionContext.php +++ b/src/PredictionContexts/ArrayPredictionContext.php @@ -5,7 +5,7 @@ namespace Antlr\Antlr4\Runtime\PredictionContexts; use Antlr\Antlr4\Runtime\Comparison\Equality; -use Antlr\Antlr4\Runtime\Comparison\Hasher; +use Antlr\Antlr4\Runtime\Comparison\MurmurHash; final class ArrayPredictionContext extends PredictionContext { @@ -86,11 +86,18 @@ public function equals(object $other): bool return false; } - if ($this->returnStates === $other->returnStates) { + // Cheap rejection first, as Java does: contexts that hash differently + // cannot be equal, and this comparison runs constantly during merging. + if (($this->cachedHashCode ?? $this->hashCode()) !== ($other->cachedHashCode ?? $other->hashCode())) { return false; } - return Equality::equals($this->parents, $other->parents); + // This condition used to be inverted — `if ($returnStates === $other->returnStates) return false;` + // — which declared identical contexts unequal and judged differing ones + // on their parents alone. It stayed hidden for as long as + // `ATNConfigSet::add()` was failing to merge contexts at all. + return $this->returnStates === $other->returnStates + && Equality::equals($this->parents, $other->parents); } public function __toString(): string @@ -125,6 +132,12 @@ public function __toString(): string protected function computeHashCode(): int { - return Hasher::hash($this->parents, $this->returnStates); + // `PredictionContext.calculateHashCode(parents, returnStates)`: every + // parent, then every return state, folded into one accumulation seeded + // with INITIAL_HASH and finished with `2 * count`. + return MurmurHash::hash( + [...$this->parents, ...$this->returnStates], + PredictionContext::INITIAL_HASH, + ); } } diff --git a/src/PredictionContexts/EmptyPredictionContext.php b/src/PredictionContexts/EmptyPredictionContext.php index 6a26f4b..3b98bc2 100644 --- a/src/PredictionContexts/EmptyPredictionContext.php +++ b/src/PredictionContexts/EmptyPredictionContext.php @@ -28,7 +28,12 @@ public function getParent(int $index): ?PredictionContext public function equals(object $other): bool { - return $other instanceof self; + // Java's is `this == o`, which is safe there because the constructor is + // private and `Instance` is the only instance. `PredictionContext::empty()` + // is the only construction site here, so identity is equally safe — and + // `instanceof self` would wrongly equate two distinct empty contexts if + // one were ever created. + return $this === $other; } public function __toString(): string diff --git a/src/PredictionContexts/PredictionContext.php b/src/PredictionContexts/PredictionContext.php index 5e2c97c..2652b8d 100644 --- a/src/PredictionContexts/PredictionContext.php +++ b/src/PredictionContexts/PredictionContext.php @@ -11,6 +11,7 @@ use Antlr\Antlr4\Runtime\LoggerProvider; use Antlr\Antlr4\Runtime\RuleContext; use Antlr\Antlr4\Runtime\Utils\DoubleKeyMap; +use Antlr\Antlr4\Runtime\Utils\Map; abstract class PredictionContext implements Hashable { @@ -26,6 +27,12 @@ abstract class PredictionContext implements Hashable */ public const EMPTY_RETURN_STATE = 0x7FFFFFFF; + /** + * The seed every prediction-context hash starts from, matching + * `PredictionContext.INITIAL_HASH` in the reference runtime. + */ + public const INITIAL_HASH = 1; + /** * Stores the computed hash code of this {@see PredictionContext}. The hash * code is computed in parts to match the following reference algorithm. @@ -53,17 +60,17 @@ public function __construct() $this->id = self::$globalNodeCount++; } + private static ?EmptyPredictionContext $empty = null; + public static function empty(): EmptyPredictionContext { - static $empty; - - if ($empty === null) { + if (self::$empty === null) { self::$globalNodeCount--; - $empty = new EmptyPredictionContext(); - $empty->id = 0; + self::$empty = new EmptyPredictionContext(); + self::$empty->id = 0; } - return $empty; + return self::$empty; } /** @@ -255,7 +262,10 @@ public static function mergeSingletons( // see if we can collapse parents due to $+x parents if local ctx $singleParent = null; - if ($a === $b || ($a->parent !== null && $a->parent === $b->parent)) { + // Java collapses on `a.parent.equals(b.parent)`; comparing by identity + // missed every equal-but-distinct parent, so `ax + bx = [a,b]x` never + // fired and the context graph grew where Java's stayed flat. + if ($a === $b || ($a->parent !== null && $b->parent !== null && $a->parent->equals($b->parent))) { // ax + // bx = // [a,b]x @@ -511,9 +521,12 @@ public static function mergeArrays( $M = new ArrayPredictionContext($mergedParents, $mergedReturnStates); - // if we created same array as a or b, return that instead + // If we created the same array as a or b, return that instead. + // Java compares with `equals()`; identity can never hold here because `M` + // was just constructed, so the fast path never fired and every merge + // allocated a fresh context. // TODO: track whether this is possible above during merge sort for speed - if ($M === $a) { + if ($M->equals($a)) { if ($mergeCache !== null) { $mergeCache->set($a, $b, $a); } @@ -529,7 +542,7 @@ public static function mergeArrays( return $a; } - if ($M === $b) { + if ($M->equals($b)) { if ($mergeCache !== null) { $mergeCache->set($a, $b, $b); } @@ -551,7 +564,9 @@ public static function mergeArrays( if (ParserATNSimulator::$traceAtnSimulation) { LoggerProvider::getLogger() - ->debug('mergeArrays a={a},b={b} -> M', [ + // `{M}` — the placeholder was missing its braces, so the trace + // printed a literal "M" instead of the merged context. + ->debug('mergeArrays a={a},b={b} -> {M}', [ 'a' => $a->__toString(), 'b' => $b->__toString(), 'M' => $M->__toString(), @@ -562,36 +577,52 @@ public static function mergeArrays( } /** - * @param array $parents + * Makes a pass over all `M` parents and merges any that are `equals()`. + * + * @param array $parents */ protected static function combineCommonParents(array &$parents): void { - $uniqueParents = new \SplObjectStorage(); + // `SplObjectStorage` keys on object *identity*, so the previous version + // could only ever collapse a parent onto itself — the canonicalisation + // never happened and equal-but-distinct parents accumulated. Java uses a + // `HashMap`, whose whole purpose here is `equals()`-based lookup. + /** @var Map $uniqueParents */ + $uniqueParents = new Map(); foreach ($parents as $parent) { - if (!$uniqueParents->contains($parent)) { - $uniqueParents[$parent] = $parent; + // Java's `HashMap` tolerates a null key; a null parent simply has + // nothing to canonicalise against. + if ($parent !== null && !$uniqueParents->contains($parent)) { + $uniqueParents->put($parent, $parent); // don't replace } } foreach ($parents as $i => $parent) { - $parents[$i] = $uniqueParents[$parent]; + if ($parent !== null) { + $parents[$i] = $uniqueParents->get($parent); + } } } /** - * @param array $visited + * @param \SplObjectStorage $visited + * an identity map, mirroring Java's `IdentityHashMap` */ public static function getCachedPredictionContext( PredictionContext $context, PredictionContextCache $contextCache, - array &$visited, + \SplObjectStorage $visited, ): self { if ($context->isEmpty()) { return $context; } - $existing = $visited[\spl_object_id($context)] ?? null; + // Identity, deliberately: this map memoises "which object did I already + // rewrite?" during one traversal. Keying it on `spl_object_id()` in a + // plain array — as this used to — is unsafe, because ids are reused once + // an object is collected, so an id could alias two different contexts. + $existing = $visited[$context] ?? null; if ($existing !== null) { return $existing; @@ -600,7 +631,7 @@ public static function getCachedPredictionContext( $existing = $contextCache->get($context); if ($existing !== null) { - $visited[\spl_object_id($context)] = $existing; + $visited[$context] = $existing; return $existing; } @@ -616,7 +647,11 @@ public static function getCachedPredictionContext( $parent = self::getCachedPredictionContext($parentContext, $contextCache, $visited); - if ($changed || !$parent->equals($parentContext)) { + // Identity, as Java has it: the cache exists to canonicalise + // instances, so "did this parent get replaced?" is an identity + // question. `equals()` would report no change for an equal-but- + // distinct instance and leave the un-canonicalised one in place. + if ($changed || $parent !== $parentContext) { if (!$changed) { $parents = []; @@ -634,7 +669,7 @@ public static function getCachedPredictionContext( if (!$changed) { $contextCache->add($context); - $visited[\spl_object_id($context)] = $context; + $visited[$context] = $context; return $context; } @@ -654,8 +689,8 @@ public static function getCachedPredictionContext( } $contextCache->add($updated); - $visited[\spl_object_id($updated)] = $updated; - $visited[\spl_object_id($context)] = $updated; + $visited[$updated] = $updated; + $visited[$context] = $updated; return $updated; } diff --git a/src/PredictionContexts/SingletonPredictionContext.php b/src/PredictionContexts/SingletonPredictionContext.php index c90aa82..6798469 100644 --- a/src/PredictionContexts/SingletonPredictionContext.php +++ b/src/PredictionContexts/SingletonPredictionContext.php @@ -4,8 +4,7 @@ namespace Antlr\Antlr4\Runtime\PredictionContexts; -use Antlr\Antlr4\Runtime\Comparison\Equality; -use Antlr\Antlr4\Runtime\Comparison\Hasher; +use Antlr\Antlr4\Runtime\Comparison\MurmurHash; /** * Used to cache {@see PredictionContext} objects. Its used for @@ -65,15 +64,34 @@ public function equals(object $other): bool return true; } - if (!$other instanceof static) { + // `self`, not `static`: Java checks `instanceof SingletonPredictionContext`, + // so a singleton and an `EmptyPredictionContext` remain comparable in both + // directions. With `static` the comparison was asymmetric. + if (!$other instanceof self) { return false; } + // Java is `returnState == s.returnState && (parent != null && parent.equals(s.parent))`, + // guarded by a hash comparison. Two null-parent singletons are + // deliberately *not* equal there, and `parent.equals(null)` is false — + // hence both null checks. + // + // The return state is compared before the hash guard: both are + // necessary conditions so the outcome is identical, but in PHP the hash + // is two method calls where the return state is an int comparison. if ($this->returnState !== $other->returnState) { return false; } - return Equality::equals($this->parent, $other->parent); + if ($this->parent === null || $other->parent === null) { + return false; + } + + if (($this->cachedHashCode ?? $this->hashCode()) !== ($other->cachedHashCode ?? $other->hashCode())) { + return false; + } + + return $this->parent->equals($other->parent); } public function __toString(): string @@ -93,10 +111,9 @@ public function __toString(): string protected function computeHashCode(): int { - if ($this->parent === null) { - return Hasher::hash(0); - } - - return Hasher::hash($this->parent, $this->returnState); + // `PredictionContext.calculateEmptyHashCode()` / `calculateHashCode()`. + return $this->parent === null + ? MurmurHash::hash([], PredictionContext::INITIAL_HASH) + : MurmurHash::hash([$this->parent, $this->returnState], PredictionContext::INITIAL_HASH); } } diff --git a/src/Recognizer.php b/src/Recognizer.php index 55d04df..8b6dacb 100644 --- a/src/Recognizer.php +++ b/src/Recognizer.php @@ -17,8 +17,15 @@ abstract class Recognizer /** @var array */ public array $log = []; - /** @var array> */ - private static array $tokenTypeMapCache = []; + /** + * Java keys this on the vocabulary itself in a `WeakHashMap`, so a cached + * map dies with the vocabulary it describes. `WeakMap` is the exact + * equivalent: keying on identity like `SplObjectStorage`, but without + * pinning every vocabulary the process has ever seen in memory. + * + * @var \WeakMap>|null + */ + private static ?\WeakMap $tokenTypeMapCache = null; /** @var array */ private array $listeners; @@ -51,8 +58,8 @@ public function getTokenTypeMap(): array { $vocabulary = $this->getVocabulary(); - $key = \spl_object_hash($vocabulary); - $result = self::$tokenTypeMapCache[$key] ?? null; + self::$tokenTypeMapCache ??= new \WeakMap(); + $result = self::$tokenTypeMapCache[$vocabulary] ?? null; if ($result === null) { $result = []; @@ -73,7 +80,7 @@ public function getTokenTypeMap(): array $result['EOF'] = Token::EOF; - self::$tokenTypeMapCache[$key] = $result; + self::$tokenTypeMapCache[$vocabulary] = $result; } return $result; diff --git a/src/RuleContext.php b/src/RuleContext.php index 998d68b..59e2cf1 100644 --- a/src/RuleContext.php +++ b/src/RuleContext.php @@ -76,11 +76,11 @@ public function __construct(?RuleContext $parent, ?int $invokingState = null) $this->invokingState = $invokingState ?? -1; } + private static ?ParserRuleContext $emptyContext = null; + public static function emptyContext(): ParserRuleContext { - static $empty; - - return $empty ?? ($empty = new ParserRuleContext(null)); + return self::$emptyContext ??= new ParserRuleContext(null); } public function depth(): int diff --git a/src/StdoutMessageLogger.php b/src/StdoutMessageLogger.php index 58d2af4..68e3626 100644 --- a/src/StdoutMessageLogger.php +++ b/src/StdoutMessageLogger.php @@ -19,7 +19,7 @@ public function log($level, \Stringable|string $message, array $context = []): v } /** - * @param array $context + * @param array $context */ private static function formatMessage(\Stringable|string $message, array $context): string { diff --git a/src/Tree/ParseTreeWalker.php b/src/Tree/ParseTreeWalker.php index a8374e8..485f7c7 100644 --- a/src/Tree/ParseTreeWalker.php +++ b/src/Tree/ParseTreeWalker.php @@ -8,11 +8,11 @@ class ParseTreeWalker { + private static ?self $default = null; + public static function default(): self { - static $instance; - - return $instance ?? ($instance = new self()); + return self::$default ??= new self(); } public function walk(ParseTreeListener $listener, ParseTree $tree): void diff --git a/src/Utils/BitSet.php b/src/Utils/BitSet.php index c7f11af..b8bbcc4 100644 --- a/src/Utils/BitSet.php +++ b/src/Utils/BitSet.php @@ -4,7 +4,7 @@ namespace Antlr\Antlr4\Runtime\Utils; -use Antlr\Antlr4\Runtime\Comparison\Hasher; +use Antlr\Antlr4\Runtime\Comparison\MurmurHash; final class BitSet { @@ -32,27 +32,40 @@ public function contains(int $value): bool } /** + * The set bits, in ascending order. + * + * A `BitSet` is ordered by bit position by definition, but the backing array + * is keyed by bit index and so iterates in *insertion* order. Returning that + * raw order made `{2, 1}` and `{1, 2}` distinguishable — visible to users, + * because `DiagnosticErrorListener` formats an alternative set straight into + * the `ambigAlts=` of a parser message. + * * @return array */ public function values(): array { - return \array_keys($this->data); + $values = \array_keys($this->data); + + \sort($values); + + return $values; } public function minValue(): int { - $values = $this->values(); - - if (\count($values) === 0) { + if ($this->data === []) { throw new \LogicException('BitSet is empty'); } - return \min($values); + // `nextSetBit(0)`: the lowest set bit, independent of insertion order. + return \min(\array_keys($this->data)); } public function hashCode(): int { - return Hasher::hash(...$this->values()); + // Hashed over the ordered bits so that equal sets hash alike — the same + // hash/equals contract that `equals()` below now honours. + return MurmurHash::hash($this->values()); } public function equals(object $other): bool @@ -61,10 +74,22 @@ public function equals(object $other): bool return true; } - return $other instanceof self - && $this->data === $other->data; + if (!$other instanceof self) { + return false; + } + + // `===` on the backing arrays compares key *order* as well as content, so + // two identical alternative sets built in different orders compared + // unequal. Java's `BitSet.equals` is positional and order-free. + return $this->values() === $other->values(); } + /** + * The number of set bits — Java's `cardinality()`. + * + * Note this is not Java's `BitSet.length()`, which is the highest set bit + * plus one. Every call site here means cardinality. + */ public function length(): int { return \count($this->data); diff --git a/src/Utils/Map.php b/src/Utils/Map.php index 5bab247..4ec7ecb 100644 --- a/src/Utils/Map.php +++ b/src/Utils/Map.php @@ -12,6 +12,8 @@ /** * @template K of Hashable * @template V + * + * @implements \IteratorAggregate */ final class Map implements Equatable, \Countable, \IteratorAggregate { diff --git a/src/Utils/Pair.php b/src/Utils/Pair.php index 3a3c5a3..b7e800d 100644 --- a/src/Utils/Pair.php +++ b/src/Utils/Pair.php @@ -6,7 +6,7 @@ use Antlr\Antlr4\Runtime\Comparison\Equality; use Antlr\Antlr4\Runtime\Comparison\Equatable; -use Antlr\Antlr4\Runtime\Comparison\Hasher; +use Antlr\Antlr4\Runtime\Comparison\MurmurHash; final class Pair implements Equatable { @@ -33,13 +33,14 @@ public function equals(object $other): bool public function hashCode(): int { - return Hasher::hash($this->a, $this->b); + return MurmurHash::hash([$this->a, $this->b]); } public function __toString(): string { + // Java's `String.format("(%s, %s)", a, b)` includes the parentheses. return \sprintf( - '%s, %s', + '(%s, %s)', $this->a === null ? 'null' : ($this->a instanceof \Stringable ? (string) $this->a : $this->a::class), diff --git a/src/Utils/Set.php b/src/Utils/Set.php index 1072d96..6f198ad 100644 --- a/src/Utils/Set.php +++ b/src/Utils/Set.php @@ -11,6 +11,8 @@ /** * @template T of Hashable + * + * @implements \IteratorAggregate */ final class Set implements Equatable, \IteratorAggregate, \Countable { @@ -140,6 +142,16 @@ public function add(Hashable $value): bool return true; } + /** + * Empties the set while keeping its equivalence, mirroring + * `Array2DHashSet.clear()`. + */ + public function clear(): void + { + $this->table = []; + $this->size = 0; + } + /** * @param T $value */ @@ -152,7 +164,7 @@ public function remove(Hashable $value): void } foreach ($this->table[$hash] as $index => $entry) { - if ($this->equivalence->equivalent($value, $entry)) { + if (!$this->equivalence->equivalent($value, $entry)) { continue; } @@ -176,24 +188,19 @@ public function equals(object $other): bool return true; } - if (!$other instanceof self - || $this->size !== $other->size - || !$this->equivalence->equals($other)) { + if (!$other instanceof self || $this->size !== $other->size) { return false; } - foreach ($this->table as $hash => $bucket) { - if (!isset($other->table[$hash]) || \count($bucket) !== \count($other->table[$hash])) { + // Set equality is membership, not layout: `Array2DHashSet.equals()` is + // `size() == other.size() && containsAll(other)`. Walking buckets pairwise + // by index — as this used to — makes equality depend on insertion order + // and on the hash distribution, so two sets with the same elements could + // compare unequal. + foreach ($other as $value) { + if (!$this->contains($value)) { return false; } - - $otherBucket = $other->table[$hash]; - - foreach ($bucket as $index => $value) { - if (!$value->equals($otherBucket[$index])) { - return false; - } - } } return true; diff --git a/src/VocabularyImpl.php b/src/VocabularyImpl.php index 16a76eb..bb8d25e 100644 --- a/src/VocabularyImpl.php +++ b/src/VocabularyImpl.php @@ -61,11 +61,11 @@ public function __construct(array $literalNames = [], array $symbolicNames = [], * {@see Vocabulary::getDisplayName()} returns the numeric value for * all tokens except {@see Token::EOF}. */ + private static ?self $emptyVocabulary = null; + public static function emptyVocabulary(): self { - static $empty; - - return $empty ?? ($empty = new self()); + return self::$emptyVocabulary ??= new self(); } /** From 4db05a9cf17da229a768fabb8e7155cb5492fe5f Mon Sep 17 00:00:00 2001 From: Marcos Passos Date: Sat, 15 Aug 2026 16:43:59 -0300 Subject: [PATCH 2/2] Fix static analysis and code style issues --- src/Atn/AltAndContextEquivalence.php | 2 ++ src/Comparison/MurmurHash.php | 2 +- src/Recognizer.php | 39 +++++++++++++++++----------- 3 files changed, 27 insertions(+), 16 deletions(-) diff --git a/src/Atn/AltAndContextEquivalence.php b/src/Atn/AltAndContextEquivalence.php index 31b98f0..f303c90 100644 --- a/src/Atn/AltAndContextEquivalence.php +++ b/src/Atn/AltAndContextEquivalence.php @@ -25,6 +25,8 @@ final class AltAndContextEquivalence implements Equivalence private function __construct() { + // Private so the class can only be reached through `instance()`. It + // holds no state, so one instance serves every caller. } public static function instance(): self diff --git a/src/Comparison/MurmurHash.php b/src/Comparison/MurmurHash.php index 1df5673..953ed6c 100644 --- a/src/Comparison/MurmurHash.php +++ b/src/Comparison/MurmurHash.php @@ -198,7 +198,7 @@ private static function rotateLeft(int $value, int $bits): int { $value &= self::MASK; - return (($value << $bits) | ($value >> 32 - $bits)) & self::MASK; + return ($value << $bits | $value >> 32 - $bits) & self::MASK; } private static function toSigned(int $value): int diff --git a/src/Recognizer.php b/src/Recognizer.php index 8b6dacb..cd604aa 100644 --- a/src/Recognizer.php +++ b/src/Recognizer.php @@ -57,32 +57,41 @@ abstract public function getVocabulary(): Vocabulary; public function getTokenTypeMap(): array { $vocabulary = $this->getVocabulary(); + $cache = self::$tokenTypeMapCache; - self::$tokenTypeMapCache ??= new \WeakMap(); - $result = self::$tokenTypeMapCache[$vocabulary] ?? null; + if ($cache === null) { + /** @var \WeakMap> $cache */ + $cache = new \WeakMap(); - if ($result === null) { - $result = []; + self::$tokenTypeMapCache = $cache; + } + + $cached = $cache[$vocabulary] ?? null; - for ($i = 0; $i <= $this->getATN()->maxTokenType; $i++) { - $literalName = $vocabulary->getLiteralName($i); + if ($cached !== null) { + return $cached; + } - if ($literalName !== null) { - $result[$literalName] = $i; - } + $result = []; - $symbolicName = $vocabulary->getSymbolicName($i); + for ($i = 0; $i <= $this->getATN()->maxTokenType; $i++) { + $literalName = $vocabulary->getLiteralName($i); - if ($symbolicName !== null) { - $result[$symbolicName] = $i; - } + if ($literalName !== null) { + $result[$literalName] = $i; } - $result['EOF'] = Token::EOF; + $symbolicName = $vocabulary->getSymbolicName($i); - self::$tokenTypeMapCache[$vocabulary] = $result; + if ($symbolicName !== null) { + $result[$symbolicName] = $i; + } } + $result['EOF'] = Token::EOF; + + $cache[$vocabulary] = $result; + return $result; }