From 86bfffc260eeb426c42746b28a6649a37084e52d Mon Sep 17 00:00:00 2001 From: tannevaled Date: Thu, 25 Jun 2026 22:53:37 +0200 Subject: [PATCH] Close real-world front-end gaps: ::-paths, and/or/not, masgn consts, ..., %r/%x/%s, keyword method names MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Confronting rbgo with Rails/Puppet/real gems showed the dominant blockers to running real Ruby were parser gaps. This closes the highest-leverage ones, all verified for parity against MRI 4.0.5 (`ruby -c`): P1 — `::` scope resolution in constant-path positions. `::`-paths (and a leading `::`) now parse as a class/module name (`class Foo::Bar`, `module A::B`, `class ::Top`), a superclass (`class E < Foo::Bar`, `class G < ::StandardError`), and a value (`::TopLevel`, `defined?(::Foo)`, `puts ::Foo`). ClassDef/ModuleDef gain NamePath, ClassDef gains SuperExpr; ScopedConst gains Global for the leading-`::` form. The all-bare-constant case keeps the existing string fields (NamePath/SuperExpr stay nil) so downstream consumers are unaffected. P2 — `and`/`or`/`not` low-precedence keyword operators, below `=` and above the trailing if/unless/while/until modifiers. They desugar to the existing `&&`/`||`/`!` nodes; `a or return` / `x = 1 and break` admit the jump keywords as operands. Wired into conditions, parens, and modifiers. P3 — multiple assignment to constant targets (`A, B = 1, 2`, `A, *B = …`). MultiAssign gains a parallel Targets slice, left nil for the all-locals fast path so existing consumers keep working. P4 — argument forwarding `...`: `def f(...)` / `def f(a, ...)` set MethodDef.Forward; `g(...)` / `g(1, ...)` produce the new ForwardArgs node. P5 — `%r{…}flags` (regexp), `%x{…}` (backtick command, new XStr node), and `%s{…}` (symbol) percent literals, with nesting/escape semantics matching MRI (`%r` keeps the backslash in source; `%x`/`%s` are string-like). P6 — keyword-named methods in def position (`def do`, `def then`, `def in`, …), and panic hardening: the token cursor now clamps at EOF, so malformed input (e.g. an unterminated string interpolation) returns a clean parse error instead of indexing past the slice and panicking. 100% coverage (CI gate), gofmt + go vet clean, CGO=0. Co-Authored-By: Claude Opus 4.8 --- arg_forwarding_test.go | 112 +++++++++++++++++++ ast/ast.go | 153 ++++++++++++++++---------- keyword_logical_test.go | 178 ++++++++++++++++++++++++++++++ keyword_method_test.go | 94 ++++++++++++++++ lexer/lexer.go | 92 +++++++++++++++- masgn_const_test.go | 77 +++++++++++++ parser.go | 230 ++++++++++++++++++++++++++++++++------- percent_rxs_test.go | 165 ++++++++++++++++++++++++++++ scope_resolution_test.go | 174 +++++++++++++++++++++++++++++ token/token.go | 28 +++-- 10 files changed, 1187 insertions(+), 116 deletions(-) create mode 100644 arg_forwarding_test.go create mode 100644 keyword_logical_test.go create mode 100644 keyword_method_test.go create mode 100644 masgn_const_test.go create mode 100644 percent_rxs_test.go create mode 100644 scope_resolution_test.go diff --git a/arg_forwarding_test.go b/arg_forwarding_test.go new file mode 100644 index 0000000..575f553 --- /dev/null +++ b/arg_forwarding_test.go @@ -0,0 +1,112 @@ +package parser_test + +import ( + "testing" + + "github.com/go-ruby-parser/parser" + "github.com/go-ruby-parser/parser/ast" +) + +func methodDef(t *testing.T, src string) *ast.MethodDef { + t.Helper() + m, ok := parseOne(t, src).(*ast.MethodDef) + if !ok { + t.Fatalf("Parse(%q): node = %T, want *ast.MethodDef", src, parseOne(t, src)) + } + return m +} + +// firstCall returns the first *ast.Call found in a method body. +func firstCall(t *testing.T, m *ast.MethodDef) *ast.Call { + t.Helper() + for _, n := range m.Body { + if c, ok := n.(*ast.Call); ok { + return c + } + } + t.Fatalf("no call in method body") + return nil +} + +func TestForwardDefAndCall(t *testing.T) { + m := methodDef(t, "def f(...); g(...); end") + if !m.Forward { + t.Error("MethodDef.Forward = false, want true") + } + if len(m.Params) != 0 { + t.Errorf("Params = %v, want empty", m.Params) + } + call := firstCall(t, m) + if len(call.Args) != 1 { + t.Fatalf("call args = %d, want 1", len(call.Args)) + } + if _, ok := call.Args[0].(*ast.ForwardArgs); !ok { + t.Errorf("call arg = %T, want *ast.ForwardArgs", call.Args[0]) + } +} + +func TestForwardWithLeadingParam(t *testing.T) { + m := methodDef(t, "def f(a, ...); g(a, ...); end") + if !m.Forward { + t.Error("Forward = false, want true") + } + if len(m.Params) != 1 || m.Params[0] != "a" { + t.Errorf("Params = %v, want [a]", m.Params) + } + call := firstCall(t, m) + if len(call.Args) != 2 { + t.Fatalf("call args = %d, want 2", len(call.Args)) + } + if _, ok := call.Args[1].(*ast.ForwardArgs); !ok { + t.Errorf("last arg = %T, want *ast.ForwardArgs", call.Args[1]) + } +} + +func TestForwardWithPositionalThenForward(t *testing.T) { + m := methodDef(t, "def f(...); g(1, ...); end") + call := firstCall(t, m) + if len(call.Args) != 2 { + t.Fatalf("call args = %d, want 2", len(call.Args)) + } + if _, ok := call.Args[0].(*ast.IntLit); !ok { + t.Errorf("arg0 = %T, want *ast.IntLit", call.Args[0]) + } + if _, ok := call.Args[1].(*ast.ForwardArgs); !ok { + t.Errorf("arg1 = %T, want *ast.ForwardArgs", call.Args[1]) + } +} + +func TestForwardDefOnly(t *testing.T) { + m := methodDef(t, "def f(...); end") + if !m.Forward { + t.Error("Forward = false, want true") + } +} + +func TestForwardParenlessDef(t *testing.T) { + m := methodDef(t, "def f ...\nend") + if !m.Forward { + t.Error("paren-less Forward = false, want true") + } +} + +func TestForwardDoesNotBreakRange(t *testing.T) { + for _, src := range []string{"(1...5)", "g(1...5)", "a = 1...10", "x[1...3]"} { + if _, err := parser.Parse(src); err != nil { + t.Errorf("Parse(%q) returned error: %v", src, err) + } + } +} + +func TestForwardValidPrograms(t *testing.T) { + for _, src := range []string{ + "def f(...); g(...); end", + "def f(a, ...); g(a, ...); end", + "def f(...); g(1, ...); end", + "def f(...); end", + } { + if _, err := parser.Parse(src); err != nil { + t.Errorf("Parse(%q) returned error: %v", src, err) + } + } +} diff --git a/ast/ast.go b/ast/ast.go index 58eb5a3..ec72b53 100644 --- a/ast/ast.go +++ b/ast/ast.go @@ -34,6 +34,10 @@ type RegexpLit struct { Flags string } +// XStr is a backtick command literal (`cmd` or %x{cmd}): its (Phase-0, +// non-interpolated) Command source is run by the shell, yielding its stdout. +type XStr struct{ Command string } + // ArrayLit is an array literal [a, b, c]. type ArrayLit struct{ Elems []Node } @@ -129,16 +133,17 @@ type While struct { // MethodDef defines a method on the current self. type MethodDef struct { - Name string - Params []string - Defaults []Node // parallel to Params; nil for a required param - SplatIndex int // index of the *splat param in Params, or -1 + Name string + Params []string + Defaults []Node // parallel to Params; nil for a required param + SplatIndex int // index of the *splat param in Params, or -1 KwParams []KwParam // keyword parameters (a:, b: default), after positionals KwRest string // name of the **rest keyword-splat param, or "" if none BlockParam string // name of the &block param, or "" if none Singleton bool // def self.foo — a singleton (class) method Recv Node // def recv.foo — explicit non-self receiver (nil otherwise) - Body []Node + Forward bool // def f(...) / def f(a, ...) — accepts forwarded arguments + Body []Node } // KwParam is a single keyword parameter. Default is nil for a required keyword @@ -155,10 +160,12 @@ type Return struct{ Value Node } // Value may be nil type ConstRef struct{ Name string } // ScopedConst is a constant looked up through ::, e.g. Math::PI or Foo::BAR. -// Recv evaluates to the module/class whose constant table is consulted. +// Recv evaluates to the module/class whose constant table is consulted. A nil +// Recv with Global true is a leading-`::` top-level lookup (`::Foo`). type ScopedConst struct { - Recv Node - Name string + Recv Node + Name string + Global bool // true for a leading `::Name` (top-level constant lookup) } // GVarRef references a global variable by name ("$~", "$1", "$stdout", …). @@ -172,8 +179,16 @@ type GVarAssign struct { // MultiAssign is a destructuring assignment to local targets: a, b = 1, 2 or // a, *b = list. SplatIndex is the index of the *splat target in Names, or -1. +// +// Targets, when non-nil, is parallel to Names and holds the general LHS target +// node for each position (e.g. a *ConstRef for a constant target `A, B = …`). +// It is left nil for the all-locals case so consumers that only understand local +// targets keep working; when any target is not a plain local, Targets is +// populated for every position and Names mirrors it (the local name, or "" for a +// non-local target whose splat capture, if any, has no local name). type MultiAssign struct { Names []string + Targets []Node // nil for all-locals; else parallel to Names SplatIndex int Values []Node } @@ -203,16 +218,27 @@ type CVarAssign struct { } // ClassDef defines or reopens a class. Super is the optional superclass name. +// +// Name carries the simple constant name; for a scope-resolution path +// (`class Foo::Bar`) or a leading-`::` name (`class ::Bar`) NamePath holds the +// full ScopedConst and Name is the trailing segment. SuperExpr, when non-nil, +// holds a superclass that is a `::`-path or otherwise not a bare constant; for a +// bare-constant superclass Super holds its name and SuperExpr is nil. type ClassDef struct { - Name string - Super string // "" if none - Body []Node + Name string + NamePath Node // *ScopedConst for `class A::B` / `class ::B`, else nil + Super string // "" if none and SuperExpr is nil + SuperExpr Node // non-nil superclass expression (e.g. a `::`-path), else nil + Body []Node } -// ModuleDef defines or reopens a module. +// ModuleDef defines or reopens a module. NamePath mirrors ClassDef.NamePath: it +// is non-nil for a scope-resolution or leading-`::` module name, in which case +// Name is the trailing segment. type ModuleDef struct { - Name string - Body []Node + Name string + NamePath Node // *ScopedConst for `module A::B` / `module ::B`, else nil + Body []Node } // Super calls the same-named method in the ancestor chain. Forward is true for a @@ -360,6 +386,11 @@ type SplatArg struct{ Value Node } // and passed as the call's block. It only ever appears last in a Call's args. type BlockPass struct{ Value Node } +// ForwardArgs is the `...` forwarding argument inside a call (`g(...)`): it +// splices the enclosing method's forwarded positional, keyword, and block +// arguments into this call. It only appears in a method that declared `...`. +type ForwardArgs struct{} + // Begin is `begin BODY (rescue …)* [else …] [ensure …] end`. type Begin struct { Body []Node @@ -376,53 +407,55 @@ type RescueClause struct { Body []Node } -func (*Program) node() {} -func (*ScopedConst) node() {} -func (*IntLit) node() {} -func (*BignumLit) node() {} -func (*FloatLit) node() {} -func (*StringLit) node() {} -func (*SymbolLit) node() {} -func (*RegexpLit) node() {} -func (*ArrayLit) node() {} -func (*HashLit) node() {} -func (*RangeLit) node() {} -func (*BoolLit) node() {} -func (*NilLit) node() {} -func (*SelfLit) node() {} -func (*VarRef) node() {} -func (*Assign) node() {} -func (*BinaryExpr) node() {} -func (*UnaryExpr) node() {} -func (*Call) node() {} -func (*If) node() {} -func (*While) node() {} -func (*MethodDef) node() {} -func (*Return) node() {} -func (*ConstRef) node() {} -func (*ConstAssign) node() {} -func (*GVarRef) node() {} -func (*GVarAssign) node() {} -func (*CVarRef) node() {} -func (*CVarAssign) node() {} -func (*MultiAssign) node() {} +func (*Program) node() {} +func (*ScopedConst) node() {} +func (*IntLit) node() {} +func (*BignumLit) node() {} +func (*FloatLit) node() {} +func (*StringLit) node() {} +func (*SymbolLit) node() {} +func (*RegexpLit) node() {} +func (*XStr) node() {} +func (*ArrayLit) node() {} +func (*HashLit) node() {} +func (*RangeLit) node() {} +func (*BoolLit) node() {} +func (*NilLit) node() {} +func (*SelfLit) node() {} +func (*VarRef) node() {} +func (*Assign) node() {} +func (*BinaryExpr) node() {} +func (*UnaryExpr) node() {} +func (*Call) node() {} +func (*If) node() {} +func (*While) node() {} +func (*MethodDef) node() {} +func (*Return) node() {} +func (*ConstRef) node() {} +func (*ConstAssign) node() {} +func (*GVarRef) node() {} +func (*GVarAssign) node() {} +func (*CVarRef) node() {} +func (*CVarAssign) node() {} +func (*MultiAssign) node() {} func (*MatchPattern) node() {} -func (*IvarRef) node() {} -func (*IvarAssign) node() {} -func (*ClassDef) node() {} -func (*ModuleDef) node() {} -func (*Super) node() {} -func (*Yield) node() {} -func (*Break) node() {} -func (*Next) node() {} -func (*OpAssign) node() {} -func (*Begin) node() {} -func (*StrInterp) node() {} -func (*Case) node() {} -func (*Retry) node() {} -func (*SplatArg) node() {} -func (*BlockPass) node() {} -func (*CaseIn) node() {} +func (*IvarRef) node() {} +func (*IvarAssign) node() {} +func (*ClassDef) node() {} +func (*ModuleDef) node() {} +func (*Super) node() {} +func (*Yield) node() {} +func (*Break) node() {} +func (*Next) node() {} +func (*OpAssign) node() {} +func (*Begin) node() {} +func (*StrInterp) node() {} +func (*Case) node() {} +func (*Retry) node() {} +func (*SplatArg) node() {} +func (*BlockPass) node() {} +func (*ForwardArgs) node() {} +func (*CaseIn) node() {} func (*ValuePattern) pattern() {} func (*BindPattern) pattern() {} diff --git a/keyword_logical_test.go b/keyword_logical_test.go new file mode 100644 index 0000000..49d95da --- /dev/null +++ b/keyword_logical_test.go @@ -0,0 +1,178 @@ +package parser_test + +import ( + "testing" + + "github.com/go-ruby-parser/parser" + "github.com/go-ruby-parser/parser/ast" +) + +// binExpr asserts n is a *ast.BinaryExpr with the given operator. +func binExpr(t *testing.T, n ast.Node, op string) *ast.BinaryExpr { + t.Helper() + be, ok := n.(*ast.BinaryExpr) + if !ok { + t.Fatalf("node = %T, want *ast.BinaryExpr", n) + } + if be.Op != op { + t.Fatalf("BinaryExpr.Op = %q, want %q", be.Op, op) + } + return be +} + +// unaryExpr asserts n is a *ast.UnaryExpr with the given operator. +func unaryExpr(t *testing.T, n ast.Node, op string) *ast.UnaryExpr { + t.Helper() + ue, ok := n.(*ast.UnaryExpr) + if !ok { + t.Fatalf("node = %T, want *ast.UnaryExpr", n) + } + if ue.Op != op { + t.Fatalf("UnaryExpr.Op = %q, want %q", ue.Op, op) + } + return ue +} + +func TestKeywordAndDesugarsToAndAnd(t *testing.T) { + // `x = true and false` is `(x = true) && false` (and binds below =). + be := binExpr(t, parseOne(t, "x = true and false"), "&&") + if _, ok := be.Left.(*ast.Assign); !ok { + t.Errorf("Left = %T, want *ast.Assign (= binds tighter than and)", be.Left) + } +} + +func TestKeywordOrDesugarsToOrOr(t *testing.T) { + binExpr(t, parseOne(t, "a or b"), "||") +} + +func TestKeywordAndOrLeftAssociative(t *testing.T) { + // `a and b or c` is `(a and b) or c`. + top := binExpr(t, parseOne(t, "a and b or c"), "||") + binExpr(t, top.Left, "&&") +} + +func TestKeywordNotPrefixStatement(t *testing.T) { + unaryExpr(t, parseOne(t, "not x"), "!") +} + +func TestKeywordNotInAssignRHS(t *testing.T) { + a, ok := parseOne(t, "x = not true").(*ast.Assign) + if !ok { + t.Fatalf("node = %T, want *ast.Assign", parseOne(t, "x = not true")) + } + unaryExpr(t, a.Value, "!") +} + +func TestKeywordNotDoubled(t *testing.T) { + outer := unaryExpr(t, parseOne(t, "not not x"), "!") + unaryExpr(t, outer.Operand, "!") +} + +func TestKeywordNotLowPrecedence(t *testing.T) { + // `not a == b` is `not (a == b)`. + ue := unaryExpr(t, parseOne(t, "not a == b"), "!") + binExpr(t, ue.Operand, "==") +} + +func TestKeywordNotInCallArg(t *testing.T) { + call := parseOne(t, "foo(not flag)").(*ast.Call) + unaryExpr(t, call.Args[0], "!") +} + +func TestKeywordOrReturnOperand(t *testing.T) { + // `a or return` — the jump keyword is a valid or-operand. + be := binExpr(t, parseOne(t, "a or return"), "||") + if _, ok := be.Right.(*ast.Return); !ok { + t.Errorf("Right = %T, want *ast.Return", be.Right) + } +} + +func TestKeywordAndBreakOperand(t *testing.T) { + be := binExpr(t, parseOne(t, "a and break"), "&&") + if _, ok := be.Right.(*ast.Break); !ok { + t.Errorf("Right = %T, want *ast.Break", be.Right) + } +} + +func TestKeywordOrNextOperand(t *testing.T) { + be := binExpr(t, parseOne(t, "a or next"), "||") + if _, ok := be.Right.(*ast.Next); !ok { + t.Errorf("Right = %T, want *ast.Next", be.Right) + } +} + +func TestKeywordAndReturnWithValue(t *testing.T) { + // `x = 1 and return v` — return takes a value as the and-operand. + be := binExpr(t, parseOne(t, "x = 1 and return 2"), "&&") + ret := be.Right.(*ast.Return) + if ret.Value == nil { + t.Errorf("return value = nil, want a value") + } +} + +func TestKeywordLogicalInCondition(t *testing.T) { + ifn := parseOne(t, "if a and b then 1 end").(*ast.If) + binExpr(t, ifn.Cond, "&&") +} + +func TestKeywordLogicalInUnless(t *testing.T) { + // `unless a or b` desugars cond to `!(a || b)`. + ifn := parseOne(t, "unless a or b then 1 end").(*ast.If) + ue := unaryExpr(t, ifn.Cond, "!") + binExpr(t, ue.Operand, "||") +} + +func TestKeywordLogicalInElsif(t *testing.T) { + ifn := parseOne(t, "if a\n1\nelsif b and c\n2\nend").(*ast.If) + binExpr(t, ifn.Elsifs[0].Cond, "&&") +} + +func TestKeywordLogicalInWhile(t *testing.T) { + w := parseOne(t, "while a or b; c; end").(*ast.While) + binExpr(t, w.Cond, "||") +} + +func TestKeywordLogicalInUntil(t *testing.T) { + w := parseOne(t, "until a and b; c; end").(*ast.While) + unaryExpr(t, w.Cond, "!") // until desugars to while !cond +} + +func TestKeywordLogicalInParens(t *testing.T) { + binExpr(t, parseOne(t, "(a and b)"), "&&") +} + +func TestKeywordNotInParens(t *testing.T) { + unaryExpr(t, parseOne(t, "(not x)"), "!") +} + +func TestKeywordLogicalModifierIf(t *testing.T) { + // `x if a and b` — the modifier condition admits and/or. + ifn := parseOne(t, "x if a and b").(*ast.If) + binExpr(t, ifn.Cond, "&&") +} + +func TestKeywordLogicalModifierWhile(t *testing.T) { + w := parseOne(t, "x while a or b").(*ast.While) + binExpr(t, w.Cond, "||") +} + +func TestKeywordLogicalModifierUntil(t *testing.T) { + w := parseOne(t, "x until a and b").(*ast.While) + unaryExpr(t, w.Cond, "!") +} + +func TestKeywordLogicalValidPrograms(t *testing.T) { + for _, src := range []string{ + "x = true and false", + "a or return", + "not x", + "x = not not x", + "a and b or c", + "if x and y then 1 end", + "while a or b; c; end", + } { + if _, err := parser.Parse(src); err != nil { + t.Errorf("Parse(%q) returned error: %v", src, err) + } + } +} diff --git a/keyword_method_test.go b/keyword_method_test.go new file mode 100644 index 0000000..433c7df --- /dev/null +++ b/keyword_method_test.go @@ -0,0 +1,94 @@ +package parser_test + +import ( + "testing" + + "github.com/go-ruby-parser/parser" + "github.com/go-ruby-parser/parser/ast" +) + +func TestKeywordMethodNames(t *testing.T) { + for _, tc := range []struct{ src, name string }{ + {"def do; end", "do"}, + {"def then; end", "then"}, + {"def in; end", "in"}, + {"def class; end", "class"}, + {"def begin; end", "begin"}, + {"def if; end", "if"}, + {"def while; end", "while"}, + {"def nil; end", "nil"}, + {"def end; end", "end"}, + } { + m, ok := parseOne(t, tc.src).(*ast.MethodDef) + if !ok { + t.Errorf("Parse(%q): node = %T, want *ast.MethodDef", tc.src, parseOne(t, tc.src)) + continue + } + if m.Name != tc.name { + t.Errorf("Parse(%q): Name = %q, want %q", tc.src, m.Name, tc.name) + } + } +} + +func TestKeywordMethodSingleton(t *testing.T) { + m := parseOne(t, "def self.do; end").(*ast.MethodDef) + if !m.Singleton || m.Name != "do" { + t.Errorf("got Singleton=%v Name=%q, want true/do", m.Singleton, m.Name) + } +} + +func TestKeywordMethodOnReceiver(t *testing.T) { + m := parseOne(t, "def obj.then; end").(*ast.MethodDef) + if m.Name != "then" { + t.Errorf("Name = %q, want then", m.Name) + } + if _, ok := m.Recv.(*ast.VarRef); !ok { + t.Errorf("Recv = %T, want *ast.VarRef", m.Recv) + } +} + +// TestNoPanicOnUnterminatedInterpolation guards the parser against panicking on +// malformed input: an unterminated string interpolation must yield a clean +// error, never a runtime panic (the EOF-clamped cursor makes this safe). +func TestNoPanicOnUnterminatedInterpolation(t *testing.T) { + for _, src := range []string{ + `"#{`, + `"#{a`, + `a("#{b`, + `"#{1 +`, + `"#{x}#{`, + } { + assertCleanError(t, src) + } +} + +// TestNoPanicOnTruncatedInput exercises a broad set of truncated/garbled inputs; +// each must return a clean parse error rather than panic. +func TestNoPanicOnTruncatedInput(t *testing.T) { + for _, src := range []string{ + `:"#{x`, + `def f(`, + `class`, + `[1,`, + `{a:`, + `(`, + `foo(`, + `if`, + } { + assertCleanError(t, src) + } +} + +// assertCleanError parses src expecting a returned error and explicitly fails if +// Parse panics instead of returning. +func assertCleanError(t *testing.T, src string) { + t.Helper() + defer func() { + if r := recover(); r != nil { + t.Fatalf("Parse(%q) panicked: %v", src, r) + } + }() + if _, err := parser.Parse(src); err == nil { + t.Errorf("Parse(%q) = no error, want a clean parse error", src) + } +} diff --git a/lexer/lexer.go b/lexer/lexer.go index 19e5b01..3d85380 100644 --- a/lexer/lexer.go +++ b/lexer/lexer.go @@ -138,6 +138,9 @@ func (l *Lexer) lexToken() token.Token { case c == '%' && l.percentBeginsLiteral(spaceBefore) && l.atPercentArray(): // %w[…] / %i[…] / %W[…] / %I[…] word- and symbol-array literals. return l.lexPercentArray(spaceBefore, line, col) + case c == '%' && l.percentBeginsLiteral(spaceBefore) && l.atPercentRXS(): + // %r{…}flags regexp, %x{…} backtick command, %s{…} symbol literals. + return l.lexPercentRXS(spaceBefore, line, col) case c == '%' && l.percentBeginsLiteral(spaceBefore) && l.atPercentString(): // %q(…) / %Q(…) / %(…) / %W(…) string literals. return l.lexPercentString(spaceBefore, line, col) @@ -512,7 +515,7 @@ func (l *Lexer) lexNumber(spaceBefore bool, line, col int) token.Token { // Underscores between digits are allowed. With no digits after the prefix it // returns ILLEGAL. func (l *Lexer) lexRadixInt(spaceBefore bool, line, col int) token.Token { - l.advance() // '0' + l.advance() // '0' kind := l.advance() | 0x20 // letter, lower-cased ok := radixDigit(kind) var digits []byte @@ -898,6 +901,91 @@ func isPercentDelim(b byte) bool { return false } +// atPercentRXS reports whether the cursor (at '%') begins a %r (regexp), +// %x (backtick command), or %s (symbol) literal: the kind letter must be +// followed by a delimiter. +func (l *Lexer) atPercentRXS() bool { + switch l.peek2() { + case 'r', 'x', 's': + default: + return false + } + return l.pos+2 < len(l.src) && isPercentDelim(l.src[l.pos+2]) +} + +// lexPercentRXS lexes a %r{…}flags regexp, %x{…} backtick command, or %s{…} +// symbol literal. As with the existing /…/ regexp form, the body is kept as raw +// source (interpolation markers are not expanded here). Bracket delimiters nest; +// a backslash keeps its following byte verbatim so an escaped delimiter does not +// close the literal. +func (l *Lexer) lexPercentRXS(spaceBefore bool, line, col int) token.Token { + l.advance() // % + kind := l.advance() + open := l.advance() + closing := percentDelimClose(open) + depth := 1 + var body []byte + for { + c := l.peek() + if c == 0 { + return token.Token{Type: token.ILLEGAL, Lit: "unterminated %-literal", Line: line, Col: col, SpaceBefore: spaceBefore} + } + if c == '\\' { + l.advance() + esc := l.peek() + if esc == 0 { + body = append(body, '\\') + break + } + l.advance() + switch { + case kind == 'r': + // %r keeps the escape pair verbatim for the regexp engine — even an + // escaped delimiter (MRI: `%r{a\}b}.source == "a\\}b"`). The escape + // only prevents the delimiter from closing the literal. + body = append(body, '\\', esc) + case esc == open || esc == closing || esc == '\\': + // %x/%s are string-like: an escaped delimiter or backslash becomes the + // literal character (the backslash is dropped). + body = append(body, esc) + default: + body = append(body, '\\', esc) + } + continue + } + if open != closing && c == open { + depth++ + } else if c == closing { + depth-- + if depth == 0 { + l.advance() // closing delimiter + break + } + } + body = append(body, l.advance()) + } + l.state = exprEnd + switch kind { + case 'r': + var flags []byte + for { + c := l.peek() + if c < 'a' || c > 'z' { + break + } + l.advance() + if c == 'i' || c == 'm' || c == 'x' { + flags = append(flags, c) + } + } + return token.Token{Type: token.REGEXP, Lit: string(body), Flags: string(flags), Line: line, Col: col, SpaceBefore: spaceBefore} + case 'x': + return token.Token{Type: token.XSTRING, Lit: string(body), Line: line, Col: col, SpaceBefore: spaceBefore} + default: // 's' — a symbol; its name is the (un-interpolated) body + return token.Token{Type: token.SYMBOL, Lit: string(body), Line: line, Col: col, SpaceBefore: spaceBefore} + } +} + // atPercentString reports whether the cursor (at '%') begins a %q/%Q/%(…) // string literal: %q or %Q followed by a delimiter, or a bare % directly // followed by a delimiter (== %Q). @@ -1232,7 +1320,7 @@ func squiggleDedent(body string) string { // a double-quoted string. func wrapHeredocDQ(body string) string { var b strings.Builder - depth := 0 // brace nesting inside an active #{ … } + depth := 0 // brace nesting inside an active #{ … } escaped := false // the previous byte was a backslash escaping this one for i := 0; i < len(body); i++ { c := body[i] diff --git a/masgn_const_test.go b/masgn_const_test.go new file mode 100644 index 0000000..9ca259f --- /dev/null +++ b/masgn_const_test.go @@ -0,0 +1,77 @@ +package parser_test + +import ( + "testing" + + "github.com/go-ruby-parser/parser" + "github.com/go-ruby-parser/parser/ast" +) + +func masgn(t *testing.T, src string) *ast.MultiAssign { + t.Helper() + m, ok := parseOne(t, src).(*ast.MultiAssign) + if !ok { + t.Fatalf("Parse(%q): node = %T, want *ast.MultiAssign", src, parseOne(t, src)) + } + return m +} + +func TestMasgnConstTargets(t *testing.T) { + m := masgn(t, "A, B = 1, 2") + if len(m.Targets) != 2 { + t.Fatalf("Targets len = %d, want 2", len(m.Targets)) + } + for i, want := range []string{"A", "B"} { + cr, ok := m.Targets[i].(*ast.ConstRef) + if !ok || cr.Name != want { + t.Errorf("Targets[%d] = %#v, want ConstRef %q", i, m.Targets[i], want) + } + if m.Names[i] != "" { + t.Errorf("Names[%d] = %q, want empty for a constant target", i, m.Names[i]) + } + } +} + +func TestMasgnMixedConstLocal(t *testing.T) { + m := masgn(t, "A, b = 1, 2") + if _, ok := m.Targets[0].(*ast.ConstRef); !ok { + t.Errorf("Targets[0] = %T, want *ast.ConstRef", m.Targets[0]) + } + vr, ok := m.Targets[1].(*ast.VarRef) + if !ok || vr.Name != "b" { + t.Errorf("Targets[1] = %#v, want VarRef b", m.Targets[1]) + } + if m.Names[1] != "b" { + t.Errorf("Names[1] = %q, want b", m.Names[1]) + } +} + +func TestMasgnConstSplat(t *testing.T) { + m := masgn(t, "a, *B = 1, 2, 3") + if m.SplatIndex != 1 { + t.Errorf("SplatIndex = %d, want 1", m.SplatIndex) + } + if _, ok := m.Targets[1].(*ast.ConstRef); !ok { + t.Errorf("Targets[1] = %T, want *ast.ConstRef", m.Targets[1]) + } +} + +func TestMasgnAllLocalsTargetsNil(t *testing.T) { + // The all-locals fast path leaves Targets nil so existing consumers are + // unaffected. + m := masgn(t, "a, b = 1, 2") + if m.Targets != nil { + t.Errorf("all-locals Targets = %#v, want nil", m.Targets) + } + if m.Names[0] != "a" || m.Names[1] != "b" { + t.Errorf("Names = %v, want [a b]", m.Names) + } +} + +func TestMasgnConstValidPrograms(t *testing.T) { + for _, src := range []string{"A, B = 1, 2", "A, b = 1, 2", "a, *B = 1, 2, 3", "X, Y, Z = foo"} { + if _, err := parser.Parse(src); err != nil { + t.Errorf("Parse(%q) returned error: %v", src, err) + } + } +} diff --git a/parser.go b/parser.go index 938364b..789109e 100644 --- a/parser.go +++ b/parser.go @@ -127,11 +127,29 @@ func Parse(src string) (prog *ast.Program, err error) { // --- token cursor --- -func (p *Parser) cur() token.Token { return p.toks[p.pos] } -// peekTok returns the token after the cursor. It is only ever called with the -// cursor on an IDENT (which is never the trailing EOF), so pos+1 is in range. +// The cursor clamps at the trailing EOF: once exhausted, cur/peekTok keep +// returning EOF and advance is a no-op, so a parser that over-reads on malformed +// input (e.g. an unterminated string interpolation) fails cleanly via expect +// instead of indexing past the slice and panicking. +func (p *Parser) cur() token.Token { + if p.pos >= len(p.toks) { + return p.toks[len(p.toks)-1] // the EOF token + } + return p.toks[p.pos] +} + +// peekTok returns the token after the cursor. Every caller first checks the +// cursor is a specific non-EOF token, and the trailing EOF is always present, so +// the cursor is at most the second-to-last token and pos+1 stays in range. func (p *Parser) peekTok() token.Token { return p.toks[p.pos+1] } -func (p *Parser) advance() token.Token { t := p.toks[p.pos]; p.pos++; return t } + +func (p *Parser) advance() token.Token { + t := p.cur() + if p.pos < len(p.toks) { + p.pos++ + } + return t +} func (p *Parser) is(tt token.Type) bool { return p.cur().Type == tt } @@ -199,12 +217,12 @@ func (p *Parser) barewordValue(name string) ast.Node { // --- statements --- var ( - bodyEnd = map[token.Type]bool{token.END: true} + bodyEnd = map[token.Type]bool{token.END: true} braceBlockEnd = map[token.Type]bool{token.RBRACE: true} - beginBodyEnd = map[token.Type]bool{token.RESCUE: true, token.ELSE: true, token.ENSURE: true, token.END: true} - caseBodyEnd = map[token.Type]bool{token.WHEN: true, token.ELSE: true, token.END: true} - inBodyEnd = map[token.Type]bool{token.IN: true, token.ELSE: true, token.END: true} - ifBodyEnd = map[token.Type]bool{token.END: true, token.ELSE: true, token.ELSIF: true} + beginBodyEnd = map[token.Type]bool{token.RESCUE: true, token.ELSE: true, token.ENSURE: true, token.END: true} + caseBodyEnd = map[token.Type]bool{token.WHEN: true, token.ELSE: true, token.END: true} + inBodyEnd = map[token.Type]bool{token.IN: true, token.ELSE: true, token.END: true} + ifBodyEnd = map[token.Type]bool{token.END: true, token.ELSE: true, token.ELSIF: true} // rangeHiEnds marks tokens that cannot begin a range's high endpoint, making // the range endless (`1..`, `arr[2..]`). rangeHiEnds = map[token.Type]bool{token.RBRACKET: true, token.RPAREN: true, token.RBRACE: true, token.NEWLINE: true, token.EOF: true, token.COMMA: true, token.END: true, token.THEN: true, token.DO: true} @@ -253,8 +271,44 @@ func (p *Parser) parseStatement() ast.Node { p.advance() return p.applyModifiers(&ast.Retry{}) default: - return p.applyModifiers(p.parseOneLineMatch(p.parseExprOrAssign())) + return p.applyModifiers(p.parseOneLineMatch(p.parseKeywordLogical())) + } +} + +// parseKeywordLogical parses the low-precedence keyword operators `and`, `or`, +// and the prefix `not`. They bind looser than `=` (and everything else except +// the trailing if/unless/while/until modifiers), so they sit between an +// assignment and a statement. `and`/`or` are left-associative and desugar to the +// `&&`/`||` BinaryExpr nodes; `not` desugars to a `!` UnaryExpr. (`p(1 and 2)` +// is itself invalid Ruby, so this layer only appears in statement/assignment +// positions, never inside a paren-less argument.) +func (p *Parser) parseKeywordLogical() ast.Node { + left := p.parseExprOrAssign() + for p.is(token.AND) || p.is(token.OR) { + op := "&&" + if p.is(token.OR) { + op = "||" + } + p.advance() + left = &ast.BinaryExpr{Op: op, Left: left, Right: p.parseKeywordOperand()} + } + return left +} + +// parseKeywordOperand parses an operand of `and`/`or`. Besides an ordinary +// expression (a leading `not` is handled by parseExprOrAssign) it accepts the +// jump keywords `return`/`break`/`next` (`a or return`, `x = 1 and break`), +// which are valid in this position in MRI. +func (p *Parser) parseKeywordOperand() ast.Node { + switch p.cur().Type { + case token.RETURN: + return p.parseReturn() + case token.BREAK: + return p.parseBreak() + case token.NEXT: + return p.parseNext() } + return p.parseExprOrAssign() } // parseOneLineMatch wraps a statement-level expression in a one-line pattern @@ -277,16 +331,16 @@ func (p *Parser) applyModifiers(node ast.Node) ast.Node { switch p.cur().Type { case token.IF: p.advance() - node = &ast.If{Cond: p.parseExprOrAssign(), Then: []ast.Node{node}} + node = &ast.If{Cond: p.parseCond(), Then: []ast.Node{node}} case token.UNLESS: p.advance() - node = &ast.If{Cond: not(p.parseExprOrAssign()), Then: []ast.Node{node}} + node = &ast.If{Cond: not(p.parseCond()), Then: []ast.Node{node}} case token.WHILE: p.advance() - node = &ast.While{Cond: p.parseExprOrAssign(), Body: []ast.Node{node}} + node = &ast.While{Cond: p.parseCond(), Body: []ast.Node{node}} case token.UNTIL: p.advance() - node = &ast.While{Cond: not(p.parseExprOrAssign()), Body: []ast.Node{node}} + node = &ast.While{Cond: not(p.parseCond()), Body: []ast.Node{node}} default: return node } @@ -295,28 +349,64 @@ func (p *Parser) applyModifiers(node ast.Node) ast.Node { func not(n ast.Node) ast.Node { return &ast.UnaryExpr{Op: "!", Operand: n} } +// parseConstPath parses a constant path in a name/superclass position: a bare +// constant (`Foo`), a scope-resolution path (`Foo::Bar::Baz`), or a leading-`::` +// path (`::Foo`, `::Foo::Bar`). It returns the trailing segment name and, when +// the path is more than a bare constant, the full *ScopedConst node (else nil). +func (p *Parser) parseConstPath() (name string, path ast.Node) { + var node ast.Node + if p.accept(token.SCOPE) { // leading `::Name` + name = p.expect(token.CONST).Lit + node = &ast.ScopedConst{Name: name, Global: true} + } else { + name = p.expect(token.CONST).Lit + node = &ast.ConstRef{Name: name} + } + scoped := node // becomes the *ScopedConst once a `::CONST` segment is seen + for p.is(token.SCOPE) && p.peekTok().Type == token.CONST { + p.advance() // :: + name = p.advance().Lit + scoped = &ast.ScopedConst{Recv: scoped, Name: name} + } + if _, ok := scoped.(*ast.ConstRef); ok { + return name, nil // a bare constant: no path node + } + return name, scoped +} + func (p *Parser) parseClass() ast.Node { p.expect(token.CLASS) - name := p.expect(token.CONST).Lit + // `class << self` (singleton-class body) is out of this front-end's scope; a + // SHOVEL here is not a constant path. + if p.is(token.SHOVEL) { + p.fail("singleton class (class << ...) is not supported") + } + name, path := p.parseConstPath() super := "" + var superExpr ast.Node if p.accept(token.LT) { - super = p.expect(token.CONST).Lit + sname, spath := p.parseConstPath() + if spath != nil { + superExpr = spath + } else { + super = sname + } } p.pushScope() // a class body has its own local scope body := p.parseStatements(bodyEnd) p.popScope() p.expect(token.END) - return &ast.ClassDef{Name: name, Super: super, Body: body} + return &ast.ClassDef{Name: name, NamePath: path, Super: super, SuperExpr: superExpr, Body: body} } func (p *Parser) parseModule() ast.Node { p.expect(token.MODULE) - name := p.expect(token.CONST).Lit + name, path := p.parseConstPath() p.pushScope() // a module body has its own local scope body := p.parseStatements(bodyEnd) p.popScope() p.expect(token.END) - return &ast.ModuleDef{Name: name, Body: body} + return &ast.ModuleDef{Name: name, NamePath: path, Body: body} } func (p *Parser) parseDef() ast.Node { @@ -346,19 +436,20 @@ func (p *Parser) parseDef() ast.Node { var defaults []ast.Node var kwParams []ast.KwParam var kwRest, blockParam string + forward := false splat := -1 if p.accept(token.LPAREN) { - params, defaults, splat, kwParams, kwRest, blockParam = p.parseDefParams(token.RPAREN) + params, defaults, splat, kwParams, kwRest, blockParam, forward = p.parseDefParams(token.RPAREN) p.expect(token.RPAREN) - } else if (p.is(token.IDENT) || p.is(token.LABEL) || p.is(token.AMPER)) && !p.is(token.NEWLINE) { + } else if (p.is(token.IDENT) || p.is(token.LABEL) || p.is(token.AMPER) || p.is(token.DOTDOTDOT)) && !p.is(token.NEWLINE) { // paren-less params: def foo a, b / def foo a:, b: 2 / def foo &blk - params, defaults, splat, kwParams, kwRest, blockParam = p.parseDefParams(token.NEWLINE) + params, defaults, splat, kwParams, kwRest, blockParam, forward = p.parseDefParams(token.NEWLINE) } // Endless method definition: def name(params) = expr (no body/end). if p.accept(token.ASSIGN) { body := []ast.Node{p.parseExprOrAssign()} p.popScope() - return &ast.MethodDef{Name: name, Params: params, Defaults: defaults, SplatIndex: splat, KwParams: kwParams, KwRest: kwRest, BlockParam: blockParam, Singleton: singleton, Recv: recv, Body: body} + return &ast.MethodDef{Name: name, Params: params, Defaults: defaults, SplatIndex: splat, KwParams: kwParams, KwRest: kwRest, BlockParam: blockParam, Singleton: singleton, Recv: recv, Forward: forward, Body: body} } body := p.parseStatements(beginBodyEnd) // A method body may carry rescue/ensure clauses without an explicit begin. @@ -367,7 +458,7 @@ func (p *Parser) parseDef() ast.Node { } p.popScope() p.expect(token.END) - return &ast.MethodDef{Name: name, Params: params, Defaults: defaults, SplatIndex: splat, KwParams: kwParams, KwRest: kwRest, BlockParam: blockParam, Singleton: singleton, Recv: recv, Body: body} + return &ast.MethodDef{Name: name, Params: params, Defaults: defaults, SplatIndex: splat, KwParams: kwParams, KwRest: kwRest, BlockParam: blockParam, Singleton: singleton, Recv: recv, Forward: forward, Body: body} } // parseDefName reads the name in a `def`: an identifier/constant, an operator @@ -395,6 +486,11 @@ func (p *Parser) parseDefName() (string, bool) { } return "[]", true } + // A keyword used as a method name: def do / def then / def in / def class … + // Ruby permits any reserved word in def-name position. + if _, isKeyword := token.Keywords[p.cur().Lit]; isKeyword { + return p.advance().Lit, true + } return "", false } @@ -402,12 +498,16 @@ func (p *Parser) parseDefName() (string, bool) { // default`. Each parameter is declared before its (and later) defaults are // parsed, so a default may reference earlier parameters. defaults is parallel to // params, nil for a required parameter. -func (p *Parser) parseDefParams(until token.Type) (params []string, defaults []ast.Node, splat int, kwParams []ast.KwParam, kwRest, blockParam string) { +func (p *Parser) parseDefParams(until token.Type) (params []string, defaults []ast.Node, splat int, kwParams []ast.KwParam, kwRest, blockParam string, forward bool) { splat = -1 if p.is(until) || p.is(token.NEWLINE) { - return params, defaults, splat, kwParams, kwRest, blockParam + return params, defaults, splat, kwParams, kwRest, blockParam, forward } for { + if p.accept(token.DOTDOTDOT) { // `...` argument-forwarding param (always last) + forward = true + break + } if p.accept(token.AMPER) { // &block param (always last) blockParam = p.expect(token.IDENT).Lit p.declareLocal(blockParam) @@ -453,18 +553,23 @@ func (p *Parser) parseDefParams(until token.Type) (params []string, defaults []a break } } - return params, defaults, splat, kwParams, kwRest, blockParam + return params, defaults, splat, kwParams, kwRest, blockParam, forward } +// parseCond parses an if/unless/while/until condition, where the low-precedence +// keyword operators `and`/`or`/`not` are permitted (`if a and b`, `while x or +// y`, `unless not done`). +func (p *Parser) parseCond() ast.Node { return p.parseKeywordLogical() } + func (p *Parser) parseIf() ast.Node { p.expect(token.IF) - cond := p.parseExprOrAssign() + cond := p.parseCond() p.accept(token.THEN) then := p.parseStatements(ifBodyEnd) node := &ast.If{Cond: cond, Then: then} for p.is(token.ELSIF) { p.advance() - c := p.parseExprOrAssign() + c := p.parseCond() p.accept(token.THEN) b := p.parseStatements(ifBodyEnd) node.Elsifs = append(node.Elsifs, ast.Elsif{Cond: c, Body: b}) @@ -479,7 +584,7 @@ func (p *Parser) parseIf() ast.Node { // parseUnless desugars `unless c ... else ... end` to `if !c ... else ... end`. func (p *Parser) parseUnless() ast.Node { p.expect(token.UNLESS) - cond := p.parseExprOrAssign() + cond := p.parseCond() p.accept(token.THEN) then := p.parseStatements(ifBodyEnd) node := &ast.If{Cond: not(cond), Then: then} @@ -504,7 +609,7 @@ func (p *Parser) parseWhile() ast.Node { func (p *Parser) parseLoopCond() ast.Node { saved := p.noDo p.noDo = true - cond := p.parseExprOrAssign() + cond := p.parseCond() p.noDo = saved return cond } @@ -911,8 +1016,9 @@ func (p *Parser) buildArrayOrFind(constName ast.Node, elems []arrayElem) ast.Pat // --- expressions --- // looksLikeMlhs scans ahead (without consuming) for a multiple-assignment left -// side: `[*]IDENT (, [*]IDENT)+ =` — at least one comma, every target a local -// name. It deliberately does not handle ivar/const/attribute targets. +// side: `[*]TARGET (, [*]TARGET)+ =` — at least one comma, every target a local +// name (IDENT) or a constant (CONST). It deliberately does not handle +// ivar/attribute targets. func (p *Parser) looksLikeMlhs() bool { i := p.pos sawComma := false @@ -920,7 +1026,7 @@ func (p *Parser) looksLikeMlhs() bool { if p.toks[i].Type == token.STAR { i++ } - if p.toks[i].Type != token.IDENT { + if p.toks[i].Type != token.IDENT && p.toks[i].Type != token.CONST { return false } i++ @@ -936,31 +1042,53 @@ func (p *Parser) looksLikeMlhs() bool { } } -// parseMlhs parses a multiple assignment whose targets are local names, with at -// most one *splat target. +// parseMlhs parses a multiple assignment whose targets are local names and/or +// constants, with at most one *splat target. When every target is a local the +// result uses only Names (and Targets stays nil); when any target is a constant +// it additionally fills Targets (parallel to Names) with the per-position LHS +// node so a consumer can store into a constant. func (p *Parser) parseMlhs() ast.Node { var names []string + var targets []ast.Node + hasConst := false splat := -1 for { if p.accept(token.STAR) { splat = len(names) } - name := p.expect(token.IDENT).Lit - names = append(names, name) - p.declareLocal(name) + if p.is(token.CONST) { + name := p.advance().Lit + names = append(names, "") + targets = append(targets, &ast.ConstRef{Name: name}) + hasConst = true + } else { + name := p.expect(token.IDENT).Lit + names = append(names, name) + targets = append(targets, &ast.VarRef{Name: name}) + p.declareLocal(name) + } if !p.accept(token.COMMA) { break } } + if !hasConst { + targets = nil // all-locals fast path: Names alone suffices + } p.expect(token.ASSIGN) values := []ast.Node{p.parseTernary()} for p.accept(token.COMMA) { values = append(values, p.parseTernary()) } - return &ast.MultiAssign{Names: names, SplatIndex: splat, Values: values} + return &ast.MultiAssign{Names: names, Targets: targets, SplatIndex: splat, Values: values} } func (p *Parser) parseExprOrAssign() ast.Node { + // A `not EXPR` prefix in value position (assignment RHS, call argument, array + // element): `x = not true`, `foo(not flag)`. `not` binds looser than the + // binary operators, so it wraps the whole following expression. + if p.accept(token.NOT) { + return not(p.parseExprOrAssign()) + } // Multiple assignment to local targets: a, b = … / a, *b = … . if p.looksLikeMlhs() { return p.parseMlhs() @@ -1559,6 +1687,9 @@ func (p *Parser) parsePrimary() ast.Node { case token.REGEXP: p.advance() return &ast.RegexpLit{Source: t.Lit, Flags: t.Flags} + case token.XSTRING: + p.advance() + return &ast.XStr{Command: t.Lit} case token.WORDS, token.SYMBOLS: p.advance() elems := []ast.Node{} @@ -1595,7 +1726,9 @@ func (p *Parser) parsePrimary() ast.Node { case token.LPAREN: p.advance() p.skipNewlines() - e := p.parseOneLineMatch(p.parseExprOrAssign()) + // A parenthesised group permits the low-precedence keyword operators + // (`(a and b)`, `(not x)`) just like a statement does. + e := p.parseOneLineMatch(p.parseKeywordLogical()) p.skipNewlines() p.expect(token.RPAREN) return e @@ -1610,6 +1743,10 @@ func (p *Parser) parsePrimary() ast.Node { return &ast.Call{Name: t.Lit, Args: args} } return &ast.ConstRef{Name: t.Lit} + case token.SCOPE: + // Leading `::Name` — a top-level constant lookup (`::Foo`, `defined?(::Foo)`). + p.advance() + return &ast.ScopedConst{Name: p.expect(token.CONST).Lit, Global: true} case token.IVAR: p.advance() return &ast.IvarRef{Name: t.Lit} @@ -1712,10 +1849,14 @@ func (p *Parser) canStartCommandArg() bool { switch t.Type { case token.INT, token.FLOAT, token.STRING, token.STRBEG, token.SYMBOL, token.IDENT, token.CONST, token.IVAR, token.CVAR, token.GVAR, token.TRUE, token.FALSE, token.NIL, token.SELF, token.BANG, token.TILDE, - token.LPAREN, token.LBRACKET, token.ARROW, token.WORDS, token.SYMBOLS, token.REGEXP, + token.LPAREN, token.LBRACKET, token.ARROW, token.WORDS, token.SYMBOLS, token.REGEXP, token.XSTRING, token.BEGIN, token.CASE: // Value-producing keywords: `p begin; 1; end`, `p case x; when 1; 2; end`. return true + case token.SCOPE: + // A leading `::Name` (top-level constant) as a command argument: `puts ::Foo`. + // Only when the `::` hugs a following constant, not the postfix `a ::b` form. + return p.peekTok().Type == token.CONST case token.MINUS, token.PLUS: // Unary-style argument: `foo -1` (operand hugs the sign), not `foo - 1`. return !p.peekTok().SpaceBefore @@ -1764,6 +1905,11 @@ func (p *Parser) parseCallArgs(until token.Type) []ast.Node { // parseOneCallArg parses a single call argument, routing `*splat` and positional // expressions into args, and `label: value` / `expr => value` pairs into kw. func (p *Parser) parseOneCallArg(args *[]ast.Node, kw **ast.HashLit) { + if p.is(token.DOTDOTDOT) { // `...` — forward the enclosing method's arguments + p.advance() + *args = append(*args, &ast.ForwardArgs{}) + return + } if p.accept(token.AMPER) { // &expr — block-pass (coerced to a Proc) *args = append(*args, &ast.BlockPass{Value: p.parseExprOrAssign()}) return diff --git a/percent_rxs_test.go b/percent_rxs_test.go new file mode 100644 index 0000000..0c66fff --- /dev/null +++ b/percent_rxs_test.go @@ -0,0 +1,165 @@ +package parser_test + +import ( + "testing" + + "github.com/go-ruby-parser/parser" + "github.com/go-ruby-parser/parser/ast" +) + +func TestPercentRegexp(t *testing.T) { + rx, ok := parseOne(t, "%r{ab}i").(*ast.RegexpLit) + if !ok { + t.Fatalf("node = %T, want *ast.RegexpLit", parseOne(t, "%r{ab}i")) + } + if rx.Source != "ab" { + t.Errorf("Source = %q, want ab", rx.Source) + } + if rx.Flags != "i" { + t.Errorf("Flags = %q, want i", rx.Flags) + } +} + +func TestPercentRegexpParenDelim(t *testing.T) { + rx := parseOne(t, "%r(foo)").(*ast.RegexpLit) + if rx.Source != "foo" || rx.Flags != "" { + t.Errorf("got %q/%q, want foo/(empty)", rx.Source, rx.Flags) + } +} + +func TestPercentRegexpBracketFlag(t *testing.T) { + rx := parseOne(t, "%r[bar]m").(*ast.RegexpLit) + if rx.Source != "bar" || rx.Flags != "m" { + t.Errorf("got %q/%q, want bar/m", rx.Source, rx.Flags) + } +} + +func TestPercentRegexpKeepsInterpolationRaw(t *testing.T) { + // Matching the existing /…/ behaviour, interpolation markers stay raw source. + rx := parseOne(t, "%r{a#{x}b}").(*ast.RegexpLit) + if rx.Source != "a#{x}b" { + t.Errorf("Source = %q, want a#{x}b", rx.Source) + } +} + +func TestPercentRegexpNoFlags(t *testing.T) { + rx := parseOne(t, "%r{ab}").(*ast.RegexpLit) + if rx.Flags != "" { + t.Errorf("Flags = %q, want empty", rx.Flags) + } +} + +func TestPercentXString(t *testing.T) { + xs, ok := parseOne(t, "%x{ls}").(*ast.XStr) + if !ok { + t.Fatalf("node = %T, want *ast.XStr", parseOne(t, "%x{ls}")) + } + if xs.Command != "ls" { + t.Errorf("Command = %q, want ls", xs.Command) + } +} + +func TestPercentXStringBracket(t *testing.T) { + xs := parseOne(t, "%x[cmd]").(*ast.XStr) + if xs.Command != "cmd" { + t.Errorf("Command = %q, want cmd", xs.Command) + } +} + +func TestPercentSymbol(t *testing.T) { + sym, ok := parseOne(t, "%s{foo}").(*ast.SymbolLit) + if !ok { + t.Fatalf("node = %T, want *ast.SymbolLit", parseOne(t, "%s{foo}")) + } + if sym.Name != "foo" { + t.Errorf("Name = %q, want foo", sym.Name) + } +} + +func TestPercentSymbolWithSpace(t *testing.T) { + sym := parseOne(t, "%s(with space)").(*ast.SymbolLit) + if sym.Name != "with space" { + t.Errorf("Name = %q, want 'with space'", sym.Name) + } +} + +func TestPercentRXSNestedDelimiters(t *testing.T) { + rx := parseOne(t, "%r{a{b}c}").(*ast.RegexpLit) + if rx.Source != "a{b}c" { + t.Errorf("Source = %q, want a{b}c", rx.Source) + } +} + +func TestPercentRXSEscapedDelimiter(t *testing.T) { + // An escaped closing delimiter does not terminate the literal; %x drops the + // backslash for the escaped delimiter. + xs := parseOne(t, "%x{a\\}b}").(*ast.XStr) + if xs.Command != "a}b" { + t.Errorf("Command = %q, want a}b", xs.Command) + } +} + +func TestPercentRegexpEscapedDelimiter(t *testing.T) { + // MRI keeps the backslash in a %r source for an escaped delimiter: + // `%r{a\}b}.source == "a\\}b"`. The escape only stops `}` from closing it. + rx := parseOne(t, "%r{a\\}b}").(*ast.RegexpLit) + if rx.Source != "a\\}b" { + t.Errorf("Source = %q, want a\\}b", rx.Source) + } +} + +func TestPercentRegexpKeepsOtherEscape(t *testing.T) { + // A non-delimiter escape is kept verbatim for the regexp engine. + rx := parseOne(t, "%r{a\\db}").(*ast.RegexpLit) + if rx.Source != "a\\db" { + t.Errorf("Source = %q, want a\\db", rx.Source) + } +} + +func TestPercentXStringKeepsNonDelimiterEscape(t *testing.T) { + // A non-delimiter escape in %x/%s is kept verbatim (the backslash stays). + xs := parseOne(t, "%x{a\\db}").(*ast.XStr) + if xs.Command != "a\\db" { + t.Errorf("Command = %q, want a\\db", xs.Command) + } + sym := parseOne(t, "%s{a\\db}").(*ast.SymbolLit) + if sym.Name != "a\\db" { + t.Errorf("Name = %q, want a\\db", sym.Name) + } +} + +func TestPercentRXSAsCommandArg(t *testing.T) { + call := parseOne(t, "puts %x{ls}").(*ast.Call) + if _, ok := call.Args[0].(*ast.XStr); !ok { + t.Errorf("arg = %T, want *ast.XStr", call.Args[0]) + } +} + +func TestPercentRXSUnterminated(t *testing.T) { + for _, src := range []string{"%r{ab", "%x{ls", "%s{foo"} { + if _, err := parser.Parse(src); err == nil { + t.Errorf("Parse(%q) = no error, want a clean error", src) + } + } +} + +func TestPercentRXSTrailingBackslash(t *testing.T) { + // A backslash at end-of-input inside the literal keeps the lone backslash and + // ends the token rather than reading past EOF. + rx := parseOne(t, "%r{a\\").(*ast.RegexpLit) + if rx.Source != "a\\" { + t.Errorf("Source = %q, want a\\", rx.Source) + } + xs := parseOne(t, "%x{a\\").(*ast.XStr) + if xs.Command != "a\\" { + t.Errorf("Command = %q, want a\\", xs.Command) + } +} + +func TestPercentRXSValidPrograms(t *testing.T) { + for _, src := range []string{"%r{ab}i", "%x{ls}", "%s{foo}", "%r(foo)", "%r[bar]m", "%s(with space)", "%x[cmd]"} { + if _, err := parser.Parse(src); err != nil { + t.Errorf("Parse(%q) returned error: %v", src, err) + } + } +} diff --git a/scope_resolution_test.go b/scope_resolution_test.go new file mode 100644 index 0000000..1d203c0 --- /dev/null +++ b/scope_resolution_test.go @@ -0,0 +1,174 @@ +package parser_test + +import ( + "testing" + + "github.com/go-ruby-parser/parser" + "github.com/go-ruby-parser/parser/ast" +) + +// scopedConst asserts n is a *ast.ScopedConst with the given trailing name and +// Global flag, returning it for further receiver inspection. +func scopedConst(t *testing.T, n ast.Node, name string, global bool) *ast.ScopedConst { + t.Helper() + sc, ok := n.(*ast.ScopedConst) + if !ok { + t.Fatalf("node = %T, want *ast.ScopedConst", n) + } + if sc.Name != name { + t.Errorf("ScopedConst.Name = %q, want %q", sc.Name, name) + } + if sc.Global != global { + t.Errorf("ScopedConst.Global = %v, want %v", sc.Global, global) + } + return sc +} + +func TestScopeResolutionClassName(t *testing.T) { + n := parseOne(t, "class Foo::Bar; end") + cd, ok := n.(*ast.ClassDef) + if !ok { + t.Fatalf("node = %T, want *ast.ClassDef", n) + } + if cd.Name != "Bar" { + t.Errorf("Name = %q, want Bar", cd.Name) + } + sc := scopedConst(t, cd.NamePath, "Bar", false) + if cr, ok := sc.Recv.(*ast.ConstRef); !ok || cr.Name != "Foo" { + t.Errorf("NamePath.Recv = %#v, want ConstRef Foo", sc.Recv) + } +} + +func TestScopeResolutionClassDeepName(t *testing.T) { + n := parseOne(t, "class A::B::C; end") + cd := n.(*ast.ClassDef) + if cd.Name != "C" { + t.Errorf("Name = %q, want C", cd.Name) + } + outer := scopedConst(t, cd.NamePath, "C", false) + inner := scopedConst(t, outer.Recv, "B", false) + if cr, ok := inner.Recv.(*ast.ConstRef); !ok || cr.Name != "A" { + t.Errorf("innermost recv = %#v, want ConstRef A", inner.Recv) + } +} + +func TestScopeResolutionLeadingClassName(t *testing.T) { + n := parseOne(t, "class ::Top; end") + cd := n.(*ast.ClassDef) + if cd.Name != "Top" { + t.Errorf("Name = %q, want Top", cd.Name) + } + sc := scopedConst(t, cd.NamePath, "Top", true) + if sc.Recv != nil { + t.Errorf("leading-:: NamePath.Recv = %#v, want nil", sc.Recv) + } +} + +func TestScopeResolutionBareClassNameNoPath(t *testing.T) { + n := parseOne(t, "class Foo; end") + cd := n.(*ast.ClassDef) + if cd.Name != "Foo" { + t.Errorf("Name = %q, want Foo", cd.Name) + } + if cd.NamePath != nil { + t.Errorf("bare class NamePath = %#v, want nil", cd.NamePath) + } +} + +func TestScopeResolutionSuperclassPath(t *testing.T) { + n := parseOne(t, "class E < Foo::Bar; end") + cd := n.(*ast.ClassDef) + if cd.Name != "E" || cd.NamePath != nil { + t.Errorf("Name/NamePath = %q/%#v, want E/nil", cd.Name, cd.NamePath) + } + if cd.Super != "" { + t.Errorf("Super = %q, want empty (path goes to SuperExpr)", cd.Super) + } + scopedConst(t, cd.SuperExpr, "Bar", false) +} + +func TestScopeResolutionSuperclassLeading(t *testing.T) { + n := parseOne(t, "class G < ::StandardError; end") + cd := n.(*ast.ClassDef) + sc := scopedConst(t, cd.SuperExpr, "StandardError", true) + if sc.Recv != nil { + t.Errorf("SuperExpr.Recv = %#v, want nil", sc.Recv) + } +} + +func TestScopeResolutionSuperclassBare(t *testing.T) { + n := parseOne(t, "class E < Base; end") + cd := n.(*ast.ClassDef) + if cd.Super != "Base" { + t.Errorf("Super = %q, want Base", cd.Super) + } + if cd.SuperExpr != nil { + t.Errorf("bare superclass SuperExpr = %#v, want nil", cd.SuperExpr) + } +} + +func TestScopeResolutionModuleName(t *testing.T) { + n := parseOne(t, "module A::B; end") + md, ok := n.(*ast.ModuleDef) + if !ok { + t.Fatalf("node = %T, want *ast.ModuleDef", n) + } + if md.Name != "B" { + t.Errorf("Name = %q, want B", md.Name) + } + scopedConst(t, md.NamePath, "B", false) +} + +func TestScopeResolutionModuleBareNoPath(t *testing.T) { + md := parseOne(t, "module M; end").(*ast.ModuleDef) + if md.Name != "M" || md.NamePath != nil { + t.Errorf("Name/NamePath = %q/%#v, want M/nil", md.Name, md.NamePath) + } +} + +func TestScopeResolutionLeadingValue(t *testing.T) { + sc := scopedConst(t, parseOne(t, "::TopLevel"), "TopLevel", true) + if sc.Recv != nil { + t.Errorf("Recv = %#v, want nil", sc.Recv) + } +} + +func TestScopeResolutionLeadingChainedValue(t *testing.T) { + outer := scopedConst(t, parseOne(t, "::Foo::Bar"), "Bar", false) + inner := scopedConst(t, outer.Recv, "Foo", true) + if inner.Recv != nil { + t.Errorf("inner Recv = %#v, want nil (leading ::)", inner.Recv) + } +} + +func TestScopeResolutionDefinedLeading(t *testing.T) { + n := parseOne(t, "defined?(::Foo)") + call, ok := n.(*ast.Call) + if !ok || call.Name != "defined?" { + t.Fatalf("node = %#v, want defined? call", n) + } + scopedConst(t, call.Args[0], "Foo", true) +} + +func TestScopeResolutionCommandArg(t *testing.T) { + // `puts ::Foo` is `puts(::Foo)`, not `(puts)::Foo`. + call := parseOne(t, "puts ::Foo").(*ast.Call) + if call.Recv != nil || call.Name != "puts" || len(call.Args) != 1 { + t.Fatalf("node = %#v, want puts(::Foo)", call) + } + scopedConst(t, call.Args[0], "Foo", true) +} + +func TestScopeResolutionPostfixStillWorks(t *testing.T) { + // A space-less `Math::PI` remains a postfix scoped const. + sc := scopedConst(t, parseOne(t, "Math::PI"), "PI", false) + if cr, ok := sc.Recv.(*ast.ConstRef); !ok || cr.Name != "Math" { + t.Errorf("Recv = %#v, want ConstRef Math", sc.Recv) + } +} + +func TestScopeResolutionSingletonClassRejected(t *testing.T) { + if _, err := parser.Parse("class << self; end"); err == nil { + t.Fatal("class << self should be rejected (out of scope), got no error") + } +} diff --git a/token/token.go b/token/token.go index b66d157..bdd8166 100644 --- a/token/token.go +++ b/token/token.go @@ -16,17 +16,18 @@ const ( INT FLOAT STRING - STRBEG // "…#{ (start of an interpolated string; Lit = literal prefix) - STRMID // }…#{ (literal between two interpolations) - STREND // }…" (literal suffix ending an interpolated string) - IDENT // local variable or method name (lowercase / _ leading) - CONST // Capitalized identifier - IVAR // @instance_variable - CVAR // @@class_variable - GVAR // $global / $~ / $1 + STRBEG // "…#{ (start of an interpolated string; Lit = literal prefix) + STRMID // }…#{ (literal between two interpolations) + STREND // }…" (literal suffix ending an interpolated string) + IDENT // local variable or method name (lowercase / _ leading) + CONST // Capitalized identifier + IVAR // @instance_variable + CVAR // @@class_variable + GVAR // $global / $~ / $1 SYMBOL // :name LABEL // name: in a hash literal REGEXP // /pattern/flags (Lit = pattern source, Flags = matched flag letters) + XSTRING // `cmd` / %x{cmd} backtick command literal (Lit = raw command source) WORDS // %w[…] word-array literal (Lit = raw whitespace-separated content) SYMBOLS // %i[…] symbol-array literal (Lit = raw whitespace-separated content) @@ -59,6 +60,9 @@ const ( SELF SUPER YIELD + AND // `and` low-precedence logical-and keyword + OR // `or` low-precedence logical-or keyword + NOT // `not` low-precedence logical-not keyword // Operators and delimiters. PLUS @@ -69,7 +73,7 @@ const ( PERCENT ASSIGN EQ - EQQ // === + EQQ // === MATCH // =~ NEQ LT @@ -107,12 +111,12 @@ const ( var typeNames = map[Type]string{ EOF: "EOF", ILLEGAL: "ILLEGAL", NEWLINE: "NEWLINE", INT: "INT", FLOAT: "FLOAT", - STRING: "STRING", STRBEG: "STRBEG", STRMID: "STRMID", STREND: "STREND", IDENT: "IDENT", CONST: "CONST", IVAR: "IVAR", CVAR: "CVAR", GVAR: "GVAR", SYMBOL: "SYMBOL", LABEL: "LABEL", REGEXP: "REGEXP", WORDS: "WORDS", SYMBOLS: "SYMBOLS", + STRING: "STRING", STRBEG: "STRBEG", STRMID: "STRMID", STREND: "STREND", IDENT: "IDENT", CONST: "CONST", IVAR: "IVAR", CVAR: "CVAR", GVAR: "GVAR", SYMBOL: "SYMBOL", LABEL: "LABEL", REGEXP: "REGEXP", XSTRING: "XSTRING", WORDS: "WORDS", SYMBOLS: "SYMBOLS", DEF: "def", CLASS: "class", MODULE: "module", END: "end", IF: "if", ELSIF: "elsif", ELSE: "else", UNLESS: "unless", WHILE: "while", UNTIL: "until", RETURN: "return", BREAK: "break", NEXT: "next", BEGIN: "begin", RESCUE: "rescue", ENSURE: "ensure", CASE: "case", WHEN: "when", IN: "in", RETRY: "retry", THEN: "then", DO: "do", TRUE: "true", FALSE: "false", NIL: "nil", SELF: "self", - SUPER: "super", YIELD: "yield", + SUPER: "super", YIELD: "yield", AND: "and", OR: "or", NOT: "not", PLUS: "+", MINUS: "-", STAR: "*", POW: "**", SLASH: "/", PERCENT: "%", ASSIGN: "=", EQ: "==", EQQ: "===", MATCH: "=~", NEQ: "!=", LT: "<", GT: ">", LE: "<=", GE: ">=", BANG: "!", SPACESHIP: "<=>", SHOVEL: "<<", ANDAND: "&&", OROR: "||", OPASSIGN: "op=", QUESTION: "?", COLON: ":", SCOPE: "::", @@ -138,7 +142,7 @@ var Keywords = map[string]Type{ "case": CASE, "when": WHEN, "in": IN, "retry": RETRY, "then": THEN, "do": DO, "true": TRUE, "false": FALSE, "nil": NIL, "self": SELF, "super": SUPER, - "yield": YIELD, + "yield": YIELD, "and": AND, "or": OR, "not": NOT, } // Token is a single lexed token.