diff --git a/alias_undef_test.go b/alias_undef_test.go new file mode 100644 index 0000000..9225f04 --- /dev/null +++ b/alias_undef_test.go @@ -0,0 +1,83 @@ +package parser + +import ( + "testing" + + "github.com/go-ruby-parser/parser/ast" +) + +// `alias` accepts symbol, operator-symbol, bare-name, and global-variable items. +func TestAlias(t *testing.T) { + cases := []struct { + src string + newN, oldName string + }{ + {`alias :== :eql?`, "==", "eql?"}, + {`alias foo bar`, "foo", "bar"}, + {`alias == eql?`, "==", "eql?"}, + {`alias new_name old_name`, "new_name", "old_name"}, + {`alias $x $y`, "$x", "$y"}, + } + for _, c := range cases { + prog, err := Parse(c.src) + if err != nil { + t.Fatalf("Parse(%q) error: %v", c.src, err) + } + a, ok := prog.Body[0].(*ast.Alias) + if !ok { + t.Fatalf("%q: expected *ast.Alias, got %T", c.src, prog.Body[0]) + } + if a.NewName != c.newN || a.OldName != c.oldName { + t.Fatalf("%q: got (%q, %q), want (%q, %q)", c.src, a.NewName, a.OldName, c.newN, c.oldName) + } + } +} + +// `undef` accepts one or more comma-separated method names. +func TestUndef(t *testing.T) { + prog, err := Parse(`undef :foo, :bar, baz`) + if err != nil { + t.Fatalf("Parse error: %v", err) + } + u, ok := prog.Body[0].(*ast.Undef) + if !ok { + t.Fatalf("expected *ast.Undef, got %T", prog.Body[0]) + } + want := []string{"foo", "bar", "baz"} + if len(u.Names) != len(want) { + t.Fatalf("names=%v, want %v", u.Names, want) + } + for i, w := range want { + if u.Names[i] != w { + t.Fatalf("name %d=%q, want %q", i, u.Names[i], w) + } + } +} + +// `alias_method` (a plain identifier, not the keyword) is unaffected. +func TestAliasMethodIdentifierUnaffected(t *testing.T) { + if _, err := Parse(`alias_method :a, :b`); err != nil { + t.Fatalf("Parse error: %v", err) + } +} + +// A chained multiple-assignment right-hand side (`a, b = c = expr`) is a further +// assignment whose value is then destructured. +func TestChainedMasgnRhs(t *testing.T) { + for _, src := range []string{ + `a, b = c = [1, 2]`, + `_, headers, _ = response = call(x)`, + } { + prog, err := Parse(src) + if err != nil { + t.Fatalf("Parse(%q) error: %v", src, err) + } + ma, ok := prog.Body[0].(*ast.MultiAssign) + if !ok { + t.Fatalf("%q: expected *ast.MultiAssign, got %T", src, prog.Body[0]) + } + if _, ok := ma.Values[0].(*ast.Assign); !ok { + t.Fatalf("%q: expected value0 *ast.Assign, got %T", src, ma.Values[0]) + } + } +} diff --git a/ast/ast.go b/ast/ast.go index 355c611..ca63f5e 100644 --- a/ast/ast.go +++ b/ast/ast.go @@ -199,6 +199,13 @@ type ConstAssign struct { Value Node } +// ScopedConstAssign assigns to a scope-resolved constant: `A::B = value` or the +// top-level `::A = value`. Target is the *ScopedConst that names the constant. +type ScopedConstAssign struct { + Target Node + Value Node +} + // IvarRef reads an instance variable (@name) of self. type IvarRef struct{ Name string } @@ -384,6 +391,18 @@ type FindPattern struct { PostName string } +// Alias is `alias NewName OldName` — it makes NewName an alias of an existing +// method (or global variable). Each name is the bare method/symbol/global text +// without a leading colon. +type Alias struct { + NewName string + OldName string +} + +// Undef is `undef name [, name…]` — it removes the named method(s) from the +// current class/module. Names hold the bare method names. +type Undef struct{ Names []string } + // Retry restarts the enclosing begin body from inside a rescue clause. type Retry struct{} @@ -442,6 +461,7 @@ func (*MethodDef) node() {} func (*Return) node() {} func (*ConstRef) node() {} func (*ConstAssign) node() {} +func (*ScopedConstAssign) node() {} func (*GVarRef) node() {} func (*GVarAssign) node() {} func (*CVarRef) node() {} @@ -462,6 +482,8 @@ func (*Begin) node() {} func (*StrInterp) node() {} func (*Case) node() {} func (*Retry) node() {} +func (*Alias) node() {} +func (*Undef) node() {} func (*SplatArg) node() {} func (*BlockPass) node() {} func (*ForwardArgs) node() {} diff --git a/block_comment_test.go b/block_comment_test.go new file mode 100644 index 0000000..59a8442 --- /dev/null +++ b/block_comment_test.go @@ -0,0 +1,73 @@ +package parser + +import ( + "testing" + + "github.com/go-ruby-parser/parser/ast" +) + +// `=begin` … `=end` block comments (at column 0) are skipped wholesale, with the +// surrounding code parsing normally. +func TestBlockComment(t *testing.T) { + cases := []string{ + "=begin\nthis is a comment\nmore\n=end\nputs 1", + "x = 1\n=begin\ncomment\n=end\ny = 2", + "=begin doc\nfoo\n=end", + "=begin\nunterminated comment to EOF", + } + for _, src := range cases { + if _, err := Parse(src); err != nil { + t.Fatalf("Parse(%q) error: %v", src, err) + } + } +} + +// A `=begin` not at column 0 is not a block comment: `x = begin … end` is an +// ordinary begin expression assigned to x. +func TestBlockCommentNotAtLineStart(t *testing.T) { + prog, err := Parse("x = begin\n 1\nend") + if err != nil { + t.Fatalf("Parse error: %v", err) + } + asn, ok := prog.Body[0].(*ast.Assign) + if !ok { + t.Fatalf("expected *ast.Assign, got %T", prog.Body[0]) + } + if _, ok := asn.Value.(*ast.Begin); !ok { + t.Fatalf("expected RHS *ast.Begin, got %T", asn.Value) + } +} + +// An identifier that merely starts with "begin" is unaffected. +func TestBlockCommentIdentifierPrefix(t *testing.T) { + if _, err := Parse("result = beginning"); err != nil { + t.Fatalf("Parse error: %v", err) + } +} + +// The bitwise op-assignments |=, &=, ^=, >>= now lex (they were previously +// missing, so `mode |= x` failed). Confirm each produces an OpAssign. +func TestBitwiseOpAssign(t *testing.T) { + for _, c := range []struct { + src string + op string + }{ + {"mode |= x", "|"}, + {"a &= b", "&"}, + {"c ^= d", "^"}, + {"x >>= 2", ">>"}, + {"y <<= 1", "<<"}, + } { + prog, err := Parse(c.src) + if err != nil { + t.Fatalf("Parse(%q) error: %v", c.src, err) + } + oa, ok := prog.Body[0].(*ast.OpAssign) + if !ok { + t.Fatalf("%q: expected *ast.OpAssign, got %T", c.src, prog.Body[0]) + } + if oa.Op != c.op { + t.Fatalf("%q: op=%q, want %q", c.src, oa.Op, c.op) + } + } +} diff --git a/command_do_block_test.go b/command_do_block_test.go new file mode 100644 index 0000000..8dd6e7e --- /dev/null +++ b/command_do_block_test.go @@ -0,0 +1,54 @@ +package parser + +import ( + "testing" + + "github.com/go-ruby-parser/parser/ast" +) + +// A `do…end` block following a paren-less command call on a receiver binds to +// that command call, mirroring MRI (`obj.foo bar do … end`). Previously the +// receiver command-call path returned before it could attach the block. +func TestReceiverCommandDoBlock(t *testing.T) { + cases := []string{ + `recv.set_callback :work, prepend: true do |x| end`, + `Capybara.register_driver name do |app| end`, + `obj.instrument "x", y do end`, + `ActiveSupport.on_load(:action_cable) do; end`, + } + for _, src := range cases { + if _, err := Parse(src); err != nil { + t.Fatalf("Parse(%q) error: %v", src, err) + } + } +} + +// The block-bearing receiver command call can itself be chained (the block +// attaches to the command, then the postfix chain continues). +func TestReceiverCommandDoBlockChained(t *testing.T) { + prog, err := Parse(`app.foo bar do end.baz`) + if err != nil { + t.Fatalf("Parse error: %v", err) + } + outer, ok := prog.Body[0].(*ast.Call) + if !ok || outer.Name != "baz" { + t.Fatalf("expected outer .baz call, got %T", prog.Body[0]) + } + inner, ok := outer.Recv.(*ast.Call) + if !ok || inner.Name != "foo" || inner.Block == nil { + t.Fatalf("expected inner foo call with block, got %#v", outer.Recv) + } +} + +// A braced block, by contrast, binds tighter (to the nearest call/arg), and a +// `do` inside a while/until condition still belongs to the loop, not the call. +func TestReceiverCommandBlockBoundaries(t *testing.T) { + for _, src := range []string{ + `while obj.foo bar do end`, + `recv.each x { |y| y }`, + } { + if _, err := Parse(src); err != nil { + t.Fatalf("Parse(%q) error: %v", src, err) + } + } +} diff --git a/def_extras_test.go b/def_extras_test.go new file mode 100644 index 0000000..c4fa5ca --- /dev/null +++ b/def_extras_test.go @@ -0,0 +1,83 @@ +package parser + +import ( + "testing" + + "github.com/go-ruby-parser/parser/ast" +) + +// A `def` may name an explicit receiver that is an instance/class/global +// variable, not just self/local/constant. +func TestDefVariableReceiver(t *testing.T) { + cases := []struct { + src string + kind string + }{ + {"def @controller.foo(&b)\nend", "ivar"}, + {"def @@reg.foo\nend", "cvar"}, + {"def $g.foo\nend", "gvar"}, + } + for _, c := range cases { + prog, err := Parse(c.src) + if err != nil { + t.Fatalf("Parse(%q) error: %v", c.src, err) + } + md := prog.Body[0].(*ast.MethodDef) + switch c.kind { + case "ivar": + if _, ok := md.Recv.(*ast.IvarRef); !ok { + t.Fatalf("%q: recv=%T, want *ast.IvarRef", c.src, md.Recv) + } + case "cvar": + if _, ok := md.Recv.(*ast.CVarRef); !ok { + t.Fatalf("%q: recv=%T, want *ast.CVarRef", c.src, md.Recv) + } + case "gvar": + if _, ok := md.Recv.(*ast.GVarRef); !ok { + t.Fatalf("%q: recv=%T, want *ast.GVarRef", c.src, md.Recv) + } + } + } +} + +// A `def` is a value-producing expression, so it can be a command argument: +// `module_function def server; end`, `private def bar; end`. +func TestDefAsCommandArgument(t *testing.T) { + for _, src := range []string{ + "module_function def server\n 1\nend", + "private def bar; end", + } { + prog, err := Parse(src) + if err != nil { + t.Fatalf("Parse(%q) error: %v", src, err) + } + call, ok := prog.Body[0].(*ast.Call) + if !ok { + t.Fatalf("%q: expected *ast.Call, got %T", src, prog.Body[0]) + } + if _, ok := call.Args[0].(*ast.MethodDef); !ok { + t.Fatalf("%q: arg0=%T, want *ast.MethodDef", src, call.Args[0]) + } + } +} + +// A string interpolation body is a full statement, so it admits a trailing +// modifier and several statements. +func TestInterpolationBodyStatements(t *testing.T) { + for _, src := range []string{ + `"x#{'s' if n > 1}y"`, + `"v#{a; b}"`, + `"#{}"`, // empty interpolation + } { + if _, err := Parse(src); err != nil { + t.Fatalf("Parse(%q) error: %v", src, err) + } + } +} + +// A bare incomplete `def` is still a clean parse error (not an internal panic). +func TestBareDefError(t *testing.T) { + if _, err := Parse("def"); err == nil { + t.Fatalf("expected error for bare def") + } +} diff --git a/lambda_paren_block_test.go b/lambda_paren_block_test.go new file mode 100644 index 0000000..081e33d --- /dev/null +++ b/lambda_paren_block_test.go @@ -0,0 +1,108 @@ +package parser + +import ( + "testing" + + "github.com/go-ruby-parser/parser/ast" +) + +// A stabby lambda accepts unparenthesized parameters: `->x { }`, `-> ctx { }`, +// `->a, b { }`, `-> message do … end`, `->x, y=1 { }`. +func TestStabbyLambdaUnparenthesizedParams(t *testing.T) { + cases := []struct { + src string + params []string + }{ + {`->x { x }`, []string{"x"}}, + {`-> ctx { ctx }`, []string{"ctx"}}, + {`->a, b { a + b }`, []string{"a", "b"}}, + {`-> message do message end`, []string{"message"}}, + {`->x, y=1 { x }`, []string{"x", "y"}}, + {`-> data { queue << data }`, []string{"data"}}, + } + for _, c := range cases { + prog, err := Parse(c.src) + if err != nil { + t.Fatalf("Parse(%q) error: %v", c.src, err) + } + call := prog.Body[0].(*ast.Call) + if call.Name != "lambda" || call.Block == nil { + t.Fatalf("%q: expected lambda call with block, got %#v", c.src, call) + } + if len(call.Block.Params) != len(c.params) { + t.Fatalf("%q: params=%v, want %v", c.src, call.Block.Params, c.params) + } + for i, p := range c.params { + if call.Block.Params[i] != p { + t.Fatalf("%q: param %d=%q, want %q", c.src, i, call.Block.Params[i], p) + } + } + } +} + +// Keyword parameters are accepted in block/lambda parameter lists. +func TestBlockKeywordParams(t *testing.T) { + for _, src := range []string{ + `[1].each { |a, b:| a }`, + `foo do |channel, count: nil, timeout: 5| 1 end`, + `m { |k:| k }`, + `->a, k: 1 { a }`, + } { + if _, err := Parse(src); err != nil { + t.Fatalf("Parse(%q) error: %v", src, err) + } + } +} + +// A keyword block param is recorded with a trailing-colon sentinel name. +func TestBlockKeywordParamShape(t *testing.T) { + prog, err := Parse(`[1].each { |a, b:, c: 5| a }`) + if err != nil { + t.Fatalf("Parse error: %v", err) + } + call := prog.Body[0].(*ast.Call) + want := []string{"a", "b:", "c:"} + if len(call.Block.Params) != len(want) { + t.Fatalf("params=%v, want %v", call.Block.Params, want) + } + for i, w := range want { + if call.Block.Params[i] != w { + t.Fatalf("param %d=%q, want %q", i, call.Block.Params[i], w) + } + } +} + +// A parenthesised group accepts trailing modifiers and statement sequences, +// evaluating to its last expression. +func TestModifierAndSequenceInParens(t *testing.T) { + for _, src := range []string{ + `(expr if cond)`, + `(x if y) || z`, + `(a; b)`, + `x = (a = 1; a + 1)`, + `(do_it unless skip)`, + } { + if _, err := Parse(src); err != nil { + t.Fatalf("Parse(%q) error: %v", src, err) + } + } +} + +// A multi-statement parenthesised group becomes a Begin whose value is its last +// statement; a single-statement group returns that statement directly. +func TestParenGroupShape(t *testing.T) { + prog, err := Parse(`(a; b)`) + if err != nil { + t.Fatalf("Parse error: %v", err) + } + if _, ok := prog.Body[0].(*ast.Begin); !ok { + t.Fatalf("expected *ast.Begin for (a; b), got %T", prog.Body[0]) + } + prog, err = Parse(`(1)`) + if err != nil { + t.Fatalf("Parse error: %v", err) + } + if _, ok := prog.Body[0].(*ast.IntLit); !ok { + t.Fatalf("expected *ast.IntLit for (1), got %T", prog.Body[0]) + } +} diff --git a/lexer/lexer.go b/lexer/lexer.go index 1a44492..9ff337f 100644 --- a/lexer/lexer.go +++ b/lexer/lexer.go @@ -108,7 +108,11 @@ func isContinuationOp(t token.Type) bool { token.EQ, token.EQQ, token.MATCH, token.NEQ, token.LT, token.GT, token.LE, token.GE, token.SPACESHIP, token.ANDAND, token.OROR, token.ASSIGN, token.OPASSIGN, token.COMMA, token.HASHROCKET, - token.DOT, token.SAFEDOT, token.QUESTION: + token.DOT, token.SAFEDOT, token.QUESTION, + // Trailing low-precedence keyword operators and modifiers whose operand / + // condition may sit on the next line: `... or\n fail`, `... and\n x`, + // `do_it unless\n cond`, `foo if\n bar` (MRI joins these lines too). + token.AND, token.OR, token.IF, token.UNLESS, token.WHILE, token.UNTIL: return true } return false @@ -131,17 +135,19 @@ func (l *Lexer) lexToken() token.Token { switch { case c == 0: return mk(token.EOF, "") - case c == '/' && l.state == exprBegin: + case c == '/' && l.state == exprBegin && l.prevType != token.DEF: // At expression-begin position a '/' opens a regexp literal, not division - // (the same disambiguation MRI uses via its lexer state). + // (the same disambiguation MRI uses via its lexer state). The one exception + // is right after `def`, where `/` names the division-operator method + // (`def /(other)`) rather than opening a pattern. return l.lexRegexp(spaceBefore, line, col) - case c == '%' && l.percentBeginsLiteral(spaceBefore) && l.atPercentArray(): + case c == '%' && l.prevType != token.DEF && l.percentBeginsLiteral(spaceBefore) && l.atPercentArray(): // %w[…] / %i[…] / %W[…] / %I[…] word- and symbol-array literals. return l.lexPercentArray(spaceBefore, line, col) - case c == '%' && l.percentBeginsLiteral(spaceBefore) && l.atPercentRXS(): + case c == '%' && l.prevType != token.DEF && l.percentBeginsLiteral(spaceBefore) && l.atPercentRXS(): // %r{…}flags regexp, %x{…} backtick command, %s{…} symbol literals. return l.lexPercentRXS(spaceBefore, line, col) - case c == '%' && l.percentBeginsLiteral(spaceBefore) && l.atPercentString(): + case c == '%' && l.prevType != token.DEF && l.percentBeginsLiteral(spaceBefore) && l.atPercentString(): // %q(…) / %Q(…) / %(…) / %W(…) string literals. return l.lexPercentString(spaceBefore, line, col) case c == '\n' || c == ';': @@ -177,6 +183,8 @@ func (l *Lexer) lexToken() token.Token { return l.lexString(spaceBefore, line, col) case c == '\'': return l.lexSingleQuote(spaceBefore, line, col) + case c == '`': + return l.lexBacktick(spaceBefore, line, col) case c == '@': return l.lexIvar(spaceBefore, line, col) case c == '$': @@ -286,6 +294,11 @@ func (l *Lexer) lexToken() token.Token { l.state = exprBegin return mk(token.OROR, "||") } + if l.peek() == '=' { // |= bitwise-or assignment + l.advance() + l.state = exprBegin + return mk(token.OPASSIGN, "|") + } l.state = exprBegin return mk(token.PIPE, "|") case '&': @@ -304,6 +317,11 @@ func (l *Lexer) lexToken() token.Token { l.state = exprBegin return mk(token.SAFEDOT, "&.") } + if l.peek() == '=' { // &= bitwise-and assignment + l.advance() + l.state = exprBegin + return mk(token.OPASSIGN, "&") + } l.state = exprBegin return mk(token.AMPER, "&") case ',': @@ -385,8 +403,13 @@ func (l *Lexer) lexToken() token.Token { l.state = exprBegin return mk(token.GE, ">=") } - if l.peek() == '>' { // >> (right shift) + if l.peek() == '>' { // >> (right shift), or >>= shift-assignment l.advance() + if l.peek() == '=' { + l.advance() + l.state = exprBegin + return mk(token.OPASSIGN, ">>") + } l.state = exprBegin return mk(token.RSHIFT, ">>") } @@ -404,6 +427,11 @@ func (l *Lexer) lexToken() token.Token { l.state = exprBegin return mk(token.COLON, ":") case '^': + if l.peek() == '=' { // ^= bitwise-xor assignment + l.advance() + l.state = exprBegin + return mk(token.OPASSIGN, "^") + } l.state = exprBegin return mk(token.CARET, "^") case '~': @@ -433,12 +461,72 @@ func (l *Lexer) skipSpaceAndComments() bool { l.advance() } seen = true + case c == '=' && l.atBlockComment(): + // `=begin` … `=end` block comment: from a line that begins with `=begin` + // (at column 0) through the line that begins with `=end`, inclusive. + l.skipBlockComment() + seen = true default: return seen } } } +// atBlockComment reports whether the cursor (on a `=`) opens an `=begin` block +// comment: the `=` must be at the start of a line and be followed by `begin` and +// then a whitespace character or end of line/input. +func (l *Lexer) atBlockComment() bool { + if l.pos != 0 && l.src[l.pos-1] != '\n' { + return false + } + const kw = "=begin" + if l.pos+len(kw) > len(l.src) || string(l.src[l.pos:l.pos+len(kw)]) != kw { + return false + } + if l.pos+len(kw) == len(l.src) { + return true + } + switch l.src[l.pos+len(kw)] { + case ' ', '\t', '\r', '\n': + return true + } + return false +} + +// skipBlockComment consumes an `=begin` … `=end` block comment, leaving the +// cursor at the newline that ends the `=end` line (or at EOF). The terminator is +// a line that begins with `=end` at column 0. +func (l *Lexer) skipBlockComment() { + for { + // Consume to (but not past) the end of the current line. + for l.peek() != '\n' && l.peek() != 0 { + l.advance() + } + if l.peek() == 0 { + return // unterminated; treat the rest of the input as the comment + } + l.advance() // the newline + // A line beginning with `=end` (optionally followed by trailing text) closes + // the comment. + const end = "=end" + if l.pos+len(end) <= len(l.src) && string(l.src[l.pos:l.pos+len(end)]) == end { + isEnd := l.pos+len(end) == len(l.src) + if !isEnd { + switch l.src[l.pos+len(end)] { + case ' ', '\t', '\r', '\n': + isEnd = true + } + } + if isEnd { + for l.peek() != '\n' && l.peek() != 0 { + l.advance() + } + return + } + } + } +} + // nextLineStartsWithDot reports whether the first significant character at or // after the cursor — skipping spaces, tabs, CR, blank lines, and comments — is a // leading `.` (method-chain dot, not a `..`/`...` range) or `&.` safe-nav dot. @@ -1421,6 +1509,44 @@ func (l *Lexer) lexSingleQuote(spaceBefore bool, line, col int) token.Token { return token.Token{Type: token.STRING, Lit: string(b), Line: line, Col: col, SpaceBefore: spaceBefore} } +// lexBacktick lexes a `cmd` command literal (cursor on the opening backtick), +// the same node `%x{…}` produces: a single XSTRING whose Lit is the raw command +// source. As with the %x form the body is kept verbatim (interpolation markers +// are not expanded here); a backslash keeps its following byte so an escaped +// backtick does not close the literal. +func (l *Lexer) lexBacktick(spaceBefore bool, line, col int) token.Token { + l.advance() // opening backtick + var body []byte + for { + c := l.peek() + if c == 0 { + return token.Token{Type: token.ILLEGAL, Lit: "unterminated command literal", Line: line, Col: col, SpaceBefore: spaceBefore} + } + if c == '`' { + l.advance() // closing backtick + break + } + if c == '\\' { + l.advance() + esc := l.peek() + if esc == 0 { + // A backslash at end of input leaves the literal unterminated. + return token.Token{Type: token.ILLEGAL, Lit: "unterminated command literal", Line: line, Col: col, SpaceBefore: spaceBefore} + } + l.advance() + if esc == '`' || esc == '\\' { + body = append(body, esc) // \` and \\ → literal char + } else { + body = append(body, '\\', esc) + } + continue + } + body = append(body, l.advance()) + } + l.state = exprEnd + return token.Token{Type: token.XSTRING, Lit: string(body), Line: line, Col: col, SpaceBefore: spaceBefore} +} + // continueString resumes lexing a string after an interpolation's closing '}', // returning STRMID if another interpolation follows or STREND at the close. func (l *Lexer) continueString(line, col int) token.Token { diff --git a/lexer/lexer_round2_test.go b/lexer/lexer_round2_test.go new file mode 100644 index 0000000..fa1082a --- /dev/null +++ b/lexer/lexer_round2_test.go @@ -0,0 +1,103 @@ +package lexer + +import ( + "testing" + + "github.com/go-ruby-parser/parser/token" +) + +// countNewlines tokenizes src and returns the number of NEWLINE tokens it emits. +func countNewlines(src string) int { + n := 0 + for _, t := range New(src).Tokenize() { + if t.Type == token.NEWLINE { + n++ + } + } + return n +} + +// TestKeywordOperatorContinuation: a line ending in a trailing low-precedence +// keyword operator (`and`/`or`) or a trailing modifier keyword (`if`/`unless`/ +// `while`/`until`) joins the next line, so the intervening newline is suppressed. +func TestKeywordOperatorContinuation(t *testing.T) { + for _, src := range []string{ + "a and\nb", + "a or\nb", + "foo if\nbar", + "foo unless\nbar", + "foo while\nbar", + "foo until\nbar", + } { + if got := countNewlines(src); got != 0 { + t.Errorf("%q: %d NEWLINE tokens, want 0 (the line should continue)", src, got) + } + } +} + +// TestBacktickXString: a `cmd` literal lexes to a single XSTRING token whose Lit +// is the raw command (with \` and \\ resolved), matching the %x{} form. +func TestBacktickXString(t *testing.T) { + for _, c := range []struct{ src, lit string }{ + {"`ls`", "ls"}, + {"`a\\`b`", "a`b"}, // escaped backtick + {"`a\\\\b`", `a\b`}, // escaped backslash + } { + tok := New(c.src).Tokenize()[0] + if tok.Type != token.XSTRING || tok.Lit != c.lit { + t.Errorf("%q: got %v, want XSTRING %q", c.src, tok, c.lit) + } + } +} + +// TestBacktickUnterminated: an unterminated backtick is ILLEGAL, including when +// the input ends right after a backslash escape. +func TestBacktickUnterminated(t *testing.T) { + for _, src := range []string{"`oops", "`a\\"} { + if tok := New(src).Tokenize()[0]; tok.Type != token.ILLEGAL { + t.Errorf("%q: type=%s, want ILLEGAL", src, tok.Type) + } + } +} + +// TestBacktickEscapePassthrough: a backslash before an ordinary character keeps +// the pair verbatim (\n stays "\n" in the command source). +func TestBacktickEscapePassthrough(t *testing.T) { + tok := New("`a\\nb`").Tokenize()[0] + if tok.Type != token.XSTRING || tok.Lit != "a\\nb" { + t.Errorf("got %v, want XSTRING %q", tok, "a\\nb") + } +} + +// TestBlockCommentBoundaries exercises atBlockComment's rejection paths: a +// near-miss marker, a bare `=begin` at EOF, and `=begin` glued to more letters. +func TestBlockCommentBoundaries(t *testing.T) { + // `=begin` exactly at EOF is a (degenerate, unterminated) block comment. + if n := countNewlines("=begin"); n != 0 { + t.Errorf("`=begin` at EOF: %d newlines, want 0", n) + } + // `=beginning` is NOT a block comment (marker glued to more letters); the `=` + // lexes as an ordinary token, so the line is normal code. + toks := New("=beginning").Tokenize() + if toks[0].Type != token.ASSIGN { + t.Errorf("`=beginning`: first token=%v, want ASSIGN", toks[0]) + } + // A short near-miss `=beg` is likewise not a block comment. + if New("=beg").Tokenize()[0].Type != token.ASSIGN { + t.Errorf("`=beg`: want leading ASSIGN token") + } + // `=begin` not at column 0 is ordinary code (here `x =begin…` → x, =, begin…). + if New("x =begin\n1\n=end").Tokenize()[0].Type != token.IDENT { + t.Errorf("`x =begin`: want leading IDENT") + } +} + +// TestKeywordContinuationDoesNotEatRealNewlines: a keyword that is NOT the last +// token on its line (the usual `if cond` form) leaves the statement newline +// intact. +func TestKeywordContinuationLeavesNormalNewlines(t *testing.T) { + // `if x` ends in `x`, not `if`, so the following newline is preserved. + if got := countNewlines("if x\ny"); got != 1 { + t.Errorf("`if x\\ny`: %d NEWLINE tokens, want 1", got) + } +} diff --git a/line_continuation_test.go b/line_continuation_test.go index ec8a494..8454ba1 100644 --- a/line_continuation_test.go +++ b/line_continuation_test.go @@ -36,6 +36,15 @@ func TestLineContinuationTrailingOperator(t *testing.T) { "foo.\nbar", "foo&.\nbar", "c ?\na : b", + // Trailing low-precedence keyword operators and modifiers continue too. + "a and\nb", + "a or\nb", + "foo if\nbar", + "foo unless\nbar", + "foo while\nbar", + "foo until\nbar", + "x = 1 or\nfail", + "y =~ /re/ or\nraise", } for _, src := range oneStmt { t.Run(src, func(t *testing.T) { diff --git a/multiline_params_test.go b/multiline_params_test.go new file mode 100644 index 0000000..0ceaa5b --- /dev/null +++ b/multiline_params_test.go @@ -0,0 +1,41 @@ +package parser + +import ( + "testing" + + "github.com/go-ruby-parser/parser/ast" +) + +// A parenthesised method parameter list may span several lines, with newlines +// after the open paren, around the commas, and before the close paren — and may +// carry scoped-constant / call defaults. +func TestMultilineDefParams(t *testing.T) { + for _, src := range []string{ + "def h(\n a,\n b\n)\nend", + "def f(a,\n b = Encoding::UTF_8,\n c = false)\nend", + "def f(a,\n b = 1,\n)\nend", // trailing comma + newline + "def g(\n)\nend", // empty multiline list + "def k(x = Foo.bar, y = baz())\nend", + "def i(a = CONST::SUB)\nend", + } { + if _, err := Parse(src); err != nil { + t.Fatalf("Parse(%q) error: %v", src, err) + } + } +} + +// A paren-less parameter list still terminates at the newline (no spill into the +// body). +func TestParenlessDefParamsUnaffected(t *testing.T) { + prog, err := Parse("def m a, b\n a + b\nend") + if err != nil { + t.Fatalf("Parse error: %v", err) + } + md, ok := prog.Body[0].(*ast.MethodDef) + if !ok { + t.Fatalf("expected *ast.MethodDef, got %T", prog.Body[0]) + } + if len(md.Params) != 2 || len(md.Body) != 1 { + t.Fatalf("params=%v body=%d, want 2 params and 1 body stmt", md.Params, len(md.Body)) + } +} diff --git a/operator_def_xstr_test.go b/operator_def_xstr_test.go new file mode 100644 index 0000000..cb9f782 --- /dev/null +++ b/operator_def_xstr_test.go @@ -0,0 +1,122 @@ +package parser + +import ( + "testing" + + "github.com/go-ruby-parser/parser/ast" +) + +// A bare `cmd` backtick command literal produces the same XStr node `%x{}` does. +func TestBacktickXString(t *testing.T) { + cases := []struct { + src string + cmd string + }{ + {"`ls -la`", "ls -la"}, + {"`git rev-parse HEAD`", "git rev-parse HEAD"}, + {"`echo \\`hi\\``", "echo `hi`"}, // escaped backtick + } + for _, c := range cases { + prog, err := Parse(c.src) + if err != nil { + t.Fatalf("Parse(%q) error: %v", c.src, err) + } + x, ok := prog.Body[0].(*ast.XStr) + if !ok { + t.Fatalf("%q: expected *ast.XStr, got %T", c.src, prog.Body[0]) + } + if x.Command != c.cmd { + t.Fatalf("%q: command=%q, want %q", c.src, x.Command, c.cmd) + } + } +} + +// A backtick literal works in value position (assignment RHS) too. +func TestBacktickXStringInAssign(t *testing.T) { + prog, err := Parse("x = `whoami`") + if err != nil { + t.Fatalf("Parse error: %v", err) + } + asn := prog.Body[0].(*ast.Assign) + if x, ok := asn.Value.(*ast.XStr); !ok || x.Command != "whoami" { + t.Fatalf("expected RHS XStr `whoami`, got %#v", asn.Value) + } +} + +// An unterminated backtick command is a parse error. +func TestBacktickXStringUnterminated(t *testing.T) { + if _, err := Parse("x = `oops"); err == nil { + t.Fatalf("expected error for unterminated backtick") + } +} + +// The full set of operator method names is accepted in `def`, including the +// unary-operator forms `+@`/`-@`/`~@`/`!@` and `def /` (which must not be lexed +// as a regexp). +func TestOperatorMethodDef(t *testing.T) { + cases := []struct { + src string + name string + }{ + {"def ===(o); end", "==="}, + {"def ==(o); end", "=="}, + {"def <=>(o); end", "<=>"}, + {"def /(o); end", "/"}, + {"def %(o); end", "%"}, + {"def *(o); end", "*"}, + {"def **(o); end", "**"}, + {"def <<(o); end", "<<"}, + {"def >>(o); end", ">>"}, + {"def &(o); end", "&"}, + {"def |(o); end", "|"}, + {"def ^(o); end", "^"}, + {"def =~(o); end", "=~"}, + {"def [](i); end", "[]"}, + {"def []=(i, v); end", "[]="}, + {"def +(o); end", "+"}, + {"def -(o); end", "-"}, + {"def +@; end", "+@"}, + {"def -@; end", "-@"}, + {"def ~; end", "~"}, + {"def ~@; end", "~@"}, + {"def !; end", "!"}, + } + for _, c := range cases { + prog, err := Parse(c.src) + if err != nil { + t.Fatalf("Parse(%q) error: %v", c.src, err) + } + md, ok := prog.Body[0].(*ast.MethodDef) + if !ok { + t.Fatalf("%q: expected *ast.MethodDef, got %T", c.src, prog.Body[0]) + } + if md.Name != c.name { + t.Fatalf("%q: name=%q, want %q", c.src, md.Name, c.name) + } + } +} + +// A singleton operator method `def self.===(o)` also parses. +func TestSingletonOperatorMethodDef(t *testing.T) { + prog, err := Parse("def self.===(o); end") + if err != nil { + t.Fatalf("Parse error: %v", err) + } + md := prog.Body[0].(*ast.MethodDef) + if !md.Singleton || md.Name != "===" { + t.Fatalf("got singleton=%v name=%q, want true ===", md.Singleton, md.Name) + } +} + +// `def /` must not break ordinary regexp/division lexing elsewhere. +func TestDefSlashNoRegexpRegression(t *testing.T) { + for _, src := range []string{ + `x = a / b`, + `y = /regex/`, + `foo(/re/)`, + } { + if _, err := Parse(src); err != nil { + t.Fatalf("Parse(%q) error: %v", src, err) + } + } +} diff --git a/parser.go b/parser.go index 44beeba..8901640 100644 --- a/parser.go +++ b/parser.go @@ -285,6 +285,10 @@ func (p *Parser) parseStatement() ast.Node { case token.RETRY: p.advance() return p.applyModifiers(&ast.Retry{}) + case token.ALIAS: + return p.applyModifiers(p.parseAlias()) + case token.UNDEF: + return p.applyModifiers(p.parseUndef()) default: return p.applyModifiers(p.parseOneLineMatch(p.parseKeywordLogical())) } @@ -405,11 +409,14 @@ func (p *Parser) parseClass() ast.Node { super := "" var superExpr ast.Node if p.accept(token.LT) { - sname, spath := p.parseConstPath() - if spath != nil { - superExpr = spath + // The superclass is any expression (`class A < Base`, `class A < self`, + // `class A < Struct.new(:x)`, `class A < ns::Base`). A bare constant keeps + // the plain-name Super form; anything else is recorded as SuperExpr. + sup := p.parseTernary() + if c, ok := sup.(*ast.ConstRef); ok { + super = c.Name } else { - super = sname + superExpr = sup } } p.pushScope() // a class body has its own local scope @@ -429,23 +436,43 @@ func (p *Parser) parseModule() ast.Node { return &ast.ModuleDef{Name: name, NamePath: path, Body: body} } +// isDefRecvStart reports whether tt can begin an explicit `def` receiver +// (`def self.x`, `def obj.x`, `def Const.x`, `def @ivar.x`, `def @@c.x`, +// `def $g.x`). +func isDefRecvStart(tt token.Type) bool { + switch tt { + case token.SELF, token.IDENT, token.CONST, token.IVAR, token.CVAR, token.GVAR: + return true + } + return false +} + func (p *Parser) parseDef() ast.Node { p.expect(token.DEF) singleton := false var recv ast.Node - // A receiver before the method name: def self.foo / def obj.foo / def Const.foo. - // The current-token check guards peekTok against running past EOF. - if (p.is(token.SELF) || p.is(token.IDENT) || p.is(token.CONST)) && p.peekTok().Type == token.DOT { - switch { - case p.is(token.SELF): + // A receiver before the method name: def self.foo / def obj.foo / def Const.foo + // / def @ivar.foo / def $g.foo. The kind guard keeps peekTok in range (the + // receiver is always a single name token followed by a dot). + if isDefRecvStart(p.cur().Type) && p.peekTok().Type == token.DOT { + switch p.cur().Type { + case token.SELF: p.advance() // self singleton = true - case p.is(token.IDENT): // def obj.foo — singleton method on a local's object + case token.IDENT: // def obj.foo — singleton method on a local's object recv = &ast.VarRef{Name: p.advance().Lit} - default: // def Const.foo — class/module method + case token.CONST: // def Const.foo — class/module method recv = &ast.ConstRef{Name: p.advance().Lit} + case token.IVAR: // def @ivar.foo — singleton method on an ivar's object + recv = &ast.IvarRef{Name: p.advance().Lit} + case token.CVAR: + recv = &ast.CVarRef{Name: p.advance().Lit} + case token.GVAR: + recv = &ast.GVarRef{Name: p.advance().Lit} + } + if recv != nil || singleton { + p.advance() // . } - p.advance() // . } name, ok := p.parseDefName() if !ok { @@ -490,10 +517,22 @@ func (p *Parser) parseDefName() (string, bool) { name += "=" } return name, true + case token.PLUS, token.MINUS, token.TILDE, token.BANG: + // Binary `+`/`-` and the unary methods `~`/`!`, plus the unary-operator + // methods `+@`/`-@`/`~@`/`!@` whose `@` hugs the operator. + name := p.advance().Lit + // `+@`/`-@`/`~@`/`!@`: a bare `@` (no following name) hugs the operator. The + // lexer yields it as an ILLEGAL "@" token (bare `@` is not a valid ivar). + if p.cur().Lit == "@" && !p.cur().SpaceBefore { + p.advance() + name += "@" + } + return name, true case token.CONST, token.SPACESHIP, token.LT, token.GT, token.LE, token.GE, - token.EQ, token.NEQ, token.SHOVEL, token.PLUS, token.MINUS, - token.STAR, token.SLASH, token.PERCENT: + token.EQ, token.EQQ, token.NEQ, token.MATCH, token.SHOVEL, token.RSHIFT, + token.STAR, token.POW, token.SLASH, token.PERCENT, + token.AMPER, token.PIPE, token.CARET: return p.advance().Lit, true case token.LBRACKET: p.advance() @@ -517,10 +556,23 @@ func (p *Parser) parseDefName() (string, bool) { // params, nil for a required parameter. func (p *Parser) parseDefParams(until token.Type) (params []string, defaults []ast.Node, splat int, kwParams []ast.KwParam, kwRest, blockParam string, forward bool) { splat = -1 + // A parenthesised parameter list may span several lines, with newlines after + // the open paren and around the separating commas; a paren-less list (until == + // NEWLINE) must not skip newlines, as one terminates it. + paren := until == token.RPAREN + if paren { + p.skipNewlines() + } if p.is(until) || p.is(token.NEWLINE) { return params, defaults, splat, kwParams, kwRest, blockParam, forward } for { + if paren { + p.skipNewlines() + if p.is(until) { // a trailing comma before the close paren + break + } + } if p.accept(token.DOTDOTDOT) { // `...` argument-forwarding param (always last) forward = true break @@ -597,6 +649,9 @@ func (p *Parser) parseDefParams(until token.Type) (params []string, defaults []a break } } + if paren { + p.skipNewlines() // tolerate a newline before the close paren + } return params, defaults, splat, kwParams, kwRest, blockParam, forward } @@ -689,6 +744,39 @@ func (p *Parser) parseReturn() ast.Node { return &ast.Return{Value: &ast.ArrayLit{Elems: elems}} } +// parseAlias parses `alias NewName OldName`. Each name is a method name (a bare +// identifier/constant/keyword/operator) or a symbol, or — for global aliasing — +// a global variable. The two names are separated by whitespace, not a comma. +func (p *Parser) parseAlias() ast.Node { + p.expect(token.ALIAS) + return &ast.Alias{NewName: p.parseFitem(), OldName: p.parseFitem()} +} + +// parseUndef parses `undef name [, name…]`, removing the named methods. +func (p *Parser) parseUndef() ast.Node { + p.expect(token.UNDEF) + names := []string{p.parseFitem()} + for p.accept(token.COMMA) { + names = append(names, p.parseFitem()) + } + return &ast.Undef{Names: names} +} + +// parseFitem reads one method-name item for alias/undef: a symbol (`:foo`, +// `:==`), a global variable (`$x`, alias only), or a bare method name — an +// identifier, a constant, a reserved word, or an operator (`==`, `<=>`, `[]`). +func (p *Parser) parseFitem() string { + switch p.cur().Type { + case token.SYMBOL, token.GVAR: + return p.advance().Lit + } + if name, ok := p.parseDefName(); ok { + return name + } + p.fail("expected a method name") + return "" +} + func (p *Parser) parseBreak() ast.Node { p.expect(token.BREAK) if p.atStatementEnd() { @@ -735,9 +823,11 @@ func (p *Parser) parseRescueTail(body []ast.Node) *ast.Begin { p.advance() clause := ast.RescueClause{} if !p.is(token.NEWLINE) && !p.is(token.HASHROCKET) && !p.is(token.THEN) { - clause.Classes = append(clause.Classes, p.parseExprOrAssign()) + // A rescue class list may be a `*splat` of an exception array + // (`rescue *EXCEPTIONS => e`, `rescue *FOO, Bar`). + clause.Classes = append(clause.Classes, p.parseSplatOrExpr()) for p.accept(token.COMMA) { - clause.Classes = append(clause.Classes, p.parseExprOrAssign()) + clause.Classes = append(clause.Classes, p.parseSplatOrExpr()) } } if p.accept(token.HASHROCKET) { @@ -775,7 +865,7 @@ func (p *Parser) parseBodyWithRescue() []ast.Node { func (p *Parser) parseInterpString() ast.Node { parts := []ast.Node{&ast.StringLit{Value: p.advance().Lit}} // STRBEG for { - parts = append(parts, p.parseExprOrAssign()) + parts = append(parts, p.parseInterpBody()) t := p.advance() parts = append(parts, &ast.StringLit{Value: t.Lit}) if t.Type == token.STREND { @@ -788,6 +878,25 @@ func (p *Parser) parseInterpString() ast.Node { return &ast.StrInterp{Parts: parts} } +// interpEnd marks the tokens that close a `#{…}` interpolation body. +var interpEnd = map[token.Type]bool{token.STRMID: true, token.STREND: true} + +// parseInterpBody parses the contents of one `#{…}` interpolation, which is a +// full statement sequence (so it admits trailing modifiers, `#{'s' if n > 1}`, +// and several `;`-separated statements). Its value is the last statement; an +// empty body (`#{}`) is nil. +func (p *Parser) parseInterpBody() ast.Node { + stmts := p.parseStatements(interpEnd) + switch len(stmts) { + case 0: + return &ast.NilLit{} + case 1: + return stmts[0] + default: + return &ast.Begin{Body: stmts} + } +} + // parseCase parses either `case [subj] (when …)* [else] end` or the pattern // form `case subj (in PATTERN [guard])* [else] end`. The first clause keyword // (when vs in) selects the form. @@ -804,9 +913,11 @@ func (p *Parser) parseCase() ast.Node { node := &ast.Case{Subject: subject} for p.is(token.WHEN) { p.advance() - clause := ast.WhenClause{Conds: []ast.Node{p.parseExprOrAssign()}} + // A `when` list may contain `*splat` of an array of candidate values + // (`when *LIST`, `when 1, *rest`). + clause := ast.WhenClause{Conds: []ast.Node{p.parseSplatOrExpr()}} for p.accept(token.COMMA) { - clause.Conds = append(clause.Conds, p.parseExprOrAssign()) + clause.Conds = append(clause.Conds, p.parseSplatOrExpr()) } p.accept(token.THEN) clause.Body = p.parseStatements(caseBodyEnd) @@ -1228,13 +1339,34 @@ func (p *Parser) parseMlhs() ast.Node { targets = nil // all-locals fast path: Names alone suffices } p.expect(token.ASSIGN) - values := []ast.Node{p.parseTernary()} + values := []ast.Node{p.parseMasgnValue()} for p.accept(token.COMMA) { - values = append(values, p.parseTernary()) + values = append(values, p.parseMasgnValue()) } return &ast.MultiAssign{Names: names, Targets: targets, SplatIndex: splat, Values: values} } +// parseSplatOrExpr parses an expression that may be prefixed by a `*splat`, +// spreading an array in a position that accepts several values (a `when` +// candidate list, a `rescue` class list). +func (p *Parser) parseSplatOrExpr() ast.Node { + if p.accept(token.STAR) { + return &ast.SplatArg{Value: p.parseExprOrAssign()} + } + return p.parseExprOrAssign() +} + +// parseMasgnValue parses one right-hand value of a multiple assignment, which +// may be a `*splat` whose array is spread across the targets (`a, b = *args`, +// `x, y = 1, *rest`) or a further (chained) assignment whose value is then +// destructured (`a, b = c = [1, 2]`, `_, h, _ = resp = call(x)`). +func (p *Parser) parseMasgnValue() ast.Node { + if p.accept(token.STAR) { + return &ast.SplatArg{Value: p.parseTernary()} + } + return p.parseExprOrAssign() +} + // parseMlhsTarget parses one masgn target and returns (localName, targetNode, // isPlainLocal). For a plain local, localName is its name, targetNode is a // *VarRef, and isPlainLocal is true. For any other target (constant, ivar, @@ -1296,7 +1428,7 @@ func (p *Parser) parseExprOrAssign() ast.Node { if p.is(token.IDENT) && p.peekTok().Type == token.ASSIGN { name := p.advance().Lit p.expect(token.ASSIGN) - val := p.parseExprOrAssign() + val := p.parseAssignRhs() p.declareLocal(name) return &ast.Assign{Name: name, Value: val} } @@ -1304,25 +1436,25 @@ func (p *Parser) parseExprOrAssign() ast.Node { if p.is(token.CONST) && p.peekTok().Type == token.ASSIGN { name := p.advance().Lit p.expect(token.ASSIGN) - return &ast.ConstAssign{Name: name, Value: p.parseExprOrAssign()} + return &ast.ConstAssign{Name: name, Value: p.parseAssignRhs()} } // Instance-variable assignment: @name '=' expr. if p.is(token.IVAR) && p.peekTok().Type == token.ASSIGN { name := p.advance().Lit p.expect(token.ASSIGN) - return &ast.IvarAssign{Name: name, Value: p.parseExprOrAssign()} + return &ast.IvarAssign{Name: name, Value: p.parseAssignRhs()} } // Class-variable assignment: @@name '=' expr. if p.is(token.CVAR) && p.peekTok().Type == token.ASSIGN { name := p.advance().Lit p.expect(token.ASSIGN) - return &ast.CVarAssign{Name: name, Value: p.parseExprOrAssign()} + return &ast.CVarAssign{Name: name, Value: p.parseAssignRhs()} } // Global-variable assignment: $name '=' expr. if p.is(token.GVAR) && p.peekTok().Type == token.ASSIGN { name := p.advance().Lit p.expect(token.ASSIGN) - return &ast.GVarAssign{Name: name, Value: p.parseExprOrAssign()} + return &ast.GVarAssign{Name: name, Value: p.parseAssignRhs()} } // Compound assignment to a local / ivar: LHS OP= expr → LHS = LHS OP expr. if p.is(token.IDENT) && p.peekTok().Type == token.OPASSIGN { @@ -1391,10 +1523,49 @@ func (p *Parser) parseExprOrAssign() ast.Node { return call } } + // Scope-resolved constant assignment: `A::B = v`, `::Top = v`. + if sc, ok := left.(*ast.ScopedConst); ok { + p.advance() + return &ast.ScopedConstAssign{Target: sc, Value: p.parseAssignRhs()} + } } return p.withRescueModifier(left) } +// parseAssignRhs parses the right-hand side of a single-target assignment. It is +// usually one expression (chainable: `a = b = 1`), but a top-level comma — or a +// leading `*splat` — makes the RHS an implicit array: `x = 1, 2, 3` ≡ `x = [1, +// 2, 3]` and `a = *list, y` ≡ `a = [*list, y]`, matching MRI. +func (p *Parser) parseAssignRhs() ast.Node { + first := p.parseRhsElem() + if !p.is(token.COMMA) { + // A bare leading `*splat` with no comma still yields a one-element array + // (`a = *list` ≡ `a = [*list]`), as MRI does. + if sp, ok := first.(*ast.SplatArg); ok { + return &ast.ArrayLit{Elems: []ast.Node{sp}} + } + return first + } + elems := []ast.Node{first} + for p.accept(token.COMMA) { + // `x = 1, 2,` — a trailing comma before the terminator is allowed. + if p.atStatementEnd() { + break + } + elems = append(elems, p.parseRhsElem()) + } + return &ast.ArrayLit{Elems: elems} +} + +// parseRhsElem parses one element of an assignment right-hand side, which may be +// a `*splat`. +func (p *Parser) parseRhsElem() ast.Node { + if p.accept(token.STAR) { + return &ast.SplatArg{Value: p.parseTernary()} + } + return p.parseExprOrAssign() +} + // withRescueModifier consumes a trailing `rescue FALLBACK` on the same line (the // modifier form: `risky rescue default`), wrapping node so a StandardError it // raises yields FALLBACK instead. It binds tighter than `=` because it is @@ -1571,8 +1742,19 @@ func (p *Parser) parsePostfixTail(node ast.Node) ast.Node { } else if p.canStartCommandArg() { // Paren-less command call on a receiver: `obj.foo bar`, // `Fiber.yield 1`. The space-separated argument list terminates - // this postfix chain (it greedily consumes the rest), so return. - return &ast.Call{Recv: node, Name: name, Args: p.parseCommandArgs(), Safe: safe} + // this postfix chain (it greedily consumes the rest). A trailing + // `do…end` then binds to this command call (`obj.foo bar do … end`), + // as MRI attaches it to the outermost command rather than to an + // argument call. + call := &ast.Call{Recv: node, Name: name, Args: p.parseCommandArgs(), Safe: safe} + if p.is(token.DO) && !p.noDo { + call.Block = p.parseDoBlock() + // The block-bearing command call may itself be chained + // (`obj.foo bar do … end.baz`), so continue the postfix loop. + node = call + break + } + return call } node = &ast.Call{Recv: node, Name: name, Args: args, Safe: safe} case p.is(token.SCOPE): @@ -1655,8 +1837,15 @@ func (p *Parser) parseHashLiteral() ast.Node { } } else { k = p.parseExprOrAssign() - p.expect(token.HASHROCKET) - v = p.parseExprOrAssign() + // A quoted string key followed by `:` is a symbol key (`{'a': 1}`, + // `{"d-a-s-h": 2}`, `{"a#{x}": 3}`), the quoted form of a `name:` label. + if sym, ok := p.stringKeyColon(k); ok { + k = sym + v = p.parseExprOrAssign() + } else { + p.expect(token.HASHROCKET) + v = p.parseExprOrAssign() + } } h.Keys = append(h.Keys, k) h.Values = append(h.Values, v) @@ -1670,6 +1859,28 @@ func (p *Parser) parseHashLiteral() ast.Node { return h } +// stringKeyColon recognises the quoted-symbol-key shorthand: a string-literal +// key immediately followed by `:` denotes a symbol key, the quoted analogue of a +// `name:` label (`{'a': 1}`, `{"d-a-s-h": 2}`, `tag("@click": "f")`). It returns +// the symbol-key node and true when key is a string literal and the cursor is on +// the separating `:` (consuming it); otherwise it returns (nil, false), leaving +// the cursor untouched so the caller falls back to the `=>` form. A plain string +// becomes a SymbolLit; an interpolated string becomes a dynamic `"…".to_sym`. +func (p *Parser) stringKeyColon(key ast.Node) (ast.Node, bool) { + if !p.is(token.COLON) { + return nil, false + } + switch s := key.(type) { + case *ast.StringLit: + p.advance() // ':' + return &ast.SymbolLit{Name: s.Value}, true + case *ast.StrInterp: + p.advance() // ':' + return &ast.Call{Recv: s, Name: "to_sym"}, true + } + return nil, false +} + // parseLambda parses a stabby lambda `->(params) { body }` / `-> { body }` / // `->(params) do body end`, desugaring it to `lambda { |params| body }`. func (p *Parser) parseLambda() ast.Node { @@ -1685,6 +1896,12 @@ func (p *Parser) parseLambda() ast.Node { params, defaults, prepends, splat, blockParam = p.parseBlockParams(token.RPAREN) p.expect(token.RPAREN) p.scope().explicitParams = true + } else if p.is(token.IDENT) || p.is(token.STAR) || p.is(token.POW) || p.is(token.AMPER) { + // Unparenthesized parameters: `->x { }`, `-> ctx { }`, `->a, b { }`, + // `-> message do … end`. The list runs up to the block opener (`{`/`do`); + // parseBlockParams stops at the first token that is not a parameter. + params, defaults, prepends, splat, blockParam = p.parseBlockParams(token.LBRACE) + p.scope().explicitParams = true } bs := p.scope() var body []ast.Node @@ -1789,6 +2006,27 @@ func (p *Parser) parseBlockParams(until token.Type) (names []string, defaults, p } continue } + if p.is(token.LABEL) { // keyword block param: |a, b:| / |c: nil, d: 5| + // Block keyword parameters are recorded with a trailing-colon sentinel + // name ("b:") so the shape round-trips distinctly from a positional one; + // the local is declared under its bare name. A value after the label is + // the keyword default, else the keyword is required. + name := p.advance().Lit + p.declareLocal(name) + names = append(names, name+":") + if p.is(token.COMMA) || p.is(until) || p.is(token.NEWLINE) { + defaults = append(defaults, nil) + } else { + saved := p.noPipe + p.noPipe = until == token.PIPE + defaults = append(defaults, p.parseExprOrAssign()) + p.noPipe = saved + } + if !p.accept(token.COMMA) { + break + } + continue + } if p.is(token.STAR) { // top-level rest param: |*rest| / |a, *rest| / (*rest) p.advance() if splat >= 0 { @@ -1957,13 +2195,18 @@ func (p *Parser) parsePrimary() ast.Node { return p.parseYield() case token.LPAREN: p.advance() - p.skipNewlines() - // A parenthesised group permits the low-precedence keyword operators - // (`(a and b)`, `(not x)`) just like a statement does. - e := p.parseOneLineMatch(p.parseKeywordLogical()) - p.skipNewlines() + // A parenthesised group is a full statement sequence: it permits the + // low-precedence keyword operators (`(a and b)`, `(not x)`), trailing + // modifiers (`(expr if cond)`, `(x if y) || z`), and several + // semicolon/newline-separated statements (`(a; b)`), evaluating to the + // last. A single statement returns directly; a sequence is wrapped in a + // Begin (whose value is its last expression). + stmts := p.parseStatements(map[token.Type]bool{token.RPAREN: true}) p.expect(token.RPAREN) - return e + if len(stmts) == 1 { + return stmts[0] + } + return &ast.Begin{Body: stmts} case token.IDENT: return p.parseIdentExpr() case token.CONST: @@ -1988,6 +2231,10 @@ func (p *Parser) parsePrimary() ast.Node { case token.CVAR: p.advance() return &ast.CVarRef{Name: t.Lit} + case token.DEF: + // `def` in expression position evaluates to the defined method's name as a + // symbol; it appears as a command argument (`private def foo; end`). + return p.parseDef() case token.BEGIN: return p.parseBegin() case token.CASE: @@ -2082,8 +2329,10 @@ func (p *Parser) canStartCommandArg() bool { case token.INT, token.FLOAT, token.STRING, token.STRBEG, token.SYMBOL, token.IDENT, token.CONST, token.IVAR, token.CVAR, token.GVAR, token.TRUE, token.FALSE, token.NIL, token.SELF, token.BANG, token.TILDE, token.LPAREN, token.LBRACKET, token.ARROW, token.WORDS, token.SYMBOLS, token.REGEXP, token.XSTRING, - token.BEGIN, token.CASE: - // Value-producing keywords: `p begin; 1; end`, `p case x; when 1; 2; end`. + token.BEGIN, token.CASE, token.DEF: + // Value-producing keywords: `p begin; 1; end`, `p case x; when 1; 2; end`, + // and `def` (which evaluates to the method's name symbol), as in + // `module_function def foo; end` / `private def bar; end`. return true case token.LABEL: // Keyword/hash argument without parens: `render json: x`, `delegate to: :c`. @@ -2190,6 +2439,12 @@ func (p *Parser) parseOneCallArg(args *[]ast.Node, kw **ast.HashLit) { p.addKwPair(kw, node, p.parseExprOrAssign()) return } + // A quoted string key followed by `:` is a symbol-keyed pair (the quoted form + // of a `key:` keyword argument): `tag(:div, "@click": "f()")`. + if sym, ok := p.stringKeyColon(node); ok { + p.addKwPair(kw, sym, p.parseExprOrAssign()) + return + } *args = append(*args, node) } diff --git a/quoted_symbol_key_test.go b/quoted_symbol_key_test.go new file mode 100644 index 0000000..68cb051 --- /dev/null +++ b/quoted_symbol_key_test.go @@ -0,0 +1,84 @@ +package parser + +import ( + "testing" + + "github.com/go-ruby-parser/parser/ast" +) + +// A quoted string key followed by `:` is a symbol key, the quoted form of a +// `name:` label: `{'desc': 'x'}`, `{"d-a-s-h": ""}`. +func TestQuotedSymbolKeyInHash(t *testing.T) { + cases := []struct { + src string + key string + }{ + {`{'desc': 'x'}`, "desc"}, + {`{"d-a-s-h": ""}`, "d-a-s-h"}, + {`{'0': {}}`, "0"}, + {`{"a": 1, "b": 2}`, "a"}, + } + for _, c := range cases { + prog, err := Parse(c.src) + if err != nil { + t.Fatalf("Parse(%q) error: %v", c.src, err) + } + h, ok := prog.Body[0].(*ast.HashLit) + if !ok { + t.Fatalf("%q: expected *ast.HashLit, got %T", c.src, prog.Body[0]) + } + sym, ok := h.Keys[0].(*ast.SymbolLit) + if !ok { + t.Fatalf("%q: key0 = %T, want *ast.SymbolLit", c.src, h.Keys[0]) + } + if sym.Name != c.key { + t.Fatalf("%q: key0 name=%q, want %q", c.src, sym.Name, c.key) + } + } +} + +// The same shorthand works in a method call's keyword arguments. +func TestQuotedSymbolKeyInCall(t *testing.T) { + prog, err := Parse(`tag(:div, "@click": "f()")`) + if err != nil { + t.Fatalf("Parse error: %v", err) + } + call := prog.Body[0].(*ast.Call) + last := call.Args[len(call.Args)-1] + h, ok := last.(*ast.HashLit) + if !ok { + t.Fatalf("expected trailing *ast.HashLit, got %T", last) + } + if sym, ok := h.Keys[0].(*ast.SymbolLit); !ok || sym.Name != "@click" { + t.Fatalf("expected symbol key @click, got %#v", h.Keys[0]) + } +} + +// An interpolated string key becomes a dynamic `"…".to_sym` symbol key. +func TestInterpolatedSymbolKey(t *testing.T) { + prog, err := Parse(`{"with_#{name}": 1}`) + if err != nil { + t.Fatalf("Parse error: %v", err) + } + h := prog.Body[0].(*ast.HashLit) + call, ok := h.Keys[0].(*ast.Call) + if !ok || call.Name != "to_sym" { + t.Fatalf("expected key0 to be a .to_sym call, got %#v", h.Keys[0]) + } + if _, ok := call.Recv.(*ast.StrInterp); !ok { + t.Fatalf("expected to_sym receiver to be *ast.StrInterp, got %T", call.Recv) + } +} + +// A ternary whose condition/branches are strings is unaffected: the `:` belongs +// to the ternary, not a symbol key. +func TestQuotedSymbolKeyTernaryNoRegression(t *testing.T) { + for _, src := range []string{ + `x = cond ? "a" : "b"`, + `foo("a" ? 1 : 2)`, + } { + if _, err := Parse(src); err != nil { + t.Fatalf("Parse(%q) error: %v", src, err) + } + } +} diff --git a/rhs_array_assign_test.go b/rhs_array_assign_test.go new file mode 100644 index 0000000..61ab6c1 --- /dev/null +++ b/rhs_array_assign_test.go @@ -0,0 +1,88 @@ +package parser + +import ( + "testing" + + "github.com/go-ruby-parser/parser/ast" +) + +// A single assignment target with multiple comma-separated right-hand values is +// an implicit array assignment: `x = 1, 2, 3` ≡ `x = [1, 2, 3]` (MRI). The RHS +// may also carry a `*splat`. +func TestRhsArrayAssign(t *testing.T) { + cases := []struct { + src string + elems int + }{ + {`args = "-t", "--server", master`, 3}, + {`x = 1, 2, 3`, 3}, + {`a = *x, y`, 2}, + {`a = 1, 2,`, 2}, // trailing comma + } + for _, c := range cases { + prog, err := Parse(c.src) + if err != nil { + t.Fatalf("Parse(%q) error: %v", c.src, err) + } + asn, ok := prog.Body[0].(*ast.Assign) + if !ok { + t.Fatalf("%q: expected *ast.Assign, got %T", c.src, prog.Body[0]) + } + arr, ok := asn.Value.(*ast.ArrayLit) + if !ok { + t.Fatalf("%q: expected RHS *ast.ArrayLit, got %T", c.src, asn.Value) + } + if len(arr.Elems) != c.elems { + t.Fatalf("%q: got %d elems, want %d", c.src, len(arr.Elems), c.elems) + } + } +} + +// A bare leading `*splat` with no comma still yields a one-element array +// (`a = *list` ≡ `a = [*list]`). +func TestRhsArrayAssignBareSplat(t *testing.T) { + prog, err := Parse(`a = *list`) + if err != nil { + t.Fatalf("Parse error: %v", err) + } + asn := prog.Body[0].(*ast.Assign) + arr, ok := asn.Value.(*ast.ArrayLit) + if !ok || len(arr.Elems) != 1 { + t.Fatalf("expected one-element array, got %#v", asn.Value) + } + if _, ok := arr.Elems[0].(*ast.SplatArg); !ok { + t.Fatalf("expected SplatArg element, got %T", arr.Elems[0]) + } +} + +// RHS array assignment applies to constant/ivar/cvar/gvar targets too. +func TestRhsArrayAssignOtherTargets(t *testing.T) { + for _, src := range []string{ + `FOO = 1, 2`, + `@x = 1, 2`, + `@@x = 1, 2`, + `$x = 1, 2`, + } { + if _, err := Parse(src); err != nil { + t.Fatalf("Parse(%q) error: %v", src, err) + } + } +} + +// Ordinary chained and single-value assignments are unaffected. +func TestRhsArrayAssignNoRegression(t *testing.T) { + for _, src := range []string{ + `a = b = 1`, + `x = foo(1, 2)`, + `h = {a: 1}`, + } { + prog, err := Parse(src) + if err != nil { + t.Fatalf("Parse(%q) error: %v", src, err) + } + asn := prog.Body[0].(*ast.Assign) + if _, isArr := asn.Value.(*ast.ArrayLit); isArr { + t.Fatalf("%q: RHS should not be an array", src) + } + } +} diff --git a/round2_coverage_test.go b/round2_coverage_test.go new file mode 100644 index 0000000..af8475a --- /dev/null +++ b/round2_coverage_test.go @@ -0,0 +1,38 @@ +package parser + +import "testing" + +// A non-string hash key followed by a bare `:` (not `=>`) is a parse error — it +// exercises stringKeyColon's fall-through (a colon present, but the key is not a +// string), after which the `=>` requirement fails. +func TestNonStringKeyColonError(t *testing.T) { + for _, src := range []string{ + `{1 + 1 : 2}`, + `{foo : 2}`, + } { + if _, err := Parse(src); err == nil { + t.Fatalf("expected a parse error for %q", src) + } + } +} + +// alias/undef with a non-name item is a parse error (parseFitem's fail path). +func TestAliasUndefBadItemError(t *testing.T) { + for _, src := range []string{ + `alias 1 2`, + `undef 99`, + `alias`, // missing both items + } { + if _, err := Parse(src); err == nil { + t.Fatalf("expected a parse error for %q", src) + } + } +} + +// An interpolated quoted symbol key in a call argument hits stringKeyColon's +// StrInterp branch (a `.to_sym` dynamic key) in the call path too. +func TestInterpolatedSymbolKeyInCall(t *testing.T) { + if _, err := Parse(`foo("x#{y}": 1)`); err != nil { + t.Fatalf("Parse error: %v", err) + } +} diff --git a/scoped_assign_super_test.go b/scoped_assign_super_test.go new file mode 100644 index 0000000..6ca0b78 --- /dev/null +++ b/scoped_assign_super_test.go @@ -0,0 +1,59 @@ +package parser + +import ( + "testing" + + "github.com/go-ruby-parser/parser/ast" +) + +// Assignment to a scope-resolved constant (`A::B = v`, `::Top::X = v`). +func TestScopedConstAssign(t *testing.T) { + for _, src := range []string{ + `A::B = 1`, + `Foo::BAR = baz`, + `::Rack::Cache::MetaStore::RAILS = self`, + } { + prog, err := Parse(src) + if err != nil { + t.Fatalf("Parse(%q) error: %v", src, err) + } + sca, ok := prog.Body[0].(*ast.ScopedConstAssign) + if !ok { + t.Fatalf("%q: expected *ast.ScopedConstAssign, got %T", src, prog.Body[0]) + } + if _, ok := sca.Target.(*ast.ScopedConst); !ok { + t.Fatalf("%q: target=%T, want *ast.ScopedConst", src, sca.Target) + } + } +} + +// A class superclass may be any expression: a bare constant keeps the plain +// Super name; self / scoped / call superclasses go into SuperExpr. +func TestClassSuperclassExpression(t *testing.T) { + bare, err := Parse("class A < B\nend") + if err != nil { + t.Fatalf("Parse error: %v", err) + } + cd := bare.Body[0].(*ast.ClassDef) + if cd.Super != "B" || cd.SuperExpr != nil { + t.Fatalf("bare super: got Super=%q SuperExpr=%v, want B/nil", cd.Super, cd.SuperExpr) + } + + for _, c := range []struct { + src string + kind string + }{ + {"class A < self\nend", "self"}, + {"class A < Struct.new(:x)\nend", "call"}, + {"class A < ns::Base\nend", "scoped"}, + } { + prog, err := Parse(c.src) + if err != nil { + t.Fatalf("Parse(%q) error: %v", c.src, err) + } + cd := prog.Body[0].(*ast.ClassDef) + if cd.SuperExpr == nil || cd.Super != "" { + t.Fatalf("%q: expected SuperExpr set and Super empty, got Super=%q SuperExpr=%T", c.src, cd.Super, cd.SuperExpr) + } + } +} diff --git a/splat_lists_test.go b/splat_lists_test.go new file mode 100644 index 0000000..c76ee6b --- /dev/null +++ b/splat_lists_test.go @@ -0,0 +1,57 @@ +package parser + +import ( + "testing" + + "github.com/go-ruby-parser/parser/ast" +) + +// A `*splat` is accepted in a rescue class list, a when candidate list, and a +// multiple-assignment right-hand side. +func TestSplatInRescueWhenMasgn(t *testing.T) { + for _, src := range []string{ + "begin\nrescue *EXCEPTIONS => e\nend", + "begin\nrescue *FOO, Bar => e\nend", + "case x\nwhen *LIST\n 1\nend", + "case x\nwhen 1, *rest\n 1\nend", + "path, headers = *args", + "a, b = 1, *rest", + } { + if _, err := Parse(src); err != nil { + t.Fatalf("Parse(%q) error: %v", src, err) + } + } +} + +// The rescue/when/masgn splats are represented as SplatArg nodes. +func TestSplatNodeShapes(t *testing.T) { + // rescue + prog, err := Parse("begin\nrescue *EX => e\nend") + if err != nil { + t.Fatalf("rescue parse: %v", err) + } + beg := prog.Body[0].(*ast.Begin) + if _, ok := beg.Rescues[0].Classes[0].(*ast.SplatArg); !ok { + t.Fatalf("rescue class0 = %T, want *ast.SplatArg", beg.Rescues[0].Classes[0]) + } + + // when + prog, err = Parse("case x\nwhen *L\n 1\nend") + if err != nil { + t.Fatalf("when parse: %v", err) + } + cs := prog.Body[0].(*ast.Case) + if _, ok := cs.Whens[0].Conds[0].(*ast.SplatArg); !ok { + t.Fatalf("when cond0 = %T, want *ast.SplatArg", cs.Whens[0].Conds[0]) + } + + // masgn RHS + prog, err = Parse("a, b = *args") + if err != nil { + t.Fatalf("masgn parse: %v", err) + } + ma := prog.Body[0].(*ast.MultiAssign) + if _, ok := ma.Values[0].(*ast.SplatArg); !ok { + t.Fatalf("masgn value0 = %T, want *ast.SplatArg", ma.Values[0]) + } +} diff --git a/supplement2_test.go b/supplement2_test.go index 1010a2e..af02ba1 100644 --- a/supplement2_test.go +++ b/supplement2_test.go @@ -73,21 +73,21 @@ var extraValid2 = []string{ // parseErrorCorpus exercises the parser's and lexer's error paths. var parseErrorCorpus = []string{ - ":", // bare colon - "\"#{1 2}\"", // two expressions in one interpolation - "p(/abc", // unterminated regexp - "p(%w[a b c)", // unterminated %w - "x = :\"unterminated", // unterminated quoted symbol - "def", // def with no name - "class", // class with no name - "if", // if with no condition/body - "case 1\nin [a, *b, *c]\nend", // two splats in array pattern - "case 1\nin {**a, **nil}\nend", // **rest with **nil - "1 +", // dangling binary operator - "foo(", // unterminated call args - "[1, 2", // unterminated array - "{a: 1", // unterminated hash - "begin\n x", // begin without end + ":", // bare colon + "\"#{1 2}\"", // two expressions in one interpolation + "p(/abc", // unterminated regexp + "p(%w[a b c)", // unterminated %w + "x = :\"unterminated", // unterminated quoted symbol + "def", // def with no name + "class", // class with no name + "if", // if with no condition/body + "case 1\nin [a, *b, *c]\nend", // two splats in array pattern + "case 1\nin {**a, **nil}\nend", // **rest with **nil + "1 +", // dangling binary operator + "foo(", // unterminated call args + "[1, 2", // unterminated array + "{a: 1", // unterminated hash + "begin\n x", // begin without end } func TestExtraValid2(t *testing.T) { diff --git a/supplement5_test.go b/supplement5_test.go index 86bd078..f130b47 100644 --- a/supplement5_test.go +++ b/supplement5_test.go @@ -8,33 +8,33 @@ import ( // errorBranches drives the parser's and lexer's expect()/raise paths. var errorBranches = []string{ - "def 1", // bad def name - "def f(a", // unterminated def params - "def f(1)", // non-identifier param - "class lower", // class name must be a constant - "class 1", // bad class name - "module 1", // bad module name - "case 1\nin\nend", // in with no pattern - "[1].each { |a", // unterminated block params - "x = ->(a", // unterminated lambda params - "x = (1", // unterminated parenthesized expr - "foo[1", // unterminated index - "x = @", // ivar with no name - "until", // until with no condition - "unless", // unless with no condition - "while", // while with no condition - "super(", // unterminated super args - "yield(", // unterminated yield args - "x = a ? b", // ternary missing ':' - "begin\nrescue =>\nend", // rescue binding with no var - "def f\nrescue =>\nend", // method rescue, bad binding - "{ 1 => }", // hash pair missing value - "[1,", // trailing comma, then EOF - "x = !", // unary bang with no operand - "x = -", // unary minus with no operand - "1 2", // two primaries, no operator - "x = %Q", // %Q at EOF - "<<~", // squiggly heredoc marker, nothing + "def 1", // bad def name + "def f(a", // unterminated def params + "def f(1)", // non-identifier param + "class lower", // class name must be a constant + "class 1", // bad class name + "module 1", // bad module name + "case 1\nin\nend", // in with no pattern + "[1].each { |a", // unterminated block params + "x = ->(a", // unterminated lambda params + "x = (1", // unterminated parenthesized expr + "foo[1", // unterminated index + "x = @", // ivar with no name + "until", // until with no condition + "unless", // unless with no condition + "while", // while with no condition + "super(", // unterminated super args + "yield(", // unterminated yield args + "x = a ? b", // ternary missing ':' + "begin\nrescue =>\nend", // rescue binding with no var + "def f\nrescue =>\nend", // method rescue, bad binding + "{ 1 => }", // hash pair missing value + "[1,", // trailing comma, then EOF + "x = !", // unary bang with no operand + "x = -", // unary minus with no operand + "1 2", // two primaries, no operator + "x = %Q", // %Q at EOF + "<<~", // squiggly heredoc marker, nothing } func TestErrorBranches(t *testing.T) { diff --git a/supplement8_test.go b/supplement8_test.go index ff92f65..c3987c5 100644 --- a/supplement8_test.go +++ b/supplement8_test.go @@ -10,32 +10,32 @@ import ( // coverage profile (implicit params, pattern atoms, block-param groups, // attribute op-assign, nested-brace interpolation, regexp/heredoc edges). var extraValid8 = []string{ - "x = _1", // _1 in a hard (top-level) scope -> plain ident - "[1].each { |x| _1 }", // _1 with explicit params -> plain ident - "o = nil\no.x += 1", // compound attribute assignment - "x = -2 ** 3", // negative literal with ** (precedence) - "[1].each { |(a, *b)| a }", // splat inside a grouped block param - "def f()\n 1\nend", // empty parameter list - "f = ->() do 1 end", // lambda with a do/end block - "case 1\nin (1)\n :a\nend", // parenthesised (grouped) pattern - "case [1]\nin [a,]\n a\nend", // array pattern with a trailing comma + "x = _1", // _1 in a hard (top-level) scope -> plain ident + "[1].each { |x| _1 }", // _1 with explicit params -> plain ident + "o = nil\no.x += 1", // compound attribute assignment + "x = -2 ** 3", // negative literal with ** (precedence) + "[1].each { |(a, *b)| a }", // splat inside a grouped block param + "def f()\n 1\nend", // empty parameter list + "f = ->() do 1 end", // lambda with a do/end block + "case 1\nin (1)\n :a\nend", // parenthesised (grouped) pattern + "case [1]\nin [a,]\n a\nend", // array pattern with a trailing comma "case 1\nin x unless false\n :a\nend", // pattern guard with `unless` - "def f\n super 1, 2\nend", // super with command args (no parens) - "foo(:a => 1)", // hashrocket call argument - "x = /abc/imx", // regexp flags - "x = %w[a [b] c]", // nested brackets in a %w literal - "x = 1<<2", // shift (atHeredoc false path) - "p 99999999999999999999999999999", // bignum literal - "x = \"#{ {a: 1} }\"", // nested braces in string interpolation - "x = :\"a#{ {b: 1} }\"", // nested braces in quoted-symbol interpolation - "x = < 1)", // hashrocket call argument + "x = /abc/imx", // regexp flags + "x = %w[a [b] c]", // nested brackets in a %w literal + "x = 1<<2", // shift (atHeredoc false path) + "p 99999999999999999999999999999", // bignum literal + "x = \"#{ {a: 1} }\"", // nested braces in string interpolation + "x = :\"a#{ {b: 1} }\"", // nested braces in quoted-symbol interpolation + "x = <", LE: "<=", GE: ">=", BANG: "!", SPACESHIP: "<=>", SHOVEL: "<<", ANDAND: "&&", OROR: "||", OPASSIGN: "op=", QUESTION: "?", COLON: ":", SCOPE: "::", @@ -143,6 +146,7 @@ var Keywords = map[string]Type{ "then": THEN, "do": DO, "true": TRUE, "false": FALSE, "nil": NIL, "self": SELF, "super": SUPER, "yield": YIELD, "and": AND, "or": OR, "not": NOT, + "alias": ALIAS, "undef": UNDEF, } // Token is a single lexed token.