Skip to content

Commit 0ccf760

Browse files
tannevaledclaude
andcommitted
feat: ast.Program.Lines — the source line of every statement
The AST carried no position information at all: 55 node types, zero line/column fields. A consumer that needs to say WHERE a frame is executing — a backtrace, Ruby's __LINE__, Module#const_source_location — had nothing to read, and go-embedded-ruby/ruby, extracted from this parser, has been reporting line 0 everywhere for exactly that reason (101 ruby/spec examples across five directories). Program.Lines maps each STATEMENT node to the 1-based line it began on. It is populated by Parse itself, on Program, rather than behind a second entry point: every existing consumer then gets positions without changing a call site, which matters because the consumers thread a *ast.Program through several layers before anything looks at a line. Why statement granularity: MRI attaches a line to every node and compiles a per-instruction table from it (compile.c v3_4_0: ADD_INSN records nd_line(node); iseq.c:673 rb_iseq_insns_info_encode_positions builds insns_info + positions). But that table is COMPRESSED deliberately — one entry per line CHANGE, not per instruction — and every reader goes through rb_iseq_line_no (iseq.c v3_4_0:2314), which resolves a pc to the nearest preceding entry. Statement boundaries are where the line changes in all but pathological source, so this map holds the same information at the granularity a consumer can observe, without a field on 55 structs. The stamp is first-write-wins, which is what makes it precise rather than merely cheap: an inner statement is recorded when its own parseStatement returns, before the enclosing one finishes, so a construct that hands its body's node straight back (`begin; foo; end`) keeps foo's line. MRI reaches the same answer from the other side — the node it compiles IS foo's node, and nd_line(foo) is foo's line. Verified against MRI 4.0.5 (+PRISM): for `puts(__LINE__ +\n 1)` opening on line 5, ruby prints 6 — __LINE__ is the line the STATEMENT started on, not the line the operand sits on. Cost, measured on a 2570-line Ruby source, 200 iterations x 5, Apple M4 Max (medians): v0.2.0 1.631 ms/op 5 288 396 B/op 13 870 allocs/op this change 1.728 ms/op 5 397 636 B/op 13 880 allocs/op +6.0% +2.1% +10 The map is sized up front from the lexer's NEWLINE count — a tight upper bound on the number of statements — which is what keeps it to +10 allocations instead of the rehash chain (1.782 ms, +22 allocs unsized). Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
1 parent 6404526 commit 0ccf760

3 files changed

Lines changed: 275 additions & 3 deletions

File tree

‎ast/ast.go‎

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,27 @@ import "math/big"
1010
type Node interface{ node() }
1111

1212
// Program is the top-level sequence of expressions.
13-
type Program struct{ Body []Node }
13+
//
14+
// Lines gives the 1-based source line each STATEMENT node in the tree began
15+
// on, keyed by the node itself (every Node is a distinct pointer, so the map is
16+
// exact). It is how a consumer says WHERE something is: a backtrace, Ruby's
17+
// __LINE__, Module#const_source_location. A node with no entry reads back as
18+
// line 0, which is what MRI's rb_iseq_line_no returns for a pc its own table
19+
// cannot place (iseq.c v3_4_0:2314).
20+
//
21+
// The granularity is deliberate. MRI attaches a line to every node and compiles
22+
// a per-instruction table from it (compile.c v3_4_0: ADD_INSN records
23+
// nd_line(node); iseq.c:673 rb_iseq_insns_info_encode_positions builds
24+
// insns_info + positions), but that table is COMPRESSED — one entry per line
25+
// CHANGE, not per instruction — and every reader resolves a pc to the nearest
26+
// preceding entry. Statement boundaries are where the line changes in all but
27+
// pathological source, so this map holds the same information at the
28+
// granularity a consumer can observe, without a position field on all 55 node
29+
// types.
30+
type Program struct {
31+
Body []Node
32+
Lines map[Node]int
33+
}
1434

1535
// IntLit is an integer literal that fits in int64.
1636
type IntLit struct{ Value int64 }

‎lines_test.go‎

Lines changed: 218 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,218 @@
1+
// Copyright (c) the go-ruby-parser authors
2+
//
3+
// SPDX-License-Identifier: BSD-3-Clause
4+
5+
package parser
6+
7+
import (
8+
"testing"
9+
10+
"github.com/go-ruby-parser/parser/ast"
11+
)
12+
13+
// lineOf walks the statement list and returns the recorded line of the nth
14+
// top-level statement.
15+
func lineOf(t *testing.T, lines map[ast.Node]int, n ast.Node) int {
16+
t.Helper()
17+
l, ok := lines[n]
18+
if !ok {
19+
t.Fatalf("no line recorded for %T", n)
20+
}
21+
return l
22+
}
23+
24+
// TestParsedLinesTopLevel pins the line of each top-level statement,
25+
// including the blank lines and comments MRI skips over.
26+
func TestParsedLinesTopLevel(t *testing.T) {
27+
src := "a = 1\n\n# comment\nb = 2\nc = 3; d = 4\n"
28+
prog, err := Parse(src)
29+
if err != nil {
30+
t.Fatal(err)
31+
}
32+
lines := prog.Lines
33+
want := []int{1, 4, 5, 5}
34+
if len(prog.Body) != len(want) {
35+
t.Fatalf("got %d statements, want %d", len(prog.Body), len(want))
36+
}
37+
for i, w := range want {
38+
if got := lineOf(t, lines, prog.Body[i]); got != w {
39+
t.Errorf("statement %d: line %d, want %d", i, got, w)
40+
}
41+
}
42+
}
43+
44+
// TestParsedLinesNested checks that a body nested inside a def, a class and
45+
// a block is stamped with its own line, not the enclosing construct's.
46+
func TestParsedLinesNested(t *testing.T) {
47+
src := "def foo\n bar\nend\nclass C\n def baz\n qux\n end\nend\n[1].each do |v|\n p v\nend\n"
48+
prog, err := Parse(src)
49+
if err != nil {
50+
t.Fatal(err)
51+
}
52+
lines := prog.Lines
53+
def, ok := prog.Body[0].(*ast.MethodDef)
54+
if !ok {
55+
t.Fatalf("statement 0 is %T, want *ast.MethodDef", prog.Body[0])
56+
}
57+
if got := lineOf(t, lines, def); got != 1 {
58+
t.Errorf("def line %d, want 1", got)
59+
}
60+
if got := lineOf(t, lines, def.Body[0]); got != 2 {
61+
t.Errorf("def body line %d, want 2", got)
62+
}
63+
64+
cls, ok := prog.Body[1].(*ast.ClassDef)
65+
if !ok {
66+
t.Fatalf("statement 1 is %T, want *ast.ClassDef", prog.Body[1])
67+
}
68+
if got := lineOf(t, lines, cls); got != 4 {
69+
t.Errorf("class line %d, want 4", got)
70+
}
71+
inner, ok := cls.Body[0].(*ast.MethodDef)
72+
if !ok {
73+
t.Fatalf("class body 0 is %T, want *ast.MethodDef", cls.Body[0])
74+
}
75+
if got := lineOf(t, lines, inner); got != 5 {
76+
t.Errorf("inner def line %d, want 5", got)
77+
}
78+
if got := lineOf(t, lines, inner.Body[0]); got != 6 {
79+
t.Errorf("inner def body line %d, want 6", got)
80+
}
81+
82+
call, ok := prog.Body[2].(*ast.Call)
83+
if !ok {
84+
t.Fatalf("statement 2 is %T, want *ast.Call", prog.Body[2])
85+
}
86+
if got := lineOf(t, lines, call); got != 9 {
87+
t.Errorf("each call line %d, want 9", got)
88+
}
89+
if got := lineOf(t, lines, call.Block.Body[0]); got != 10 {
90+
t.Errorf("block body line %d, want 10", got)
91+
}
92+
}
93+
94+
// TestParsedLinesFirstWriteWins covers the pass-through construct: `begin;
95+
// foo; end` hands foo's node straight back, and the stamp must stay foo's line
96+
// (the inner one, written first) rather than the begin's.
97+
func TestParsedLinesFirstWriteWins(t *testing.T) {
98+
src := "x = 0\nbegin\n foo\nend\n"
99+
prog, err := Parse(src)
100+
if err != nil {
101+
t.Fatal(err)
102+
}
103+
lines := prog.Lines
104+
begin, ok := prog.Body[1].(*ast.Begin)
105+
if !ok {
106+
t.Fatalf("statement 1 is %T, want *ast.Begin", prog.Body[1])
107+
}
108+
if got := lineOf(t, lines, begin.Body[0]); got != 3 {
109+
t.Errorf("begin body line %d, want 3", got)
110+
}
111+
}
112+
113+
// TestParsedLinesMultiline pins the rule a consumer depends on: a statement
114+
// spanning several lines carries the line it STARTED on, which is the line MRI
115+
// attributes to the call node too.
116+
func TestParsedLinesMultiline(t *testing.T) {
117+
src := "foo(1,\n 2,\n 3)\n"
118+
prog, err := Parse(src)
119+
if err != nil {
120+
t.Fatal(err)
121+
}
122+
lines := prog.Lines
123+
if got := lineOf(t, lines, prog.Body[0]); got != 1 {
124+
t.Errorf("multi-line call line %d, want 1", got)
125+
}
126+
}
127+
128+
// TestParsedLinesModifier checks a statement rewritten by applyModifiers:
129+
// `foo if bar` becomes an *ast.If, and that If is what carries the line.
130+
func TestParsedLinesModifier(t *testing.T) {
131+
src := "\n\nfoo if bar\n"
132+
prog, err := Parse(src)
133+
if err != nil {
134+
t.Fatal(err)
135+
}
136+
lines := prog.Lines
137+
if _, ok := prog.Body[0].(*ast.If); !ok {
138+
t.Fatalf("statement 0 is %T, want *ast.If", prog.Body[0])
139+
}
140+
if got := lineOf(t, lines, prog.Body[0]); got != 3 {
141+
t.Errorf("modifier-if line %d, want 3", got)
142+
}
143+
}
144+
145+
// TestParsedLinesKeywordStatements covers the non-default arms of
146+
// parseStatement1 (return/break/next/retry/alias/undef), each of which is
147+
// stamped through the same wrapper.
148+
func TestParsedLinesKeywordStatements(t *testing.T) {
149+
src := "def m\n return 1\nend\n[1].each do\n break\nend\n[1].each do\n next\nend\nbegin\n retry\nrescue\nend\nalias a b\nundef c\n"
150+
prog, err := Parse(src)
151+
if err != nil {
152+
t.Fatal(err)
153+
}
154+
lines := prog.Lines
155+
def := prog.Body[0].(*ast.MethodDef)
156+
if got := lineOf(t, lines, def.Body[0]); got != 2 {
157+
t.Errorf("return line %d, want 2", got)
158+
}
159+
brk := prog.Body[1].(*ast.Call)
160+
if got := lineOf(t, lines, brk.Block.Body[0]); got != 5 {
161+
t.Errorf("break line %d, want 5", got)
162+
}
163+
nxt := prog.Body[2].(*ast.Call)
164+
if got := lineOf(t, lines, nxt.Block.Body[0]); got != 8 {
165+
t.Errorf("next line %d, want 8", got)
166+
}
167+
bg := prog.Body[3].(*ast.Begin)
168+
if got := lineOf(t, lines, bg.Body[0]); got != 11 {
169+
t.Errorf("retry line %d, want 11", got)
170+
}
171+
if got := lineOf(t, lines, prog.Body[4]); got != 14 {
172+
t.Errorf("alias line %d, want 14", got)
173+
}
174+
if got := lineOf(t, lines, prog.Body[5]); got != 15 {
175+
t.Errorf("undef line %d, want 15", got)
176+
}
177+
}
178+
179+
// TestParsedLinesError checks that a malformed input yields no program and no
180+
// map, like Parse's error contract.
181+
func TestParsedLinesError(t *testing.T) {
182+
prog, err := Parse("def\n")
183+
if err == nil {
184+
t.Fatal("want a parse error")
185+
}
186+
if prog != nil {
187+
t.Errorf("got prog=%v, want nil", prog)
188+
}
189+
}
190+
191+
// TestParsedLinesEmptyProgram pins that an empty source still yields a usable
192+
// (non-nil, empty) map rather than one a consumer must nil-check.
193+
func TestParsedLinesEmptyProgram(t *testing.T) {
194+
prog, err := Parse("\n# just a comment\n")
195+
if err != nil {
196+
t.Fatal(err)
197+
}
198+
if prog.Lines == nil {
199+
t.Error("Lines is nil for an empty program")
200+
}
201+
if len(prog.Lines) != 0 {
202+
t.Errorf("Lines has %d entries, want 0", len(prog.Lines))
203+
}
204+
}
205+
206+
// TestParsedLinesInternalError drives the recover path of the shared parse
207+
// core through ParsedLines, so the nil-map assignment there is exercised.
208+
func TestParsedLinesInternalError(t *testing.T) {
209+
parseHook = func() { panic("boom") }
210+
defer func() { parseHook = nil }()
211+
prog, err := Parse("a = 1\n")
212+
if err == nil {
213+
t.Fatal("want an internal parse error")
214+
}
215+
if prog != nil {
216+
t.Errorf("got prog=%v, want nil", prog)
217+
}
218+
}

‎parser.go‎

Lines changed: 36 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -146,6 +146,9 @@ type Parser struct {
146146
// next line (`f(`⏎` k:`⏎` v)`) is then a continued pair, not a value-omitted
147147
// shorthand; at top level (depth 0) a newline after `key:` ends the command.
148148
bracketDepth int
149+
// lines collects the 1-based source line each statement node began on; it
150+
// becomes ast.Program.Lines.
151+
lines map[ast.Node]int
149152
}
150153

151154
// parseHook, when non-nil, runs at the start of Parse. It exists only so a
@@ -157,8 +160,19 @@ var parseHook func()
157160
// yields a parse error, and any unexpected internal panic is also surfaced as a
158161
// parse error rather than propagating to the caller.
159162
func Parse(src string) (prog *ast.Program, err error) {
163+
160164
toks := lexer.New(src).Tokenize()
161-
p := &Parser{toks: toks, scopes: []*scope{newScope(true)}, argParenEnd: -1}
165+
// One statement per source line is the dense case, and the lexer emits one
166+
// NEWLINE per line (and per `;`), so the terminator count is a tight upper
167+
// bound on the entries: sizing the map by it removes the rehash growth that
168+
// otherwise makes the table cost several times what it stores.
169+
terms := 0
170+
for _, t := range toks {
171+
if t.Type == token.NEWLINE {
172+
terms++
173+
}
174+
}
175+
p := &Parser{toks: toks, scopes: []*scope{newScope(true)}, argParenEnd: -1, lines: make(map[ast.Node]int, terms)}
162176
defer func() {
163177
if r := recover(); r != nil {
164178
if pe, ok := r.(parseError); ok {
@@ -176,7 +190,7 @@ func Parse(src string) (prog *ast.Program, err error) {
176190
}
177191
body := p.parseStatements(map[token.Type]bool{})
178192
p.expect(token.EOF)
179-
return &ast.Program{Body: body}, nil
193+
return &ast.Program{Body: body, Lines: p.lines}, nil
180194
}
181195

182196
// --- token cursor ---
@@ -347,7 +361,27 @@ func (p *Parser) parseStatements(stop map[token.Type]bool) []ast.Node {
347361
return body
348362
}
349363

364+
// parseStatement parses one statement and records the line it started on into
365+
// Program.Lines.
366+
//
367+
// The stamp is first-write-wins, and that is what makes it precise: an inner
368+
// statement is stamped when ITS parseStatement returns, before the enclosing one
369+
// finishes, so a construct that yields its body's node straight through
370+
// (`begin; foo; end`) keeps foo's own line rather than the begin's. MRI reaches
371+
// the same answer from the other side — the node it compiles IS foo's node, and
372+
// nd_line(foo) is foo's line.
350373
func (p *Parser) parseStatement() ast.Node {
374+
line := p.cur().Line
375+
n := p.parseStatement1()
376+
if n != nil {
377+
if _, seen := p.lines[n]; !seen {
378+
p.lines[n] = line
379+
}
380+
}
381+
return n
382+
}
383+
384+
func (p *Parser) parseStatement1() ast.Node {
351385
switch p.cur().Type {
352386
case token.RETURN:
353387
return p.applyModifiers(p.parseReturn())

0 commit comments

Comments
 (0)