diff --git a/lexer/lexer.go b/lexer/lexer.go index dd7ac07..3b3cec3 100644 --- a/lexer/lexer.go +++ b/lexer/lexer.go @@ -1426,9 +1426,15 @@ func (l *Lexer) atPercentArray() bool { } // lexPercentArray lexes a %w/%i/%W/%I array literal. The non-interpolating -// %w/%i forms become a single WORDS/SYMBOLS token whose Lit the parser splits. -// The interpolating %W/%I forms are spliced into the equivalent array of -// (interpolated) string/symbol elements. Bracket delimiters nest. +// %w/%i forms become a WORDS/SYMBOLS token carrying both the raw body (Lit) and +// the words it splits into (Words). The interpolating %W/%I forms are spliced +// into the equivalent array of (interpolated) string/symbol elements. Bracket +// delimiters nest. +// +// A backslash escapes the character after it, which is therefore neither a +// nesting bracket nor the terminator: `%w[a\]b]` is the single word `a]b`, not +// an unterminated list (MRI parse.y, tokadd_string — the `c == '\\'` branch +// consumes the next character before the term/paren tests can see it). func (l *Lexer) lexPercentArray(spaceBefore bool, line, col int) token.Token { l.advance() // % kind := l.advance() @@ -1436,10 +1442,21 @@ func (l *Lexer) lexPercentArray(spaceBefore bool, line, col int) token.Token { closing := percentDelimClose(open) depth := 1 var content []byte + unterminated := token.Token{Type: token.ILLEGAL, Lit: "unterminated %-array literal", Line: line, Col: col, SpaceBefore: spaceBefore} for { c := l.peek() if c == 0 { - return token.Token{Type: token.ILLEGAL, Lit: "unterminated %-array literal", Line: line, Col: col, SpaceBefore: spaceBefore} + return unterminated + } + if c == '\\' { + // Keep both bytes raw; the escape is resolved when the body is + // split into words, which is where the delimiters are still known. + content = append(content, l.advance()) + if l.peek() == 0 { + return unterminated + } + content = append(content, l.advance()) + continue } if open != closing && c == open { depth++ @@ -1460,7 +1477,79 @@ func (l *Lexer) lexPercentArray(spaceBefore bool, line, col int) token.Token { if kind == 'i' { tt = token.SYMBOLS } - return token.Token{Type: tt, Lit: string(content), Line: line, Col: col, SpaceBefore: spaceBefore} + return token.Token{ + Type: tt, + Lit: string(content), + Words: splitWordList(string(content), open, closing), + Line: line, + Col: col, + SpaceBefore: spaceBefore, + } +} + +// isWordListSpace reports whether c separates two words of a %-list. It is +// MRI's ISSPACE: space, tab, newline, vertical tab, form feed, carriage return. +func isWordListSpace(c byte) bool { + switch c { + case ' ', '\t', '\n', '\v', '\f', '\r': + return true + } + return false +} + +// splitWordList splits the raw body of a NON-interpolating %w/%i list into its +// words, resolving escapes as MRI does (parse.y, tokadd_string with +// func = str_sword: STR_FUNC_QWORDS, no STR_FUNC_EXPAND, no STR_FUNC_ESCAPE). +// +// Two rules matter and both follow from the backslash branch being tested +// before the word-splitting one: +// +// - Only UNESCAPED whitespace separates words. `%w[a\ b c]` is the two words +// "a b" and "c" — the escaped space is an ordinary character of the first +// word. (ruby -e 'p %w[a\ b c]' => ["a b", "c"]) +// - The backslash itself survives EXCEPT before whitespace, a delimiter, or +// another backslash, where it escapes a literal character instead. So +// `%w[a\tb]` is the four characters a \ t b and NOT a tab, while +// `%w[a\\b]` is the three characters a \ b. +// (ruby -e 'p %w[a\tb]' => ["a\\tb"]; ruby -e 'p %w[a\\b]' => ["a\\b"]) +func splitWordList(body string, open, closing byte) []string { + words := []string{} + var cur []byte + flush := func() { + if len(cur) > 0 { + words = append(words, string(cur)) + cur = nil + } + } + for i := 0; i < len(body); i++ { + c := body[i] + if c == '\\' && i+1 < len(body) { + d := body[i+1] + switch { + case isWordListSpace(d): + // The backslash is dropped and the whitespace becomes a + // literal character of the current word, never a separator. + cur = append(cur, d) + case d == '\\': + cur = append(cur, '\\') + case d == closing || (open != closing && d == open): + cur = append(cur, d) + default: + // Everything else keeps the backslash verbatim; there is no + // escape set here, only the four cases above. + cur = append(cur, '\\', d) + } + i++ + continue + } + if isWordListSpace(c) { + flush() + continue + } + cur = append(cur, c) + } + flush() + return words } // isPercentDelim reports whether b can open a %-literal. MRI accepts any @@ -1670,7 +1759,7 @@ func (l *Lexer) splicePercentInterp(content string, symbols, spaceBefore bool, l toks = append(toks, token.Token{Type: token.COMMA, Lit: ",", Line: line, Col: col}) } if symbols && !strings.Contains(w, "#{") { - toks = append(toks, token.Token{Type: token.SYMBOL, Lit: w, Line: line, Col: col}) + toks = append(toks, token.Token{Type: token.SYMBOL, Lit: unescapeInterpWord(w), Line: line, Col: col}) continue } for _, t := range New(`"` + wrapHeredocDQ(w) + `"`).Tokenize() { @@ -1689,8 +1778,31 @@ func (l *Lexer) splicePercentInterp(content string, symbols, spaceBefore bool, l return toks[0] } +// unescapeInterpWord resolves the escapes of a %I word that carries no +// interpolation, so `%I[a\ b]` is the symbol :"a b" and not the four characters +// a \ space b. A %I list interpolates, so its escape set is the double-quoted +// one; the word is re-lexed as a double-quoted string to reuse that decoder. +// (ruby -e 'p %I[a\ b]' => [:"a b"]) +func unescapeInterpWord(w string) string { + if !strings.Contains(w, `\`) { + return w + } + // The caller has already established that the word holds no interpolation, + // and the list scanner never leaves an unpaired backslash in it, so the + // re-lex always yields one string token followed by EOF. + return New(`"` + wrapHeredocDQ(w) + `"`).Tokenize()[0].Lit +} + // splitPercentWords splits a %W/%I body on whitespace, keeping whitespace that -// falls inside a #{…} interpolation as part of the word. +// is escaped, or that falls inside a #{…} interpolation, as part of the word. +// +// Escapes are left in place rather than resolved: each word is re-lexed as a +// double-quoted string, whose decoder already implements the full escape set +// MRI applies to an interpolating list (parse.y, str_dword: read_escape). The +// one divergence is a backslash before a newline — MRI's QWORDS test precedes +// its EXPAND test, so in %W that is a literal newline rather than the line +// continuation it would be in a string — so it is rewritten to `\n` here. +// (ruby -e 'p %W[a\b]' => ["a\nb"]) func splitPercentWords(s string) []string { var words []string var cur []byte @@ -1703,7 +1815,16 @@ func splitPercentWords(s string) []string { } for i := 0; i < len(s); i++ { c := s[i] - if depth == 0 && (c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == '\f' || c == '\v') { + if depth == 0 && c == '\\' && i+1 < len(s) { + if d := s[i+1]; d == '\n' { + cur = append(cur, '\\', 'n') + } else { + cur = append(cur, '\\', d) + } + i++ + continue + } + if depth == 0 && isWordListSpace(c) { flush() continue } diff --git a/parser.go b/parser.go index 99fb211..f3170f0 100644 --- a/parser.go +++ b/parser.go @@ -2917,7 +2917,9 @@ func (p *Parser) parsePrimary() ast.Node { case token.WORDS, token.SYMBOLS: p.advance() elems := []ast.Node{} - for _, w := range strings.Fields(t.Lit) { + // The lexer splits the body: separating whitespace can only be told + // from an escaped literal space there, where the delimiters are known. + for _, w := range t.Words { if t.Type == token.SYMBOLS { elems = append(elems, &ast.SymbolLit{Name: w}) } else { diff --git a/percent_word_escape_test.go b/percent_word_escape_test.go new file mode 100644 index 0000000..7cc5038 --- /dev/null +++ b/percent_word_escape_test.go @@ -0,0 +1,153 @@ +package parser_test + +import ( + "testing" + + "github.com/go-ruby-parser/parser" +) + +// The tables below pin the escape rules of the four word-list forms. Every +// `want` is the value the installed MRI prints for the same source — each case +// was run through `ruby -e 'p '` and compared byte for byte — so a +// disagreement here is a divergence from Ruby, not a matter of taste. +// +// The rules, from MRI's parse.y (tokadd_string): a backslash escapes the +// character after it, which therefore neither separates two words nor opens or +// closes the list. In a NON-interpolating %w/%i the backslash is otherwise KEPT +// literally — it escapes a character only before whitespace, a delimiter, or +// another backslash. In an interpolating %W/%I the full double-quote escape set +// applies instead, except that a backslash before a newline is a literal +// newline rather than a line continuation. + +// TestPercentWordListEscapes covers %w (non-interpolating word list). +func TestPercentWordListEscapes(t *testing.T) { + for _, tc := range []struct { + src string + want []string + }{ + // An escaped space is a literal space inside the word, never a + // separator. This is the case ruby/spec's Dir fixtures depend on: a + // directory literally named `special/test +()[]{}`. + {`x = %w[a\ b c]`, []string{"a b", "c"}}, // ["a b", "c"] + {`x = %w[special/test\ +()[]{} other]`, []string{"special/test +()[]{}", "other"}}, // ["special/test +()[]{}", "other"] + {`x = %w|a\ b c|`, []string{"a b", "c"}}, // ["a b", "c"] + {`x = %w[ a\ b c ]`, []string{"a b", "c"}}, // ["a b", "c"] + {`x = %w[a\ b\ c]`, []string{"a b c"}}, // ["a b c"] + {`x = %w[a\ b]`, []string{"a ", "b"}}, // ["a ", "b"] + {`x = %w[\ ]`, []string{" "}}, // [" "] + {"x = %w[a\\\tb]", []string{"a\tb"}}, // ["a\tb"] (escaped TAB) + {"x = %w[a\\\nb]", []string{"a\nb"}}, // ["a\nb"] (escaped newline) + + // An escaped delimiter is a literal character: it neither closes the + // list nor opens a nested level. + {`x = %w[a\]b]`, []string{"a]b"}}, // ["a]b"] + {`x = %w[a\[b]`, []string{"a[b"}}, // ["a[b"] + {`x = %w(a\)b)`, []string{"a)b"}}, // ["a)b"] + {`x = %w(a\(b)`, []string{"a(b"}}, // ["a(b"] + {`x = %w{a\}b}`, []string{"a}b"}}, // ["a}b"] + {`x = %w{a\{b}`, []string{"a{b"}}, // ["a{b"] + {`x = %wb>`, []string{"a>b"}}, // ["a>b"] + {`x = %w!a\!b!`, []string{"a!b"}}, // ["a!b"] + + // An escaped backslash is ONE literal backslash. + {`x = %w[a\\b]`, []string{`a\b`}}, // ["a\\b"] + {`x = %w[\\]`, []string{`\`}}, // ["\\"] + {`x = %w[a\b\\c]`, []string{`a\b\c`}}, // ["a\\b\\c"] + + // Every other backslash is kept verbatim: %w has no escape set, so + // `\t` is the two characters `\` and `t`, NOT a tab. + {`x = %w[a\tb]`, []string{`a\tb`}}, // ["a\\tb"] + {`x = %w[a\nb]`, []string{`a\nb`}}, // ["a\\nb"] + {`x = %w[a\ub]`, []string{`a\ub`}}, // ["a\\ub"] + {`x = %w[a\#b]`, []string{`a\#b`}}, // ["a\\#b"] + {`x = %w[a\x41b]`, []string{`a\x41b`}}, // ["a\\x41b"] + + // Unescaped whitespace still separates, and brackets still nest. + {`x = %w[a b c]`, []string{"a", "b", "c"}}, // ["a", "b", "c"] + {"x = %w[a\tb]", []string{"a", "b"}}, // ["a", "b"] + {"x = %w[a\nb]", []string{"a", "b"}}, // ["a", "b"] + {`x = %w[a[b]c]`, []string{"a[b]c"}}, // ["a[b]c"] + {`x = %w[a[b]c d]`, []string{"a[b]c", "d"}}, // ["a[b]c", "d"] + {`x = %w(a(b)c)`, []string{"a(b)c"}}, // ["a(b)c"] + {`x = %w[]`, nil}, // [] + {`x = %w[ ]`, nil}, // [] + } { + wantStrings(t, tc.src, parseRHS(t, tc.src), tc.want) + } +} + +// TestPercentInterpWordListEscapes covers %W (interpolating word list), where +// the full double-quote escape set applies. +func TestPercentInterpWordListEscapes(t *testing.T) { + for _, tc := range []struct { + src string + want []string + }{ + {`x = %W[a\ b]`, []string{"a b"}}, // ["a b"] + {`x = %W[\ ]`, []string{" "}}, // [" "] + {`x = %W[a\]b]`, []string{"a]b"}}, // ["a]b"] + {`x = %W{a\}b}`, []string{"a}b"}}, // ["a}b"] + {`x = %W[a\\b]`, []string{`a\b`}}, // ["a\\b"] + {`x = %W[a\tb]`, []string{"a\tb"}}, // ["a\tb"] + {`x = %W[a\nb]`, []string{"a\nb"}}, // ["a\nb"] + {`x = %W[a\sb]`, []string{"a b"}}, // ["a b"] + {`x = %W[a\eb]`, []string{"a\x1bb"}}, // ["a\eb"] + {`x = %W[a\#b]`, []string{"a#b"}}, // ["a#b"] + {`x = %W[a\x41b]`, []string{"aAb"}}, // ["aAb"] + {`x = %W[a\101b]`, []string{"aAb"}}, // ["aAb"] + // An escaped `#{` is literal text, not an interpolation. + {`x = %W[a\#{1}b]`, []string{"a#{1}b"}}, // ["a\#{1}b"] + // A backslash before a newline is a literal newline here, NOT the line + // continuation it would be inside a double-quoted string: MRI tests + // STR_FUNC_QWORDS before STR_FUNC_EXPAND. + {"x = %W[a\\\nb]", []string{"a\nb"}}, // ["a\nb"] + {"x = %W[a\\\tb]", []string{"a\tb"}}, // ["a\tb"] + {`x = %W[a b c]`, []string{"a", "b", "c"}}, + } { + wantStrings(t, tc.src, parseRHS(t, tc.src), tc.want) + } +} + +// TestPercentSymbolListEscapes covers %i and %I, which split exactly as %w and +// %W do and differ only in producing symbols. +func TestPercentSymbolListEscapes(t *testing.T) { + for _, tc := range []struct { + src string + want []string + }{ + // %i: non-interpolating, so the backslash survives except before + // whitespace, a delimiter, or another backslash. + {`x = %i[a\ b]`, []string{"a b"}}, // [:"a b"] + {`x = %i[\ ]`, []string{" "}}, // [:" "] + {`x = %i[a\]b c]`, []string{"a]b", "c"}}, // [:"a]b", :c] + {`x = %i[a\\b]`, []string{`a\b`}}, // [:"a\\b"] + {`x = %i[a\tb]`, []string{`a\tb`}}, // [:"a\\tb"] + {`x = %i[a b c]`, []string{"a", "b", "c"}}, + + // %I: interpolating, so the double-quote escape set applies. + {`x = %I[a\ b]`, []string{"a b"}}, // [:"a b"] + {`x = %I[x\ y\ z]`, []string{"x y z"}}, // [:"x y z"] + {`x = %I[a\]b]`, []string{"a]b"}}, // [:"a]b"] + {`x = %I[a\tb]`, []string{"a\tb"}}, // [:"a\tb"] + {`x = %I[a b c]`, []string{"a", "b", "c"}}, + } { + wantSymbols(t, tc.src, parseRHS(t, tc.src), tc.want) + } +} + +// TestPercentWordListUnterminated guards the scanner's end-of-input handling +// now that a backslash consumes the character after it: a list that ends on a +// trailing backslash is unterminated, not silently accepted. +func TestPercentWordListUnterminated(t *testing.T) { + for _, src := range []string{ + `x = %w[a\`, + `x = %W[a\`, + `x = %i[a\`, + `x = %I[a\`, + `x = %w[a\]`, // the `\]` is escaped, so the list never closes + } { + if _, err := parser.Parse(src); err == nil { + t.Errorf("expected a parse error for %q, got none", src) + } + } +} diff --git a/token/token.go b/token/token.go index f7408e6..2fa50f3 100644 --- a/token/token.go +++ b/token/token.go @@ -28,8 +28,8 @@ const ( LABEL // name: in a hash literal REGEXP // /pattern/flags (Lit = pattern source, Flags = matched flag letters) XSTRING // `cmd` / %x{cmd} backtick command literal (Lit = raw command source) - WORDS // %w[…] word-array literal (Lit = raw whitespace-separated content) - SYMBOLS // %i[…] symbol-array literal (Lit = raw whitespace-separated content) + WORDS // %w[…] word-array literal (Lit = raw body, Words = the split words) + SYMBOLS // %i[…] symbol-array literal (Lit = raw body, Words = the split words) // Keywords. DEF @@ -155,7 +155,8 @@ var Keywords = map[string]Type{ type Token struct { Type Type Lit string - Flags string // regexp flag letters in source order (i, m, x, o and the encoding flags n, u, e, s), only set for REGEXP tokens + Flags string // regexp flag letters in source order (i, m, x, o and the encoding flags n, u, e, s), only set for REGEXP tokens + Words []string // the words of a WORDS/SYMBOLS literal, already split and unescaped; only set for those two types Line int Col int SpaceBefore bool // whitespace immediately preceded this token (MRI spaceSeen)