Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions ast/ast.go
Original file line number Diff line number Diff line change
Expand Up @@ -36,8 +36,9 @@ type StringLit struct{ Value string }
// SymbolLit is a symbol literal (:name); Name excludes the leading colon.
type SymbolLit struct{ Name string }

// RegexpLit is a regexp literal /source/flags. Flags holds the subset of the
// flag letters i, m, x that were present.
// RegexpLit is a regexp literal /source/flags. Flags holds, in source order,
// the subset of the flag letters i, m, x, o and the encoding flags n (ASCII-8BIT),
// u (UTF-8), e (EUC-JP) and s (Windows-31J) that were present.
type RegexpLit struct {
Source string
Flags string
Expand Down
11 changes: 7 additions & 4 deletions lexer/lexer.go
Original file line number Diff line number Diff line change
Expand Up @@ -1025,8 +1025,9 @@ func (l *Lexer) lexIvar(spaceBefore bool, line, col int) token.Token {
// cursor. Escapes are preserved verbatim into the source (so \d, \. and the
// like reach the engine untouched) except that an escaped delimiter \/ becomes
// a literal '/'. Trailing flag letters i, m, x and o (the "interpolate once"
// flag) are collected into Flags; any other trailing letters (e.g. the encoding
// flags n/u/e/s) are ignored gracefully (consumed but not recorded).
// flag) plus the encoding flags n (ASCII-8BIT), u (UTF-8), e (EUC-JP) and s
// (Windows-31J) are collected into Flags in source order; any other trailing
// lowercase letters are consumed but not recorded (ignored gracefully).
func (l *Lexer) lexRegexp(spaceBefore bool, line, col int) token.Token {
l.advance() // opening '/'
var src []byte
Expand Down Expand Up @@ -1071,7 +1072,8 @@ func (l *Lexer) lexRegexp(spaceBefore bool, line, col int) token.Token {
break
}
l.advance()
if c == 'i' || c == 'm' || c == 'x' || c == 'o' {
if c == 'i' || c == 'm' || c == 'x' || c == 'o' ||
c == 'n' || c == 'u' || c == 'e' || c == 's' {
flags = append(flags, c)
}
}
Expand Down Expand Up @@ -1563,7 +1565,8 @@ func (l *Lexer) lexPercentRXS(spaceBefore bool, line, col int) token.Token {
break
}
l.advance()
if c == 'i' || c == 'm' || c == 'x' || c == 'o' {
if c == 'i' || c == 'm' || c == 'x' || c == 'o' ||
c == 'n' || c == 'u' || c == 'e' || c == 's' {
flags = append(flags, c)
}
}
Expand Down
38 changes: 32 additions & 6 deletions percent_rxs_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -65,12 +65,38 @@ func TestPercentRegexpOnceFlag(t *testing.T) {
}
}

func TestRegexpEncodingFlagsStillDropped(t *testing.T) {
// The encoding flags n/u/e/s remain consumed-but-unrecorded (as before), so a
// bare /x/n exposes no flags — only i/m/x/o are meaningful to the VM.
rx := parseOne(t, "/x/n").(*ast.RegexpLit)
if rx.Flags != "" {
t.Errorf("Flags = %q, want empty (encoding flag n dropped)", rx.Flags)
func TestRegexpEncodingFlagsRecorded(t *testing.T) {
// The encoding flags n/u/e/s are now recorded (matching MRI, which records an
// ARG_ENCODING for each): n=ASCII-8BIT, u=UTF-8, e=EUC-JP, s=Windows-31J. They
// are collected into Flags in source order alongside i/m/x/o.
cases := []struct{ src, want string }{
{"/x/n", "n"}, // n -> ASCII-8BIT (BINARY)
{"/x/u", "u"}, // u -> UTF-8
{"/x/e", "e"}, // e -> EUC-JP
{"/x/s", "s"}, // s -> Windows-31J (SJIS)
{"/a/mn", "mn"}, // encoding flag recorded after i/m/x/o, in source order
{"/a/un", "un"}, // multiple trailing letters preserved in the order written
}
for _, c := range cases {
rx := parseOne(t, c.src).(*ast.RegexpLit)
if rx.Flags != c.want {
t.Errorf("%s: Flags = %q, want %q", c.src, rx.Flags, c.want)
}
}
}

func TestPercentRegexpEncodingFlagsRecorded(t *testing.T) {
// The %r{…} form records the same encoding flags as the /…/ form.
cases := []struct{ src, want string }{
{"%r{x}n", "n"},
{"%r{x}u", "u"},
{"%r{x}mn", "mn"},
}
for _, c := range cases {
rx := parseOne(t, c.src).(*ast.RegexpLit)
if rx.Flags != c.want {
t.Errorf("%s: Flags = %q, want %q", c.src, rx.Flags, c.want)
}
}
}

Expand Down
2 changes: 1 addition & 1 deletion token/token.go
Original file line number Diff line number Diff line change
Expand Up @@ -155,7 +155,7 @@ var Keywords = map[string]Type{
type Token struct {
Type Type
Lit string
Flags string // regexp flag letters (i, m, x), only set for REGEXP tokens
Flags string // regexp flag letters in source order (i, m, x, o and the encoding flags n, u, e, s), only set for REGEXP tokens
Line int
Col int
SpaceBefore bool // whitespace immediately preceded this token (MRI spaceSeen)
Expand Down