From 972ec5956f27b9ddb65a26ad6337e13d10d4d1ff Mon Sep 17 00:00:00 2001 From: tannevaled Date: Sun, 6 Sep 2026 20:48:23 +0200 Subject: [PATCH] regexp: consume the n/u/e/s encoding modifiers on literal regexps go-ruby-parser v0.1.8 (#23) now records the encoding modifiers n/u/e/s in RegexpLit.Flags alongside i/m/x/o. compileRegexp maps them onto the Regexp's FIXEDENCODING/NOENCODING state and source encoding (MRI re.c char_to_option / rb_reg_initialize): n=ASCII-8BIT (NOENCODING), u=UTF-8, e=EUC-JP, s=Windows-31J (u/e/s FIXEDENCODING); last encoding letter wins. i/m/x/o are unaffected and sortIMX still excludes the new letters from the engine prefix. Verified byte-for-byte vs MRI 4.0.5: /x/n.options 32, /x/u.options 16, /x/u.encoding UTF-8, /x/e EUC-JP, /x/s Windows-31J, /x/n.encoding US-ASCII, /x/u.fixed_encoding? true, /a/un.options 32. Ratchet: core/regexp 229->241 (+12), language/regexp 192->198 (+6). Bumps go-ruby-parser 0.1.7 -> 0.1.8. Co-Authored-By: Claude Opus 4.8 --- go.mod | 2 +- go.sum | 2 + internal/vm/regexp.go | 28 +++++++++++++- .../regexp_encflags_wave18_internal_test.go | 38 +++++++++++++++++++ 4 files changed, 68 insertions(+), 2 deletions(-) create mode 100644 internal/vm/regexp_encflags_wave18_internal_test.go diff --git a/go.mod b/go.mod index 2e7ef9a9..61e070c5 100644 --- a/go.mod +++ b/go.mod @@ -129,7 +129,7 @@ require ( github.com/go-ruby-pagy/pagy v0.0.0-20260707193354-a84fe1d16a86 github.com/go-ruby-paper-trail/paper-trail v0.0.0-20260707193452-4e10f2b4d0b3 github.com/go-ruby-parquet/parquet v0.0.0-20260704170648-c7f0507946f7 - github.com/go-ruby-parser/parser v0.1.7 + github.com/go-ruby-parser/parser v0.1.8 github.com/go-ruby-pathname/pathname v0.0.0-20260629151955-d8d2c4e5f81b github.com/go-ruby-pg/pg v0.0.0-20260702135906-e5650264cc5d github.com/go-ruby-prawn/prawn v0.0.0-20260728194101-a5f2d786bf88 diff --git a/go.sum b/go.sum index 6167fe03..44a48b24 100644 --- a/go.sum +++ b/go.sum @@ -547,6 +547,8 @@ github.com/go-ruby-parser/parser v0.1.6 h1:q6GEdtf6Dkl0BiiNvrGmHePLIPfFEfjEd6aRr github.com/go-ruby-parser/parser v0.1.6/go.mod h1:NRbX4dYcQ6CgcaSaMmu1dh95QuKGq+RdenUoWU73214= github.com/go-ruby-parser/parser v0.1.7 h1:drBCs/DNLgzs+IB2Pqifpl1arwh05PyDj069QdFWkG8= github.com/go-ruby-parser/parser v0.1.7/go.mod h1:NRbX4dYcQ6CgcaSaMmu1dh95QuKGq+RdenUoWU73214= +github.com/go-ruby-parser/parser v0.1.8 h1:Tf/SzpqNqo5s9UGx4rQbQ/JAfLW51pcjHNcx8jqj8hE= +github.com/go-ruby-parser/parser v0.1.8/go.mod h1:NRbX4dYcQ6CgcaSaMmu1dh95QuKGq+RdenUoWU73214= github.com/go-ruby-pathname/pathname v0.0.0-20260629151955-d8d2c4e5f81b h1:/ZN1eLOg0oH4wF8qmd6NOyB4ENYthx6wFwOpZiPhRIE= github.com/go-ruby-pathname/pathname v0.0.0-20260629151955-d8d2c4e5f81b/go.mod h1:IvW5q5Kyl5l3hrYwiHylOEmk5hpGSo7XqcmUXBHF5H0= github.com/go-ruby-pg/pg v0.0.0-20260702135906-e5650264cc5d h1:JKIZD+pJQ64/nla4DHdzMLnc4/v45MyYwnjACwS6dyM= diff --git a/internal/vm/regexp.go b/internal/vm/regexp.go index b718a040..a6717910 100644 --- a/internal/vm/regexp.go +++ b/internal/vm/regexp.go @@ -422,7 +422,33 @@ func (vm *VM) compileRegexp(source, flags string) object.Value { if err != nil { raise("RegexpError", "%s: /%s/", mapRegexpEngineError(err.Error()), source) } - return &Regexp{re: re, source: source, flags: flags, nameMap: nameMap} + r := &Regexp{re: re, source: source, flags: flags, nameMap: nameMap} + applyRegexpEncodingFlags(r, flags) + return r +} + +// applyRegexpEncodingFlags sets a literal regexp's FIXEDENCODING / NOENCODING +// state and source encoding from its trailing encoding modifiers, which the +// parser now records alongside i/m/x/o (go-ruby-parser >= v0.1.8). Mapping per +// MRI re.c char_to_option / rb_reg_initialize: n = ASCII-8BIT (ARG_ENCODING_NONE, +// the NOENCODING bit), u = UTF-8, e = EUC-JP, s = Windows-31J — the last three +// pin the pattern (ARG_ENCODING_FIXED). i/m/x/o carry no encoding. At most one +// encoding letter is meaningful; if several appear the last wins. The canonical +// registry names are used so encodingName()/isFixedEncoding()/optionBits() report +// the same values as the runtime Regexp.new(str, opts) path. +func applyRegexpEncodingFlags(r *Regexp, flags string) { + for i := 0; i < len(flags); i++ { + switch flags[i] { + case 'n': + r.noEnc, r.fixedEnc, r.srcEnc = true, false, "" + case 'u': + r.fixedEnc, r.noEnc, r.srcEnc = true, false, "UTF-8" + case 'e': + r.fixedEnc, r.noEnc, r.srcEnc = true, false, "EUC-JP" + case 's': + r.fixedEnc, r.noEnc, r.srcEnc = true, false, "Windows-31J" + } + } } // mapRegexpEngineError rewrites the go-ruby-regexp engine's diagnostic for the diff --git a/internal/vm/regexp_encflags_wave18_internal_test.go b/internal/vm/regexp_encflags_wave18_internal_test.go new file mode 100644 index 00000000..3b52d994 --- /dev/null +++ b/internal/vm/regexp_encflags_wave18_internal_test.go @@ -0,0 +1,38 @@ +// Copyright (c) the go-embedded-ruby/ruby authors +// +// SPDX-License-Identifier: BSD-3-Clause + +package vm + +import "testing" + +// TestApplyRegexpEncodingFlags locks the literal-regexp encoding-modifier mapping +// (go-ruby-parser >= v0.1.8 records n/u/e/s in Flags): n = NOENCODING/ASCII-8BIT, +// u = UTF-8, e = EUC-JP, s = Windows-31J (u/e/s FIXEDENCODING); i/m/x/o carry no +// encoding; the last encoding letter wins. Verified byte-for-byte against MRI +// 4.0.5 (/x/n.options == 32, /x/u.encoding == UTF-8, /a/un.options == 32, …). +func TestApplyRegexpEncodingFlags(t *testing.T) { + cases := []struct { + flags string + fixed, noEnc bool + srcEnc string + }{ + {"", false, false, ""}, + {"imxo", false, false, ""}, // non-encoding letters are ignored + {"n", false, true, ""}, + {"u", true, false, "UTF-8"}, + {"e", true, false, "EUC-JP"}, + {"s", true, false, "Windows-31J"}, + {"mn", false, true, ""}, // m ignored, n applies + {"un", false, true, ""}, // last wins: n after u + {"nu", true, false, "UTF-8"}, // last wins: u after n + } + for _, c := range cases { + r := &Regexp{} + applyRegexpEncodingFlags(r, c.flags) + if r.fixedEnc != c.fixed || r.noEnc != c.noEnc || r.srcEnc != c.srcEnc { + t.Errorf("flags %q: got fixed=%v noEnc=%v srcEnc=%q, want fixed=%v noEnc=%v srcEnc=%q", + c.flags, r.fixedEnc, r.noEnc, r.srcEnc, c.fixed, c.noEnc, c.srcEnc) + } + } +}