diff --git a/internal/vm/autoload.go b/internal/vm/autoload.go index 1ad3263e..69d73890 100644 --- a/internal/vm/autoload.go +++ b/internal/vm/autoload.go @@ -125,6 +125,35 @@ func (vm *VM) autoloadPathFor(cls *RClass, name string, inherit bool) object.Val return object.NilV } +// autoloadVisible reports whether a pending autoload of name on c makes the +// constant answer "defined" to `defined?` and #const_defined?. It does NOT, +// while the registered feature is already loaded or is being loaded right now: +// MRI resolves an autoload entry through check_autoload_required, which calls +// rb_feature_provided(file, &loading) and treats an entry whose feature is +// loading as having no autoload left to run (ruby/ruby v3_4_0 variable.c +// check_autoload_required / autoload_defined_p / rb_autoload_at_p). That is +// what makes a file required DIRECTLY, while an autoload for it is registered, +// see its own constant as undefined: +// +// MZ.autoload :K, path +// require path # inside the file: defined?(MZ::K) is nil, +// # MZ.const_defined?(:K) is false, MZ.autoload?(:K) is nil, +// # but MZ.constants(false) still includes :K +// +// The constant list is deliberately NOT filtered this way — MRI keeps the +// reserved entry in Module#constants throughout — so only the two "is it +// defined?" questions consult this. +func (vm *VM) autoloadVisible(c *RClass, name string) bool { + if c == nil || c.autoloads == nil { + return false + } + path, ok := c.autoloads[name] + if !ok { + return false + } + return !vm.featureLoaded(path) +} + // registerAutoloadOn records (or replaces) a pending autoload for name on cls. // When the constant is already defined in cls's own table the registration is // dropped, matching MRI where autoload of a defined constant is inert. A fresh @@ -172,9 +201,45 @@ func (vm *VM) tryAutoload(cls *RClass, name string) bool { // Kernel#require is what runs — ruby/spec exercises exactly that. vm.send(vm.main, "require", []object.Value{object.NewString(path)}, nil) done = true + vm.warnAutoloadDidNotDefine(cls, name, path) return true } +// warnAutoloadDidNotDefine emits MRI's diagnostic for an autoload whose file +// completed without defining the constant in the module the autoload was +// registered on — the case where lookup then carries on into the enclosing +// lexical and ancestor scopes. ruby/ruby v3_4_0 variable.c autoload_try_load: +// +// // After we loaded the feature, if the constant is not defined, we remove +// // it completely: +// rb_const_entry_t *ce = rb_const_lookup(arguments->module, arguments->name); +// if (!ce || UNDEF_P(ce->value)) { +// result = Qfalse; +// rb_const_remove(arguments->module, arguments->name); +// if (arguments->module == rb_cObject) { +// rb_warning("Expected %"PRIsVALUE" to define %"PRIsVALUE" but it didn't", …); +// } else { +// rb_warning("Expected %"PRIsVALUE" to define %"PRIsVALUE"::%"PRIsVALUE" but it didn't", …); +// } +// } +// +// It goes through rb_warning, not rb_warn, so only $VERBOSE == true shows it — +// the same gate warnCircularRequire applies — and it is emitted at most once +// per registration because the caller has already retired the entry. +func (vm *VM) warnAutoloadDidNotDefine(cls *RClass, name, path string) { + if _, defined := cls.consts[name]; defined { + return + } + if v, ok := vm.globals["$VERBOSE"].(object.Bool); !ok || !bool(v) { + return + } + qualified := name + if cls != vm.cObject { + qualified = vm.moduleToSStr(cls) + "::" + name + } + vm.curStderr().writeStr("warning: Expected " + path + " to define " + qualified + " but it didn't\n") +} + // featureLoaded reports whether path names a file that has already been loaded, // resolved the way require would resolve it. $LOADED_FEATURES is the authority // (doRequire records each file there and featureDropped retires a cache entry diff --git a/internal/vm/defined.go b/internal/vm/defined.go index 1fa6a765..720e3b75 100644 --- a/internal/vm/defined.go +++ b/internal/vm/defined.go @@ -25,7 +25,7 @@ func (vm *VM) hasScopedConst(cls *RClass, name string) bool { continue } } - if _, ok := c.autoloads[name]; ok { + if vm.autoloadVisible(c, name) { return true } } @@ -37,19 +37,18 @@ func (vm *VM) hasScopedConst(cls *RClass, name string) bool { // require — used by defined? to report a registered-but-unloaded constant. func (vm *VM) autoloadPending(cref *RClass, name string) bool { for _, c := range vm.nesting(cref) { - if _, ok := c.autoloads[name]; ok { + if vm.autoloadVisible(c, name) { return true } } if cref != nil { for _, c := range vm.ancestors(cref) { - if _, ok := c.autoloads[name]; ok { + if vm.autoloadVisible(c, name) { return true } } } - _, ok := vm.cObject.autoloads[name] - return ok + return vm.autoloadVisible(vm.cObject, name) } // gvarDefined reports whether a global variable is set. User globals live in diff --git a/internal/vm/module_residuals.go b/internal/vm/module_residuals.go index 9801871e..422e2120 100644 --- a/internal/vm/module_residuals.go +++ b/internal/vm/module_residuals.go @@ -254,7 +254,7 @@ func (vm *VM) constSegDefined(mod *RClass, name string, inherit, isFirst bool) ( if v, ok := mod.consts[name]; ok { return v, true } - return object.NilVal(), hasAutoload(mod, name) + return object.NilVal(), vm.autoloadVisible(mod, name) } if v, ok := vm.constInAncestors(mod, name); ok { return v, true @@ -266,7 +266,7 @@ func (vm *VM) constSegDefined(mod *RClass, name string, inherit, isFirst bool) ( if v, ok := vm.cObject.consts[name]; ok { return v, true } - if hasAutoload(vm.cObject, name) { + if vm.autoloadVisible(vm.cObject, name) { return object.NilVal(), true } } @@ -296,7 +296,7 @@ func (vm *VM) autoloadPendingInAncestors(mod *RClass, name string) bool { if (c == vm.cObject || c == vm.cBasicObject) && mod != vm.cObject && mod != vm.cBasicObject { continue } - if hasAutoload(c, name) { + if vm.autoloadVisible(c, name) { return true } } diff --git a/internal/vm/pack.go b/internal/vm/pack.go index bf3576f4..38ce3c60 100644 --- a/internal/vm/pack.go +++ b/internal/vm/pack.go @@ -81,13 +81,15 @@ func (vm *VM) registerPackUnpack() { } return object.NewStringBytesEnc(out, enc) }) - vm.cString.define("unpack", func(_ *VM, self object.Value, args []object.Value, _ *Proc) object.Value { + vm.cString.define("unpack", func(vm *VM, self object.Value, args []object.Value, _ *Proc) object.Value { data := self.(*object.String).Bytes() - return object.NewArrayFromSlice(unpackElems(data, vm.packFormat(args))) + fmtStr := vm.packFormat(args) + return object.NewArrayFromSlice(unpackElems(data[vm.unpackOffset(args, len(data)):], fmtStr)) }) - vm.cString.define("unpack1", func(_ *VM, self object.Value, args []object.Value, _ *Proc) object.Value { + vm.cString.define("unpack1", func(vm *VM, self object.Value, args []object.Value, _ *Proc) object.Value { data := self.(*object.String).Bytes() - elems := unpackElems(data, vm.packFormat(args)) + fmtStr := vm.packFormat(args) + elems := unpackElems(data[vm.unpackOffset(args, len(data)):], fmtStr) if len(elems) == 0 { return object.NilV } @@ -95,6 +97,49 @@ func (vm *VM) registerPackUnpack() { }) } +// unpackOffset reads the `offset:` keyword of String#unpack / #unpack1 and +// returns the BYTE offset unpacking starts from, validating it the way MRI's +// pack_unpack_internal does (ruby/ruby v3_4_0 pack.c): +// +// if (offset < 0) rb_raise(rb_eArgError, "offset can't be negative"); +// len = RSTRING_LEN(str); +// if (offset > len) rb_raise(rb_eArgError, "offset outside of string"); +// +// The bound is EXCLUSIVE of the error only at offset == len: that is accepted +// and leaves nothing to unpack, so "a".unpack("C", offset: 1) is [nil] and +// "a".unpack1("C", offset: 1) is nil, while offset 2 raises. The offset counts +// BYTES, not characters, so "\u0608".unpack1("C", offset: 1) reads the second +// byte of the two-byte character. +// +// No keyword at all yields 0. Any keyword other than :offset is rejected the +// way rb_get_kwargs does. +func (vm *VM) unpackOffset(args []object.Value, byteLen int) int { + h, ok := trailingHash(args) + if !ok || len(args) < 2 { + return 0 + } + off := object.Value(nil) + for _, k := range h.Keys { + sym, isSym := k.(object.Symbol) + if isSym && string(sym) == "offset" { + off, _ = h.Get(k) + continue + } + raise("ArgumentError", "unknown keyword: %s", k.Inspect()) + } + if off == nil || object.IsNil(off) { + return 0 + } + n := vm.toIntCoerce(off) + if n < 0 { + raise("ArgumentError", "offset can't be negative") + } + if n > int64(byteLen) { + raise("ArgumentError", "offset outside of string") + } + return int(n) +} + // packFormat extracts the mandatory format argument, which is a String or any // object that responds to #to_str. func (vm *VM) packFormat(args []object.Value) string { diff --git a/internal/vm/refinement.go b/internal/vm/refinement.go index 1c223e66..7082bc03 100644 --- a/internal/vm/refinement.go +++ b/internal/vm/refinement.go @@ -53,6 +53,9 @@ func (vm *VM) registerRefinements() { return self.(*RClass).refinedClass }) + // Refinement#import_methods (see refinement_import.go). + vm.installRefinementImport() + // Module#refinements returns this module's own refinement modules (not those // of included modules). vm.cModule.define("refinements", func(_ *VM, self object.Value, _ []object.Value, _ *Proc) object.Value { diff --git a/internal/vm/refinement_import.go b/internal/vm/refinement_import.go new file mode 100644 index 00000000..419a8743 --- /dev/null +++ b/internal/vm/refinement_import.go @@ -0,0 +1,148 @@ +// Copyright (c) the go-embedded-ruby/ruby authors +// +// SPDX-License-Identifier: BSD-3-Clause + +package vm + +import ( + "sort" + + "github.com/go-embedded-ruby/ruby/internal/object" +) + +// Refinement#import_methods — copying Ruby-defined instance methods of one or +// more plain modules INTO a refinement, so they behave as if they had been +// written in the refine block itself. +// +// MRI (ruby/ruby v3_4_0 eval.c refinement_import_methods): +// +// rb_check_arity(argc, 1, UNLIMITED_ARGUMENTS); +// for (i = 0; i < argc; i++) { +// Check_Type(argv[i], T_MODULE); +// if (RCLASS_SUPER(argv[i])) { +// rb_warn("%"PRIsVALUE" has ancestors, but Refinement#import_methods " +// "doesn't import their methods", rb_class_path(argv[i])); +// } +// } +// arg.cref = rb_vm_cref_replace_with_duplicated_cref(); +// arg.refinement = refinement; +// for (i = 0; i < argc; i++) { … rb_id_table_foreach(m_tbl, …); } +// return refinement; +// +// and refinement_import_methods_i, which does the copying: +// +// if (me->def->type != VM_METHOD_TYPE_ISEQ) { +// rb_raise(rb_eArgError, "Can't import method which is not defined with " +// "Ruby code: %"PRIsVALUE"#%"PRIsVALUE, …); +// } +// rb_cref_t *new_cref = rb_vm_cref_dup_without_refinements(me->def->body.iseq.cref); +// CREF_REFINEMENTS_SET(new_cref, CREF_REFINEMENTS(arg->cref)); +// rb_add_method_iseq(arg->refinement, key, me->def->body.iseq.iseqptr, +// new_cref, METHOD_ENTRY_VISI(me)); +// +// Four consequences the ruby/spec suite pins, each reproduced below: +// +// - Every argument is type-checked BEFORE anything is copied, so a bad +// argument late in the list imports nothing at all; but the +// "not defined with Ruby code" check happens DURING the copy, so the +// modules listed before the offending one keep their imports. +// - Only the module's OWN instance method table is walked: neither its +// included/prepended modules (hence the warning) nor its class methods. +// - The copy keeps its source cref for constants but takes the refinement's +// refinements, so an imported method sees the other imported methods and +// the rest of the refine block. Here the copy keeps the source method's +// lexScope (rbgo's cref for constant lookup) and takes the refinement as +// its owner, which is what this VM's refinement dispatch keys on. +// - Visibility is carried over with the method. + +// installRefinementImport defines Refinement#import_methods. It is private, as +// MRI defines it, so it is reached as a bare call with the refinement as self — +// which is exactly how a refine block body invokes it. +func (vm *VM) installRefinementImport() { + m := &Method{ + name: "import_methods", + native: refinementImportMethods, + owner: vm.cRefinement, + vis: visPrivate, + } + vm.cRefinement.methods["import_methods"] = m +} + +// refinementImportMethods is the Refinement#import_methods body. +func refinementImportMethods(vm *VM, self object.Value, args []object.Value, _ *Proc) object.Value { + r, ok := self.(*RClass) + if !ok || !r.isRefinement { + raise("TypeError", "wrong argument type %s (expected Refinement)", vm.classOf(self).name) + } + if len(args) == 0 { + raise("ArgumentError", "wrong number of arguments (given 0, expected 1+)") + } + // Pass 1: Check_Type every argument first, so one bad argument leaves the + // refinement untouched, then warn for each module that has ancestors + // (RCLASS_SUPER non-NULL for a module means it includes or prepends one). + mods := make([]*RClass, len(args)) + for i, a := range args { + mod, isMod := a.(*RClass) + if !isMod || !mod.isModule { + raise("TypeError", "wrong argument type %s (expected Module)", vm.classOf(a).name) + } + mods[i] = mod + } + for _, mod := range mods { + if len(mod.includes) > 0 || len(mod.prepends) > 0 { + vm.rbWarn("warning: %s has ancestors, but Refinement#import_methods doesn't import their methods", + vm.moduleToSStr(mod)) + } + } + // Pass 2: copy. Modules are walked in argument order, so a name defined in + // several of them ends up holding the LAST one's definition. + for _, mod := range mods { + for _, name := range sortedMethodNames(mod.methods) { + src := mod.methods[name] + if src.undefined { + continue + } + if !definedWithRubyCode(src) { + raise("ArgumentError", "Can't import method which is not defined with Ruby code: %s#%s", + vm.moduleToSStr(mod), name) + } + cp := *src + // The refinement owns the copy: this VM reads a frame's refinement + // scope off the running method's owner, so an imported method that + // calls another imported method (or anything else the refine block + // defines) resolves it through the refinement, as MRI's + // CREF_REFINEMENTS_SET arranges. + cp.owner = r + // Constants keep resolving where the method was WRITTEN + // (rb_vm_cref_dup_without_refinements preserves the source cref), so + // pin lexScope to the source module unless the source already carried + // an explicit one. + if cp.lexScope == nil { + cp.lexScope = src.owner + } + r.methods[name] = &cp + } + } + return r +} + +// definedWithRubyCode reports whether m is VM_METHOD_TYPE_ISEQ — a method +// written as Ruby source with `def`. A native (C-function) method, an +// attr_reader/writer and a define_method block body are all other types, which +// refinement_import_methods_i refuses. +func definedWithRubyCode(m *Method) bool { + return m.iseq != nil && m.native == nil && m.proc == nil && m.attrKind == 0 +} + +// sortedMethodNames returns tbl's keys in a stable order. MRI's +// rb_id_table_foreach has no defined order; sorting makes which method of a +// module is reported in the "not defined with Ruby code" ArgumentError — and +// how far the import got before raising — reproducible run to run. +func sortedMethodNames(tbl map[string]*Method) []string { + out := make([]string, 0, len(tbl)) + for k := range tbl { + out = append(out, k) + } + sort.Strings(out) + return out +} diff --git a/internal/vm/reflection.go b/internal/vm/reflection.go index 1830d119..099cee8d 100644 --- a/internal/vm/reflection.go +++ b/internal/vm/reflection.go @@ -65,6 +65,47 @@ func (u *UnboundMethod) ToS() string { func (u *UnboundMethod) Inspect() string { return u.ToS() } func (u *UnboundMethod) Truthy() bool { return true } +// callerRefinedInstanceMethod returns the method a refinement active at the CALL +// SITE contributes for instances of mod, or nil when ordinary lookup should +// answer. It is the module-level counterpart of refinedMethod (which keys on a +// receiver): MRI's *_with_refinements entry points read the refinements off the +// current cref, which for a native that pushes no frame of its own is the cref +// on top of frameCrefs — the same read Module.used_refinements performs. +// +// A refinement only wins at the exact class it refines and only ahead of that +// class's OWN definition, so the ancestor chain is walked and the search stops +// at the first level that defines the name itself. +// +// Reference: ruby/ruby v3_4_0 proc.c mnew_unbound → +// rb_method_entry_with_refinements(klass, id, &iclass). +func (vm *VM) callerRefinedInstanceMethod(mod *RClass, name string) *Method { + if !vm.anyRefinements || mod == nil { + return nil + } + n := len(vm.frameCrefs) + if n == 0 || vm.frameCrefs[n-1] == nil { + return nil + } + refs := vm.activeRefinements(vm.frameCrefs[n-1]) + if len(refs) == 0 { + return nil + } + for _, k := range vm.ancestors(mod) { + for _, r := range refs { + if r.refinedClass != k { + continue + } + if m := lookupOwnOrIncluded(r, name); m != nil && !m.undefined { + return m + } + } + if m, ok := k.methods[name]; ok && !m.undefined { + return nil + } + } + return nil +} + // registerReflection installs the reflection API: Module#instance_method, // Object#method/#singleton_class, the UnboundMethod class, and Method#unbind. It // also teaches define_method to accept a Method/UnboundMethod body. @@ -72,11 +113,18 @@ func (vm *VM) registerReflection() { cUnbound := newClass("UnboundMethod", vm.cObject) vm.consts["UnboundMethod"] = cUnbound - // Module#instance_method(:m) → UnboundMethod resolved up the ancestor chain. + // Module#instance_method(:m) → UnboundMethod resolved up the ancestor chain, + // with the refinements active at the CALL SITE consulted first: MRI's + // rb_mod_instance_method goes through mnew_unbound, which resolves the entry + // with rb_method_entry_with_refinements(klass, id, &iclass) (ruby/ruby + // v3_4_0 proc.c), so `using R; C.instance_method(:m)` hands back R's method. vm.cModule.define("instance_method", func(vm *VM, self object.Value, args []object.Value, _ *Proc) object.Value { mod := self.(*RClass) name := vm.coerceNameArg(args[0]) - m := vm.lookupForModuleOp(mod, name) + m := vm.callerRefinedInstanceMethod(mod, name) + if m == nil { + m = vm.lookupForModuleOp(mod, name) + } if m == nil || m.undefined { vm.raiseNameError("undefined method '"+name+"' for class '"+mod.name+"'", name) } diff --git a/internal/vm/regexp.go b/internal/vm/regexp.go index cc193169..4db2d2d4 100644 --- a/internal/vm/regexp.go +++ b/internal/vm/regexp.go @@ -156,6 +156,13 @@ func (r *Regexp) encodingName() string { if r.fixedEnc && r.srcEnc != "" { return r.srcEnc } + // MRI measures the code range of the PREPROCESSED pattern, not of the + // source text: /\xFF/n is four ASCII characters as written, but the byte + // it denotes is 0xFF, so the pattern is not 7-bit and a /n regexp reports + // BINARY (re.c rb_reg_preprocess → rb_reg_initialize). + if r.noEnc && sourceHasNonASCIIByteEscape(r.source) { + return "ASCII-8BIT" + } return "US-ASCII" } if r.noEnc { @@ -185,6 +192,27 @@ func (r *Regexp) isFixedEncoding() bool { return !asciiOnly([]byte(r.source)) && !r.noEnc } +// sourceHasNonASCIIByteEscape reports whether the pattern source contains a +// \xHH escape denoting a byte at or above 0x80. Such an escape makes the +// preprocessed pattern non-7-bit even though every character of the source text +// is ASCII, which is what decides a /n regexp's encoding (see encodingName). A +// backslash that is itself escaped (\\x41) does not begin an escape, so the scan +// skips two bytes after every backslash it does not consume. +func sourceHasNonASCIIByteEscape(src string) bool { + for i := 0; i+1 < len(src); i++ { + if src[i] != '\\' { + continue + } + if src[i+1] == 'x' && i+3 < len(src) && allHex(src[i+2:i+4]) { + if v, err := strconv.ParseUint(src[i+2:i+4], 16, 16); err == nil && v >= 0x80 { + return true + } + } + i++ + } + return false +} + // sourceHasNonASCIIUnicodeEscape reports whether the pattern source contains a // \uHHHH or \u{…} escape that resolves to a code point above U+007F. MRI fixes // such a pattern to UTF-8 encoding (an all-ASCII \u escape like A leaves the @@ -417,8 +445,27 @@ func (vm *VM) compileRegexp(source, flags string) object.Value { // the original source) and, when needed, rewrite named groups to synthetic // ASCII names — both no-ops for sources that do not use those features. engineSrc := translateUnicodeEscapes(prefix+source, source) + engineSrc = translateCharEscapes(engineSrc) engineSrc, nameMap := rewriteNamedGroups(engineSrc) - re, err := onig.Compile(engineSrc) + engineSrc = rewriteBeginLineAnchor(engineSrc) + // A /n pattern is ARG_ENCODING_NONE — ASCII-8BIT — so its input-advancing + // atoms step ONE BYTE, not one UTF-8 character: /./n.match("\303\251") + // captures just "\303" (re.c char_to_option, rb_reg_initialize). Everything + // else compiles in the engine's UTF-8 mode. + // + // KNOWN GAP: MRI does not stop there. rb_reg_prepare_re RE-COMPILES the + // pattern in the SUBJECT's encoding whenever the two differ, so the same + // /./n matched against a UTF-8 string steps a whole character again (and + // rb_reg_prepare_enc emits its "historical binary regexp match /.../n + // against UTF-8 string" warning for exactly that case). A Regexp here holds + // one compiled program, so the binary form is used for every subject. The + // binary-subject case — the one ruby/spec measures — is now right; the + // re-compilation is tracked separately. + engineEnc := onig.UTF8 + if strings.ContainsRune(flags, 'n') { + engineEnc = onig.ASCII8BIT + } + re, err := onig.CompileEnc(engineSrc, engineEnc) if err != nil { raise("RegexpError", "%s: /%s/", mapRegexpEngineError(err.Error()), source) } @@ -427,6 +474,76 @@ func (vm *VM) compileRegexp(source, flags string) object.Value { return r } +// beginLineEquivalent is the engine pattern that reproduces Onigmo's +// OP_BEGIN_LINE exactly: a "^" matches at the start of the string, or right +// after a newline PROVIDED that is not the end of the string. The engine this +// VM links drops that second condition and so fires "^" once more, at the very +// end of a string that ends in a newline: +// +// "Text\n".gsub(/^/, " ") => " Text\n " (MRI: " Text\n") +// "a\nb\n".gsub(/^/, "-") => "-a\n-b\n-" (MRI: "-a\n-b\n") +// +// The engine's own "^" is kept (so its anchor optimisation still applies) and +// guarded with (?!\z); the \A alternative restores the one case the guard +// would otherwise remove, the empty string, whose start IS its end. +// +// Reference: ruby/ruby v3_4_0 regexec.c OP_BEGIN_LINE — +// +// if (ON_STR_BEGIN(s)) { if (IS_NOTBOL(...)) goto fail; ... } +// else if (ONIGENC_IS_MBC_NEWLINE(encode, sprev, end) && !ON_STR_END(s)) ... +// goto fail; +const beginLineEquivalent = `(?:^(?!\z)|\A)` + +// rewriteBeginLineAnchor replaces every "^" that is a line anchor with +// beginLineEquivalent, leaving alone the "^" that are not anchors: one escaped +// with a backslash, one inside a character class (whether the negation slot of +// "[^…]" or a literal "[a^b]"), and one inside a "(?#…)" comment — a comment +// runs to its first ")", which the replacement text contains. +// +// Character-class nesting is tracked by depth so Onigmo's POSIX brackets and +// nested classes ("[[:alpha:]]", "[a-z&&[^b]]") keep their inner "^" literal. +// A source with no "^" at all is returned unchanged. +func rewriteBeginLineAnchor(src string) string { + if !strings.ContainsRune(src, '^') { + return src + } + var b strings.Builder + b.Grow(len(src) + 16) + depth := 0 // character-class nesting depth + for i := 0; i < len(src); i++ { + c := src[i] + switch { + case c == '\\': + b.WriteByte(c) + if i+1 < len(src) { + i++ + b.WriteByte(src[i]) + } + case depth == 0 && c == '(' && strings.HasPrefix(src[i:], "(?#"): + if j := strings.IndexByte(src[i:], ')'); j >= 0 { + b.WriteString(src[i : i+j+1]) + i += j + } else { + b.WriteString(src[i:]) + i = len(src) + } + case c == '[': + depth++ + b.WriteByte(c) + case c == ']': + if depth > 0 { + depth-- + } + b.WriteByte(c) + case c == '^' && depth == 0: + b.WriteString(beginLineEquivalent) + default: + b.WriteByte(c) + } + } + return b.String() +} + // applyRegexpEncodingFlags sets a literal regexp's FIXEDENCODING / NOENCODING // state and source encoding from its trailing encoding modifiers, which the // parser now records alongside i/m/x/o (go-ruby-parser >= v0.1.8). Mapping per @@ -952,7 +1069,7 @@ func strMatchRegexp(v object.Value) *Regexp { case *Regexp: return x case *object.String: - re, err := onig.Compile(x.Str()) + re, err := onig.Compile(rewriteBeginLineAnchor(x.Str())) if err != nil { raise("RegexpError", "%s: /%s/", err.Error(), x.Str()) } @@ -963,14 +1080,171 @@ func strMatchRegexp(v object.Value) *Regexp { } } -// checkSubjectEncoding raises ArgumentError when the match subject is a String -// whose bytes are not valid in its own encoding, matching MRI's -// rb_reg_prepare_enc ("invalid byte sequence in "). A non-String subject -// (Symbol, or an object coerced via #to_str) is left to the caller's coercion. -func (vm *VM) checkSubjectEncoding(v object.Value) { - if s, ok := v.(*object.String); ok && !validInEncoding(s.Bytes(), s.EncName()) { - raise("ArgumentError", "invalid byte sequence in %s", s.EncName()) +// encInspectName renders an encoding the way MRI's rb_enc_inspect_name does in +// the regexp-match incompatibility message, where ASCII-8BIT is shown under +// both of its names. +func encInspectName(name string) string { + if name == "ASCII-8BIT" { + return "BINARY (ASCII-8BIT)" + } + return name +} + +// raiseRegexpEncError is re.c reg_enc_error: the Encoding::CompatibilityError a +// match between a regexp and a string of irreconcilable encodings raises. +// +// rb_raise(rb_eEncCompatError, +// "incompatible encoding regexp match (%s regexp with %s string)", +// rb_enc_inspect_name(rb_enc_get(re)), rb_enc_inspect_name(rb_enc_get(str))); +func raiseRegexpEncError(reEnc, strEnc string) { + raise("Encoding::CompatibilityError", "incompatible encoding regexp match (%s regexp with %s string)", + encInspectName(reEnc), encInspectName(strEnc)) +} + +// checkSubjectEncoding decides whether re may be matched against the subject v +// at all, reproducing ruby/ruby v3_4_0 re.c rb_reg_prepare_enc: +// +// if (cr == ENC_CODERANGE_BROKEN) +// rb_raise(rb_eArgError, "invalid byte sequence in %s", …); +// enc = rb_enc_get(str); +// if (RREGEXP_PTR(re)->enc == enc) { } +// else if (cr == ENC_CODERANGE_7BIT && RREGEXP_PTR(re)->enc == rb_usascii_encoding()) +// enc = RREGEXP_PTR(re)->enc; +// else if (!rb_enc_asciicompat(enc)) reg_enc_error(re, str); +// else if (rb_reg_fixed_encoding_p(re)) { +// if (!rb_enc_asciicompat(RREGEXP_PTR(re)->enc) || cr != ENC_CODERANGE_7BIT) +// reg_enc_error(re, str); +// } +// else if (warn && (RBASIC(re)->flags & REG_ENCODING_NONE) && +// enc != rb_ascii8bit_encoding() && cr != ENC_CODERANGE_7BIT) +// rb_warn("historical binary regexp match /.../n against %s string", …); +// +// The last clause is rb_warn, so $VERBOSE == nil silences it but false does +// not. A non-String subject (a Symbol, or an object coerced through #to_str) is +// left to the caller's coercion, and a nil re only performs the broken-bytes +// check. +func (vm *VM) checkSubjectEncoding(re *Regexp, v object.Value) { + str, ok := v.(*object.String) + if !ok { + return + } + bytes, strEnc := str.Bytes(), str.EncName() + if !validInEncoding(bytes, strEnc) { + raise("ArgumentError", "invalid byte sequence in %s", strEnc) + } + reEnc := re.encodingName() + if reEnc == strEnc { + return + } + // ENC_CODERANGE_7BIT is only ever reported for an ASCII-COMPATIBLE encoding + // (coderange_scan short-circuits otherwise), so a UTF-16LE string of ASCII + // code points is VALID, not 7BIT — which is what makes it incompatible with + // a US-ASCII regexp rather than silently promoted. + sevenBit := asciiOnly(bytes) && encIsASCIICompat(strEnc) + if sevenBit && reEnc == "US-ASCII" { + return + } + if !encIsASCIICompat(strEnc) { + raiseRegexpEncError(reEnc, strEnc) + } + if re.isFixedEncoding() { + if !encIsASCIICompat(reEnc) || !sevenBit { + raiseRegexpEncError(reEnc, strEnc) + } + return + } + if re.noEnc && strEnc != "ASCII-8BIT" && !sevenBit { + vm.rbWarn("warning: historical binary regexp match /.../n against %s string", strEnc) + } +} + +// patternNeedsLeftContext reports whether matching src from a byte offset can +// depend on the text BEFORE that offset — i.e. whether handing the engine only +// the tail subject[pos:] would change the answer. That is the case for the +// beginning-of-line/string anchors (^, \A), the word-boundary assertions (\b, +// \B), the keep operator (\K) and look-behind ((?<=…), (? len(s) { + return nil, pos } + m := r.matcher() + if !patternNeedsLeftContext(r.source) { + return m.Match(s[pos:]), pos + } + for p := pos; p <= len(s); { + if hit := m.MatchAt(s, p); hit != nil { + return hit, 0 + } + if p == len(s) { + break + } + _, w := utf8.DecodeRuneInString(s[p:]) + p += w + } + return nil, 0 } // runMatch matches re against subject, returning a MatchData value or nil. It @@ -1001,12 +1275,12 @@ func (vm *VM) runMatchFrom(re *Regexp, subject string, pos int64) object.Value { return object.NilV } byteOff := charToByte(subject, int(pos)) - md := re.matcher().Match(subject[byteOff:]) + md, base := re.searchFrom(subject, byteOff) if md == nil { vm.lastMatch = object.NilV return object.NilV } - m := &MatchData{md: md, subject: subject, re: re, byteOff: byteOff} + m := &MatchData{md: md, subject: subject, re: re, byteOff: base} vm.lastMatch = m return m } @@ -1017,13 +1291,13 @@ func (vm *VM) runMatchFrom(re *Regexp, subject string, pos int64) object.Value { // index of the match start, or nil. off may equal the character count. func (vm *VM) strIndexRegexp(re *Regexp, subject string, off int) object.Value { byteOff := charToByte(subject, off) - md := re.matcher().Match(subject[byteOff:]) // leftmost match in the tail; \G pins to its start + md, base := re.searchFrom(subject, byteOff) // leftmost match at or after the cursor if md == nil { vm.lastMatch = object.NilV return object.NilV } - vm.lastMatch = &MatchData{md: md, subject: subject, re: re, byteOff: byteOff} - return object.IntValue(int64(byteToChar(subject, byteOff+md.Begin(0)))) + vm.lastMatch = &MatchData{md: md, subject: subject, re: re, byteOff: base} + return object.IntValue(int64(byteToChar(subject, base+md.Begin(0)))) } // lastRegexpMatch returns the match of re that begins at the largest byte offset @@ -1389,7 +1663,7 @@ func (vm *VM) scan(re *Regexp, subject string, self object.Value, blk *Proc) obj last := object.Value(object.NilV) // $~ after the call: last match, or nil when none pos := 0 for pos <= len(subject) { - md := re.matcher().Match(subject[pos:]) + md, base := re.searchFrom(subject, pos) if md == nil { break } @@ -1398,7 +1672,7 @@ func (vm *VM) scan(re *Regexp, subject string, self object.Value, blk *Proc) obj // absolute byteOff so MatchData#string, #begin and #offset are correct). // MRI leaves $~ set to the last match after scan returns, even if a block // reassigned it, so re-set it after the block runs. - cur := &MatchData{md: md, subject: subject, re: re, byteOff: pos} + cur := &MatchData{md: md, subject: subject, re: re, byteOff: base} vm.lastMatch = cur last = cur if blk != nil { @@ -1407,17 +1681,17 @@ func (vm *VM) scan(re *Regexp, subject string, self object.Value, blk *Proc) obj } else { results = append(results, elem) } - matchEnd := md.End(0) // byte offset within subject[pos:] - if matchEnd == md.Begin(0) { + matchEnd := base + md.End(0) // absolute byte offset into subject + if md.End(0) == md.Begin(0) { // Empty match: emit here, then step one character forward. - pos += matchEnd + pos = matchEnd if pos >= len(subject) { break } _, w := utf8.DecodeRuneInString(subject[pos:]) pos += w } else { - pos += matchEnd + pos = matchEnd } } // $~ is the last match (or nil when there was none), as MRI leaves it. @@ -1555,12 +1829,12 @@ func splitRegexp(re *Regexp, subject string, limit int, enc string) object.Value if limit > 0 && pieces+1 == limit { break } - md := re.matcher().Match(subject[search:]) + md, base := re.searchFrom(subject, search) if md == nil { break } - mBegin := search + md.Begin(0) - mEnd := search + md.End(0) + mBegin := base + md.Begin(0) + mEnd := base + md.End(0) if mEnd == mBegin { // Empty match: an empty match at the very start is skipped; otherwise // it ends the current character field. Advance one character. @@ -1685,18 +1959,19 @@ func (vm *VM) gsub(re *Regexp, subject, repl string, blk *Proc, global bool) obj search := 0 // byte cursor where the next search begins last := object.Value(object.NilV) // $~ after the call: last match, or nil when there is none for search <= len(subject) { - md := re.matcher().Match(subject[search:]) + md, base := re.searchFrom(subject, search) if md == nil { break } - mBegin := search + md.Begin(0) - mEnd := search + md.End(0) + mBegin := base + md.Begin(0) + mEnd := base + md.End(0) b.WriteString(subject[pos:mBegin]) // literal text before the match // Expose this match through $~ / $1.. so a replacement block sees the - // captures. md's offsets are relative to the searched slice, so carry the - // FULL subject with byteOff=search — then MatchData#string is the whole - // receiver and #offset/#begin are absolute, as MRI reports inside the block. - cur := &MatchData{md: md, subject: subject, re: re, byteOff: search} + // captures. md's offsets are relative to base (the offset searchFrom + // matched from), so carry the FULL subject with byteOff=base — then + // MatchData#string is the whole receiver and #offset/#begin are absolute, + // as MRI reports inside the block. + cur := &MatchData{md: md, subject: subject, re: re, byteOff: base} vm.lastMatch = cur last = cur if blk != nil { @@ -1740,16 +2015,16 @@ func (vm *VM) gsubHash(re *Regexp, subject string, h *object.Hash, global bool) search := 0 // byte cursor where the next search begins last := object.Value(object.NilV) // $~ after the call: last match, or nil when there is none for search <= len(subject) { - md := re.matcher().Match(subject[search:]) + md, base := re.searchFrom(subject, search) if md == nil { break } - mBegin := search + md.Begin(0) - mEnd := search + md.End(0) + mBegin := base + md.Begin(0) + mEnd := base + md.End(0) b.WriteString(subject[pos:mBegin]) // literal text before the match - // Carry the FULL subject with byteOff=search so $~ reports absolute + // Carry the FULL subject with byteOff=base so $~ reports absolute // offsets and the whole receiver (see gsub). - cur := &MatchData{md: md, subject: subject, re: re, byteOff: search} + cur := &MatchData{md: md, subject: subject, re: re, byteOff: base} vm.lastMatch = cur last = cur // Look the match up with Hash#[] (not a bare Get) so a missing key runs the @@ -2133,8 +2408,8 @@ func (vm *VM) installRegexp() { if _, isNil := args[0].(object.Nil); isNil { return object.False } - vm.checkSubjectEncoding(args[0]) re := reArg(self) + vm.checkSubjectEncoding(re, args[0]) subject := strArg(args[0]) // match?(str, pos): probe from character offset pos, without touching $~ // (the predicate form has no match-data side effect). @@ -2147,7 +2422,10 @@ func (vm *VM) installRegexp() { if pos < 0 || pos > nChars { return object.False } - return object.Bool(re.matcher().MatchString(subject[charToByte(subject, int(pos)):])) + // Search from the cursor with the WHOLE subject visible, as + // rb_reg_search does — /\A/.match?("hello", 2) is false, not true. + md, _ := re.searchFrom(subject, charToByte(subject, int(pos))) + return object.Bool(md != nil) } return object.Bool(re.matcher().MatchString(subject)) }) @@ -2159,7 +2437,7 @@ func (vm *VM) installRegexp() { vm.lastMatch = object.NilV return object.NilV } - vm.checkSubjectEncoding(args[0]) + vm.checkSubjectEncoding(re, args[0]) // The subject is coerced like any Regexp operand: a Symbol yields its name, // anything else is taken via #to_str (Integer/Exception raise TypeError). subject := vm.regexpOperandStr(args[0]) @@ -2519,6 +2797,7 @@ func (vm *VM) regexpMatchIndex(re *Regexp, subject object.Value) object.Value { if !ok { raise("TypeError", "no implicit conversion of %s into String", classNameOf(subject)) } + vm.checkSubjectEncoding(re, subject) md := re.matcher().Match(s) if md == nil { vm.lastMatch = object.NilV @@ -2804,3 +3083,143 @@ func namedKey(name string, symbolize bool) object.Value { } return object.NewString(name) } + +// onigMeaningfulEscapeLetters are the ASCII letters that carry a meaning after +// a backslash in a Ruby (Onigmo) pattern: the character types and properties +// (d D h H p P s S w W R X), the anchors and assertions (A b B G K z Z), the +// character escapes (a c C e f M n r t u v x o), and the group references +// (g k). Every OTHER letter is, in MRI, simply that letter — "\y" matches "y", +// "\Q" matches "Q" — which is what escapes_spec's "allows any character to be +// escaped" pins. The engine this VM links rejects them instead, so +// translateCharEscapes drops the backslash. +const onigMeaningfulEscapeLetters = "abcdefghknoprstuvwxzABCDGHKMPRSWXZ" + +// controlEscapeValue applies Onigmo's control-character rule to one byte: "?" +// denotes DEL and every other byte is masked with 0x9F, so \cA, \ca and \cc all +// give 0x01/0x03 exactly as the corresponding String escapes do ("\cA".ord == 1, +// "\c#".ord == 3). A byte at or above 0x80 is not a control escape MRI accepts +// here, and is reported as unusable. +func controlEscapeValue(c byte) (byte, bool) { + if c >= 0x80 { + return 0, false + } + if c == '?' { + return 0x7F, true + } + return c & 0x9F, true +} + +// controlEscapePayload reads the character a \c / \C- escape controls from the +// text that follows it, returning its control value and how many bytes it +// spanned. The payload may itself be backslash-escaped — /\c\\/ controls a +// backslash, giving 0x1C — which is why it is not simply the next byte. +func controlEscapePayload(rest string) (value byte, width int, ok bool) { + if len(rest) == 0 { + return 0, 0, false + } + if rest[0] == '\\' { + if len(rest) < 2 { + return 0, 0, false + } + v, good := controlEscapeValue(rest[1]) + return v, 2, good + } + v, good := controlEscapeValue(rest[0]) + return v, 1, good +} + +// hexEscape renders a byte as the \xHH escape the engine accepts in and out of +// a character class. +func hexEscape(v byte) string { + const hexDigits = "0123456789abcdef" + return `\x` + string([]byte{hexDigits[v>>4], hexDigits[v&0xf]}) +} + +// translateCharEscapes rewrites the Ruby escape sequences that denote a single +// CHARACTER but that the linked engine does not parse, into the \xHH escape it +// does parse, and drops the backslash from an escape that is just a letter: +// +// \cX and \C-X control characters — /\c#\cc\cC/, /\C-*\C-J\C-j/ +// \0, \0nn octal, and any \nnn INSIDE a character class — /[\000-\b]/ +// \y \Q \j … a letter with no meaning after a backslash — /\y/ matches "y" +// +// Outside a character class a backslash followed by 1-9 is left untouched: that +// is a back-reference, which the engine handles. Inside one it cannot be, so +// Onigmo reads it as octal there (/[\1]/ matches "\x01"), and so does this. +// +// A value above 0x7F is left as written: MRI turns such a pattern into a binary +// (ASCII-8BIT) regexp, which is a different matter from spelling one character. +// +// Character-class nesting is tracked by depth, as in rewriteBeginLineAnchor, so +// POSIX brackets and nested classes are counted correctly. A source with no +// backslash is returned unchanged. +// +// Reference: ruby/ruby v3_4_0 regparse.c fetch_escaped_value (the \c / \C- / +// \M- cases) and fetch_token's ONIG_SYN_OP_ESC_OCTAL3 branch. +func translateCharEscapes(src string) string { + if !strings.ContainsRune(src, '\\') { + return src + } + var b strings.Builder + b.Grow(len(src)) + depth := 0 // character-class nesting depth + for i := 0; i < len(src); i++ { + c := src[i] + if c == '[' { + depth++ + b.WriteByte(c) + continue + } + if c == ']' { + if depth > 0 { + depth-- + } + b.WriteByte(c) + continue + } + if c != '\\' || i+1 >= len(src) { + b.WriteByte(c) + continue + } + n := src[i+1] + switch { + case n == 'c' && i+2 < len(src): + if payload, width, ok := controlEscapePayload(src[i+2:]); ok { + b.WriteString(hexEscape(payload)) + i += 1 + width + continue + } + case n == 'C' && i+3 < len(src) && src[i+2] == '-': + if payload, width, ok := controlEscapePayload(src[i+3:]); ok { + b.WriteString(hexEscape(payload)) + i += 2 + width + continue + } + case n >= '0' && n <= '7' && (depth > 0 || n == '0'): + digits := 1 + for digits < 3 && i+1+digits < len(src) && src[i+1+digits] >= '0' && src[i+1+digits] <= '7' { + digits++ + } + v, err := strconv.ParseUint(src[i+1:i+1+digits], 8, 16) + if err == nil && v <= 0x7F { + b.WriteString(hexEscape(byte(v))) + i += digits + continue + } + case isASCIILetter(n) && !strings.ContainsRune(onigMeaningfulEscapeLetters, rune(n)): + b.WriteByte(n) + i++ + continue + } + // Not one of the rewritten forms: copy the escape through untouched. + b.WriteByte(c) + b.WriteByte(n) + i++ + } + return b.String() +} + +// isASCIILetter reports whether c is an ASCII letter. +func isASCIILetter(c byte) bool { + return c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' +} diff --git a/internal/vm/string_byteops.go b/internal/vm/string_byteops.go index 09444efa..8ef22544 100644 --- a/internal/vm/string_byteops.go +++ b/internal/vm/string_byteops.go @@ -98,13 +98,13 @@ func (vm *VM) strByteindex(self *object.String, args []object.Value) object.Valu } switch needle := args[0].(type) { case *Regexp: - md := needle.re.Match(s[off:]) + md, base := needle.searchFrom(s, off) if md == nil { vm.lastMatch = object.NilV return object.NilV } - vm.lastMatch = &MatchData{md: md, subject: s, re: needle, byteOff: off} - return object.IntValue(int64(off + md.Begin(0))) + vm.lastMatch = &MatchData{md: md, subject: s, re: needle, byteOff: base} + return object.IntValue(int64(base + md.Begin(0))) default: ns := vm.strValueArg(args[0]) vm.combinedEncName(self, ns) // raises Encoding::CompatibilityError if incompatible diff --git a/internal/vm/wave28_modreg_internal_test.go b/internal/vm/wave28_modreg_internal_test.go new file mode 100644 index 00000000..176a67ab --- /dev/null +++ b/internal/vm/wave28_modreg_internal_test.go @@ -0,0 +1,804 @@ +// Copyright (c) the go-embedded-ruby/ruby authors +// +// SPDX-License-Identifier: BSD-3-Clause + +package vm + +import ( + "os" + "strings" + "testing" + + "github.com/go-embedded-ruby/ruby/internal/object" +) + +// writeTestFile writes a fixture file for the autoload tests below. +func writeTestFile(path, body string) error { + return os.WriteFile(path, []byte(body), 0o600) +} + +// rescueRuby runs fn and reports the Ruby exception class and message it raised, +// or ("", "") when it returned normally. It lets the whitebox tests below reach +// the raise-only branches of the wave-28 helpers without going through a script. +func rescueRuby(fn func()) (class, msg string) { + defer func() { + if r := recover(); r != nil { + if re, ok := r.(RubyError); ok { + class, msg = re.Class, re.Message + return + } + panic(r) + } + }() + fn() + return "", "" +} + +func TestPatternNeedsLeftContext(t *testing.T) { + tests := []struct { + src string + want bool + }{ + {"", false}, + {"abc", false}, + {"a$", false}, + {`a\z`, false}, + {`\d+`, false}, + {"^a", true}, + {"a^b", true}, // conservative: not the class-negation slot + {"[^a]", false}, // the negation slot is not an anchor + {`\^`, false}, // an escaped caret is a literal + {`\A`, true}, // beginning-of-string + {`\b`, true}, // word boundary + {`\B`, true}, // non-boundary + {`\K`, true}, // keep operator + {`\G`, false}, // deliberately excluded (see searchFrom) + {`\\A`, false}, // an escaped backslash, then a literal A + {"(?<=a)", true}, // look-behind + {"(?a)", false}, // a NAMED group is not look-behind + {"(?:a)", false}, // a plain group is not look-behind + {`a\`, false}, // a trailing backslash consumes nothing + {"[a^b]", true}, // conservative over-report inside a class + {"(?<", false}, // truncated look-behind prefix + } + for _, tc := range tests { + if got := patternNeedsLeftContext(tc.src); got != tc.want { + t.Errorf("patternNeedsLeftContext(%q) = %v, want %v", tc.src, got, tc.want) + } + } +} + +func TestSearchFromBoundsAndPaths(t *testing.T) { + vm := New(&strings.Builder{}) + + // Out-of-range cursors yield no match and echo the cursor back as the base. + plain := vm.compileRegexp("a", "").(*Regexp) + if md, base := plain.searchFrom("abc", -1); md != nil || base != -1 { + t.Fatalf("searchFrom(-1) = %v, %d; want nil, -1", md, base) + } + if md, base := plain.searchFrom("abc", 4); md != nil || base != 4 { + t.Fatalf("searchFrom(4) = %v, %d; want nil, 4", md, base) + } + + // The fast path slices the tail, so offsets come back relative to the cursor. + md, base := plain.searchFrom("xxa", 1) + if md == nil || base != 1 || base+md.Begin(0) != 2 { + t.Fatalf("fast path: base=%d begin=%v", base, md) + } + if md, _ := plain.searchFrom("xxx", 0); md != nil { + t.Fatalf("fast path should not match") + } + + // A left-context pattern takes the anchored probe, which reports absolute + // offsets (base 0) and sees the real prefix. + anchored := vm.compileRegexp(`\A`, "").(*Regexp) + if md, base := anchored.searchFrom("abc", 1); md != nil { + t.Fatalf("\\A must not match from 1, got %v (base %d)", md, base) + } + if md, base := anchored.searchFrom("abc", 0); md == nil || base != 0 { + t.Fatalf("\\A must match at 0, got %v (base %d)", md, base) + } + // The probe must advance by whole characters and reach the very end. + bound := vm.compileRegexp(`\b`, "").(*Regexp) + if md, base := bound.searchFrom("éa", 0); md == nil || base != 0 || md.Begin(0) != 0 { + t.Fatalf("\\b over a multi-byte prefix: %v base=%d", md, base) + } + if md, _ := bound.searchFrom("...", 0); md != nil { + t.Fatalf("\\b must not match inside punctuation only") + } + // The probe must walk PAST a failing start and stop at the end of the string. + head := vm.compileRegexp(`\Ab`, "").(*Regexp) + if md, _ := head.searchFrom("ab", 0); md != nil { + t.Fatalf("\\Ab must not match \"ab\" from any start") + } +} + +func TestRewriteBeginLineAnchor(t *testing.T) { + tests := []struct{ src, want string }{ + {"abc", "abc"}, // no caret at all + {"^a", beginLineEquivalent + "a"}, // the anchor + {`\^a`, `\^a`}, // escaped: left alone + {"[^a]", "[^a]"}, // class negation + {"[[:alpha:]^]", "[[:alpha:]^]"}, // nested class keeps its caret + {"[a-z&&[^b]]", "[a-z&&[^b]]"}, // nested negation + {"(?#a^b)^c", "(?#a^b)" + beginLineEquivalent + "c"}, // a comment is copied whole + {"(?#unterminated^", "(?#unterminated^"}, // no closing paren: copy the rest + {"]^a", "]" + beginLineEquivalent + "a"}, // a stray ] must not go negative + {`a\`, `a\`}, // trailing backslash, no caret + } + for _, tc := range tests { + if got := rewriteBeginLineAnchor(tc.src); got != tc.want { + t.Errorf("rewriteBeginLineAnchor(%q) = %q, want %q", tc.src, got, tc.want) + } + } +} + +func TestBeginLineAnchorSemantics(t *testing.T) { + // The whole point of the rewrite: "^" does not fire at the end of a string + // that ends in a newline (regexec.c OP_BEGIN_LINE), but does on the empty + // string, whose start is its end. + tests := []struct{ src, want string }{ + {`p "Text\n".gsub(/^/, " ")`, "\" Text\\n\"\n"}, + {`p "a\nb\n".gsub(/^/, "-")`, "\"-a\\n-b\\n\"\n"}, + {`p "".gsub(/^/, "-")`, "\"-\"\n"}, + {`p "a\nb".gsub(/^/, "-")`, "\"-a\\n-b\"\n"}, + {`p("a\nb" =~ /^b/)`, "2\n"}, + {`p("abc" =~ /^b/)`, "nil\n"}, + {`p(/^a[b^c]d$/.to_s)`, "\"(?-mix:^a[b^c]d$)\"\n"}, + } + for _, tc := range tests { + if got := eval(t, tc.src); got != tc.want { + t.Errorf("%s => %q, want %q", tc.src, got, tc.want) + } + } +} + +func TestSearchFromScriptSemantics(t *testing.T) { + // Each of these was wrong while the engine was handed subject[pos:]. + tests := []struct{ src, want string }{ + {`p "ab cd".scan(/\b\w/)`, "[\"a\", \"c\"]\n"}, + {`p "a\nb\nc".split(/^/)`, "[\"a\\n\", \"b\\n\", \"c\"]\n"}, + {`p "hello".index(/\A/, 2)`, "nil\n"}, + {`p "hello world".match(/\Ao/, 4)`, "nil\n"}, + {`p "xay".byteindex(/\A/, 1)`, "nil\n"}, + {`p "aaa".gsub(/\A/, "-")`, "\"-aaa\"\n"}, + {`p "a1b2".gsub(/\b/, "|")`, "\"|a1b2|\"\n"}, + {`p "ab".gsub(/(?<=a)/, "-")`, "\"a-b\"\n"}, + {`p "abc".scan(/\A./)`, "[\"a\"]\n"}, + // Regexp#match?(str, pos) searches from a cursor too. + {`p(/\A/.match?("hello", 2))`, "false\n"}, + {`p(/\A/.match?("hello", 0))`, "true\n"}, + {`p(/\b\w/.match?("ab cd", 1))`, "true\n"}, + {`p(/l/.match?("hello", 4))`, "false\n"}, + {`p(/o/.match?("hello", 9))`, "false\n"}, + // The Hash form of gsub goes through the same loop. + {`p "ab cd".gsub(/\b/, "" => "|")`, "\"|ab| |cd|\"\n"}, + // And the ordinary, context-free patterns must be untouched. + {`p "a,b,,c".split(/,/)`, "[\"a\", \"b\", \"\", \"c\"]\n"}, + {`p "hello".scan(/l/)`, "[\"l\", \"l\"]\n"}, + {`p "one two".sub(/(\w+) (\w+)/, "\\2 \\1")`, "\"two one\"\n"}, + } + for _, tc := range tests { + if got := eval(t, tc.src); got != tc.want { + t.Errorf("%s => %q, want %q", tc.src, got, tc.want) + } + } +} + +func TestControlEscapeHelpers(t *testing.T) { + if _, ok := controlEscapeValue(0x80); ok { + t.Error("a byte at 0x80 is not a control escape payload") + } + if v, ok := controlEscapeValue('?'); !ok || v != 0x7F { + t.Errorf("\\c? = %#x, %v; want 0x7f, true", v, ok) + } + if v, ok := controlEscapeValue('A'); !ok || v != 0x01 { + t.Errorf("\\cA = %#x, %v; want 0x01, true", v, ok) + } + if v, ok := controlEscapeValue('#'); !ok || v != 0x03 { + t.Errorf("\\c# = %#x, %v; want 0x03, true", v, ok) + } + + if _, _, ok := controlEscapePayload(""); ok { + t.Error("an empty payload is not readable") + } + if _, _, ok := controlEscapePayload(`\`); ok { + t.Error("a lone backslash payload is not readable") + } + if v, w, ok := controlEscapePayload(`\\`); !ok || w != 2 || v != 0x1C { + t.Errorf("\\c\\ = %#x, %d, %v; want 0x1c, 2, true", v, w, ok) + } + if v, w, ok := controlEscapePayload("A"); !ok || w != 1 || v != 0x01 { + t.Errorf("\\cA = %#x, %d, %v; want 0x01, 1, true", v, w, ok) + } + if _, _, ok := controlEscapePayload("\u00e9"); ok { + t.Error("a multi-byte payload is not a control escape") + } + + if got := hexEscape(0x0a); got != `\x0a` { + t.Errorf("hexEscape(0x0a) = %q", got) + } + if !isASCIILetter('a') || !isASCIILetter('Z') || isASCIILetter('1') { + t.Error("isASCIILetter misclassified") + } +} + +func TestTranslateCharEscapes(t *testing.T) { + tests := []struct{ src, want string }{ + {"abc", "abc"}, // no backslash + {`\d+`, `\d+`}, // a meaningful letter is left alone + {`\k`, `\k`}, // group reference + {`\cA`, `\x01`}, // control + {`\C-J`, `\x0a`}, // control, \C- spelling + {`\c\\`, `\x1c`}, // the payload is itself escaped + {`\C-\\`, `\x1c`}, // + {`\0`, `\x00`}, // octal + {`\101`, `\101`}, // outside a class: a back-reference, untouched + {`[\1]`, `[\x01]`}, // inside a class: octal + {`[\000-\b]`, `[\x00-\b]`}, // + {`(a)\1`, `(a)\1`}, // a real back-reference survives + {`\y`, `y`}, // a meaningless letter loses its backslash + {`\Q`, `Q`}, // + {`\400`, `\400`}, // above 0x7f: left as written + {`a\`, `a\`}, // trailing backslash + {`\c`, `\c`}, // truncated \c + {`\C`, `\C`}, // truncated \C- + {`\C-`, `\C-`}, // + {`\-`, `\-`}, // a punctuation escape is not a letter + {`[a]\0`, `[a]\x00`}, // the class depth returns to zero + {`]\1`, `]\1`}, // a stray ] must not make the depth negative + {"\\c\u00e9", "\\c\u00e9"}, // an unusable control payload is copied + } + for _, tc := range tests { + if got := translateCharEscapes(tc.src); got != tc.want { + t.Errorf("translateCharEscapes(%q) = %q, want %q", tc.src, got, tc.want) + } + } +} + +func TestCharEscapeSemantics(t *testing.T) { + tests := []struct{ src, want string }{ + {`p(/\c#\cc\cC/ =~ "\x03\x03\x03")`, "0\n"}, + {`p(/\C-*\C-J\C-j/ =~ "\n\n\n")`, "0\n"}, + {`p(/[\000-\b]/ =~ "\x00")`, "0\n"}, + {`p(/\y/ =~ "y")`, "0\n"}, + {`p(/(a)\1/ =~ "aa")`, "0\n"}, + {`p(/\d\w\s/ =~ "1a ")`, "0\n"}, + } + for _, tc := range tests { + if got := eval(t, tc.src); got != tc.want { + t.Errorf("%s => %q, want %q", tc.src, got, tc.want) + } + } +} + +func TestUnpackOffsetKeyword(t *testing.T) { + tests := []struct{ src, want string }{ + {`p "abc".unpack("C*", offset: 1)`, "[98, 99]\n"}, + {`p "a".unpack("C", offset: 1)`, "[nil]\n"}, + {`p "a".unpack1("C", offset: 1)`, "nil\n"}, + {`p "abc".unpack("C*")`, "[97, 98, 99]\n"}, + {`p "abc".unpack1("C")`, "97\n"}, + // The offset counts bytes, not characters. + {`p "\u0608".unpack1("C", offset: 1)`, "136\n"}, + // nil is "no offset given". + {`p "abc".unpack("C", offset: nil)`, "[97]\n"}, + // An object answering #to_int is accepted, as NUM2LONG does. + {"class O; def to_int; 1; end; end\np \"abc\".unpack(\"C\", offset: O.new)", "[98]\n"}, + {`begin; "a".unpack("C", offset: 2); rescue ArgumentError => e; p e.message; end`, + "\"offset outside of string\"\n"}, + {`begin; "a".unpack1("C", offset: -1); rescue ArgumentError => e; p e.message; end`, + "\"offset can't be negative\"\n"}, + {`begin; "a".unpack("C", bogus: 1); rescue ArgumentError => e; p e.message; end`, + "\"unknown keyword: :bogus\"\n"}, + } + for _, tc := range tests { + if got := eval(t, tc.src); got != tc.want { + t.Errorf("%s => %q, want %q", tc.src, got, tc.want) + } + } +} + +func TestUnpackOffsetDirect(t *testing.T) { + vm := New(&strings.Builder{}) + // No trailing Hash, and a lone Hash argument (the format itself) are both 0. + if got := vm.unpackOffset([]object.Value{object.NewString("C")}, 3); got != 0 { + t.Errorf("no keyword = %d, want 0", got) + } + h := object.NewHash() + h.Set(object.Symbol("offset"), object.IntValue(2)) + if got := vm.unpackOffset([]object.Value{object.NewString("C"), h}, 3); got != 2 { + t.Errorf("offset keyword = %d, want 2", got) + } + // A non-Symbol key is an unknown keyword too. + bad := object.NewHash() + bad.Set(object.NewString("offset"), object.IntValue(1)) + class, _ := rescueRuby(func() { + vm.unpackOffset([]object.Value{object.NewString("C"), bad}, 3) + }) + if class != "ArgumentError" { + t.Errorf("a String key raised %q, want ArgumentError", class) + } +} + +func TestRefinementImportMethods(t *testing.T) { + tests := []struct{ name, src, want string }{ + {"imports", ` + su = Module.new do + def indent(level); " " * level + self; end + end + R = Module.new do + refine String do + import_methods su + end + end + module S; using R; p "foo".indent(3); end`, "\" foo\"\n"}, + {"last_module_wins", ` + a = Module.new { def tag; "a"; end } + b = Module.new { def tag; "b"; end } + R = Module.new { refine(String) { import_methods a, b } } + module S; using R; p "x".tag; end`, "\"b\"\n"}, + {"methods_see_each_other", ` + su = Module.new do + def indent(level); " " * level + self; end + def dotted(level); indent(level) + "."; end + end + R = Module.new { refine(String) { import_methods su } } + module S; using R; p "foo".dotted(2); end`, "\" foo.\"\n"}, + {"owner_is_the_refinement", ` + su = Module.new { def tag; "t"; end } + R = Module.new { refine(String) { import_methods su } } + module S + using R + p String.instance_method(:tag).owner == R.refinements.first + end`, "true\n"}, + {"no_class_methods", ` + su = Module.new { def self.tag; "t"; end } + R = Module.new { refine(String) { import_methods su } } + module S + using R + p String.instance_methods.include?(:tag) + end`, "false\n"}, + {"visibility_carried", ` + su = Module.new do + def pub; "p"; end + private + def priv; "q"; end + end + R = Module.new { refine(String) { import_methods su } } + module S + using R + p "x".pub + begin; "x".priv; rescue NoMethodError; p :private; end + end`, "\"p\"\n:private\n"}, + {"type_error", ` + R = Module.new do + refine String do + begin; import_methods Integer; rescue TypeError => e; p e.message; end + end + end`, "\"wrong argument type Class (expected Module)\"\n"}, + {"arity", ` + R = Module.new do + refine String do + begin; import_methods; rescue ArgumentError => e; p e.message; end + end + end`, "\"wrong number of arguments (given 0, expected 1+)\"\n"}, + {"not_ruby_code", ` + R = Module.new do + refine String do + begin; import_methods Kernel; rescue ArgumentError => e; p e.class; end + end + end`, "ArgumentError\n"}, + {"nothing_imported_when_one_arg_is_bad", ` + su = Module.new { def tag; "t"; end } + R = Module.new do + refine String do + begin; import_methods su, Integer; rescue TypeError; end + end + end + module S + using R + begin; "x".tag; rescue NoMethodError; p :none; end + end`, ":none\n"}, + {"warns_about_ancestors", ` + $VERBOSE = true + M1 = Module.new + M2 = Module.new { include M1 } + R = Module.new { refine(String) { import_methods M2 } } + p :done`, "warning: M2 has ancestors, but Refinement#import_methods doesn't import their methods\n:done\n"}, + {"undefined_entries_are_skipped", ` + su = Module.new do + def gone; end + undef_method :gone + def kept; "k"; end + end + R = Module.new { refine(String) { import_methods su } } + module S; using R; p "x".kept; end`, "\"k\"\n"}, + {"returns_the_refinement", ` + su = Module.new { def tag; "t"; end } + R = Module.new do + refine String do + p import_methods(su).equal?(self) + end + end`, "true\n"}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if got := eval(t, tc.src); got != tc.want { + t.Errorf("got %q, want %q", got, tc.want) + } + }) + } +} + +func TestImportMethodsRejectsNonRefinementSelf(t *testing.T) { + src := ` + begin + Module.new.send(:import_methods, Module.new) + rescue NoMethodError, TypeError => e + p e.class + end` + if got := eval(t, src); got != "NoMethodError\n" && got != "TypeError\n" { + t.Errorf("got %q, want NoMethodError or TypeError", got) + } +} + +func TestImportMethodsOnNonRefinementSelf(t *testing.T) { + // import_methods is only ever reached with a Refinement as self, so this + // guard is defensive: call the body directly to exercise it. + vm := New(&strings.Builder{}) + class, msg := rescueRuby(func() { + refinementImportMethods(vm, object.IntValue(1), []object.Value{vm.cString}, nil) + }) + if class != "TypeError" || !strings.Contains(msg, "expected Refinement") { + t.Errorf("non-refinement self raised %q / %q", class, msg) + } + // A plain RClass that is not a refinement takes the same branch. + class, _ = rescueRuby(func() { + refinementImportMethods(vm, vm.cString, []object.Value{vm.cString}, nil) + }) + if class != "TypeError" { + t.Errorf("a non-refinement module raised %q, want TypeError", class) + } +} + +func TestImportMethodsPinsLexicalScope(t *testing.T) { + // A method written in a NAMED module body carries no explicit lexScope, so + // the import must pin it to the source module for constant lookup. CONST is + // resolvable only from SrcMod, never from the refinement. + src := ` + module SrcMod + CONST = "from-src" + def tagged; CONST; end + end + R = Module.new { refine(String) { import_methods SrcMod } } + module S + using R + p "x".tagged + end` + if got := eval(t, src); got != "\"from-src\"\n" { + t.Errorf("imported method constant lookup = %q", got) + } +} + +func TestDefinedWithRubyCodeAndSortedNames(t *testing.T) { + if definedWithRubyCode(&Method{}) { + t.Error("a method with no iseq is not Ruby code") + } + if definedWithRubyCode(&Method{native: func(*VM, object.Value, []object.Value, *Proc) object.Value { return object.NilV }}) { + t.Error("a native method is not Ruby code") + } + if definedWithRubyCode(&Method{proc: &Proc{}}) { + t.Error("a define_method body is not Ruby code") + } + names := sortedMethodNames(map[string]*Method{"b": {}, "a": {}, "c": {}}) + if strings.Join(names, ",") != "a,b,c" { + t.Errorf("sortedMethodNames = %v, want a,b,c", names) + } + if len(sortedMethodNames(nil)) != 0 { + t.Error("sortedMethodNames(nil) must be empty") + } +} + +func TestCallerRefinedInstanceMethod(t *testing.T) { + vm := New(&strings.Builder{}) + // Nothing has ever called refine: the fast exit. + if m := vm.callerRefinedInstanceMethod(vm.cString, "upcase"); m != nil { + t.Error("no refinements anywhere must yield nil") + } + vm.anyRefinements = true + if m := vm.callerRefinedInstanceMethod(nil, "upcase"); m != nil { + t.Error("a nil module must yield nil") + } + // No frame cref recorded (a bare native call at bootstrap). + if m := vm.callerRefinedInstanceMethod(vm.cString, "upcase"); m != nil { + t.Error("no frame cref must yield nil") + } + vm.frameCrefs = append(vm.frameCrefs, nil) + if m := vm.callerRefinedInstanceMethod(vm.cString, "upcase"); m != nil { + t.Error("a nil cref must yield nil") + } + vm.frameCrefs[len(vm.frameCrefs)-1] = vm.cObject + if m := vm.callerRefinedInstanceMethod(vm.cString, "upcase"); m != nil { + t.Error("a scope with no active refinement must yield nil") + } +} + +func TestInstanceMethodHonoursRefinements(t *testing.T) { + tests := []struct{ src, want string }{ + {` + class K; end + module R; refine(K) { def foo; "r"; end }; end + module S + using R + p K.instance_method(:foo).class + p K.instance_method(:foo).owner == R.refinements.first + end`, "UnboundMethod\ntrue\n"}, + // Outside the using scope the refinement is not visible. + {` + class K; end + module R; refine(K) { def foo; "r"; end }; end + begin; K.instance_method(:foo); rescue NameError; p :none; end`, ":none\n"}, + // The class's OWN definition wins over a lower-priority refinement. + {` + class K; def foo; "k"; end; end + module R; refine(K) { def bar; "r"; end }; end + module S + using R + p K.instance_method(:foo).owner + end`, "K\n"}, + // An active refinement of a DIFFERENT class is skipped, and a name no + // refinement and no ancestor defines still raises. + {` + class K; end + class J; end + module R + refine(J) { def foo; "j"; end } + refine(K) { def bar; "k"; end } + end + module S + using R + owner = K.instance_method(:bar).owner + p(R.refinements.include?(owner) && owner.target == K) + begin; K.instance_method(:nope); rescue NameError; p :none; end + end`, "true\n:none\n"}, + } + for _, tc := range tests { + if got := eval(t, tc.src); got != tc.want { + t.Errorf("%s => %q, want %q", tc.src, got, tc.want) + } + } +} + +func TestAutoloadVisibleDirect(t *testing.T) { + vm := New(&strings.Builder{}) + if vm.autoloadVisible(nil, "X") { + t.Error("a nil class has no autoload") + } + if vm.autoloadVisible(vm.cObject, "NoSuchConstantHere") { + t.Error("a class with no autoload table has none") + } + mod := newClass("AV", nil) + mod.isModule = true + mod.autoloads = map[string]string{} + if vm.autoloadVisible(mod, "X") { + t.Error("an empty autoload table has no entry") + } + // A path that resolves to no file on disk was never loaded, so the entry is + // visible. + mod.autoloads["X"] = "rbgo_wave28_no_such_feature" + if !vm.autoloadVisible(mod, "X") { + t.Error("an unloaded feature must leave the entry visible") + } +} + +func TestAutoloadInProgressIsInvisible(t *testing.T) { + dir := t.TempDir() + feature := dir + "/rbgo_w28_al.rb" + if err := writeTestFile(feature, "$probe = [defined?(MZ::K), MZ.autoload?(:K), MZ.const_defined?(:K), MZ.constants(false).include?(:K)]\nmodule MZ; K = 1; end\n"); err != nil { + t.Fatal(err) + } + src := ` + module MZ; end + MZ.autoload :K, ` + object.NewString(feature).Inspect() + ` + require ` + object.NewString(feature).Inspect() + ` + p $probe` + // From inside the file, the constant is undefined and carries no autoload, + // but it is still listed in Module#constants. + if got := eval(t, src); got != "[nil, nil, false, true]\n" { + t.Errorf("in-progress autoload probe = %q, want [nil, nil, false, true]", got) + } +} + +func TestWarnAutoloadDidNotDefine(t *testing.T) { + dir := t.TempDir() + feature := dir + "/rbgo_w28_empty.rb" + if err := writeTestFile(feature, "# defines nothing\n"); err != nil { + t.Fatal(err) + } + lit := object.NewString(feature).Inspect() + + // Not verbose: silent. + quiet := ` + require "stringio" + $stderr = StringIO.new + module MQ; end + MQ.autoload :MISSING, ` + lit + ` + begin; MQ::MISSING; rescue NameError; end + out = $stderr.string + $stderr = STDERR + p out` + if got := eval(t, quiet); got != "\"\"\n" { + t.Errorf("non-verbose warning = %q, want empty", got) + } + + // Verbose: the module-qualified form. + loud := ` + require "stringio" + $VERBOSE = true + $stderr = StringIO.new + module ML; end + ML.autoload :MISSING, ` + lit + ` + begin; ML::MISSING; rescue NameError; end + out = $stderr.string + $stderr = STDERR + p out.include?("Expected ") && out.include?("to define ML::MISSING but it didn't")` + if got := eval(t, loud); got != "true\n" { + t.Errorf("verbose warning = %q, want true", got) + } + + // On Object the name is NOT qualified. + top := ` + require "stringio" + $VERBOSE = true + $stderr = StringIO.new + autoload :MISSING_AT_TOP, ` + lit + ` + begin; MISSING_AT_TOP; rescue NameError; end + out = $stderr.string + $stderr = STDERR + p out.include?("to define MISSING_AT_TOP but it didn't")` + if got := eval(t, top); got != "true\n" { + t.Errorf("top-level warning = %q, want true", got) + } +} + +func TestWarnAutoloadDidNotDefineSkipsWhenDefined(t *testing.T) { + vm := New(&strings.Builder{}) + mod := newClass("WD", nil) + mod.isModule = true + mod.consts["K"] = object.IntValue(1) + // The constant IS defined, so nothing is written even in verbose mode. + vm.globals["$VERBOSE"] = object.Bool(true) + vm.warnAutoloadDidNotDefine(mod, "K", "feature.rb") +} + +func TestRegexpPrepareEncoding(t *testing.T) { + tests := []struct{ name, src, want string }{ + {"broken_subject", ` + s = "\x80".dup.force_encoding("UTF-8") + begin; s =~ /./; rescue ArgumentError => e; p e.message; end`, + "\"invalid byte sequence in UTF-8\"\n"}, + {"match_incompatible", ` + begin + /\A[[:space:]]*\z/.match(" ".encode("UTF-16LE")) + rescue Encoding::CompatibilityError => e + p e.message + end`, + "\"incompatible encoding regexp match (US-ASCII regexp with UTF-16LE string)\"\n"}, + {"match_p_incompatible", ` + begin + /\A[[:space:]]*\z/.match?(" ".encode("UTF-16LE")) + rescue Encoding::CompatibilityError + p :raised + end`, ":raised\n"}, + {"tilde_incompatible", ` + begin + /\A[[:space:]]*\z/ =~ " ".encode("UTF-16LE") + rescue Encoding::CompatibilityError + p :raised + end`, ":raised\n"}, + {"fixed_non_ascii_compatible_regexp", ` + begin + Regexp.new("".dup.force_encoding("UTF-16LE"), Regexp::FIXEDENCODING) =~ " ".encode("UTF-8") + rescue Encoding::CompatibilityError => e + p e.message + end`, + "\"incompatible encoding regexp match (UTF-16LE regexp with UTF-8 string)\"\n"}, + {"fixed_regexp_non_seven_bit_string", ` + begin + Regexp.new("".dup.force_encoding("US-ASCII"), Regexp::FIXEDENCODING) =~ "\303\251".dup.force_encoding("UTF-8") + rescue Encoding::CompatibilityError => e + p e.message + end`, + "\"incompatible encoding regexp match (US-ASCII regexp with UTF-8 string)\"\n"}, + {"binary_name_in_message", ` + begin + /é/ =~ "\xC3\xA9".dup.force_encoding("BINARY") + rescue Encoding::CompatibilityError => e + p e.message + end`, + "\"incompatible encoding regexp match (UTF-8 regexp with BINARY (ASCII-8BIT) string)\"\n"}, + // The historical-binary warning, and its two silencing conditions. It goes + // through rbWarn, which this VM keeps quiet until $VERBOSE is assigned. + {"historical_binary_warning", ` + $VERBOSE = true + p(/a/n =~ "\xC3\xA9".dup.force_encoding("UTF-8"))`, + "warning: historical binary regexp match /.../n against UTF-8 string\nnil\n"}, + {"no_warning_for_binary_subject", ` + $VERBOSE = true + p(/a/n =~ "\xC3\xA9".dup.force_encoding("BINARY"))`, "nil\n"}, + {"no_warning_for_seven_bit", ` + $VERBOSE = true + p(/a/n =~ "abc")`, "0\n"}, + // Same encoding on both sides is accepted without any of the above. + {"same_encoding", `p(/é/ =~ "é")`, "0\n"}, + {"symbol_subject_is_left_to_coercion", `p(/a/ =~ :abc)`, "0\n"}, + // A FIXEDENCODING regexp whose own encoding is ASCII-compatible accepts a + // 7-bit string in another encoding without raising. + {"fixed_regexp_accepts_seven_bit", ` + re = Regexp.new("a".dup.force_encoding("UTF-8"), Regexp::FIXEDENCODING) + p(re =~ "abc".dup.force_encoding("US-ASCII"))`, "0\n"}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if got := eval(t, tc.src); got != tc.want { + t.Errorf("got %q, want %q", got, tc.want) + } + }) + } +} + +func TestRegexpBinaryEncodingFlag(t *testing.T) { + tests := []struct{ src, want string }{ + // /n steps one BYTE (the subject here is binary, as ruby/spec's + // encoding_spec is, so MRI does the same). + {`p(/./n.match("\xC3\xA9".dup.force_encoding("BINARY")).to_a.map(&:bytes))`, "[[195]]\n"}, + {`p(/#{/./}/n.match("\xC3\xA9".dup.force_encoding("BINARY")).to_a.map(&:bytes))`, "[[195]]\n"}, + // Without /n the dot is a whole character. + {`p(/./.match("é").to_a.map(&:bytes))`, "[[195, 169]]\n"}, + // A \xHH escape above 0x7F makes a /n pattern BINARY even though every + // character of the source text is ASCII. + {`p(/\xFF/n.encoding)`, "#\n"}, + {`p(/\xFF#{/./}/n.encoding)`, "#\n"}, + // An all-7-bit /n pattern stays US-ASCII. + {`p(/.#{/./}/n.encoding)`, "#\n"}, + {`p(/\x41/n.encoding)`, "#\n"}, + } + for _, tc := range tests { + if got := eval(t, tc.src); got != tc.want { + t.Errorf("%s => %q, want %q", tc.src, got, tc.want) + } + } +} + +func TestSourceHasNonASCIIByteEscape(t *testing.T) { + tests := []struct { + src string + want bool + }{ + {"abc", false}, + {`\x41`, false}, // below 0x80 + {`\xFF`, true}, // + {`\x80`, true}, // the boundary + {`\\xFF`, false}, // the backslash is itself escaped + {`\x`, false}, // truncated + {`\xZZ`, false}, // not hex + {`a\xff`, true}, // lower case hex + {`\`, false}, // a lone trailing backslash + } + for _, tc := range tests { + if got := sourceHasNonASCIIByteEscape(tc.src); got != tc.want { + t.Errorf("sourceHasNonASCIIByteEscape(%q) = %v, want %v", tc.src, got, tc.want) + } + } +} + +func TestEncInspectName(t *testing.T) { + if got := encInspectName("ASCII-8BIT"); got != "BINARY (ASCII-8BIT)" { + t.Errorf("encInspectName(ASCII-8BIT) = %q", got) + } + if got := encInspectName("UTF-8"); got != "UTF-8" { + t.Errorf("encInspectName(UTF-8) = %q", got) + } +}