diff --git a/src/coreclr/gcinfo/CMakeLists.txt b/src/coreclr/gcinfo/CMakeLists.txt index 9757b755f27173..aeabca98f04843 100644 --- a/src/coreclr/gcinfo/CMakeLists.txt +++ b/src/coreclr/gcinfo/CMakeLists.txt @@ -52,6 +52,16 @@ function(create_gcinfo_lib) set_target_definitions_to_custom_os_and_arch(${ARGN}) endfunction() +if (CLR_CMAKE_TARGET_ARCH_WASM) + create_gcinfo_lib(TARGET gcinfo_unix_wasm OS browser ARCH wasm) + install_clr(TARGETS gcinfo_unix_wasm DESTINATIONS sharedFramework COMPONENT runtime) +endif() + +# Wasm needs the interpreter's GC-info encoder, but cannot run JITs or cross-compilers. +if (CLR_CMAKE_HOST_ARCH_WASM) + return() +endif() + if (CLR_CMAKE_TARGET_UNIX) set(TARGET_OS_NAME unix) else() @@ -92,7 +102,5 @@ if (CLR_CMAKE_TARGET_ARCH_I386 AND CLR_CMAKE_TARGET_UNIX) endif (CLR_CMAKE_TARGET_ARCH_I386 AND CLR_CMAKE_TARGET_UNIX) if (CLR_CMAKE_TARGET_ARCH_WASM) - create_gcinfo_lib(TARGET gcinfo_unix_wasm OS browser ARCH wasm) create_gcinfo_lib(TARGET gcinfo_universal_wasm OS universal ARCH wasm) - install_clr(TARGETS gcinfo_unix_wasm DESTINATIONS sharedFramework COMPONENT runtime) endif (CLR_CMAKE_TARGET_ARCH_WASM) diff --git a/src/coreclr/hosts/corerun/wasm/libCorerun.js b/src/coreclr/hosts/corerun/wasm/libCorerun.js index c2d8598af73c9a..0b067a06507152 100644 --- a/src/coreclr/hosts/corerun/wasm/libCorerun.js +++ b/src/coreclr/hosts/corerun/wasm/libCorerun.js @@ -229,7 +229,7 @@ function libCoreRunFactory() { const sp = stackSave(); try { const ptrPtr = stackAlloc(4); - if (_posix_memalign(ptrPtr, 16, payloadSize)) { + if (_posix_memalign(ptrPtr, 128, payloadSize)) { throw new Error("posix_memalign failed for Webcil payload"); } if (typeof (wasmExports.__stack_pointer) === "undefined") { diff --git a/src/coreclr/inc/clrnt.h b/src/coreclr/inc/clrnt.h index f0c72b682b465e..d6734290c1b439 100644 --- a/src/coreclr/inc/clrnt.h +++ b/src/coreclr/inc/clrnt.h @@ -513,6 +513,8 @@ RtlVirtualUnwind( #endif // TARGET_RISCV64 #ifdef TARGET_WASM +#include "daccess.h" + // // Define unwind information flags. // @@ -524,7 +526,7 @@ RtlVirtualUnwind( PEXCEPTION_ROUTINE RtlVirtualUnwind ( _In_ DWORD HandlerType, - _In_ DWORD ImageBase, + _In_ DWORD GCInfoBase, _In_ DWORD ControlPc, _In_ PRUNTIME_FUNCTION FunctionEntry, __inout PT_CONTEXT ContextRecord, @@ -538,16 +540,85 @@ UINT32 DecodeULEB128AsU32(PTR_BYTE* ppData); ULONG RtlpGetFunctionEndAddress ( _In_ PT_RUNTIME_FUNCTION FunctionEntry, - _In_ TADDR ImageBase + _In_ TADDR GCInfoBase ); -#define RUNTIME_FUNCTION__IsFunclet(FunctionEntry) ((FunctionEntry)->BeginAddress & 0x80000000) -#define RUNTIME_FUNCTION__BeginAddress(FunctionEntry) ((FunctionEntry)->BeginAddress & 0x7FFFFFFF) -#define RUNTIME_FUNCTION__SetBeginAddress(FunctionEntry,address) ((FunctionEntry)->BeginAddress = (address)) +struct RUNTIME_FUNCTION_GROUP +{ + // Keep the virtual-IP fields in the first cache line and cluster sparse fields for compression. + uint32_t VirtualIPBase; + uint32_t IsFuncletBits; + uint8_t HighBitsOfUnwindOffset[16]; + uint8_t Padding[8]; + uint8_t VirtualIPOffsets[32]; + uint8_t MidBitsOfUnwindOffset[32]; + uint8_t LowBitsOfUnwindOffset[32]; +}; + +static_assert(sizeof(RUNTIME_FUNCTION_GROUP) == 128); +static_assert(offsetof(RUNTIME_FUNCTION_GROUP, VirtualIPBase) == 0); +static_assert(offsetof(RUNTIME_FUNCTION_GROUP, IsFuncletBits) == 4); +static_assert(offsetof(RUNTIME_FUNCTION_GROUP, HighBitsOfUnwindOffset) == 8); +static_assert(offsetof(RUNTIME_FUNCTION_GROUP, Padding) == 24); +static_assert(offsetof(RUNTIME_FUNCTION_GROUP, VirtualIPOffsets) == 32); +static_assert(offsetof(RUNTIME_FUNCTION_GROUP, MidBitsOfUnwindOffset) == 64); +static_assert(offsetof(RUNTIME_FUNCTION_GROUP, LowBitsOfUnwindOffset) == 96); +static_assert(sizeof(T_RUNTIME_FUNCTION) == 4); + +inline DPTR(RUNTIME_FUNCTION_GROUP) GetRUNTIME_FUNCTION_GROUP_AND_INDEX( + DPTR(T_RUNTIME_FUNCTION) function, uint32_t* index) +{ + LIMITED_METHOD_DAC_CONTRACT; + TADDR address = dac_cast(function); + *index = static_cast((address & 0x7F) >> 2); + return dac_cast(address & ~static_cast(0x7F)); +} -#define RUNTIME_FUNCTION__EndAddress(FunctionEntry, ImageBase) (RtlpGetFunctionEndAddress(FunctionEntry, (ULONG64)(ImageBase))) +inline bool RUNTIME_FUNCTION__Skip(DPTR(T_RUNTIME_FUNCTION) function) +{ + LIMITED_METHOD_DAC_CONTRACT; + uint32_t index; + DPTR(RUNTIME_FUNCTION_GROUP) group = GetRUNTIME_FUNCTION_GROUP_AND_INDEX(function, &index); + return group->VirtualIPOffsets[index] == 0xFF; +} -#define RUNTIME_FUNCTION__SetUnwindInfoAddress(prf,address) do { (prf)->UnwindData = (address); } while (0) +inline bool RUNTIME_FUNCTION__IsFunclet(DPTR(T_RUNTIME_FUNCTION) function) +{ + LIMITED_METHOD_DAC_CONTRACT; + uint32_t index; + DPTR(RUNTIME_FUNCTION_GROUP) group = GetRUNTIME_FUNCTION_GROUP_AND_INDEX(function, &index); + return (group->IsFuncletBits & (uint32_t(1) << index)) != 0; +} + +inline uint32_t RUNTIME_FUNCTION__BeginAddress(DPTR(T_RUNTIME_FUNCTION) function) +{ + LIMITED_METHOD_DAC_CONTRACT; + uint32_t index; + DPTR(RUNTIME_FUNCTION_GROUP) group = GetRUNTIME_FUNCTION_GROUP_AND_INDEX(function, &index); + _ASSERTE(group->VirtualIPOffsets[index] != 0xFF); + return group->VirtualIPBase + group->VirtualIPOffsets[index] + index * 4; +} + +inline uint32_t RUNTIME_FUNCTION__GetUnwindDataOffset(DPTR(T_RUNTIME_FUNCTION) function) +{ + LIMITED_METHOD_DAC_CONTRACT; + uint32_t index; + DPTR(RUNTIME_FUNCTION_GROUP) group = GetRUNTIME_FUNCTION_GROUP_AND_INDEX(function, &index); + _ASSERTE(group->VirtualIPOffsets[index] != 0xFF); + uint32_t high = (group->HighBitsOfUnwindOffset[index / 2] >> ((index & 1) * 4)) & 0xF; + uint32_t offset = (uint32_t(group->MidBitsOfUnwindOffset[index]) << 8) | + uint32_t(group->LowBitsOfUnwindOffset[index]); + if (high == 0xF) + { + _ASSERTE(index < 31 && group->VirtualIPOffsets[31] == 0xFF); + return (uint32_t(group->MidBitsOfUnwindOffset[31]) << 24) | + (uint32_t(group->LowBitsOfUnwindOffset[31]) << 16) | offset; + } + + return (high << 16) | offset; +} + +#define RUNTIME_FUNCTION__EndAddress(FunctionEntry, GCInfoBase) (RtlpGetFunctionEndAddress(FunctionEntry, (TADDR)(GCInfoBase))) typedef struct _UNWIND_INFO { // dummy diff --git a/src/coreclr/inc/corcompile.h b/src/coreclr/inc/corcompile.h index 16b688eaa57e2e..a83ccf077eb501 100644 --- a/src/coreclr/inc/corcompile.h +++ b/src/coreclr/inc/corcompile.h @@ -57,8 +57,7 @@ inline ReadyToRunCrossModuleInlineFlags operator &( const ReadyToRunCrossModuleI } #ifdef TARGET_WASM -// why was it defined only for x86 before? -typedef DPTR(RUNTIME_FUNCTION) PTR_RUNTIME_FUNCTION; +typedef DPTR(T_RUNTIME_FUNCTION) PTR_RUNTIME_FUNCTION; #endif #ifdef TARGET_X86 diff --git a/src/coreclr/inc/crosscomp.h b/src/coreclr/inc/crosscomp.h index de0829acd50938..953b01a3987f56 100644 --- a/src/coreclr/inc/crosscomp.h +++ b/src/coreclr/inc/crosscomp.h @@ -713,8 +713,14 @@ typedef struct _KNONVOLATILE_CONTEXT_POINTERS_EX #define T_KNONVOLATILE_CONTEXT_POINTERS KNONVOLATILE_CONTEXT_POINTERS #define PT_KNONVOLATILE_CONTEXT_POINTERS PKNONVOLATILE_CONTEXT_POINTERS +#if defined(TARGET_WASM) && !defined(HOST_WASM) +typedef struct _T_RUNTIME_FUNCTION { + DWORD Opaque; +} T_RUNTIME_FUNCTION, *PT_RUNTIME_FUNCTION; +#else #define T_RUNTIME_FUNCTION RUNTIME_FUNCTION #define PT_RUNTIME_FUNCTION PRUNTIME_FUNCTION +#endif // TARGET_WASM && !HOST_WASM #endif diff --git a/src/coreclr/inc/readytorun.h b/src/coreclr/inc/readytorun.h index 63ef3b699aae3a..23e08ca865077b 100644 --- a/src/coreclr/inc/readytorun.h +++ b/src/coreclr/inc/readytorun.h @@ -156,6 +156,7 @@ enum class ReadyToRunSectionType : uint32_t ProxyTypeMaps = 125, // Added in V18.3 TypeMapAssemblyTargets = 126, // Added in V18.3 WasmAsyncResumeInfo = 127, // Added in V29 + WasmGCInfo = 128, // If you add a new section consider whether it is a breaking or non-breaking change. // Usually it is non-breaking, but if it is preferable to have older runtimes fail diff --git a/src/coreclr/jit/CMakeLists.txt b/src/coreclr/jit/CMakeLists.txt index 3999593c2bbca1..511e7a7c3b44ba 100644 --- a/src/coreclr/jit/CMakeLists.txt +++ b/src/coreclr/jit/CMakeLists.txt @@ -1,5 +1,9 @@ set(CMAKE_INCLUDE_CURRENT_DIR ON) +if (CLR_CMAKE_HOST_ARCH_WASM) + return() +endif() + include_directories("./jitstd") include_directories("../inc") include_directories("../jitshared") diff --git a/src/coreclr/jit/gcencode.cpp b/src/coreclr/jit/gcencode.cpp index 4630cd276227ad..f1d74e4b1e06a4 100644 --- a/src/coreclr/jit/gcencode.cpp +++ b/src/coreclr/jit/gcencode.cpp @@ -4278,6 +4278,13 @@ void GCInfo::gcMakeRegPtrTable( flags = (GcSlotFlags)(flags | GC_SLOT_INTERIOR); } +#ifdef TARGET_WASM + // Pin value-type fields like scalar locals: Wasm filters and their parents + // can report the same slots, which must not be relocated twice. + // See dotnet/runtime#131374, dotnet/runtime#133932, and dotnet/runtime#134769. + flags = static_cast(flags | GC_SLOT_PINNED); +#endif // TARGET_WASM + GcStackSlotBase stackSlotBase = GC_SP_REL; if (varDsc->lvFramePointerBased) { diff --git a/src/coreclr/jit/unwindwasm.cpp b/src/coreclr/jit/unwindwasm.cpp index 51b310b581068a..15261257fb98dc 100644 --- a/src/coreclr/jit/unwindwasm.cpp +++ b/src/coreclr/jit/unwindwasm.cpp @@ -61,6 +61,7 @@ void Compiler::unwindReserveFunc(FuncInfoDsc* func) bool isFunclet = func->IsFunclet(); bool isColdCode = false; assert(func->endVirtualIP > func->startVirtualIP); + assert(func->endVirtualIP - func->startVirtualIP >= 2); ULONG encodedSize = emitter::SizeOfULEB128(func->funWasmFrameSize) + emitter::SizeOfULEB128(func->endVirtualIP - func->startVirtualIP); @@ -113,6 +114,7 @@ void Compiler::unwindEmitFunc(FuncInfoDsc* func, void* pHotCode, void* pColdCode uint8_t buffer[10]; size_t index = 0; assert(func->endVirtualIP > func->startVirtualIP); + assert(func->endVirtualIP - func->startVirtualIP >= 2); index += GetEmitter()->emitOutputULEB128(buffer + index, func->funWasmFrameSize); index += GetEmitter()->emitOutputULEB128(buffer + index, func->endVirtualIP - func->startVirtualIP); assert(index <= sizeof(buffer)); diff --git a/src/coreclr/pal/inc/pal.h b/src/coreclr/pal/inc/pal.h index ba0ec4bf8793b7..e07c232ea3e74e 100644 --- a/src/coreclr/pal/inc/pal.h +++ b/src/coreclr/pal/inc/pal.h @@ -2488,11 +2488,15 @@ enum { typedef IMAGE_ARM64_RUNTIME_FUNCTION_ENTRY RUNTIME_FUNCTION, *PRUNTIME_FUNCTION; #else // HOST_ARM64 typedef struct _RUNTIME_FUNCTION { +#ifdef HOST_WASM + DWORD Opaque; +#else // HOST_WASM DWORD BeginAddress; #ifdef HOST_AMD64 DWORD EndAddress; #endif DWORD UnwindData; +#endif // HOST_WASM } RUNTIME_FUNCTION, *PRUNTIME_FUNCTION; #endif // HOST_ARM64 diff --git a/src/coreclr/tools/Common/Compiler/ObjectWriter/WasmObjectWriter.cs b/src/coreclr/tools/Common/Compiler/ObjectWriter/WasmObjectWriter.cs index f491b8c6155c9a..b3af1864e5592c 100644 --- a/src/coreclr/tools/Common/Compiler/ObjectWriter/WasmObjectWriter.cs +++ b/src/coreclr/tools/Common/Compiler/ObjectWriter/WasmObjectWriter.cs @@ -61,6 +61,9 @@ internal abstract partial class WasmObjectWriter : ObjectWriter private readonly WasmFunctionBodyDeduplicator _functionBodyDeduplicator = new(); private readonly Dictionary _tableSlotIndices = []; private readonly List _tableSlotFunctions = []; +#if READYTORUN + private int? _runtimeFunctionTableStart; +#endif private readonly List _foldedFunctionAliases = []; /// /// Maps symbol names to their location in the object file. These definitions do not encode @@ -131,6 +134,12 @@ public override void EmitObject( _functionBodyDeduplicator.Prepare(nodes, _nodeFactory, ShouldSkip); } base.EmitObject(outputFileStream, nodes, dumper, logger); +#if READYTORUN + if (_nodeFactory.RuntimeFunctionsTable is { } runtimeFunctions) + { + logger.LogMessage($"Wasm runtime functions: {runtimeFunctions.WasmRuntimeFunctionCount} slots, {runtimeFunctions.WasmSkipCount - runtimeFunctions.WasmAlignmentSkipCount} skips, {runtimeFunctions.WasmAlignmentSkipCount} final alignment skips."); + } +#endif bool ShouldSkip(ObjectNode node) { @@ -242,6 +251,7 @@ private protected override void RecordMethodDefinition(INodeWithTypeSignature no // diagnostic output, but engine-level samples necessarily use the canonical name. _wasmSymbolManager.AddAlias(methodName, canonicalName); RegisterTableSlot(methodName, canonicalName); + RecordRuntimeFunctionPadding((ObjectNode)node, 0, canonicalName); if (!alternateName.IsNull) { @@ -255,6 +265,7 @@ private protected override void RecordMethodDefinition(INodeWithTypeSignature no WriteSignatureIndexForFunction(node); RegisterFunctionSymbol(methodName); RegisterTableSlot(methodName, methodName); + RecordRuntimeFunctionPadding((ObjectNode)node, 0, methodName); if (!alternateName.IsNull) { @@ -338,10 +349,27 @@ private void RecordFunclets(INodeWithFunclets nodeWithFunclets) Utf8String funcletName = GetFuncletName(new Utf8String(mangledNodeName), i); RegisterFunctionSymbol(funcletName); RegisterTableSlot(funcletName, funcletName); + RecordRuntimeFunctionPadding((ObjectNode)nodeWithFunclets, i + 1, funcletName); RegisterStubIndexAndSignature(funcletSignature); } } + private void RecordRuntimeFunctionPadding(ObjectNode node, int frameIndex, Utf8String functionName) + { +#if READYTORUN + if (node is DependencyAnalysis.ReadyToRun.MethodWithGCInfo method) + { + int count = _nodeFactory.RuntimeFunctionsTable.GetWasmPaddingAfter(method, frameIndex, out int ordinal); + _runtimeFunctionTableStart ??= _tableSlotFunctions.Count - 1; + Debug.Assert(_tableSlotFunctions.Count - 1 - _runtimeFunctionTableStart.Value == ordinal); + for (int i = 0; i < count; i++) + { + _tableSlotFunctions.Add(functionName); + } + } +#endif + } + private protected static Utf8String GetFuncletName(Utf8String methodName, int index) => new($"{methodName}_funclet_{index}"); diff --git a/src/coreclr/tools/Common/Internal/Runtime/ModuleHeaders.cs b/src/coreclr/tools/Common/Internal/Runtime/ModuleHeaders.cs index 5c2bd42f340552..35bf19de33e561 100644 --- a/src/coreclr/tools/Common/Internal/Runtime/ModuleHeaders.cs +++ b/src/coreclr/tools/Common/Internal/Runtime/ModuleHeaders.cs @@ -85,6 +85,7 @@ enum ReadyToRunSectionType ProxyTypeMaps = 125, // Added to CoreCLR in V18.3 TypeMapAssemblyTargets = 126, // Added in V18.3 WasmAsyncResumeInfo = 127, // Added in V29 + WasmGCInfo = 128, // // NativeAOT ReadyToRun sections diff --git a/src/coreclr/tools/aot/ILCompiler.ReadyToRun.Tests/TestCases/R2RTestSuites.cs b/src/coreclr/tools/aot/ILCompiler.ReadyToRun.Tests/TestCases/R2RTestSuites.cs index 52e9e298f3684b..e47c23801f1140 100644 --- a/src/coreclr/tools/aot/ILCompiler.ReadyToRun.Tests/TestCases/R2RTestSuites.cs +++ b/src/coreclr/tools/aot/ILCompiler.ReadyToRun.Tests/TestCases/R2RTestSuites.cs @@ -419,6 +419,43 @@ void Validate(ReadyToRunReader reader) } } + [ConditionalTheory(typeof(TestPaths), nameof(TestPaths.IsWasmTarget))] + [InlineData(false)] + [InlineData(true)] + public void WasmStructGCFieldsArePinned(bool optimize) + { + CompiledAssembly input = new() + { + AssemblyName = nameof(WasmStructGCFieldsArePinned), + SourceResourceNames = ["Webcil/WasmWebcilModule.cs"], + }; + + new R2RTestRunner(_output).Run(new R2RTestCase( + nameof(WasmStructGCFieldsArePinned), + [ + new(nameof(WasmStructGCFieldsArePinned), [new CrossgenAssembly(input)]) + { + Options = optimize ? [Crossgen2Option.Optimize] : [], + AdditionalArgs = optimize ? [] : ["--optimize-disabled"], + OutputFileExtension = ".wasm", + Validate = reader => + { + ReadyToRunMethod method = Assert.Single(R2RAssert.GetAllMethods(reader), + method => method.SignatureString.Contains(".StructGCFields(", StringComparison.Ordinal)); + ILCompiler.Reflection.ReadyToRun.Amd64.GcInfo gcInfo = + Assert.IsType(method.GcInfo); + Assert.True(gcInfo.SlotTable.NumUntracked >= 2); + Assert.Contains(gcInfo.SlotTable.GcSlots, + slot => (slot.Flags & GcSlotFlags.GC_SLOT_INTERIOR) != 0); + Assert.Contains(gcInfo.SlotTable.GcSlots, + slot => (slot.Flags & GcSlotFlags.GC_SLOT_INTERIOR) == 0); + Assert.All(gcInfo.SlotTable.GcSlots, slot => + Assert.Equal(GcSlotFlags.GC_SLOT_PINNED, slot.Flags & GcSlotFlags.GC_SLOT_PINNED)); + }, + }, + ])); + } + [ConditionalFact(typeof(TestPaths), nameof(TestPaths.IsWasmTarget))] public void WasmWebcilModule() { @@ -458,6 +495,7 @@ public void WasmWebcilModule() static void ValidateDefault(ReadyToRunReader reader) { + WasmR2RAssert.AssertRuntimeFunctionGroups(reader); var webcilReader = Assert.IsType(reader.CompositeReader); WasmR2RAssert.AssertFrameIdentityPrologs(webcilReader); Assert.True(webcilReader.IsWasmWrapped); diff --git a/src/coreclr/tools/aot/ILCompiler.ReadyToRun.Tests/TestCases/Webcil/WasmWebcilModule.cs b/src/coreclr/tools/aot/ILCompiler.ReadyToRun.Tests/TestCases/Webcil/WasmWebcilModule.cs index 222bc392d603df..d906967e476eb4 100644 --- a/src/coreclr/tools/aot/ILCompiler.ReadyToRun.Tests/TestCases/Webcil/WasmWebcilModule.cs +++ b/src/coreclr/tools/aot/ILCompiler.ReadyToRun.Tests/TestCases/Webcil/WasmWebcilModule.cs @@ -22,6 +22,30 @@ public struct GenericTokenValue public int Value; } + public ref struct GCFields + { + public object Reference; + public Span ByRef; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + public static int StructGCFields() + { + GCFields fields = new() + { + Reference = new object(), + ByRef = new int[] { 42 }, + }; + return ReadStructGCFields(ref fields); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static int ReadStructGCFields(ref GCFields fields) + { + GC.Collect(); + return fields.Reference.GetHashCode() ^ fields.ByRef[0]; + } + [MethodImpl(MethodImplOptions.NoInlining)] public static Type ReadIntrinsicTypeToken(int index) => index switch { diff --git a/src/coreclr/tools/aot/ILCompiler.ReadyToRun.Tests/TestCasesRunner/WasmR2RAssert.cs b/src/coreclr/tools/aot/ILCompiler.ReadyToRun.Tests/TestCasesRunner/WasmR2RAssert.cs index e14499a5ea25f1..a6474e40427042 100644 --- a/src/coreclr/tools/aot/ILCompiler.ReadyToRun.Tests/TestCasesRunner/WasmR2RAssert.cs +++ b/src/coreclr/tools/aot/ILCompiler.ReadyToRun.Tests/TestCasesRunner/WasmR2RAssert.cs @@ -22,6 +22,128 @@ namespace ILCompiler.ReadyToRun.Tests.TestCasesRunner; internal static class WasmR2RAssert { + public static void AssertRuntimeFunctionGroups(ReadyToRunReader reader) + { + ReadyToRunSection section = reader.ReadyToRunHeader.Sections[ReadyToRunSectionType.RuntimeFunctions]; + ReadyToRunSection gcInfo = reader.ReadyToRunHeader.Sections[ReadyToRunSectionType.WasmGCInfo]; + Assert.Equal(0, section.RelativeVirtualAddress & 127); + Assert.Equal(0, section.Size & 127); + ReadOnlySpan image = reader.Image; + int tableOffset = reader.CompositeReader.GetOffset(section.RelativeVirtualAddress); + Dictionary entries = new(); + uint? previousBegin = null; + + for (int offset = 0; offset < section.Size; offset += 128) + { + ReadOnlySpan group = image.Slice(tableOffset + offset, 128); + Assert.True(group.Slice(24, 8).SequenceEqual(new byte[8])); + uint baseIP = BinaryPrimitives.ReadUInt32LittleEndian(group); + uint funcletBits = BinaryPrimitives.ReadUInt32LittleEndian(group.Slice(4)); + bool seenSkip = false; + bool previousIsFunclet = false; + for (int index = 0; index < 32; index++) + { + bool isFunclet = (funcletBits & (1u << index)) != 0; + if (group[32 + index] == byte.MaxValue) + { + Assert.NotEqual(0, index); + Assert.Equal(previousIsFunclet, isFunclet); + seenSkip = true; + continue; + } + + Assert.False(seenSkip); + uint begin = checked(baseIP + group[32 + index] + (uint)index * 4); + if (previousBegin.HasValue) + { + Assert.True(begin >= previousBegin.Value + 4); + } + previousBegin = begin; + previousIsFunclet = isFunclet; + uint high = (uint)(group[8 + index / 2] >> ((index & 1) * 4)) & 0xF; + uint unwind = ((uint)group[64 + index] << 8) | group[96 + index]; + if (high == 0xF) + { + Assert.NotEqual(31, index); + Assert.Equal(byte.MaxValue, group[32 + 31]); + unwind |= ((uint)group[64 + 31] << 24) | ((uint)group[96 + 31] << 16); + Assert.True(unwind >= 0xF0000); + } + else + { + unwind |= high << 16; + } + Assert.InRange(unwind, 0u, checked((uint)gcInfo.Size - 1)); + entries.Add(offset / 4 + index, + (checked((int)begin), checked(gcInfo.RelativeVirtualAddress + (int)unwind), isFunclet)); + } + } + + foreach (ReadyToRunMethod method in R2RAssert.GetAllMethods(reader)) + { + foreach (RuntimeFunction function in method.RuntimeFunctions) + { + Assert.True(entries.TryGetValue(function.Id, out var expected)); + Assert.Equal(expected.Begin, function.StartAddress); + Assert.Equal(expected.Unwind, function.UnwindRVA); + Assert.Equal(expected.IsFunclet, function.WasmIsFunclet); + } + } + + AssertRuntimeFunctionEncodingBoundaries(reader, tableOffset); + } + + private static void AssertRuntimeFunctionEncodingBoundaries(ReadyToRunReader reader, int tableOffset) + { + byte[] original = reader.Image.AsSpan(tableOffset, 128).ToArray(); + try + { + foreach (int index in new[] { 0, 1, 15, 16, 30, 31 }) + { + foreach (uint unwind in new uint[] { 0, 0xEFFFF, 0xF0000, 0xFFFFF, 0x100000, 0x12345678 }) + { + Span group = reader.Image.AsSpan(tableOffset, 128); + group.Clear(); + BinaryPrimitives.WriteUInt32LittleEndian(group, 0x100); + BinaryPrimitives.WriteUInt32LittleEndian(group.Slice(4), 1u << index); + bool escaped = unwind >= 0xF0000; + uint high = escaped ? 0xF : unwind >> 16; + group[8 + index / 2] = (byte)(high << ((index & 1) * 4)); + group[64 + index] = (byte)(unwind >> 8); + group[96 + index] = (byte)unwind; + if (escaped && index != 31) + { + group[32 + 31] = byte.MaxValue; + group[64 + 31] = (byte)(unwind >> 24); + group[96 + 31] = (byte)(unwind >> 16); + } + + if (escaped && index == 31) + { + Assert.Throws(() => + reader.TryReadWasmRuntimeFunction(index, out _, out _, out _)); + continue; + } + + Assert.True(reader.TryReadWasmRuntimeFunction(index, out int begin, out int unwindRva, out bool isFunclet)); + Assert.Equal(0x100 + index * 4, begin); + Assert.Equal(reader.ReadyToRunHeader.Sections[ReadyToRunSectionType.WasmGCInfo].RelativeVirtualAddress + (int)unwind, unwindRva); + Assert.True(isFunclet); + group[32 + index] = byte.MaxValue; + Assert.False(reader.TryReadWasmRuntimeFunction(index, out _, out _, out _)); + } + } + + Assert.Throws(() => reader.TryReadWasmRuntimeFunction(-1, out _, out _, out _)); + int entryCount = reader.ReadyToRunHeader.Sections[ReadyToRunSectionType.RuntimeFunctions].Size / 4; + Assert.Throws(() => reader.TryReadWasmRuntimeFunction(entryCount, out _, out _, out _)); + } + finally + { + original.CopyTo(reader.Image.AsSpan(tableOffset, 128)); + } + } + public static void AssertFrameIdentityPrologs(WebcilImageReader reader) { ReadOnlySpan image = reader.GetEntireImage().AsSpan(); diff --git a/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/DependencyAnalysis/ArrayOfEmbeddedDataNode.cs b/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/DependencyAnalysis/ArrayOfEmbeddedDataNode.cs index 8fadcc5f3beb0f..926d8a72f8b678 100644 --- a/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/DependencyAnalysis/ArrayOfEmbeddedDataNode.cs +++ b/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/DependencyAnalysis/ArrayOfEmbeddedDataNode.cs @@ -77,13 +77,18 @@ protected virtual void GetElementDataForNodes(ref ObjectDataBuilder builder, Nod protected virtual int GetAlignmentRequirement(NodeFactory factory) { return factory.Target.PointerSize; } + protected virtual void SortNodes(List nodes, NodeFactory factory) + { + if (_sorter != null) + nodes.MergeSort(_sorter); + } + public override ObjectData GetData(NodeFactory factory, bool relocsOnly) { ObjectDataBuilder builder = new ObjectDataBuilder(factory, relocsOnly); builder.RequireInitialAlignment(GetAlignmentRequirement(factory)); - if (_sorter != null) - _nestedNodesList.MergeSort(_sorter); + SortNodes(_nestedNodesList, factory); builder.AddSymbol(this); diff --git a/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/DependencyAnalysis/ReadyToRun/MethodGCInfoNode.cs b/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/DependencyAnalysis/ReadyToRun/MethodGCInfoNode.cs index fe8c2ae0a5f3a4..1678cae99a147a 100644 --- a/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/DependencyAnalysis/ReadyToRun/MethodGCInfoNode.cs +++ b/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/DependencyAnalysis/ReadyToRun/MethodGCInfoNode.cs @@ -23,6 +23,18 @@ public MethodGCInfoNode(MethodWithGCInfo methodNode) public override int ClassCode => 892356612; + public int GetWasmDataSize() + { + Debug.Assert(_methodNode.ColdCodeNode is null); + int size = _methodNode.GCInfo?.Length ?? 0; + foreach (FrameInfo frame in _methodNode.FrameInfos) + { + size = checked(size + frame.BlobData.Length); + } + + return size; + } + public void AppendMangledName(NameMangler nameMangler, Utf8StringBuilder sb) { sb.Append("MethodGCInfoNode->"u8); diff --git a/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/DependencyAnalysis/ReadyToRun/RuntimeFunctionsGCInfoNode.cs b/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/DependencyAnalysis/ReadyToRun/RuntimeFunctionsGCInfoNode.cs index aad3718edb6938..9dbaf6e55fdf70 100644 --- a/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/DependencyAnalysis/ReadyToRun/RuntimeFunctionsGCInfoNode.cs +++ b/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/DependencyAnalysis/ReadyToRun/RuntimeFunctionsGCInfoNode.cs @@ -13,6 +13,51 @@ public RuntimeFunctionsGCInfoNode() } public HashSet Deduplicator; + private ObjectData _wasmData; + + public override ObjectData GetData(NodeFactory factory, bool relocsOnly) + { + if (!factory.Target.IsWasm || relocsOnly) + { + return base.GetData(factory, relocsOnly); + } + + return _wasmData ??= base.GetData(factory, relocsOnly); + } + + protected override void SortNodes(List nodes, NodeFactory factory) + { + if (!factory.Target.IsWasm || nodes.Count < 2) + { + base.SortNodes(nodes, factory); + return; + } + + Dictionary duplicationCounts = new Dictionary(nodes.Count, Deduplicator.Comparer); + foreach (MethodGCInfoNode node in nodes) + { + duplicationCounts.TryGetValue(node, out int count); + duplicationCounts[node] = count + 1; + } + + Dictionary frequencies = new Dictionary(nodes.Count); + Dictionary sizes = new Dictionary(nodes.Count); + foreach (MethodGCInfoNode node in nodes) + { + frequencies.Add(node, duplicationCounts[node]); + sizes.Add(node, node.GetWasmDataSize()); + } + + nodes.MergeSort((x, y) => + { + int result = frequencies[y].CompareTo(frequencies[x]); + if (result == 0) + { + result = sizes[x].CompareTo(sizes[y]); + } + return result != 0 ? result : CompilerComparer.Instance.Compare(x, y); + }); + } protected internal override int Phase => (int)ObjectNodePhase.Ordered; diff --git a/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/DependencyAnalysis/ReadyToRun/RuntimeFunctionsTableNode.cs b/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/DependencyAnalysis/ReadyToRun/RuntimeFunctionsTableNode.cs index f6967aad85279c..9a2120aec0f9f6 100644 --- a/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/DependencyAnalysis/ReadyToRun/RuntimeFunctionsTableNode.cs +++ b/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/DependencyAnalysis/ReadyToRun/RuntimeFunctionsTableNode.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. using System; +using System.Buffers.Binary; using System.Collections.Generic; using ILCompiler.ObjectWriter; @@ -19,6 +20,31 @@ public class RuntimeFunctionsTableNode : HeaderTableNode private Dictionary<(MethodWithGCInfo method, int funcletIdx), uint> _methodToVirtualIP; private readonly NodeFactory _nodeFactory; private int _tableSize = -1; + private const int WasmGroupSize = 128; + private const int WasmGroupEntryCount = 32; + private const uint WasmUnwindEscapeStart = 0xF0000; + private readonly List _wasmRuntimeFunctions = new(); + private readonly Dictionary<(MethodWithGCInfo, int), int> _wasmFrameIndices = new(); + + private readonly struct WasmRuntimeFunction + { + public WasmRuntimeFunction(uint virtualIP, uint unwindOffset, bool isFunclet, bool isSkip = false) + { + VirtualIP = virtualIP; + UnwindOffset = unwindOffset; + IsFunclet = isFunclet; + IsSkip = isSkip; + } + + public readonly uint VirtualIP; + public readonly uint UnwindOffset; + public readonly bool IsFunclet; + public readonly bool IsSkip; + } + + public int WasmSkipCount { get; private set; } + public int WasmAlignmentSkipCount { get; private set; } + public int WasmRuntimeFunctionCount => _wasmRuntimeFunctions.Count; public RuntimeFunctionsTableNode(NodeFactory nodeFactory) { @@ -50,6 +76,10 @@ public int GetIndex(MethodWithGCInfo method) private void LayoutRuntimeFunctions() { + if (_nodeFactory.Target.IsWasm) + { + _nodeFactory.RuntimeFunctionsGCInfo.GetData(_nodeFactory, relocsOnly: false); + } _methodNodes = new List(); _insertedMethodNodes = new Dictionary(); @@ -61,47 +91,114 @@ private void LayoutRuntimeFunctions() _insertedMethodNodes[method] = runtimeFunctionIndex; runtimeFunctionIndex += method.FrameInfos.Length; } + + if (_nodeFactory.Target.IsWasm) + { + LayoutWasmRuntimeFunctions(); + } } - public uint GetWasmVirtualIP(MethodWithGCInfo lookupMethod, int lookupFrameIndex) + private void LayoutWasmRuntimeFunctions() { - bool isWasm = _nodeFactory.Target.Architecture == TargetArchitecture.Wasm32; + _methodToVirtualIP = new(); + uint virtualIP = 0; + uint groupBase = 0; + uint? unwindPrefix = null; + bool precedingIsFunclet = false; - if (!isWasm) + foreach (MethodWithGCInfo method in _methodNodes) { - // For non-WASM targets, virtual IPs are not used, so throw - throw new InvalidOperationException("Virtual IPs are not used on non-WASM targets."); + Debug.Assert(method.ColdCodeNode is null); + int[] unwindOffsets = method.GCInfoNode.CalculateFuncletOffsets(_nodeFactory); + for (int frame = 0; frame < method.FrameInfos.Length; frame++) + { + uint unwindOffset = checked((uint)unwindOffsets[frame]); + int index = _wasmRuntimeFunctions.Count % WasmGroupEntryCount; + if (index == 0) + { + groupBase = virtualIP; + unwindPrefix = null; + } + + bool escapes = unwindOffset >= WasmUnwindEscapeStart; + long residual = (long)virtualIP - groupBase - index * 4; + Debug.Assert(residual >= 0); + if (residual < 0) + { + throw new InvalidOperationException("Wasm runtime functions must have at least four virtual IP units."); + } + if (residual >= byte.MaxValue || + (index == WasmGroupEntryCount - 1 && (escapes || unwindPrefix.HasValue)) || + (escapes && unwindPrefix.HasValue && unwindPrefix.Value != (unwindOffset >> 16))) + { + PadGroup(); + groupBase = virtualIP; + unwindPrefix = null; + } + + if (escapes) + { + unwindPrefix = unwindOffset >> 16; + } + + int ordinal = _wasmRuntimeFunctions.Count; + if (frame == 0) + { + _insertedMethodNodes[method] = ordinal; + } + _wasmFrameIndices.Add((method, frame), ordinal); + _methodToVirtualIP.Add((method, frame), virtualIP); + precedingIsFunclet = frame != 0; + _wasmRuntimeFunctions.Add(new WasmRuntimeFunction(virtualIP, unwindOffset, precedingIsFunclet)); + virtualIP = checked(virtualIP + GetFunctionLocalVirtualIPCountFromUnwindBlob(method.FrameInfos[frame].BlobData)); + } } - if (_methodToVirtualIP != null) + int skipsBeforeFinalPadding = WasmSkipCount; + PadGroup(); + WasmAlignmentSkipCount = WasmSkipCount - skipsBeforeFinalPadding; + + void PadGroup() { - return _methodToVirtualIP[(lookupMethod, lookupFrameIndex)]; + while (_wasmRuntimeFunctions.Count % WasmGroupEntryCount != 0) + { + _wasmRuntimeFunctions.Add(new WasmRuntimeFunction(0, 0, precedingIsFunclet, isSkip: true)); + WasmSkipCount++; + } } + } - if (_methodNodes == null) + public int GetWasmPaddingAfter(MethodWithGCInfo method, int frame, out int ordinal) + { + GetIndex(method); + ordinal = _wasmFrameIndices[(method, frame)]; + int next = ordinal + 1; + int count = 0; + while (next < _wasmRuntimeFunctions.Count && _wasmRuntimeFunctions[next].IsSkip) { - LayoutRuntimeFunctions(); + count++; + next++; } - Dictionary<(MethodWithGCInfo method, int funcletIdx), uint> methodToVirtualIP = new (); + return count; + } - // For WASM, track virtual IP index for each RUNTIME_FUNCTION entry - uint currentVirtualIP = 0; + public uint GetWasmVirtualIP(MethodWithGCInfo lookupMethod, int lookupFrameIndex) + { + bool isWasm = _nodeFactory.Target.Architecture == TargetArchitecture.Wasm32; - foreach (MethodWithGCInfo method in _methodNodes) + if (!isWasm) { - for (int frameIndex = 0; frameIndex < method.FrameInfos.Length; frameIndex++) - { - FrameInfo frameInfo = method.FrameInfos[frameIndex]; - methodToVirtualIP.Add((method, frameIndex), currentVirtualIP); + // For non-WASM targets, virtual IPs are not used, so throw + throw new InvalidOperationException("Virtual IPs are not used on non-WASM targets."); + } - // Advance the virtual IP by the number of virtual IPs for this frame - uint virtualIPCount = GetFunctionLocalVirtualIPCountFromUnwindBlob(frameInfo.BlobData); - currentVirtualIP += virtualIPCount; - } + if (_methodToVirtualIP != null) + { + return _methodToVirtualIP[(lookupMethod, lookupFrameIndex)]; } - _methodToVirtualIP = methodToVirtualIP; + LayoutRuntimeFunctions(); return _methodToVirtualIP[(lookupMethod, lookupFrameIndex)]; } @@ -123,6 +220,11 @@ public override ObjectData GetData(NodeFactory factory, bool relocsOnly = false) if (_methodNodes == null) LayoutRuntimeFunctions(); + if (factory.Target.IsWasm) + { + return GetWasmData(factory); + } + ObjectDataBuilder runtimeFunctionsBuilder = new ObjectDataBuilder(factory, relocsOnly); runtimeFunctionsBuilder.RequireInitialAlignment(4); @@ -132,8 +234,6 @@ public override ObjectData GetData(NodeFactory factory, bool relocsOnly = false) uint runtimeFunctionIndex = 0; List mapping = new List(); - bool isWasm = _nodeFactory.Target.Architecture == TargetArchitecture.Wasm32; - for (int cold = 0; cold < 2; cold++) { foreach (MethodWithGCInfo method in _methodNodes) @@ -180,20 +280,12 @@ public override ObjectData GetData(NodeFactory factory, bool relocsOnly = false) symbol = method; } - if (isWasm) + runtimeFunctionsBuilder.EmitReloc(symbol, RelocType.IMAGE_REL_BASED_ADDR32NB, frameInfo.StartOffset); + if (!relocsOnly && _nodeFactory.Target.Architecture == TargetArchitecture.X64) { - // Set high bit for frame indices greater than 0 to indicate that the RUNTIME_FUNCTION is a funclet - runtimeFunctionsBuilder.EmitUInt(GetWasmVirtualIP(method, frameIndex) | (frameIndex != 0 ? 0x80000000 : 0)); - } - else - { - runtimeFunctionsBuilder.EmitReloc(symbol, RelocType.IMAGE_REL_BASED_ADDR32NB, frameInfo.StartOffset); - if (!relocsOnly && _nodeFactory.Target.Architecture == TargetArchitecture.X64) - { - // On Amd64, the 2nd word contains the EndOffset of the runtime function - Debug.Assert(frameInfo.StartOffset != frameInfo.EndOffset); - runtimeFunctionsBuilder.EmitReloc(symbol, RelocType.IMAGE_REL_BASED_ADDR32NB, delta: frameInfo.EndOffset); - } + // On Amd64, the 2nd word contains the EndOffset of the runtime function + Debug.Assert(frameInfo.StartOffset != frameInfo.EndOffset); + runtimeFunctionsBuilder.EmitReloc(symbol, RelocType.IMAGE_REL_BASED_ADDR32NB, delta: frameInfo.EndOffset); } runtimeFunctionsBuilder.EmitReloc(factory.RuntimeFunctionsGCInfo, RelocType.IMAGE_REL_BASED_ADDR32NB, funcletOffsets[frameIndex]); runtimeFunctionIndex++; @@ -214,23 +306,63 @@ public override ObjectData GetData(NodeFactory factory, bool relocsOnly = false) // Emit sentinel entry runtimeFunctionsBuilder.EmitUInt(~0u); - if (isWasm) + _tableSize = runtimeFunctionsBuilder.CountBytes; + return runtimeFunctionsBuilder.ToObjectData(); + } + + private ObjectData GetWasmData(NodeFactory factory) + { + ObjectDataBuilder builder = new ObjectDataBuilder(factory, relocsOnly: false); + builder.RequireInitialAlignment(WasmGroupSize); + builder.AddSymbol(this); + + for (int start = 0; start < _wasmRuntimeFunctions.Count; start += WasmGroupEntryCount) { - // After sentinel, emit a WASM_TABLE_INDEX_I32 reloc pointing to the first - // compiled method. This records the min function table index in the file. - if (_methodNodes.Count > 0) + byte[] group = new byte[WasmGroupSize]; + uint virtualIPBase = _wasmRuntimeFunctions[start].VirtualIP; + BinaryPrimitives.WriteUInt32LittleEndian(group, virtualIPBase); + uint funcletBits = 0; + for (int index = 0; index < WasmGroupEntryCount; index++) { - runtimeFunctionsBuilder.EmitReloc(_methodNodes[0], RelocType.WASM_TABLE_INDEX_I32, delta: 0); - } - else - { - // If there are no compiled methods, emit a 0 for the min table index. - runtimeFunctionsBuilder.EmitUInt(0); + WasmRuntimeFunction function = _wasmRuntimeFunctions[start + index]; + group[32 + index] = function.IsSkip ? byte.MaxValue : + checked((byte)(function.VirtualIP - virtualIPBase - (uint)index * 4)); + if (function.IsFunclet) + { + funcletBits |= 1u << index; + } + if (function.IsSkip) + { + continue; + } + + uint high = function.UnwindOffset >> 16; + if (function.UnwindOffset >= WasmUnwindEscapeStart) + { + Debug.Assert(_wasmRuntimeFunctions[start + WasmGroupEntryCount - 1].IsSkip); + group[64 + WasmGroupEntryCount - 1] = (byte)(function.UnwindOffset >> 24); + group[96 + WasmGroupEntryCount - 1] = (byte)(function.UnwindOffset >> 16); + high = 0xF; + } + group[8 + index / 2] |= (byte)(high << ((index & 1) * 4)); + group[64 + index] = (byte)(function.UnwindOffset >> 8); + group[96 + index] = (byte)function.UnwindOffset; } + BinaryPrimitives.WriteUInt32LittleEndian(group.AsSpan(4), funcletBits); + builder.EmitBytes(group); } - _tableSize = runtimeFunctionsBuilder.CountBytes; - return runtimeFunctionsBuilder.ToObjectData(); + builder.EmitUInt(uint.MaxValue); + if (_methodNodes.Count != 0) + { + builder.EmitReloc(_methodNodes[0], RelocType.WASM_TABLE_INDEX_I32); + } + else + { + builder.EmitUInt(0); + } + _tableSize = builder.CountBytes; + return builder.ToObjectData(); } /// diff --git a/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/DependencyAnalysis/ReadyToRunCodegenNodeFactory.cs b/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/DependencyAnalysis/ReadyToRunCodegenNodeFactory.cs index b24cfff205429f..6709ba1459ce34 100644 --- a/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/DependencyAnalysis/ReadyToRunCodegenNodeFactory.cs +++ b/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/DependencyAnalysis/ReadyToRunCodegenNodeFactory.cs @@ -1102,6 +1102,11 @@ public void AttachToDependencyGraph(DependencyAnalyzerBase graph, I RuntimeFunctionsGCInfo = new RuntimeFunctionsGCInfoNode(); graph.AddRoot(RuntimeFunctionsGCInfo, "GC info is always generated"); + if (Target.IsWasm) + { + Header.Add(Internal.Runtime.ReadyToRunSectionType.WasmGCInfo, RuntimeFunctionsGCInfo); + } + if (!Target.IsWasm) { DelayLoadMethodCallThunks = new SymbolNodeRange("DelayLoadMethodCallThunkNodeRange"); diff --git a/src/coreclr/tools/aot/ILCompiler.Reflection.ReadyToRun/ILCompiler.Reflection.ReadyToRun.csproj b/src/coreclr/tools/aot/ILCompiler.Reflection.ReadyToRun/ILCompiler.Reflection.ReadyToRun.csproj index f52a184f66d211..21b22337ee7e8b 100644 --- a/src/coreclr/tools/aot/ILCompiler.Reflection.ReadyToRun/ILCompiler.Reflection.ReadyToRun.csproj +++ b/src/coreclr/tools/aot/ILCompiler.Reflection.ReadyToRun/ILCompiler.Reflection.ReadyToRun.csproj @@ -32,4 +32,8 @@ + + + + diff --git a/src/coreclr/tools/aot/ILCompiler.Reflection.ReadyToRun/ReadyToRunMethod.cs b/src/coreclr/tools/aot/ILCompiler.Reflection.ReadyToRun/ReadyToRunMethod.cs index 6a014fbeab897b..e76d04ef01db51 100644 --- a/src/coreclr/tools/aot/ILCompiler.Reflection.ReadyToRun/ReadyToRunMethod.cs +++ b/src/coreclr/tools/aot/ILCompiler.Reflection.ReadyToRun/ReadyToRunMethod.cs @@ -636,26 +636,32 @@ private void ParseRuntimeFunctions(bool partial) curOffset = coldOffset; runtimeFunctionId = coldRuntimeFunctionId; } - int startRva = _readyToRunReader.ImageReader.ReadInt32(ref curOffset); + int startRva; + int unwindRva; bool isFunclet = false; - if (_readyToRunReader.Machine == Machine.ArmThumb2) + int endRva = -1; + if (_readyToRunReader.Machine == WasmMachine.Wasm32) { - // The low bit of this address is set since the function contains thumb code. - // Clear this bit in order to get the "real" RVA of the start of the function. - startRva = (int)(startRva & ~1); + while (!_readyToRunReader.TryReadWasmRuntimeFunction(runtimeFunctionId, out startRva, out unwindRva, out isFunclet)) + { + runtimeFunctionId++; + } } - else if (_readyToRunReader.Machine == WasmMachine.Wasm32) + else { - // On WASM, bit 31 is the funclet flag and bits 30:0 are the virtual IP. - isFunclet = (startRva & unchecked((int)0x80000000)) != 0; - startRva = (int)(startRva & 0x7FFFFFFF); - } - int endRva = -1; - if (_readyToRunReader.Machine == Machine.Amd64) - { - endRva = _readyToRunReader.ImageReader.ReadInt32(ref curOffset); + startRva = _readyToRunReader.ImageReader.ReadInt32(ref curOffset); + if (_readyToRunReader.Machine == Machine.ArmThumb2) + { + // The low bit of this address is set since the function contains thumb code. + // Clear this bit in order to get the "real" RVA of the start of the function. + startRva = (int)(startRva & ~1); + } + if (_readyToRunReader.Machine == Machine.Amd64) + { + endRva = _readyToRunReader.ImageReader.ReadInt32(ref curOffset); + } + unwindRva = _readyToRunReader.ImageReader.ReadInt32(ref curOffset); } - int unwindRva = _readyToRunReader.ImageReader.ReadInt32(ref curOffset); int unwindOffset = _readyToRunReader.CompositeReader.GetOffset(unwindRva); BaseUnwindInfo unwindInfo = null; diff --git a/src/coreclr/tools/aot/ILCompiler.Reflection.ReadyToRun/ReadyToRunReader.cs b/src/coreclr/tools/aot/ILCompiler.Reflection.ReadyToRun/ReadyToRunReader.cs index 7a57773a932dcb..8b76f109202cdc 100644 --- a/src/coreclr/tools/aot/ILCompiler.Reflection.ReadyToRun/ReadyToRunReader.cs +++ b/src/coreclr/tools/aot/ILCompiler.Reflection.ReadyToRun/ReadyToRunReader.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. using System; +using System.Buffers.Binary; using System.Collections.Generic; using System.Collections.Immutable; using System.IO; @@ -834,6 +835,10 @@ private unsafe void EnsureExceptionInfo() /// internal int CalculateRuntimeFunctionSize() { + if (Machine == WasmMachine.Wasm32) + { + return sizeof(int); + } if (Machine == Machine.Amd64) { return 3 * sizeof(int); @@ -841,6 +846,45 @@ internal int CalculateRuntimeFunctionSize() return 2 * sizeof(int); } + internal bool TryReadWasmRuntimeFunction(int ordinal, out int beginAddress, out int unwindRva, out bool isFunclet) + { + ReadyToRunSection section = ReadyToRunHeader.Sections[ReadyToRunSectionType.RuntimeFunctions]; + if ((section.Size & 127) != 0 || ordinal < 0 || ordinal >= section.Size / sizeof(int)) + { + throw new BadImageFormatException("Invalid Wasm runtime function ordinal or group size."); + } + + int groupOffset = checked(CompositeReader.GetOffset(section.RelativeVirtualAddress) + (ordinal / 32) * 128); + ReadOnlySpan group = Image.AsSpan(groupOffset, 128); + int index = ordinal & 31; + beginAddress = 0; + unwindRva = 0; + isFunclet = (BinaryPrimitives.ReadUInt32LittleEndian(group.Slice(4)) & (1u << index)) != 0; + if (group[32 + index] == byte.MaxValue) + { + return false; + } + + beginAddress = checked((int)BinaryPrimitives.ReadUInt32LittleEndian(group) + group[32 + index] + index * 4); + uint high = (uint)(group[8 + index / 2] >> ((index & 1) * 4)) & 0xF; + uint unwindOffset = ((uint)group[64 + index] << 8) | group[96 + index]; + if (high == 0xF) + { + if (index == 31 || group[32 + 31] != byte.MaxValue) + { + throw new BadImageFormatException("Invalid Wasm runtime function unwind escape."); + } + unwindOffset |= ((uint)group[64 + 31] << 24) | ((uint)group[96 + 31] << 16); + } + else + { + unwindOffset |= high << 16; + } + + unwindRva = checked(ReadyToRunHeader.Sections[ReadyToRunSectionType.WasmGCInfo].RelativeVirtualAddress + (int)unwindOffset); + return true; + } + private uint? _wasmMinFunctionTableIndex; /// @@ -1235,7 +1279,19 @@ private void MarkResumptionStubEntryPoints(bool[] isEntryPoint, ReadyToRunSectio for (int i = 0; i < nRuntimeFunctions; i++) { int entryOffset = runtimeFunctionsOffset + i * runtimeFunctionSize; - uint beginAddress = BitConverter.ToUInt32(Image, entryOffset); + uint beginAddress; + if (Machine == WasmMachine.Wasm32) + { + if (!TryReadWasmRuntimeFunction(i, out int virtualIP, out _, out _)) + { + continue; + } + beginAddress = checked((uint)virtualIP); + } + else + { + beginAddress = BitConverter.ToUInt32(Image, entryOffset); + } stubRvaToMethodIndexMap[beginAddress] = i; } @@ -1291,7 +1347,10 @@ private void CountRuntimeFunctions(bool[] isEntryPoint, IDictionary int i = runtimeFunctionId; do { - count++; + if (Machine != WasmMachine.Wasm32 || TryReadWasmRuntimeFunction(i, out _, out _, out _)) + { + count++; + } i++; } while (i < isEntryPoint.Length && !isEntryPoint[i] && i < firstColdRuntimeFunction); diff --git a/src/coreclr/vm/codeman.cpp b/src/coreclr/vm/codeman.cpp index a039d39815a26b..505938f721c486 100644 --- a/src/coreclr/vm/codeman.cpp +++ b/src/coreclr/vm/codeman.cpp @@ -1201,12 +1201,12 @@ void IJitManager::EnumMemoryRegions(CLRDataEnumMemoryFlags flags) #endif // #ifdef DACCESS_COMPILE -PTR_VOID GetUnwindDataBlob(TADDR moduleBase, PTR_RUNTIME_FUNCTION pRuntimeFunction, /* out */ SIZE_T * pSize) +PTR_VOID GetUnwindDataBlob(TADDR unwindInfoBase, PTR_RUNTIME_FUNCTION pRuntimeFunction, /* out */ SIZE_T * pSize) { LIMITED_METHOD_CONTRACT; #if defined(TARGET_AMD64) - PTR_UNWIND_INFO pUnwindInfo(dac_cast(moduleBase + RUNTIME_FUNCTION__GetUnwindInfoAddress(pRuntimeFunction))); + PTR_UNWIND_INFO pUnwindInfo(dac_cast(unwindInfoBase + RUNTIME_FUNCTION__GetUnwindInfoAddress(pRuntimeFunction))); *pSize = ALIGN_UP(offsetof(UNWIND_INFO, UnwindCode) + sizeof(UNWIND_CODE) * pUnwindInfo->CountOfUnwindCodes + @@ -1216,7 +1216,7 @@ PTR_VOID GetUnwindDataBlob(TADDR moduleBase, PTR_RUNTIME_FUNCTION pRuntimeFuncti return pUnwindInfo; #elif defined(TARGET_X86) - PTR_UNWIND_INFO pUnwindInfo(dac_cast(moduleBase + RUNTIME_FUNCTION__GetUnwindInfoAddress(pRuntimeFunction))); + PTR_UNWIND_INFO pUnwindInfo(dac_cast(unwindInfoBase + RUNTIME_FUNCTION__GetUnwindInfoAddress(pRuntimeFunction))); *pSize = sizeof(UNWIND_INFO); @@ -1229,7 +1229,7 @@ PTR_VOID GetUnwindDataBlob(TADDR moduleBase, PTR_RUNTIME_FUNCTION pRuntimeFuncti _ASSERTE((pRuntimeFunction->UnwindData & 0x3) == 0); // compute the size of the unwind info - PTR_DWORD xdata = dac_cast(pRuntimeFunction->UnwindData + moduleBase); + PTR_DWORD xdata = dac_cast(pRuntimeFunction->UnwindData + unwindInfoBase); int size = 4; #if defined(TARGET_ARM) @@ -1268,7 +1268,7 @@ PTR_VOID GetUnwindDataBlob(TADDR moduleBase, PTR_RUNTIME_FUNCTION pRuntimeFuncti _ASSERTE((pRuntimeFunction->UnwindData & 0x3) == 0); // compute the size of the unwind info - PTR_ULONG xdata = dac_cast(pRuntimeFunction->UnwindData + moduleBase); + PTR_ULONG xdata = dac_cast(pRuntimeFunction->UnwindData + unwindInfoBase); ULONG epilogScopes = 0; ULONG unwindWords = 0; ULONG size = 0; @@ -1301,7 +1301,7 @@ PTR_VOID GetUnwindDataBlob(TADDR moduleBase, PTR_RUNTIME_FUNCTION pRuntimeFuncti return xdata; #elif defined(TARGET_WASM) - PTR_BYTE pUnwindInfo(dac_cast(moduleBase + pRuntimeFunction->UnwindData)); + PTR_BYTE pUnwindInfo(dac_cast(unwindInfoBase + RUNTIME_FUNCTION__GetUnwindDataOffset(pRuntimeFunction))); PTR_BYTE pUnwindData = pUnwindInfo; DecodeULEB128AsU32(&pUnwindData); // Skip the count of bytes to unwind DecodeULEB128AsU32(&pUnwindData); // Read the function length in virtual IP units @@ -5962,10 +5962,14 @@ TADDR ExecutionManager::GetWasmVirtualIPFromFunctionTableIndex(DWORD functionInd ReadyToRunInfo* pR2RInfo = pModule->GetReadyToRunInfo(); DWORD localIndex = functionIndex - pSection->minFunctionTableIndex; + if (RUNTIME_FUNCTION__Skip(pR2RInfo->GetRuntimeFunctions() + localIndex)) + { + return 0; + } do { PTR_RUNTIME_FUNCTION pRuntimeFunction = pR2RInfo->GetRuntimeFunctions() + localIndex; - if (RUNTIME_FUNCTION__IsFunclet(pRuntimeFunction)) + if (RUNTIME_FUNCTION__Skip(pRuntimeFunction) || RUNTIME_FUNCTION__IsFunclet(pRuntimeFunction)) { // Funclets are handled specially. Their RUNTIME_FUNCTION BeginAddress field describes the virtual IP of the funclet, but // function local virtual ips are relative to the virtual IP of the controlling function, and so the logical VirtualIPFromFunctionTableIndex @@ -5994,6 +5998,7 @@ TADDR ExecutionManager::GetWasmVirtualIPFromCoreLibRuntimeFunctionIndex(DWORD fu _ASSERTE(pR2RInfo->GetMinVirtualIP() == WasmInitialVirtualIP); _ASSERTE(functionIndex < pR2RInfo->GetRuntimeFunctionCount()); PTR_RUNTIME_FUNCTION pRuntimeFunction = pR2RInfo->GetRuntimeFunctions() + functionIndex; + _ASSERTE(!RUNTIME_FUNCTION__Skip(pRuntimeFunction)); _ASSERTE(!RUNTIME_FUNCTION__IsFunclet(pRuntimeFunction)); return WasmInitialVirtualIP + RUNTIME_FUNCTION__BeginAddress(pRuntimeFunction); } @@ -6036,29 +6041,14 @@ TADDR ExecutionManager::GetWasmFunctionTableIndexFromVirtualIP(TADDR virtualIP) DWORD localVirtualIP32 = (DWORD)localVirtualIP; PTR_RUNTIME_FUNCTION pRuntimeFunctions = pR2RInfo->GetRuntimeFunctions(); - // Find the function entry with the greatest BeginAddress <= localVirtualIP. - DWORD low = 0; - DWORD high = runtimeFunctionCount; - while (low + 1 < high) - { - DWORD mid = low + ((high - low) / 2); - DWORD beginAddress = RUNTIME_FUNCTION__BeginAddress(pRuntimeFunctions + mid); - if (beginAddress <= localVirtualIP32) - { - low = mid; - } - else - { - high = mid; - } - } - - if (RUNTIME_FUNCTION__BeginAddress(pRuntimeFunctions + low) > localVirtualIP32) + int index = NativeUnwindInfoLookupTable::LookupUnwindInfoForMethod( + localVirtualIP32, pRuntimeFunctions, 0, runtimeFunctionCount - 1); + if (index < 0) { return 0; } - return pR2RInfo->GetMinFunctionTableIndex() + low; + return pR2RInfo->GetMinFunctionTableIndex() + index; } #endif // TARGET_WASM @@ -6534,6 +6524,11 @@ static void GetFuncletStartOffsetsHelper(PCODE pCodeStart, SIZE_T size, SIZE_T o // static void EnumRuntimeFunctionEntriesToFindEntry(PTR_RUNTIME_FUNCTION pRtf, PTR_RUNTIME_FUNCTION pFirstFunctionEntry, UINT32 numFunctionEntries) { +#ifdef TARGET_WASM + // Group decoding and backward skip scans need more than the logical entry's four bytes. + DacEnumMemoryRegion(dac_cast(pFirstFunctionEntry), numFunctionEntries * sizeof(T_RUNTIME_FUNCTION)); + return; +#else // TARGET_WASM pRtf.EnumMem(); if (pFirstFunctionEntry == NULL || numFunctionEntries == 0) @@ -6574,6 +6569,7 @@ static void EnumRuntimeFunctionEntriesToFindEntry(PTR_RUNTIME_FUNCTION pRtf, PTR mid = (low + high) / 2; _ASSERTE(low <= mid && mid <= high); } +#endif // TARGET_WASM } #endif // DACCESS_COMPILE @@ -6650,6 +6646,36 @@ int NativeUnwindInfoLookupTable::LookupUnwindInfoForMethod(DWORD RelativePc, SUPPORTS_DAC; } CONTRACTL_END; +#ifdef TARGET_WASM + int first = Low; + int end = High + 1; + while (Low < end) + { + int middle = Low + (end - Low) / 2; + int candidate = middle; + while (candidate >= first && RUNTIME_FUNCTION__Skip(pRuntimeFunctionTable + candidate)) + { + candidate--; + } + + if (candidate >= first && RUNTIME_FUNCTION__BeginAddress(pRuntimeFunctionTable + candidate) <= RelativePc) + { + Low = middle + 1; + } + else + { + end = middle; + } + } + + int result = Low - 1; + while (result >= first && RUNTIME_FUNCTION__Skip(pRuntimeFunctionTable + result)) + { + result--; + } + + return result; +#else // TARGET_WASM #ifdef TARGET_ARM RelativePc |= THUMB_CODE; @@ -6695,6 +6721,7 @@ int NativeUnwindInfoLookupTable::LookupUnwindInfoForMethod(DWORD RelativePc, } return -1; +#endif // TARGET_WASM } #ifdef FEATURE_COLD_R2R_CODE @@ -6847,7 +6874,11 @@ GCInfoToken ReadyToRunJitManager::GetGCInfoToken(const METHODTOKEN& MethodToken) } CONTRACTL_END; PTR_RUNTIME_FUNCTION pRuntimeFunction = JitTokenToRuntimeFunction(MethodToken); +#ifdef TARGET_WASM + TADDR baseAddress = JitTokenToReadyToRunInfo(MethodToken)->GetGCInfoBase(); +#else // TARGET_WASM TADDR baseAddress = JitTokenToModuleRVABase(MethodToken); +#endif // TARGET_WASM SIZE_T nUnwindDataSize; PTR_VOID pUnwindData = GetUnwindDataBlob(baseAddress, pRuntimeFunction, &nUnwindDataSize); @@ -7234,7 +7265,11 @@ BOOL ReadyToRunJitManager::JitCodeToMethodInfo(RangeSection * pRangeSection, #endif // FEATURE_COLD_R2R_CODE MethodDesc *pMethodDesc; - while ((pMethodDesc = pInfo->GetMethodDescForEntryPoint(ImageBase + RUNTIME_FUNCTION__BeginAddress(pRuntimeFunctions + MethodIndex))) == NULL) + while ( +#ifdef TARGET_WASM + RUNTIME_FUNCTION__Skip(pRuntimeFunctions + MethodIndex) || +#endif // TARGET_WASM + (pMethodDesc = pInfo->GetMethodDescForEntryPoint(ImageBase + RUNTIME_FUNCTION__BeginAddress(pRuntimeFunctions + MethodIndex))) == NULL) MethodIndex--; PTR_RUNTIME_FUNCTION FunctionEntry = pRuntimeFunctions + MethodIndex; @@ -7296,6 +7331,31 @@ TADDR ReadyToRunJitManager::GetFuncletStartAddress(EECodeInfo * pCodeInfo) DWORD ReadyToRunJitManager::GetFuncletStartOffsets(const METHODTOKEN& MethodToken, DWORD* pStartFuncletOffsets, DWORD dwLength) { +#ifdef TARGET_WASM + LIMITED_METHOD_DAC_CONTRACT; + ReadyToRunInfo* info = JitTokenToReadyToRunInfo(MethodToken); + PTR_RUNTIME_FUNCTION main = JitTokenToRuntimeFunction(MethodToken); + PTR_RUNTIME_FUNCTION end = info->GetRuntimeFunctions() + info->GetRuntimeFunctionCount(); + DWORD count = 0; + for (PTR_RUNTIME_FUNCTION entry = main + 1; entry < end; entry++) + { + if (RUNTIME_FUNCTION__Skip(entry)) + { + continue; + } + if (!RUNTIME_FUNCTION__IsFunclet(entry)) + { + break; + } + if (count < dwLength) + { + pStartFuncletOffsets[count] = RUNTIME_FUNCTION__BeginAddress(entry) - RUNTIME_FUNCTION__BeginAddress(main); + } + count++; + } + + return count; +#else // TARGET_WASM PTR_RUNTIME_FUNCTION pFirstFuncletFunctionEntry = dac_cast(MethodToken.m_pCodeHeader) + 1; TADDR moduleBase = JitTokenToModuleFunctionsBase(MethodToken); @@ -7334,6 +7394,7 @@ DWORD ReadyToRunJitManager::GetFuncletStartOffsets(const METHODTOKEN& MethodToke } return nFunclets; +#endif // TARGET_WASM } BOOL ReadyToRunJitManager::LazyIsFunclet(EECodeInfo* pCodeInfo) @@ -7358,7 +7419,7 @@ BOOL ReadyToRunJitManager::LazyIsFunclet(EECodeInfo* pCodeInfo) { // This maps to a hot entry in the lookup table, so check its unwind info SIZE_T unwindSize; - PTR_VOID pUnwindData = GetUnwindDataBlob(pCodeInfo->GetModuleBase(), pCodeInfo->GetFunctionEntry(), &unwindSize); + PTR_VOID pUnwindData = GetUnwindDataBlob(pCodeInfo->GetUnwindInfoBase(), pCodeInfo->GetFunctionEntry(), &unwindSize); _ASSERTE(pUnwindData != NULL); #ifdef TARGET_AMD64 @@ -7399,7 +7460,7 @@ BOOL ReadyToRunJitManager::IsFilterFunclet(EECodeInfo * pCodeInfo) #else // Get address of the personality routine for the function being queried. SIZE_T size; - PTR_VOID pUnwindData = GetUnwindDataBlob(pCodeInfo->GetModuleBase(), pCodeInfo->GetFunctionEntry(), &size); + PTR_VOID pUnwindData = GetUnwindDataBlob(pCodeInfo->GetUnwindInfoBase(), pCodeInfo->GetFunctionEntry(), &size); _ASSERTE(pUnwindData != NULL); // Personality routine is always the last element of the unwind data @@ -7410,7 +7471,7 @@ BOOL ReadyToRunJitManager::IsFilterFunclet(EECodeInfo * pCodeInfo) if (pInfo->m_nRuntimeFunctions == 0) return FALSE; - PTR_VOID pFirstUnwindData = GetUnwindDataBlob(pCodeInfo->GetModuleBase(), pInfo->m_pRuntimeFunctions, &size); + PTR_VOID pFirstUnwindData = GetUnwindDataBlob(pCodeInfo->GetUnwindInfoBase(), pInfo->m_pRuntimeFunctions, &size); _ASSERTE(pFirstUnwindData != NULL); DWORD rvaFirstPersonalityRoutine = *(dac_cast(dac_cast(pFirstUnwindData) + size) - 1); @@ -7519,7 +7580,7 @@ void ReadyToRunJitManager::EnumMemoryRegionsForMethodUnwindInfo(CLRDataEnumMemor EnumRuntimeFunctionEntriesToFindEntry(pRtf, pReadyToRunInfo->m_pRuntimeFunctions, pReadyToRunInfo->m_nRuntimeFunctions); SIZE_T size; - PTR_VOID pUnwindData = GetUnwindDataBlob(pCodeInfo->GetModuleBase(), pRtf, &size); + PTR_VOID pUnwindData = GetUnwindDataBlob(pCodeInfo->GetUnwindInfoBase(), pRtf, &size); if (pUnwindData != NULL) DacEnumMemoryRegion(PTR_TO_TADDR(pUnwindData), size); } diff --git a/src/coreclr/vm/codeman.h b/src/coreclr/vm/codeman.h index 4a4d294f12fa7a..54118b4d4ec311 100644 --- a/src/coreclr/vm/codeman.h +++ b/src/coreclr/vm/codeman.h @@ -3259,6 +3259,16 @@ class EECodeInfo return GetJitManager()->JitTokenToModuleFunctionsBase(GetMethodToken()); } +#ifdef TARGET_WASM + TADDR GetUnwindInfoBase(); +#else // TARGET_WASM + TADDR GetUnwindInfoBase() + { + WRAPPER_NO_CONTRACT; + return GetModuleBase(); + } +#endif // TARGET_WASM + PTR_RUNTIME_FUNCTION GetFunctionEntry(); BOOL IsFunclet(); EECodeInfo GetMainFunctionInfo(); diff --git a/src/coreclr/vm/datadescriptor/datadescriptor.inc b/src/coreclr/vm/datadescriptor/datadescriptor.inc index 3acee0c9a626f7..dbd9838f96187f 100644 --- a/src/coreclr/vm/datadescriptor/datadescriptor.inc +++ b/src/coreclr/vm/datadescriptor/datadescriptor.inc @@ -1042,11 +1042,13 @@ CDAC_TYPE_END(ImageDataDirectory) CDAC_TYPE_BEGIN(RuntimeFunction) CDAC_TYPE_SIZE(sizeof(RUNTIME_FUNCTION)) +#ifndef TARGET_WASM CDAC_TYPE_FIELD(RuntimeFunction, T_UINT32, BeginAddress, offsetof(RUNTIME_FUNCTION, BeginAddress)) #ifdef TARGET_AMD64 CDAC_TYPE_FIELD(RuntimeFunction, T_UINT32, EndAddress, offsetof(RUNTIME_FUNCTION, EndAddress)) #endif CDAC_TYPE_FIELD(RuntimeFunction, T_UINT32, UnwindData, offsetof(RUNTIME_FUNCTION, UnwindData)) +#endif // TARGET_WASM CDAC_TYPE_END(RuntimeFunction) CDAC_TYPE_BEGIN(UnwindInfo) diff --git a/src/coreclr/vm/jitinterface.cpp b/src/coreclr/vm/jitinterface.cpp index 5c3388339b4d50..dc445686843a99 100644 --- a/src/coreclr/vm/jitinterface.cpp +++ b/src/coreclr/vm/jitinterface.cpp @@ -11641,6 +11641,7 @@ void CEECodeGenInfo::CompressDebugInfo(PCODE nativeEntry, NativeCodeVersion nati EE_TO_JIT_TRANSITION(); } +#ifndef TARGET_WASM void reservePersonalityRoutineSpace(uint32_t &unwindSize) { #ifdef TARGET_AMD64 @@ -11862,6 +11863,8 @@ void CEEJitInfo::allocUnwindInfo ( EE_TO_JIT_TRANSITION(); } +#endif // !TARGET_WASM + void CEEJitInfo::recordCallSite(uint32_t instrOffset, CORINFO_SIG_INFO * callSig, CORINFO_METHOD_HANDLE methodHandle) @@ -15642,6 +15645,21 @@ PTR_RUNTIME_FUNCTION EECodeInfo::GetFunctionEntry() return m_pFunctionEntry; } +#ifdef TARGET_WASM +TADDR EECodeInfo::GetUnwindInfoBase() +{ + WRAPPER_NO_CONTRACT; + + const METHODTOKEN& methodToken = GetMethodToken(); + if ((methodToken.m_pRangeSection->_flags & RangeSection::RANGE_SECTION_VIRTUALIP) != 0) + { + return methodToken.m_pRangeSection->_pR2RModule->GetReadyToRunInfo()->GetGCInfoBase(); + } + + return GetModuleBase(); +} +#endif // TARGET_WASM + BOOL EECodeInfo::IsFunclet() { WRAPPER_NO_CONTRACT; diff --git a/src/coreclr/vm/jitinterface.h b/src/coreclr/vm/jitinterface.h index 7a77b0a884230f..6cae62511cc69f 100644 --- a/src/coreclr/vm/jitinterface.h +++ b/src/coreclr/vm/jitinterface.h @@ -687,6 +687,7 @@ class CEEJitInfo final : public CEECodeGenInfo void WriteCodeBytes(); void WriteCode(EECodeGenManager * jitMgr) override; +#ifndef TARGET_WASM void reserveUnwindInfo(bool isFunclet, bool isColdCode, uint32_t unwindSize) override; void allocUnwindInfo ( @@ -698,6 +699,7 @@ class CEEJitInfo final : public CEECodeGenInfo uint8_t * pUnwindBlock, /* IN */ CorJitFuncKind funcKind /* IN */ ) override; +#endif // !TARGET_WASM void recordCallSite( uint32_t instrOffset, /* IN */ diff --git a/src/coreclr/vm/readytoruninfo.cpp b/src/coreclr/vm/readytoruninfo.cpp index b0d0d865506f7a..25ae489679336f 100644 --- a/src/coreclr/vm/readytoruninfo.cpp +++ b/src/coreclr/vm/readytoruninfo.cpp @@ -922,6 +922,22 @@ ReadyToRunInfo::ReadyToRunInfo(Module * pModule, LoaderAllocator* pLoaderAllocat } #ifdef TARGET_WASM + if (pRuntimeFunctionsDir != nullptr && + ((pRuntimeFunctionsDir->Size % 128) != 0 || (dac_cast(m_pRuntimeFunctions) & 127) != 0)) + { + ThrowHR(COR_E_BADIMAGEFORMAT); + } + m_gcInfoBase = 0; + if (m_nRuntimeFunctions > 0) + { + IMAGE_DATA_DIRECTORY* pGCInfoDir = m_pComposite->FindSection(ReadyToRunSectionType::WasmGCInfo); + if (pGCInfoDir == nullptr) + { + ThrowHR(COR_E_BADIMAGEFORMAT); + } + m_gcInfoBase = dac_cast(m_pComposite->GetLayout()->GetDirectoryData(pGCInfoDir)); + } + // For WASM, the min function table index is stored as a u32 immediately after the // sentinel entry (0xFFFFFFFF) at the end of the RUNTIME_FUNCTION table. if (m_nRuntimeFunctions > 0) @@ -2969,11 +2985,15 @@ void ReadyToRunInfo::RegisterVirtualIPRange(Module* pModule) // The last RUNTIME_FUNCTION entry's BeginAddress is the virtual IP index of that entry. // Total virtual IPs = lastEntry.BeginAddress + virtualIPCount(lastEntry) T_RUNTIME_FUNCTION* pLastEntry = &m_pRuntimeFunctions[m_nRuntimeFunctions - 1]; + while (RUNTIME_FUNCTION__Skip(pLastEntry)) + { + pLastEntry--; + } UINT32 lastEntryVirtualIPIndex = RUNTIME_FUNCTION__BeginAddress(pLastEntry); // Decode the virtual IP count from the last entry's unwind data. // Unwind format: ULEB128(frameSize) ULEB128(virtualIPCount) - PTR_BYTE pUnwindData = dac_cast(imageBase + pLastEntry->UnwindData); + PTR_BYTE pUnwindData = dac_cast(GetGCInfoBase() + RUNTIME_FUNCTION__GetUnwindDataOffset(pLastEntry)); DecodeULEB128AsU32(&pUnwindData); // skip frame size UINT32 lastEntryVIPCount = DecodeULEB128AsU32(&pUnwindData) * 2; // Multiply by 2 to force all virtual IPs to be an even number. diff --git a/src/coreclr/vm/readytoruninfo.h b/src/coreclr/vm/readytoruninfo.h index 469ca418252297..56ffa4fd456163 100644 --- a/src/coreclr/vm/readytoruninfo.h +++ b/src/coreclr/vm/readytoruninfo.h @@ -157,6 +157,7 @@ class ReadyToRunInfo DWORD m_nRuntimeFunctions; #ifdef TARGET_WASM + TADDR m_gcInfoBase; DWORD m_minFunctionTableIndex; TADDR m_minVirtualIP = 0; #endif // TARGET_WASM @@ -234,6 +235,12 @@ class ReadyToRunInfo DWORD GetRuntimeFunctionCount() const { return m_nRuntimeFunctions; } #ifdef TARGET_WASM + TADDR GetGCInfoBase() const + { + LIMITED_METHOD_DAC_CONTRACT; + _ASSERTE(m_gcInfoBase != 0); + return m_gcInfoBase; + } DWORD GetMinFunctionTableIndex() const { return m_minFunctionTableIndex; } TADDR GetMinVirtualIP() const { diff --git a/src/coreclr/vm/stackwalk.cpp b/src/coreclr/vm/stackwalk.cpp index 785f992812936d..873b2cafa6e0e3 100644 --- a/src/coreclr/vm/stackwalk.cpp +++ b/src/coreclr/vm/stackwalk.cpp @@ -491,13 +491,13 @@ PCODE Thread::VirtualUnwindCallFrame(T_CONTEXT* pContext, codeInfo.Init(uControlPc); pFunctionEntry = codeInfo.GetFunctionEntry(); - uImageBase = (UINT_PTR)codeInfo.GetModuleBase(); + uImageBase = (UINT_PTR)codeInfo.GetUnwindInfoBase(); #endif // !TARGET_UNIX } else { pFunctionEntry = pCodeInfo->GetFunctionEntry(); - uImageBase = (UINT_PTR)pCodeInfo->GetModuleBase(); + uImageBase = (UINT_PTR)pCodeInfo->GetUnwindInfoBase(); // RUNTIME_FUNCTION of cold code just points to the RUNTIME_FUNCTION of hot code. The unwinder // expects this indirection to be resolved, so we use RUNTIME_FUNCTION of the hot code even diff --git a/src/coreclr/vm/wasm/helpers.cpp b/src/coreclr/vm/wasm/helpers.cpp index 6b57b3e8c29d8b..5e06b4eb684c68 100644 --- a/src/coreclr/vm/wasm/helpers.cpp +++ b/src/coreclr/vm/wasm/helpers.cpp @@ -627,10 +627,10 @@ void FlushWriteBarrierInstructionCache() ULONG RtlpGetFunctionEndAddress ( _In_ PT_RUNTIME_FUNCTION FunctionEntry, - _In_ TADDR ImageBase + _In_ TADDR GCInfoBase ) { - PTR_BYTE pUnwindData = dac_cast(FunctionEntry->UnwindData + ImageBase); + PTR_BYTE pUnwindData = dac_cast(RUNTIME_FUNCTION__GetUnwindDataOffset(FunctionEntry) + GCInfoBase); DecodeULEB128AsU32(&pUnwindData); // Skip the count of bytes to unwind uint32_t logicalVirtualIPLength = DecodeULEB128AsU32(&pUnwindData) * 2; // Read the function length in virtual IP units, then multiply by 2 to report them as even numbers. return logicalVirtualIPLength + RUNTIME_FUNCTION__BeginAddress(FunctionEntry); @@ -1874,7 +1874,7 @@ TADDR GetWasmVirtualIPFromStackPointer(TADDR sp) } } -static void WasmUnwindStackFrameCore(TADDR* pSP, TADDR* pIP, UINT_PTR ImageBase, PRUNTIME_FUNCTION FunctionEntry, bool callerIsNative = false) +static void WasmUnwindStackFrameCore(TADDR* pSP, TADDR* pIP, UINT_PTR GCInfoBase, PRUNTIME_FUNCTION FunctionEntry, bool callerIsNative = false) { TADDR sp = *pSP; TADDR fp = GetWasmFramePointerFromStackPointer_Internal(sp); @@ -1885,7 +1885,7 @@ static void WasmUnwindStackFrameCore(TADDR* pSP, TADDR* pIP, UINT_PTR ImageBase, } else { - PTR_BYTE pUnwindData = dac_cast(FunctionEntry->UnwindData + ImageBase); + PTR_BYTE pUnwindData = dac_cast(RUNTIME_FUNCTION__GetUnwindDataOffset(FunctionEntry) + GCInfoBase); *pSP = fp + DecodeULEB128AsU32(&pUnwindData); // Unwind the frame pointer to the callers stack pointer // A reverse-pinvoke frame's caller is native, not an R2R shadow frame; leave the IP unset so the walk // continues via the Frame chain instead of reading the native caller SP as a shadow frame. @@ -1909,7 +1909,7 @@ TADDR GetWasmFramePointerFromStackPointer(TADDR sp, PCODE controlPC) PEXCEPTION_ROUTINE RtlVirtualUnwind ( _In_ DWORD HandlerType, - _In_ DWORD ImageBase, + _In_ DWORD GCInfoBase, _In_ DWORD ControlPc, _In_ PRUNTIME_FUNCTION FunctionEntry, __inout PT_CONTEXT ContextRecord, @@ -1921,7 +1921,7 @@ RtlVirtualUnwind ( _ASSERTE(FunctionEntry != NULL); _ASSERTE(HandlerType == 0); _ASSERTE(ExecutionManager::IsVirtualIP(ControlPc)); - _ASSERTE(ImageBase != 0); + _ASSERTE(GCInfoBase != 0); // CoreCLR callers currently do not use HandlerData or EstablisherFrame on WASM, // so we set them to 0. If future callers require them, proper unwinding support @@ -1945,7 +1945,7 @@ RtlVirtualUnwind ( } } - WasmUnwindStackFrameCore((TADDR*)&ContextRecord->InterpreterSP, (TADDR*)&ContextRecord->InterpreterIP, ImageBase, FunctionEntry, callerIsNative); + WasmUnwindStackFrameCore((TADDR*)&ContextRecord->InterpreterSP, (TADDR*)&ContextRecord->InterpreterIP, GCInfoBase, FunctionEntry, callerIsNative); if (ContextRecord->InterpreterSP != 0) { diff --git a/src/native/libs/Common/JavaScript/host/assets.ts b/src/native/libs/Common/JavaScript/host/assets.ts index d3e12d1756aa08..667d792af5acfc 100644 --- a/src/native/libs/Common/JavaScript/host/assets.ts +++ b/src/native/libs/Common/JavaScript/host/assets.ts @@ -90,13 +90,13 @@ async function checkWebcilResponse(webcilPromise: Promise, virtualPath return res; } -// Allocates a 16-byte-aligned buffer for the Webcil payload. The pointer is heap memory that +// Allocates a 128-byte-aligned buffer for grouped R2R runtime functions. The pointer is heap memory that // outlives the stack frame, so it can be passed as the __memory_base import. function allocWebcilPayload(payloadSize: number): number { const sp = _ems_.stackSave(); try { const ptrPtr = _ems_.stackAlloc(sizeOfPtr); - if (_ems_._posix_memalign(ptrPtr as any, 16, payloadSize)) { + if (_ems_._posix_memalign(ptrPtr as any, 128, payloadSize)) { throw new Error("posix_memalign failed for Webcil payload"); } return _ems_.HEAPU32[ptrPtr as any >>> 2];