[Wasm] Compress ReadyToRun runtime-function tables

src/coreclr/gcinfo/CMakeLists.txt

@@ -52,6 +52,16 @@ function(create_gcinfo_lib)
   set_target_definitions_to_custom_os_and_arch(${ARGN})
 endfunction()
 
+if (CLR_CMAKE_TARGET_ARCH_WASM)
+  create_gcinfo_lib(TARGET gcinfo_unix_wasm OS browser ARCH wasm)
+  install_clr(TARGETS gcinfo_unix_wasm DESTINATIONS sharedFramework COMPONENT runtime)
+endif()
+
+# Wasm needs the interpreter's GC-info encoder, but cannot run JITs or cross-compilers.
+if (CLR_CMAKE_HOST_ARCH_WASM)
+  return()
+endif()
+
 if (CLR_CMAKE_TARGET_UNIX)
 set(TARGET_OS_NAME unix)
 else()
@@ -92,7 +102,5 @@ if (CLR_CMAKE_TARGET_ARCH_I386 AND CLR_CMAKE_TARGET_UNIX)
 endif (CLR_CMAKE_TARGET_ARCH_I386 AND CLR_CMAKE_TARGET_UNIX)
 
 if (CLR_CMAKE_TARGET_ARCH_WASM)
-  create_gcinfo_lib(TARGET gcinfo_unix_wasm OS browser ARCH wasm)
   create_gcinfo_lib(TARGET gcinfo_universal_wasm OS universal ARCH wasm)
-  install_clr(TARGETS gcinfo_unix_wasm DESTINATIONS sharedFramework COMPONENT runtime)
 endif (CLR_CMAKE_TARGET_ARCH_WASM)

src/coreclr/hosts/corerun/wasm/libCorerun.js

@@ -229,7 +229,7 @@ function libCoreRunFactory() {
             const sp = stackSave();
             try {
                 const ptrPtr = stackAlloc(4);
-                if (_posix_memalign(ptrPtr, 16, payloadSize)) {
+                if (_posix_memalign(ptrPtr, 128, payloadSize)) {
                     throw new Error("posix_memalign failed for Webcil payload");
                 }
                 if (typeof (wasmExports.__stack_pointer) === "undefined") {

src/coreclr/inc/clrnt.h

@@ -513,6 +513,8 @@ RtlVirtualUnwind(
 #endif // TARGET_RISCV64
 
 #ifdef TARGET_WASM
+#include "daccess.h"
+
 //
 // Define unwind information flags.
 //
@@ -524,7 +526,7 @@ RtlVirtualUnwind(
 PEXCEPTION_ROUTINE
 RtlVirtualUnwind (
     _In_ DWORD HandlerType,
-    _In_ DWORD ImageBase,
+    _In_ DWORD GCInfoBase,
     _In_ DWORD ControlPc,
     _In_ PRUNTIME_FUNCTION FunctionEntry,
     __inout PT_CONTEXT ContextRecord,
@@ -538,16 +540,85 @@ UINT32 DecodeULEB128AsU32(PTR_BYTE* ppData);
 ULONG
 RtlpGetFunctionEndAddress (
     _In_ PT_RUNTIME_FUNCTION FunctionEntry,
-    _In_ TADDR ImageBase
+    _In_ TADDR GCInfoBase
     );
 
-#define RUNTIME_FUNCTION__IsFunclet(FunctionEntry) ((FunctionEntry)->BeginAddress & 0x80000000)
-#define RUNTIME_FUNCTION__BeginAddress(FunctionEntry)               ((FunctionEntry)->BeginAddress & 0x7FFFFFFF)
-#define RUNTIME_FUNCTION__SetBeginAddress(FunctionEntry,address)    ((FunctionEntry)->BeginAddress = (address))
+struct RUNTIME_FUNCTION_GROUP
+{
+    // Keep the virtual-IP fields in the first cache line and cluster sparse fields for compression.
+    uint32_t VirtualIPBase;
+    uint32_t IsFuncletBits;
+    uint8_t HighBitsOfUnwindOffset[16];
+    uint8_t Padding[8];
+    uint8_t VirtualIPOffsets[32];
+    uint8_t MidBitsOfUnwindOffset[32];
+    uint8_t LowBitsOfUnwindOffset[32];
+};
+
+static_assert(sizeof(RUNTIME_FUNCTION_GROUP) == 128);
+static_assert(offsetof(RUNTIME_FUNCTION_GROUP, VirtualIPBase) == 0);
+static_assert(offsetof(RUNTIME_FUNCTION_GROUP, IsFuncletBits) == 4);
+static_assert(offsetof(RUNTIME_FUNCTION_GROUP, HighBitsOfUnwindOffset) == 8);
+static_assert(offsetof(RUNTIME_FUNCTION_GROUP, Padding) == 24);
+static_assert(offsetof(RUNTIME_FUNCTION_GROUP, VirtualIPOffsets) == 32);
+static_assert(offsetof(RUNTIME_FUNCTION_GROUP, MidBitsOfUnwindOffset) == 64);
+static_assert(offsetof(RUNTIME_FUNCTION_GROUP, LowBitsOfUnwindOffset) == 96);
+static_assert(sizeof(T_RUNTIME_FUNCTION) == 4);
+
+inline DPTR(RUNTIME_FUNCTION_GROUP) GetRUNTIME_FUNCTION_GROUP_AND_INDEX(
+    DPTR(T_RUNTIME_FUNCTION) function, uint32_t* index)
+{
+    LIMITED_METHOD_DAC_CONTRACT;
+    TADDR address = dac_cast<TADDR>(function);
+    *index = static_cast<uint32_t>((address & 0x7F) >> 2);
+    return dac_cast<DPTR(RUNTIME_FUNCTION_GROUP)>(address & ~static_cast<TADDR>(0x7F));
+}
 
-#define RUNTIME_FUNCTION__EndAddress(FunctionEntry, ImageBase)      (RtlpGetFunctionEndAddress(FunctionEntry, (ULONG64)(ImageBase)))
+inline bool RUNTIME_FUNCTION__Skip(DPTR(T_RUNTIME_FUNCTION) function)
+{
+    LIMITED_METHOD_DAC_CONTRACT;
+    uint32_t index;
+    DPTR(RUNTIME_FUNCTION_GROUP) group = GetRUNTIME_FUNCTION_GROUP_AND_INDEX(function, &index);
+    return group->VirtualIPOffsets[index] == 0xFF;
+}
 
-#define RUNTIME_FUNCTION__SetUnwindInfoAddress(prf,address)         do { (prf)->UnwindData = (address); } while (0)
+inline bool RUNTIME_FUNCTION__IsFunclet(DPTR(T_RUNTIME_FUNCTION) function)
+{
+    LIMITED_METHOD_DAC_CONTRACT;
+    uint32_t index;
+    DPTR(RUNTIME_FUNCTION_GROUP) group = GetRUNTIME_FUNCTION_GROUP_AND_INDEX(function, &index);
+    return (group->IsFuncletBits & (uint32_t(1) << index)) != 0;
+}
+
+inline uint32_t RUNTIME_FUNCTION__BeginAddress(DPTR(T_RUNTIME_FUNCTION) function)
+{
+    LIMITED_METHOD_DAC_CONTRACT;
+    uint32_t index;
+    DPTR(RUNTIME_FUNCTION_GROUP) group = GetRUNTIME_FUNCTION_GROUP_AND_INDEX(function, &index);
+    _ASSERTE(group->VirtualIPOffsets[index] != 0xFF);
+    return group->VirtualIPBase + group->VirtualIPOffsets[index] + index * 4;
+}
+
+inline uint32_t RUNTIME_FUNCTION__GetUnwindDataOffset(DPTR(T_RUNTIME_FUNCTION) function)
+{
+    LIMITED_METHOD_DAC_CONTRACT;
+    uint32_t index;
+    DPTR(RUNTIME_FUNCTION_GROUP) group = GetRUNTIME_FUNCTION_GROUP_AND_INDEX(function, &index);
+    _ASSERTE(group->VirtualIPOffsets[index] != 0xFF);
+    uint32_t high = (group->HighBitsOfUnwindOffset[index / 2] >> ((index & 1) * 4)) & 0xF;
+    uint32_t offset = (uint32_t(group->MidBitsOfUnwindOffset[index]) << 8) |
+        uint32_t(group->LowBitsOfUnwindOffset[index]);
+    if (high == 0xF)
+    {
+        _ASSERTE(index < 31 && group->VirtualIPOffsets[31] == 0xFF);
+        return (uint32_t(group->MidBitsOfUnwindOffset[31]) << 24) |
+            (uint32_t(group->LowBitsOfUnwindOffset[31]) << 16) | offset;
+    }
+
+    return (high << 16) | offset;
+}
+
+#define RUNTIME_FUNCTION__EndAddress(FunctionEntry, GCInfoBase)     (RtlpGetFunctionEndAddress(FunctionEntry, (TADDR)(GCInfoBase)))
 
 typedef struct _UNWIND_INFO {
     // dummy

src/coreclr/inc/corcompile.h

@@ -57,8 +57,7 @@ inline ReadyToRunCrossModuleInlineFlags operator &( const ReadyToRunCrossModuleI
 }
 
 #ifdef TARGET_WASM
-// why was it defined only for x86 before?
-typedef DPTR(RUNTIME_FUNCTION) PTR_RUNTIME_FUNCTION;
+typedef DPTR(T_RUNTIME_FUNCTION) PTR_RUNTIME_FUNCTION;
 #endif
 
 #ifdef TARGET_X86

src/coreclr/inc/crosscomp.h

@@ -713,8 +713,14 @@ typedef struct _KNONVOLATILE_CONTEXT_POINTERS_EX
 #define T_KNONVOLATILE_CONTEXT_POINTERS KNONVOLATILE_CONTEXT_POINTERS
 #define PT_KNONVOLATILE_CONTEXT_POINTERS PKNONVOLATILE_CONTEXT_POINTERS
 
+#if defined(TARGET_WASM) && !defined(HOST_WASM)
+typedef struct _T_RUNTIME_FUNCTION {
+    DWORD Opaque;
+} T_RUNTIME_FUNCTION, *PT_RUNTIME_FUNCTION;
+#else
 #define T_RUNTIME_FUNCTION RUNTIME_FUNCTION
 #define PT_RUNTIME_FUNCTION PRUNTIME_FUNCTION
+#endif // TARGET_WASM && !HOST_WASM
 
 #endif
 

src/coreclr/inc/readytorun.h

@@ -156,6 +156,7 @@ enum class ReadyToRunSectionType : uint32_t
     ProxyTypeMaps               = 125, // Added in V18.3
     TypeMapAssemblyTargets      = 126, // Added in V18.3
     WasmAsyncResumeInfo         = 127, // Added in V29
+    WasmGCInfo                  = 128,
 
     // If you add a new section consider whether it is a breaking or non-breaking change.
     // Usually it is non-breaking, but if it is preferable to have older runtimes fail

src/coreclr/jit/CMakeLists.txt

@@ -1,5 +1,9 @@
 set(CMAKE_INCLUDE_CURRENT_DIR ON)
 
+if (CLR_CMAKE_HOST_ARCH_WASM)
+  return()
+endif()
+
 include_directories("./jitstd")
 include_directories("../inc")
 include_directories("../jitshared")

src/coreclr/jit/gcencode.cpp

@@ -4278,6 +4278,13 @@ void GCInfo::gcMakeRegPtrTable(
                     flags = (GcSlotFlags)(flags | GC_SLOT_INTERIOR);
                 }
 
+#ifdef TARGET_WASM
+                // Pin value-type fields like scalar locals: Wasm filters and their parents
+                // can report the same slots, which must not be relocated twice.
+                // See dotnet/runtime#131374, dotnet/runtime#133932, and dotnet/runtime#134769.
+                flags = static_cast<GcSlotFlags>(flags | GC_SLOT_PINNED);
+#endif // TARGET_WASM
+
                 GcStackSlotBase stackSlotBase = GC_SP_REL;
                 if (varDsc->lvFramePointerBased)
                 {

src/coreclr/jit/unwindwasm.cpp

@@ -61,6 +61,7 @@ void Compiler::unwindReserveFunc(FuncInfoDsc* func)
     bool isFunclet  = func->IsFunclet();
     bool isColdCode = false;
     assert(func->endVirtualIP > func->startVirtualIP);
+    assert(func->endVirtualIP - func->startVirtualIP >= 2);
     ULONG encodedSize = emitter::SizeOfULEB128(func->funWasmFrameSize) +
                         emitter::SizeOfULEB128(func->endVirtualIP - func->startVirtualIP);
 
@@ -113,6 +114,7 @@ void Compiler::unwindEmitFunc(FuncInfoDsc* func, void* pHotCode, void* pColdCode
     uint8_t buffer[10];
     size_t  index = 0;
     assert(func->endVirtualIP > func->startVirtualIP);
+    assert(func->endVirtualIP - func->startVirtualIP >= 2);
     index += GetEmitter()->emitOutputULEB128(buffer + index, func->funWasmFrameSize);
     index += GetEmitter()->emitOutputULEB128(buffer + index, func->endVirtualIP - func->startVirtualIP);
     assert(index <= sizeof(buffer));

src/coreclr/pal/inc/pal.h

@@ -2488,11 +2488,15 @@ enum {
 typedef IMAGE_ARM64_RUNTIME_FUNCTION_ENTRY RUNTIME_FUNCTION, *PRUNTIME_FUNCTION;
 #else // HOST_ARM64
 typedef struct _RUNTIME_FUNCTION {
+#ifdef HOST_WASM
+    DWORD Opaque;
+#else // HOST_WASM
     DWORD BeginAddress;
 #ifdef HOST_AMD64
     DWORD EndAddress;
 #endif
     DWORD UnwindData;
+#endif // HOST_WASM
 } RUNTIME_FUNCTION, *PRUNTIME_FUNCTION;
 #endif // HOST_ARM64
 

src/coreclr/tools/Common/Compiler/ObjectWriter/WasmObjectWriter.cs

@@ -61,6 +61,9 @@ internal abstract partial class WasmObjectWriter : ObjectWriter
         private readonly WasmFunctionBodyDeduplicator _functionBodyDeduplicator = new();
         private readonly Dictionary<Utf8String, int> _tableSlotIndices = [];
         private readonly List<Utf8String> _tableSlotFunctions = [];
+#if READYTORUN
+        private int? _runtimeFunctionTableStart;
+#endif
         private readonly List<FoldedFunctionAlias> _foldedFunctionAliases = [];
         /// <summary>
         /// Maps symbol names to their location in the object file. These definitions do not encode
@@ -131,6 +134,12 @@ public override void EmitObject(
                 _functionBodyDeduplicator.Prepare(nodes, _nodeFactory, ShouldSkip);
             }
             base.EmitObject(outputFileStream, nodes, dumper, logger);
+#if READYTORUN
+            if (_nodeFactory.RuntimeFunctionsTable is { } runtimeFunctions)
+            {
+                logger.LogMessage($"Wasm runtime functions: {runtimeFunctions.WasmRuntimeFunctionCount} slots, {runtimeFunctions.WasmSkipCount - runtimeFunctions.WasmAlignmentSkipCount} skips, {runtimeFunctions.WasmAlignmentSkipCount} final alignment skips.");
+            }
+#endif
 
             bool ShouldSkip(ObjectNode node)
             {
@@ -242,6 +251,7 @@ private protected override void RecordMethodDefinition(INodeWithTypeSignature no
                 // diagnostic output, but engine-level samples necessarily use the canonical name.
                 _wasmSymbolManager.AddAlias(methodName, canonicalName);
                 RegisterTableSlot(methodName, canonicalName);
+                RecordRuntimeFunctionPadding((ObjectNode)node, 0, canonicalName);
 
                 if (!alternateName.IsNull)
                 {
@@ -255,6 +265,7 @@ private protected override void RecordMethodDefinition(INodeWithTypeSignature no
             WriteSignatureIndexForFunction(node);
             RegisterFunctionSymbol(methodName);
             RegisterTableSlot(methodName, methodName);
+            RecordRuntimeFunctionPadding((ObjectNode)node, 0, methodName);
 
             if (!alternateName.IsNull)
             {
@@ -338,10 +349,27 @@ private void RecordFunclets(INodeWithFunclets nodeWithFunclets)
                 Utf8String funcletName = GetFuncletName(new Utf8String(mangledNodeName), i);
                 RegisterFunctionSymbol(funcletName);
                 RegisterTableSlot(funcletName, funcletName);
+                RecordRuntimeFunctionPadding((ObjectNode)nodeWithFunclets, i + 1, funcletName);
                 RegisterStubIndexAndSignature(funcletSignature);
             }
         }
 
+        private void RecordRuntimeFunctionPadding(ObjectNode node, int frameIndex, Utf8String functionName)
+        {
+#if READYTORUN
+            if (node is DependencyAnalysis.ReadyToRun.MethodWithGCInfo method)
+            {
+                int count = _nodeFactory.RuntimeFunctionsTable.GetWasmPaddingAfter(method, frameIndex, out int ordinal);
+                _runtimeFunctionTableStart ??= _tableSlotFunctions.Count - 1;
+                Debug.Assert(_tableSlotFunctions.Count - 1 - _runtimeFunctionTableStart.Value == ordinal);
+                for (int i = 0; i < count; i++)
+                {
+                    _tableSlotFunctions.Add(functionName);
+                }
+            }
+#endif
+        }
+
         private protected static Utf8String GetFuncletName(Utf8String methodName, int index) =>
             new($"{methodName}_funclet_{index}");
 

src/coreclr/tools/Common/Internal/Runtime/ModuleHeaders.cs

@@ -85,6 +85,7 @@ enum ReadyToRunSectionType
         ProxyTypeMaps               = 125, // Added to CoreCLR in V18.3
         TypeMapAssemblyTargets      = 126, // Added in V18.3
         WasmAsyncResumeInfo         = 127, // Added in V29
+        WasmGCInfo                  = 128,
 
         //
         // NativeAOT ReadyToRun sections

src/coreclr/tools/aot/ILCompiler.ReadyToRun.Tests/TestCases/R2RTestSuites.cs

@@ -419,6 +419,43 @@ void Validate(ReadyToRunReader reader)
         }
     }
 
+    [ConditionalTheory(typeof(TestPaths), nameof(TestPaths.IsWasmTarget))]
+    [InlineData(false)]
+    [InlineData(true)]
+    public void WasmStructGCFieldsArePinned(bool optimize)
+    {
+        CompiledAssembly input = new()
+        {
+            AssemblyName = nameof(WasmStructGCFieldsArePinned),
+            SourceResourceNames = ["Webcil/WasmWebcilModule.cs"],
+        };
+
+        new R2RTestRunner(_output).Run(new R2RTestCase(
+            nameof(WasmStructGCFieldsArePinned),
+            [
+                new(nameof(WasmStructGCFieldsArePinned), [new CrossgenAssembly(input)])
+                {
+                    Options = optimize ? [Crossgen2Option.Optimize] : [],
+                    AdditionalArgs = optimize ? [] : ["--optimize-disabled"],
+                    OutputFileExtension = ".wasm",
+                    Validate = reader =>
+                    {
+                        ReadyToRunMethod method = Assert.Single(R2RAssert.GetAllMethods(reader),
+                            method => method.SignatureString.Contains(".StructGCFields(", StringComparison.Ordinal));
+                        ILCompiler.Reflection.ReadyToRun.Amd64.GcInfo gcInfo =
+                            Assert.IsType<ILCompiler.Reflection.ReadyToRun.Amd64.GcInfo>(method.GcInfo);
+                        Assert.True(gcInfo.SlotTable.NumUntracked >= 2);
+                        Assert.Contains(gcInfo.SlotTable.GcSlots,
+                            slot => (slot.Flags & GcSlotFlags.GC_SLOT_INTERIOR) != 0);
+                        Assert.Contains(gcInfo.SlotTable.GcSlots,
+                            slot => (slot.Flags & GcSlotFlags.GC_SLOT_INTERIOR) == 0);
+                        Assert.All(gcInfo.SlotTable.GcSlots, slot =>
+                            Assert.Equal(GcSlotFlags.GC_SLOT_PINNED, slot.Flags & GcSlotFlags.GC_SLOT_PINNED));
+                    },
+                },
+            ]));
+    }
+
     [ConditionalFact(typeof(TestPaths), nameof(TestPaths.IsWasmTarget))]
     public void WasmWebcilModule()
     {
@@ -458,6 +495,7 @@ public void WasmWebcilModule()
 
         static void ValidateDefault(ReadyToRunReader reader)
         {
+            WasmR2RAssert.AssertRuntimeFunctionGroups(reader);
             var webcilReader = Assert.IsType<WebcilImageReader>(reader.CompositeReader);
             WasmR2RAssert.AssertFrameIdentityPrologs(webcilReader);
             Assert.True(webcilReader.IsWasmWrapped);

src/coreclr/tools/aot/ILCompiler.ReadyToRun.Tests/TestCases/Webcil/WasmWebcilModule.cs

@@ -22,6 +22,30 @@ public struct GenericTokenValue
         public int Value;
     }
 
+    public ref struct GCFields
+    {
+        public object Reference;
+        public Span<int> ByRef;
+    }
+
+    [MethodImpl(MethodImplOptions.NoInlining)]
+    public static int StructGCFields()
+    {
+        GCFields fields = new()
+        {
+            Reference = new object(),
+            ByRef = new int[] { 42 },
+        };
+        return ReadStructGCFields(ref fields);
+    }
+
+    [MethodImpl(MethodImplOptions.NoInlining)]
+    private static int ReadStructGCFields(ref GCFields fields)
+    {
+        GC.Collect();
+        return fields.Reference.GetHashCode() ^ fields.ByRef[0];
+    }
+
     [MethodImpl(MethodImplOptions.NoInlining)]
     public static Type ReadIntrinsicTypeToken(int index) => index switch
     {