From dc2976521cc0abf9fa694273b50b2e3302159376 Mon Sep 17 00:00:00 2001 From: kuba Date: Mon, 26 Oct 2026 05:00:58 +0100 Subject: [PATCH 01/82] [GPU] The command processor write history records only when its diagnostics switch is on The history was always on: every deferred write built a kind string, took an interlocked slot and a timestamp, and a DMA copy read the guest source only for the record. Record is now a branch while SHARPEMU_CP_HISTORY, SHARPEMU_CP_WATCH or SHARPEMU_GPU_CLOBBER_DIAG is unset, and the call sites skip the string and the guest read. The stall, clobber and shader-code dumps read the history when it is on. Tests enable it for the whole run. --- .../CommandProcessorWriteHistory.cs | 19 +++++++- .../GpuCommandInterpreter.Labels.cs | 12 ++++- .../GpuCommandInterpreter.Memory.cs | 35 ++++++++++---- .../Gpu/GpuCommands/GpuCommandInterpreter.cs | 2 +- .../CommandProcessorWriteHistoryFixture.cs | 46 +++++++++++++++++++ 5 files changed, 100 insertions(+), 14 deletions(-) create mode 100644 tests/SharpEmu.Libs.Tests/Gpu/GpuCommands/CommandProcessorWriteHistoryFixture.cs diff --git a/src/SharpEmu.Libs/Gpu/GpuCommands/CommandProcessorWriteHistory.cs b/src/SharpEmu.Libs/Gpu/GpuCommands/CommandProcessorWriteHistory.cs index c2f4d1e5a..3a080bb96 100644 --- a/src/SharpEmu.Libs/Gpu/GpuCommands/CommandProcessorWriteHistory.cs +++ b/src/SharpEmu.Libs/Gpu/GpuCommands/CommandProcessorWriteHistory.cs @@ -7,13 +7,25 @@ namespace SharpEmu.Libs.Gpu.GpuCommands; /// The last few thousand guest writes the command processor made or deferred to the GPU: /// labels, write-data packets, end-of-pipe values. A wait that never completes is either /// waiting for a write that has not happened or for one that happened and was overwritten -/// before the waiter looked; this is what tells the two apart. Diagnostics only. +/// before the waiter looked; this is what tells the two apart. Diagnostics only: nothing +/// records unless , so the callers pay one branch per write otherwise. +/// SHARPEMU_CP_HISTORY=1 turns it on, as do SHARPEMU_CP_WATCH and SHARPEMU_GPU_CLOBBER_DIAG +/// (the clobber report prints the history); the stall and shader-code dumps read it when on. /// public static class CommandProcessorWriteHistory { public readonly record struct Entry( long Ticks, int QueueId, ulong Address, int Size, ulong Value, string Kind, bool Deferred); + // Off by default: a record costs a timestamp, an interlocked add and, at some call sites, a + // formatted string or a guest read made only for the record. Tests set it directly. + public static bool Enabled { get; set; } = EnabledByEnvironment(Environment.GetEnvironmentVariable); + + internal static bool EnabledByEnvironment(Func getEnvironmentVariable) => + getEnvironmentVariable("SHARPEMU_CP_HISTORY") == "1" || + !string.IsNullOrWhiteSpace(getEnvironmentVariable("SHARPEMU_CP_WATCH")) || + getEnvironmentVariable("SHARPEMU_GPU_CLOBBER_DIAG") == "1"; + private const int Capacity = 4096; private static readonly Entry[] _entries = new Entry[Capacity]; private static long _next; @@ -33,6 +45,11 @@ private static ulong[] ParseWatched(string? text) => string.IsNullOrWhiteSpace(t public static void Record(int queueId, ulong address, int size, ulong value, string kind, bool deferred) { + if (!Enabled) + { + return; + } + foreach (var watched in Watched) { if (watched >= address && watched < address + (ulong)Math.Max(size, 1)) diff --git a/src/SharpEmu.Libs/Gpu/GpuCommands/GpuCommandInterpreter.Labels.cs b/src/SharpEmu.Libs/Gpu/GpuCommands/GpuCommandInterpreter.Labels.cs index b5c69eba5..1b6408fa5 100644 --- a/src/SharpEmu.Libs/Gpu/GpuCommands/GpuCommandInterpreter.Labels.cs +++ b/src/SharpEmu.Libs/Gpu/GpuCommands/GpuCommandInterpreter.Labels.cs @@ -184,7 +184,11 @@ private void WriteEndOfPipeDword(ulong destination, uint value, EndOfPipeWriteKi { if (_host.DefersEndOfPipeLabel(kind)) { - CommandProcessorWriteHistory.Record(QueueId, destination, sizeof(uint), value, kind.ToString(), deferred: true); + if (CommandProcessorWriteHistory.Enabled) + { + CommandProcessorWriteHistory.Record(QueueId, destination, sizeof(uint), value, kind.ToString(), deferred: true); + } + PendingLabelRecorded?.Invoke(destination, sizeof(uint), value); _lastWriteLength = 0; } @@ -195,7 +199,11 @@ private void WriteEndOfPipeQword(ulong destination, ulong value, EndOfPipeWriteK { if (_host.DefersEndOfPipeLabel(kind)) { - CommandProcessorWriteHistory.Record(QueueId, destination, sizeof(ulong), value, kind.ToString(), deferred: true); + if (CommandProcessorWriteHistory.Enabled) + { + CommandProcessorWriteHistory.Record(QueueId, destination, sizeof(ulong), value, kind.ToString(), deferred: true); + } + PendingLabelRecorded?.Invoke(destination, sizeof(ulong), value); _lastWriteLength = 0; } diff --git a/src/SharpEmu.Libs/Gpu/GpuCommands/GpuCommandInterpreter.Memory.cs b/src/SharpEmu.Libs/Gpu/GpuCommands/GpuCommandInterpreter.Memory.cs index 819e1d207..a976ef5a9 100644 --- a/src/SharpEmu.Libs/Gpu/GpuCommands/GpuCommandInterpreter.Memory.cs +++ b/src/SharpEmu.Libs/Gpu/GpuCommands/GpuCommandInterpreter.Memory.cs @@ -79,8 +79,12 @@ internal void WriteCommandProcessorBytes(ulong destination, ReadOnlySpan s : source.Length >= sizeof(uint) ? BinaryPrimitives.ReadUInt32LittleEndian(source) : 0UL; using (RenderPhaseProfile.MeasureDetail(RenderPhaseProfile.Phase.CommandCpBookkeeping)) { - CommandProcessorWriteHistory.Record(QueueId, destination, source.Length, - source.Length == sizeof(uint) ? (uint)first : first, kind, deferred: true); + if (CommandProcessorWriteHistory.Enabled) + { + CommandProcessorWriteHistory.Record(QueueId, destination, source.Length, + source.Length == sizeof(uint) ? (uint)first : first, kind, deferred: true); + } + NotePendingBytes(destination, source); } @@ -142,10 +146,13 @@ private void WriteCommandProcessorQwords(ulong destination, int count, ulong str using (RenderPhaseProfile.MeasureDetail(RenderPhaseProfile.Phase.CommandCpBookkeeping)) { - for (var index = 0; index < count; index++) + if (CommandProcessorWriteHistory.Enabled) { - CommandProcessorWriteHistory.Record(QueueId, destination + (ulong)index * strideBytes, - sizeof(ulong), value, kind, deferred: true); + for (var index = 0; index < count; index++) + { + CommandProcessorWriteHistory.Record(QueueId, destination + (ulong)index * strideBytes, + sizeof(ulong), value, kind, deferred: true); + } } if (PendingQwordsRecorded is { } noteRun) @@ -230,7 +237,11 @@ private void RecordReferenceClock(ulong destination, uint byteCount, ulong clock { var is64Bit = byteCount == sizeof(ulong); var value = is64Bit ? clock : (uint)clock; - CommandProcessorWriteHistory.Record(QueueId, destination, (int)byteCount, value, "COPY_DATA_CLOCK", deferred: true); + if (CommandProcessorWriteHistory.Enabled) + { + CommandProcessorWriteHistory.Record(QueueId, destination, (int)byteCount, value, "COPY_DATA_CLOCK", deferred: true); + } + PendingLabelRecorded?.Invoke(destination, (int)byteCount, value); _lastWriteLength = 0; RecordHostEndOfPipe(new EndOfPipeWrite( @@ -294,8 +305,12 @@ internal void TransferData( if (!destinationIsGds && DefersCommandProcessorWrites) { var fill = (uint)sourceOrImmediate; - CommandProcessorWriteHistory.Record(QueueId, destination, (int)Math.Min(byteCount, int.MaxValue), - fill, "DMA_FILL", deferred: true); + if (CommandProcessorWriteHistory.Enabled) + { + CommandProcessorWriteHistory.Record(QueueId, destination, (int)Math.Min(byteCount, int.MaxValue), + fill, "DMA_FILL", deferred: true); + } + if (byteCount <= MaxNotedPendingBytes && byteCount % sizeof(uint) == 0) { Span pattern = stackalloc uint[MaxNotedPendingBytes / sizeof(uint)]; @@ -308,7 +323,7 @@ internal void TransferData( return; } - if (!destinationIsGds) + if (!destinationIsGds && CommandProcessorWriteHistory.Enabled) { CommandProcessorWriteHistory.Record(QueueId, destination, (int)Math.Min(byteCount, int.MaxValue), (uint)sourceOrImmediate, "DMA_FILL", deferred: false); @@ -334,7 +349,7 @@ internal void TransferData( } var deferredCopy = !destinationIsGds && !sourceIsGds && DefersCommandProcessorWrites; - if (!destinationIsGds) + if (!destinationIsGds && CommandProcessorWriteHistory.Enabled) { // The value is the source's, read now for the record only: a copy may land later. ulong copied = 0; diff --git a/src/SharpEmu.Libs/Gpu/GpuCommands/GpuCommandInterpreter.cs b/src/SharpEmu.Libs/Gpu/GpuCommands/GpuCommandInterpreter.cs index 39c76f859..a9c9ec120 100644 --- a/src/SharpEmu.Libs/Gpu/GpuCommands/GpuCommandInterpreter.cs +++ b/src/SharpEmu.Libs/Gpu/GpuCommands/GpuCommandInterpreter.cs @@ -913,7 +913,7 @@ internal void WriteBytes(ulong address, ReadOnlySpan source, string kind = } GuestMemoryWritten?.Invoke(address, (ulong)source.Length); - if (source.Length is sizeof(uint) or sizeof(ulong)) + if (CommandProcessorWriteHistory.Enabled && source.Length is sizeof(uint) or sizeof(ulong)) { CommandProcessorWriteHistory.Record(QueueId, address, source.Length, source.Length == sizeof(uint) diff --git a/tests/SharpEmu.Libs.Tests/Gpu/GpuCommands/CommandProcessorWriteHistoryFixture.cs b/tests/SharpEmu.Libs.Tests/Gpu/GpuCommands/CommandProcessorWriteHistoryFixture.cs new file mode 100644 index 000000000..7b31f00e7 --- /dev/null +++ b/tests/SharpEmu.Libs.Tests/Gpu/GpuCommands/CommandProcessorWriteHistoryFixture.cs @@ -0,0 +1,46 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using System.Runtime.CompilerServices; +using SharpEmu.Libs.Gpu.GpuCommands; +using Xunit; + +namespace SharpEmu.Libs.Tests.Gpu.GpuCommands; + +/// +/// The write history records only when its diagnostics switch is on (SHARPEMU_CP_HISTORY=1); the +/// tests that read it back from the command processor turn it on for the whole run. +/// +internal static class CommandProcessorWriteHistoryFixture +{ + [ModuleInitializer] + internal static void Initialize() => CommandProcessorWriteHistory.Enabled = true; +} + +public sealed class CommandProcessorWriteHistorySwitchTests +{ + private static bool Enabled(params (string Name, string Value)[] environment) => + CommandProcessorWriteHistory.EnabledByEnvironment(name => + environment.Where(pair => pair.Name == name).Select(pair => pair.Value).FirstOrDefault()); + + [Fact] + public void TheHistoryIsOffUnlessADiagnosticsSwitchAsksForIt() + { + Assert.False(Enabled()); + Assert.False(Enabled(("SHARPEMU_CP_HISTORY", "0"))); + Assert.True(Enabled(("SHARPEMU_CP_HISTORY", "1"))); + Assert.True(Enabled(("SHARPEMU_CP_WATCH", "7F0000001000"))); + Assert.True(Enabled(("SHARPEMU_GPU_CLOBBER_DIAG", "1"))); + } + + [Fact] + public void AnEnabledHistoryReturnsWhatWasRecorded() + { + Assert.True(CommandProcessorWriteHistory.Enabled); + const ulong address = 0x7F00_1230_0000UL; + CommandProcessorWriteHistory.Record(0, address, 4, 0x22, "SWITCH_ON", deferred: false); + var entry = Assert.Single(CommandProcessorWriteHistory.Find(address, 4)); + Assert.Equal("SWITCH_ON", entry.Kind); + Assert.Equal(0x22UL, entry.Value); + } +} From 418f6bf73910e568a88ee5b77f9a4e9bf6bbd508 Mon Sep 17 00:00:00 2001 From: kuba Date: Mon, 26 Oct 2026 05:00:58 +0100 Subject: [PATCH 02/82] [GPU] Fan, quad, 8-bit and restart index expansion writes into per-thread scratch Each draw allocated a read buffer, a widened array, a List, its copy and the output bytes (about 30 B per index, LOH above ~21k indices). The passes now read into and write from reused per-thread buffers (grown to the largest draw, not retained above 8 MB) and the upload takes exactly the byte count. Output is checked equal to the previous allocating forms in IndexExpansionTests. --- .../RenderExecutor.IndexExpansion.cs | 310 ++++++++++++++++++ .../Gpu/Rendering/RenderExecutor.Resources.cs | 2 +- .../Gpu/Rendering/RenderExecutor.cs | 154 +-------- .../Gpu/Rendering/IndexExpansionTests.cs | 286 ++++++++++++++++ 4 files changed, 608 insertions(+), 144 deletions(-) create mode 100644 src/SharpEmu.Libs/Gpu/Rendering/RenderExecutor.IndexExpansion.cs create mode 100644 tests/SharpEmu.Libs.Tests/Gpu/Rendering/IndexExpansionTests.cs diff --git a/src/SharpEmu.Libs/Gpu/Rendering/RenderExecutor.IndexExpansion.cs b/src/SharpEmu.Libs/Gpu/Rendering/RenderExecutor.IndexExpansion.cs new file mode 100644 index 000000000..2268a8a42 --- /dev/null +++ b/src/SharpEmu.Libs/Gpu/Rendering/RenderExecutor.IndexExpansion.cs @@ -0,0 +1,310 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using System.Buffers.Binary; +using SharpEmu.Libs.Gpu.Abstractions; + +namespace SharpEmu.Libs.Gpu.Rendering; + +// The index buffers a draw has to rewrite before the host can draw them: fans, polygons and quad lists as triangle +// lists (Metal has no fan primitive), 8-bit indices widened to 16, and a custom restart index mapped to the +// all-ones one. They used to allocate several arrays per draw (about 30 bytes per index, the large object heap +// above roughly 21k indices); the passes below write into per-thread scratch that the draw's upload consumes +// before the next draw, so a draw allocates nothing once the scratch has grown to its largest index count. +public sealed partial class RenderExecutor +{ + private byte[] ReadIndexBytes(ulong indexAddress, int byteCount, string what, uint indexCount) + { + var source = IndexExpansionScratch.Current.Read(byteCount); + if (indexAddress == 0 || !_host.TryReadGuest(indexAddress, source.AsSpan(0, byteCount))) + { + throw _host.Fatal($"The {what} index data is unreadable: address=0x{indexAddress:X16} count={indexCount}."); + } + + return source; + } + + // The draw's guest indices, widened to 32 bits, in scratch that the next scratch call may overwrite. + private ReadOnlySpan ReadGuestIndicesScratch(ulong indexAddress, uint indexCount, GuestIndexType type) + { + var stride = type switch + { + GuestIndexType.Index8 => 1, + GuestIndexType.Index16 => 2, + GuestIndexType.Index32 => 4, + _ => throw _host.Fatal($"The index type and size is unknown: indexTypeAndSize={(uint)type}."), + }; + var source = ReadIndexBytes(indexAddress, checked((int)indexCount * stride), "fan", indexCount); + var indices = IndexExpansionScratch.Current.Vertices(checked((int)indexCount)).AsSpan(0, (int)indexCount); + IndexExpansion.Widen(source, stride, indices); + return indices; + } + + // The vertices 0..count-1 of a non-indexed fan, in scratch. + private static ReadOnlySpan SequentialVerticesScratch(uint vertexCount) + { + var vertices = IndexExpansionScratch.Current.Vertices(checked((int)vertexCount)).AsSpan(0, (int)vertexCount); + for (var index = 0; index < vertices.Length; index++) + { + vertices[index] = (uint)index; + } + + return vertices; + } + + // The triangle list the primitive assembler makes of these vertices (see IndexExpansion.TriangulateFans), + // in scratch separate from the vertices'. + private ReadOnlySpan TriangulateFansScratch(GuestPrimitiveType type, ReadOnlySpan vertices, uint? restartIndex) + { + if (type == GuestPrimitiveType.QuadListLegacy && (vertices.Length & 0x3) != 0) + { + throw _host.Fatal($"A legacy quad list count is not a multiple of four: count={vertices.Length}."); + } + + var capacity = IndexExpansion.MaxTriangleListCount(type, vertices.Length); + var output = IndexExpansionScratch.Current.List(capacity).AsSpan(0, capacity); + var written = IndexExpansion.TriangulateFans(type, vertices, restartIndex, output); + return output[..written]; + } + + // The list's indices as little-endian bytes in scratch; the array is longer than the returned byte count. + private byte[] ListIndexBytesScratch(ReadOnlySpan list, bool wide, out int byteCount) + { + byteCount = checked(list.Length * (wide ? sizeof(uint) : sizeof(ushort))); + var bytes = IndexExpansionScratch.Current.Output(byteCount); + if (!IndexExpansion.WriteListIndexBytes(list, wide, bytes)) + { + throw _host.Fatal("A triangle fan converted to a triangle list has a 32-bit vertex index of 0xFFFFFFFF."); + } + + return bytes; + } + + private byte[] ConvertRestartIndicesScratch(ulong indexAddress, uint indexCount, GpuIndexType indexType, uint restartIndex, out int byteCount) + { + var sourceStride = indexType == GpuIndexType.Uint16 ? sizeof(ushort) : sizeof(uint); + var source = ReadIndexBytes(indexAddress, checked((int)indexCount * sourceStride), "restart", indexCount); + // Widen 16-bit indices so their maximum value remains an ordinary vertex. + byteCount = checked((int)indexCount * sizeof(uint)); + var converted = IndexExpansionScratch.Current.Output(byteCount); + if (!IndexExpansion.ConvertRestart(source.AsSpan(0, (int)indexCount * sourceStride), sourceStride, restartIndex, converted)) + { + throw _host.Fatal("A custom restart index conflicts with a 32-bit vertex index of 0xFFFFFFFF."); + } + + return converted; + } + + private byte[] ExpandIndex8Scratch(ulong indexAddress, uint indexCount, bool primitiveRestart, uint restartIndex, out int byteCount) + { + if (indexAddress == 0) + { + throw _host.Fatal($"An 8-bit index draw has no index address: count={indexCount}."); + } + + var source = ReadIndexBytes(indexAddress, checked((int)indexCount), "8-bit", indexCount); + byteCount = checked((int)indexCount * 2); + var expanded = IndexExpansionScratch.Current.Output(byteCount); + IndexExpansion.ExpandIndex8(source.AsSpan(0, (int)indexCount), primitiveRestart, restartIndex, expanded); + return expanded; + } +} + +/// The index rewrites as pure functions over spans, so a test can compare them with a straightforward form. +internal static class IndexExpansion +{ + /// Widens 8-, 16- or 32-bit little-endian indices to 32 bits. + public static void Widen(ReadOnlySpan source, int stride, Span destination) + { + switch (stride) + { + case 1: + for (var index = 0; index < destination.Length; index++) + { + destination[index] = source[index]; + } + + break; + case 2: + for (var index = 0; index < destination.Length; index++) + { + destination[index] = BinaryPrimitives.ReadUInt16LittleEndian(source[(index * 2)..]); + } + + break; + default: + for (var index = 0; index < destination.Length; index++) + { + destination[index] = BinaryPrimitives.ReadUInt32LittleEndian(source[(index * 4)..]); + } + + break; + } + } + + /// The most indices writes for this many vertices. + public static int MaxTriangleListCount(GuestPrimitiveType type, int vertexCount) => + type == GuestPrimitiveType.QuadListLegacy + ? checked((vertexCount / 4) * 6) + : Math.Max(0, checked((vertexCount - 2) * 3)); + + /// + /// The triangle list the primitive assembler makes of these vertices, in its vertex order: fan + /// triangle i is (v[i+1], v[i+2], v[0]), the order DI_PT_TRIFAN assembles (the first-vertex + /// provoking vertex is v[i+1], as radv's VK fans report it), and a restart index ends the fan, the + /// next index starting a new one. A polygon is one fan. Each quad of a quad list is (q0, q1, q2) + /// and (q0, q2, q3), both taking q0 as their provoking vertex. Returns the index count written; + /// the destination holds indices, and a quad count that is not + /// a whole number of quads is the caller's to refuse. + /// + public static int TriangulateFans(GuestPrimitiveType type, ReadOnlySpan vertices, uint? restartIndex, Span output) + { + if (type == GuestPrimitiveType.QuadListLegacy) + { + var written = 0; + for (var quad = 0; quad + 3 < vertices.Length; quad += 4, written += 6) + { + output[written] = vertices[quad]; + output[written + 1] = vertices[quad + 1]; + output[written + 2] = vertices[quad + 2]; + output[written + 3] = vertices[quad]; + output[written + 4] = vertices[quad + 2]; + output[written + 5] = vertices[quad + 3]; + } + + return written; + } + + var count = 0; + var hub = 0; + for (var index = 0; index < vertices.Length; index++) + { + if (restartIndex is { } restart && vertices[index] == restart) + { + hub = index + 1; + continue; + } + + if (index - hub >= 2) + { + output[count] = vertices[index - 1]; + output[count + 1] = vertices[index]; + output[count + 2] = vertices[hub]; + count += 3; + } + } + + return count; + } + + /// + /// Writes the indices as 16- or 32-bit little-endian values. A 32-bit all-ones index is refused + /// (false): Metal reserves it for primitive restart. A 16-bit value that does not fit throws. + /// + public static bool WriteListIndexBytes(ReadOnlySpan list, bool wide, Span destination) + { + for (var index = 0; index < list.Length; index++) + { + var value = list[index]; + if (wide) + { + if (value == uint.MaxValue) + { + return false; + } + + BinaryPrimitives.WriteUInt32LittleEndian(destination[(index * sizeof(uint))..], value); + } + else + { + BinaryPrimitives.WriteUInt16LittleEndian(destination[(index * sizeof(ushort))..], checked((ushort)value)); + } + } + + return true; + } + + /// 8-bit indices as 16-bit ones, the restart value (compared on its low byte) becoming 0xFFFF. + public static void ExpandIndex8(ReadOnlySpan source, bool primitiveRestart, uint restartIndex, Span destination) + { + var restart = restartIndex & 0xFF; + for (var i = 0; i < source.Length; i++) + { + var value = source[i]; + var index = primitiveRestart && value == restart ? (ushort)0xFFFF : value; + destination[i * 2] = (byte)index; + destination[(i * 2) + 1] = (byte)(index >> 8); + } + } + + /// + /// 16- or 32-bit indices as 32-bit ones with a custom restart index mapped to the all-ones one. + /// False when a vertex index already is all-ones, which the mapping would turn into a restart. + /// + public static bool ConvertRestart(ReadOnlySpan source, int sourceStride, uint restartIndex, Span destination) + { + var count = source.Length / sourceStride; + for (var index = 0; index < count; index++) + { + var value = sourceStride == sizeof(ushort) + ? BinaryPrimitives.ReadUInt16LittleEndian(source[(index * sourceStride)..]) + : BinaryPrimitives.ReadUInt32LittleEndian(source[(index * sourceStride)..]); + if (value == uint.MaxValue) + { + return false; + } + + BinaryPrimitives.WriteUInt32LittleEndian(destination[(index * sizeof(uint))..], + value == restartIndex ? uint.MaxValue : value); + } + + return true; + } +} + +/// +/// The buffers the index rewrites use, one set per thread. A buffer grows to the largest request and is +/// kept up to a limit; a larger one is allocated for its draw alone, so one huge draw does not pin memory. +/// Every request is valid until the next request for the same buffer on this thread. +/// +internal sealed class IndexExpansionScratch +{ + private const int RetainLimitBytes = 8 << 20; + + [ThreadStatic] + private static IndexExpansionScratch? _current; + + public static IndexExpansionScratch Current => _current ??= new IndexExpansionScratch(); + + private byte[] _read = []; + private byte[] _output = []; + private uint[] _vertices = []; + private uint[] _list = []; + + public byte[] Read(int bytes) => Take(ref _read, bytes); + + public byte[] Output(int bytes) => Take(ref _output, bytes); + + public uint[] Vertices(int count) => Take(ref _vertices, count); + + public uint[] List(int count) => Take(ref _list, count); + + private static T[] Take(ref T[] slot, int count) where T : unmanaged + { + if (slot.Length >= count) + { + return slot; + } + + var elementBytes = (long)System.Runtime.CompilerServices.Unsafe.SizeOf(); + var doubled = Math.Max((long)count, (long)slot.Length * 2); + var size = doubled * elementBytes <= RetainLimitBytes ? doubled : count; + var grown = new T[size]; + if (size * elementBytes <= RetainLimitBytes) + { + slot = grown; + } + + return grown; + } +} diff --git a/src/SharpEmu.Libs/Gpu/Rendering/RenderExecutor.Resources.cs b/src/SharpEmu.Libs/Gpu/Rendering/RenderExecutor.Resources.cs index fd8d185e2..60a03da0c 100644 --- a/src/SharpEmu.Libs/Gpu/Rendering/RenderExecutor.Resources.cs +++ b/src/SharpEmu.Libs/Gpu/Rendering/RenderExecutor.Resources.cs @@ -181,7 +181,7 @@ private PreparedIndexBuffer AcquireIndexBuffer(in IndexSource source) } var binding = source.HostData is { } hostData - ? _host.UploadTransient(hostData, TransientIndexAlignment) + ? _host.UploadTransient(hostData.AsSpan(0, checked((int)source.Size)), TransientIndexAlignment) : _host.ObtainBuffer(source.Address, source.Size, isWritten: false); return new PreparedIndexBuffer(binding, source.Size, source.Type); } diff --git a/src/SharpEmu.Libs/Gpu/Rendering/RenderExecutor.cs b/src/SharpEmu.Libs/Gpu/Rendering/RenderExecutor.cs index e758ae925..129758d80 100644 --- a/src/SharpEmu.Libs/Gpu/Rendering/RenderExecutor.cs +++ b/src/SharpEmu.Libs/Gpu/Rendering/RenderExecutor.cs @@ -495,6 +495,7 @@ void TraceDisposition(string disposition) var draw = new DrawCall("DrawIndexed", RecordedOperation.DrawIndex, arguments.IndexCount, arguments.InstanceCount, arguments.FirstInstance); byte[]? expandedIndices = null; + var expandedLength = 0; var triangleListIndices = 0u; if (TriangulatesFans(userConfig)) { @@ -503,14 +504,14 @@ void TraceDisposition(string disposition) var restartIndex = primitiveRestart ? banks.Context.PrimitiveResetIndex & IndexMask((GuestIndexType)arguments.IndexTypeAndSize) : (uint?)null; - var vertices = ReadGuestIndices(arguments.IndexAddress, arguments.IndexCount, (GuestIndexType)arguments.IndexTypeAndSize); - var list = TriangulateFans((GuestPrimitiveType)userConfig.PrimitiveType, vertices, restartIndex); + var vertices = ReadGuestIndicesScratch(arguments.IndexAddress, arguments.IndexCount, (GuestIndexType)arguments.IndexTypeAndSize); + var list = TriangulateFansScratch((GuestPrimitiveType)userConfig.PrimitiveType, vertices, restartIndex); if (list.Length == 0) { return; } - expandedIndices = ListIndexBytes(list, wide: true); + expandedIndices = ListIndexBytesScratch(list, wide: true, out expandedLength); triangleListIndices = (uint)list.Length; indexType = GpuIndexType.Uint32; topology = GpuPrimitiveTopology.TriangleList; @@ -518,7 +519,7 @@ void TraceDisposition(string disposition) } else if (expandIndex8) { - expandedIndices = ExpandIndex8(arguments.IndexAddress, arguments.IndexCount, primitiveRestart, banks.Context.PrimitiveResetIndex); + expandedIndices = ExpandIndex8Scratch(arguments.IndexAddress, arguments.IndexCount, primitiveRestart, banks.Context.PrimitiveResetIndex, out expandedLength); } else if (primitiveRestart) { @@ -526,7 +527,7 @@ void TraceDisposition(string disposition) var restartIndex = banks.Context.PrimitiveResetIndex & indexMask; if (restartIndex != indexMask) { - expandedIndices = ConvertRestartIndices(arguments.IndexAddress, arguments.IndexCount, indexType, restartIndex); + expandedIndices = ConvertRestartIndicesScratch(arguments.IndexAddress, arguments.IndexCount, indexType, restartIndex, out expandedLength); indexType = GpuIndexType.Uint32; } } @@ -535,7 +536,7 @@ void TraceDisposition(string disposition) true, arguments.IndexAddress, expandedIndices, - expandedIndices is null ? indexSize : (ulong)expandedIndices.Length, + expandedIndices is null ? indexSize : (ulong)expandedLength, indexType); ref var state = ref BeginDrawState(); if (!TryResolveDrawTargets(banks, in draw, ref state)) @@ -816,13 +817,8 @@ private void DrawAutoCore(ulong submitId, RegisterBanks banks, in DrawAutoArgume throw _host.Fatal("A triangle fan to convert to a triangle list still has its counts in guest memory."); } - var vertices = new uint[arguments.VertexCount]; - for (var index = 0u; index < arguments.VertexCount; index++) - { - vertices[index] = index; - } - - var list = TriangulateFans((GuestPrimitiveType)userConfig.PrimitiveType, vertices, restartIndex: null); + var vertices = SequentialVerticesScratch(arguments.VertexCount); + var list = TriangulateFansScratch((GuestPrimitiveType)userConfig.PrimitiveType, vertices, restartIndex: null); if (list.Length == 0) { _host.ResetBindings(); @@ -830,8 +826,8 @@ private void DrawAutoCore(ulong submitId, RegisterBanks banks, in DrawAutoArgume } var wide = arguments.VertexCount > ushort.MaxValue; - var bytes = ListIndexBytes(list, wide); - var listSource = new IndexSource(true, 0, bytes, (ulong)bytes.Length, wide ? GpuIndexType.Uint32 : GpuIndexType.Uint16); + var bytes = ListIndexBytesScratch(list, wide, out var byteCount); + var listSource = new IndexSource(true, 0, bytes, (ulong)byteCount, wide ? GpuIndexType.Uint32 : GpuIndexType.Uint16); var listEmission = new DrawEmission(true, vertexOffset, 0, firstInstance, TriangleListIndices: (uint)list.Length); RecordDraw(submitId, banks, in draw, ref state, GpuPrimitiveTopology.TriangleList, in listEmission, in listSource, primitiveRestart: false, setBindDebug: false, setAutoDebug: true); @@ -858,81 +854,6 @@ private bool TriangulatesFans(UserConfigRegisters userConfig) => (GuestPrimitiveType)userConfig.PrimitiveType is GuestPrimitiveType.TriangleFan or GuestPrimitiveType.Polygon or GuestPrimitiveType.QuadListLegacy; - // The triangle list the primitive assembler makes of these vertices, in its vertex order: - // fan triangle i is (v[i+1], v[i+2], v[0]), the order DI_PT_TRIFAN assembles (the - // first-vertex provoking vertex is v[i+1], as radv's VK fans report it), and a restart - // index ends the fan, the next index starting a new one. A polygon is one fan. Each - // quad of a quad list is (q0, q1, q2) and (q0, q2, q3), both taking q0 as their - // provoking vertex; a count that is not a whole number of quads is refused. - private uint[] TriangulateFans(GuestPrimitiveType type, uint[] vertices, uint? restartIndex) - { - if (type == GuestPrimitiveType.QuadListLegacy) - { - if ((vertices.Length & 0x3) != 0) - { - throw _host.Fatal($"A legacy quad list count is not a multiple of four: count={vertices.Length}."); - } - - var quads = new uint[checked((vertices.Length / 4) * 6)]; - for (int quad = 0, output = 0; quad < vertices.Length; quad += 4, output += 6) - { - quads[output] = vertices[quad]; - quads[output + 1] = vertices[quad + 1]; - quads[output + 2] = vertices[quad + 2]; - quads[output + 3] = vertices[quad]; - quads[output + 4] = vertices[quad + 2]; - quads[output + 5] = vertices[quad + 3]; - } - - return quads; - } - - var triangles = new List(Math.Max(0, (vertices.Length - 2) * 3)); - var hub = 0; - for (var index = 0; index < vertices.Length; index++) - { - if (restartIndex is { } restart && vertices[index] == restart) - { - hub = index + 1; - continue; - } - - if (index - hub >= 2) - { - triangles.Add(vertices[index - 1]); - triangles.Add(vertices[index]); - triangles.Add(vertices[hub]); - } - } - - return [.. triangles]; - } - - private byte[] ListIndexBytes(uint[] list, bool wide) - { - var bytes = new byte[checked(list.Length * (wide ? sizeof(uint) : sizeof(ushort)))]; - for (var index = 0; index < list.Length; index++) - { - var value = list[index]; - if (wide) - { - // Metal reserves the all-ones index for primitive restart. - if (value == uint.MaxValue) - { - throw _host.Fatal("A triangle fan converted to a triangle list has a 32-bit vertex index of 0xFFFFFFFF."); - } - - BinaryPrimitives.WriteUInt32LittleEndian(bytes.AsSpan(index * sizeof(uint)), value); - } - else - { - BinaryPrimitives.WriteUInt16LittleEndian(bytes.AsSpan(index * sizeof(ushort)), checked((ushort)value)); - } - } - - return bytes; - } - private static uint IndexMask(GuestIndexType type) => type switch { GuestIndexType.Index8 => 0xFFu, @@ -970,59 +891,6 @@ private uint[] ReadGuestIndices(ulong indexAddress, uint indexCount, GuestIndexT return indices; } - private byte[] ConvertRestartIndices(ulong indexAddress, uint indexCount, GpuIndexType indexType, uint restartIndex) - { - var sourceStride = indexType == GpuIndexType.Uint16 ? sizeof(ushort) : sizeof(uint); - var source = new byte[checked((int)indexCount * sourceStride)]; - if (indexAddress == 0 || !_host.TryReadGuest(indexAddress, source)) - { - throw _host.Fatal($"The restart index data is unreadable: address=0x{indexAddress:X16} count={indexCount}."); - } - - // Widen 16-bit indices so their maximum value remains an ordinary vertex. - var converted = new byte[checked((int)indexCount * sizeof(uint))]; - for (var index = 0; index < (int)indexCount; index++) - { - var value = sourceStride == sizeof(ushort) - ? BinaryPrimitives.ReadUInt16LittleEndian(source.AsSpan(index * sourceStride)) - : BinaryPrimitives.ReadUInt32LittleEndian(source.AsSpan(index * sourceStride)); - if (value == uint.MaxValue) - { - throw _host.Fatal("A custom restart index conflicts with a 32-bit vertex index of 0xFFFFFFFF."); - } - - BinaryPrimitives.WriteUInt32LittleEndian(converted.AsSpan(index * sizeof(uint)), - value == restartIndex ? uint.MaxValue : value); - } - - return converted; - } - - private byte[] ExpandIndex8(ulong indexAddress, uint indexCount, bool primitiveRestart, uint restartIndex) - { - if (indexAddress == 0) - { - throw _host.Fatal($"An 8-bit index draw has no index address: count={indexCount}."); - } - - var source = new byte[indexCount]; - if (!_host.TryReadGuest(indexAddress, source)) - { - throw _host.Fatal($"The 8-bit index data is unreadable: address=0x{indexAddress:X16} count={indexCount}."); - } - - var expanded = new byte[indexCount * 2]; - for (var i = 0; i < indexCount; i++) - { - var value = source[i]; - ushort index = primitiveRestart && value == (restartIndex & 0xFF) ? (ushort)0xFFFF : value; - expanded[i * 2] = (byte)index; - expanded[(i * 2) + 1] = (byte)(index >> 8); - } - - return expanded; - } - // Vertex offset: the index offset register, or the embedded fetch scalar when that register is zero. public static int ResolveVertexOffset(uint indexOffset, VertexInputInfo vertexInput) { diff --git a/tests/SharpEmu.Libs.Tests/Gpu/Rendering/IndexExpansionTests.cs b/tests/SharpEmu.Libs.Tests/Gpu/Rendering/IndexExpansionTests.cs new file mode 100644 index 000000000..d4ed3d178 --- /dev/null +++ b/tests/SharpEmu.Libs.Tests/Gpu/Rendering/IndexExpansionTests.cs @@ -0,0 +1,286 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using System.Buffers.Binary; +using SharpEmu.Libs.Gpu.Rendering; +using Xunit; + +namespace SharpEmu.Libs.Tests.Gpu.Rendering; + +// The index rewrites of a draw (fans and quad lists as triangle lists, 8-bit widening, the custom +// restart index) write into reused scratch; each must produce what the straightforward allocating form +// did, whatever the scratch held before. +public sealed class IndexExpansionTests +{ + // The allocating forms the executor used before the scratch. + private static uint[] ReferenceTriangulate(GuestPrimitiveType type, uint[] vertices, uint? restartIndex) + { + if (type == GuestPrimitiveType.QuadListLegacy) + { + var quads = new uint[checked((vertices.Length / 4) * 6)]; + for (int quad = 0, output = 0; quad < vertices.Length; quad += 4, output += 6) + { + quads[output] = vertices[quad]; + quads[output + 1] = vertices[quad + 1]; + quads[output + 2] = vertices[quad + 2]; + quads[output + 3] = vertices[quad]; + quads[output + 4] = vertices[quad + 2]; + quads[output + 5] = vertices[quad + 3]; + } + + return quads; + } + + var triangles = new List(Math.Max(0, (vertices.Length - 2) * 3)); + var hub = 0; + for (var index = 0; index < vertices.Length; index++) + { + if (restartIndex is { } restart && vertices[index] == restart) + { + hub = index + 1; + continue; + } + + if (index - hub >= 2) + { + triangles.Add(vertices[index - 1]); + triangles.Add(vertices[index]); + triangles.Add(vertices[hub]); + } + } + + return [.. triangles]; + } + + private static byte[] ReferenceListBytes(uint[] list, bool wide) + { + var bytes = new byte[list.Length * (wide ? sizeof(uint) : sizeof(ushort))]; + for (var index = 0; index < list.Length; index++) + { + if (wide) + { + BinaryPrimitives.WriteUInt32LittleEndian(bytes.AsSpan(index * sizeof(uint)), list[index]); + } + else + { + BinaryPrimitives.WriteUInt16LittleEndian(bytes.AsSpan(index * sizeof(ushort)), checked((ushort)list[index])); + } + } + + return bytes; + } + + private static byte[] ReferenceIndex8(byte[] source, bool primitiveRestart, uint restartIndex) + { + var expanded = new byte[source.Length * 2]; + for (var i = 0; i < source.Length; i++) + { + var value = source[i]; + ushort index = primitiveRestart && value == (restartIndex & 0xFF) ? (ushort)0xFFFF : value; + expanded[i * 2] = (byte)index; + expanded[(i * 2) + 1] = (byte)(index >> 8); + } + + return expanded; + } + + private static byte[] ReferenceRestart(byte[] source, int stride, uint restartIndex) + { + var count = source.Length / stride; + var converted = new byte[count * sizeof(uint)]; + for (var index = 0; index < count; index++) + { + var value = stride == sizeof(ushort) + ? BinaryPrimitives.ReadUInt16LittleEndian(source.AsSpan(index * stride)) + : BinaryPrimitives.ReadUInt32LittleEndian(source.AsSpan(index * stride)); + BinaryPrimitives.WriteUInt32LittleEndian(converted.AsSpan(index * sizeof(uint)), + value == restartIndex ? uint.MaxValue : value); + } + + return converted; + } + + private static uint[] Vertices(System.Random random, int count, uint? restart, uint ceiling) + { + var vertices = new uint[count]; + for (var index = 0; index < count; index++) + { + vertices[index] = restart is { } r && random.Next(5) == 0 ? r : (uint)random.NextInt64(0, ceiling); + } + + return vertices; + } + + // The scratch the executor holds is dirty with a longer, different draw before each case. + private static void Dirty(Span list, Span bytes) + { + list.Fill(0xDEAD_BEEF); + bytes.Fill(0xA5); + } + + [Theory] + [InlineData(GuestPrimitiveType.TriangleFan, false)] + [InlineData(GuestPrimitiveType.Polygon, false)] + [InlineData(GuestPrimitiveType.TriangleFan, true)] + [InlineData(GuestPrimitiveType.QuadListLegacy, false)] + public void TriangulateFans_MatchesTheAllocatingForm(GuestPrimitiveType type, bool withRestart) + { + var random = new System.Random(1234 + (int)type); + foreach (var count in new[] { 0, 1, 2, 3, 4, 5, 8, 9, 64, 1000, 5001 }) + { + if (type == GuestPrimitiveType.QuadListLegacy && (count & 3) != 0) + { + continue; + } + + uint? restart = withRestart ? 0xFFFFu : null; + var vertices = Vertices(random, count, restart, 60_000); + var expected = ReferenceTriangulate(type, vertices, restart); + var capacity = IndexExpansion.MaxTriangleListCount(type, count); + Assert.True(expected.Length <= capacity); + + var output = new uint[capacity + 7]; + Dirty(output, []); + var written = IndexExpansion.TriangulateFans(type, vertices, restart, output.AsSpan(0, capacity)); + + Assert.Equal(expected, output.AsSpan(0, written).ToArray()); + } + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void ListIndexBytes_MatchTheAllocatingForm(bool wide) + { + var random = new System.Random(77); + foreach (var count in new[] { 0, 3, 6, 255, 4096 }) + { + var list = Vertices(random, count, null, wide ? 4_000_000_000u : 65_536u); + var expected = ReferenceListBytes(list, wide); + var bytes = new byte[expected.Length + 16]; + Dirty([], bytes); + + Assert.True(IndexExpansion.WriteListIndexBytes(list, wide, bytes)); + Assert.Equal(expected, bytes.AsSpan(0, expected.Length).ToArray()); + } + } + + [Fact] + public void ListIndexBytes_RefuseAnAllOnesIndexAndANarrowOverflow() + { + Assert.False(IndexExpansion.WriteListIndexBytes([1u, uint.MaxValue], wide: true, new byte[8])); + Assert.Throws(() => IndexExpansion.WriteListIndexBytes([0x1_0000u], wide: false, new byte[2])); + } + + [Theory] + [InlineData(false, 0xFFu)] + [InlineData(true, 0xFFu)] + [InlineData(true, 0x1_0007u)] + public void ExpandIndex8_MatchesTheAllocatingForm(bool primitiveRestart, uint restartIndex) + { + var random = new System.Random(5); + foreach (var count in new[] { 0, 1, 7, 256, 3000 }) + { + var source = new byte[count]; + random.NextBytes(source); + if (count > 3) + { + source[1] = (byte)(restartIndex & 0xFF); + } + + var expected = ReferenceIndex8(source, primitiveRestart, restartIndex); + var destination = new byte[expected.Length + 5]; + Dirty([], destination); + + IndexExpansion.ExpandIndex8(source, primitiveRestart, restartIndex, destination); + + Assert.Equal(expected, destination.AsSpan(0, expected.Length).ToArray()); + } + } + + [Theory] + [InlineData(2, 0xFFFEu)] + [InlineData(2, 0x7u)] + [InlineData(4, 0xFFFF_FFFEu)] + [InlineData(4, 0x12u)] + public void ConvertRestart_MatchesTheAllocatingForm(int stride, uint restartIndex) + { + var random = new System.Random(99); + foreach (var count in new[] { 0, 1, 5, 400 }) + { + var source = new byte[count * stride]; + for (var index = 0; index < count; index++) + { + var value = random.Next(3) == 0 ? restartIndex : (uint)random.NextInt64(0, stride == 2 ? 0xFFFF : 0xFFFF_FFFEL); + if (stride == 2) + { + BinaryPrimitives.WriteUInt16LittleEndian(source.AsSpan(index * 2), (ushort)value); + } + else + { + BinaryPrimitives.WriteUInt32LittleEndian(source.AsSpan(index * 4), value); + } + } + + var expected = ReferenceRestart(source, stride, restartIndex); + var destination = new byte[expected.Length + 9]; + Dirty([], destination); + + Assert.True(IndexExpansion.ConvertRestart(source, stride, restartIndex, destination)); + Assert.Equal(expected, destination.AsSpan(0, expected.Length).ToArray()); + } + } + + [Fact] + public void ConvertRestart_RefusesAnIndexThatIsAlreadyAllOnes() + { + var source = new byte[8]; + BinaryPrimitives.WriteUInt32LittleEndian(source.AsSpan(4), uint.MaxValue); + Assert.False(IndexExpansion.ConvertRestart(source, sizeof(uint), 7, new byte[8])); + } + + [Theory] + [InlineData(1)] + [InlineData(2)] + [InlineData(4)] + public void Widen_ReadsLittleEndianIndicesOfEachStride(int stride) + { + var random = new System.Random(stride); + var count = 300; + var source = new byte[count * stride]; + random.NextBytes(source); + var widened = new uint[count + 3]; + widened.AsSpan().Fill(0xFFFF_FFFF); + + IndexExpansion.Widen(source, stride, widened.AsSpan(0, count)); + + for (var index = 0; index < count; index++) + { + var expected = stride switch + { + 1 => source[index], + 2 => BinaryPrimitives.ReadUInt16LittleEndian(source.AsSpan(index * 2)), + _ => BinaryPrimitives.ReadUInt32LittleEndian(source.AsSpan(index * 4)), + }; + Assert.Equal(expected, widened[index]); + } + + Assert.Equal(0xFFFF_FFFFu, widened[count]); + } + + [Fact] + public void TheScratchGrowsAndKeepsItsBuffersUnlessAnOversizedDraw() + { + var scratch = new IndexExpansionScratch(); + var small = scratch.Vertices(100); + Assert.Same(small, scratch.Vertices(50)); + var grown = scratch.Vertices(1000); + Assert.True(grown.Length >= 1000); + Assert.Same(grown, scratch.Vertices(1000)); + + // A request past the retention limit is served and then forgotten, so one huge draw pins nothing. + var huge = scratch.Vertices(4 << 20); + Assert.True(huge.Length >= 4 << 20); + Assert.Same(grown, scratch.Vertices(1000)); + } +} From e8dfe3df147e8d89ca7dec3382c5d9a988f9a520 Mon Sep 17 00:00:00 2001 From: kuba Date: Mon, 26 Oct 2026 05:00:58 +0100 Subject: [PATCH 03/82] [GPU] A register snapshot is handed out again while no register write has happened since SHARPEMU_REGISTER_SNAPSHOT_REUSE=0 takes a fresh snapshot every time. --- .../GpuCommands/Registers/RegisterBanks.cs | 28 +++++++++ .../Registers/RegisterBanksSnapshotTests.cs | 63 +++++++++++++++++++ 2 files changed, 91 insertions(+) diff --git a/src/SharpEmu.Libs/Gpu/GpuCommands/Registers/RegisterBanks.cs b/src/SharpEmu.Libs/Gpu/GpuCommands/Registers/RegisterBanks.cs index d514bcf58..5c1e31364 100644 --- a/src/SharpEmu.Libs/Gpu/GpuCommands/Registers/RegisterBanks.cs +++ b/src/SharpEmu.Libs/Gpu/GpuCommands/Registers/RegisterBanks.cs @@ -65,16 +65,39 @@ public RegisterBanks Clone() private bool _computeShared; private bool _userConfigShared; + // The last snapshot, handed out again while no writer has prepared a write since: its parts are + // still the live ones, so a second copy would be the same banks in another object. A snapshot is + // read-only, so sharing one between draws changes nothing a draw can see. The live banks' + // draw-time markers are not covered by the write hooks, so they are compared. + // SHARPEMU_REGISTER_SNAPSHOT_REUSE=0 takes a fresh snapshot every time. + private static readonly bool SnapshotReuseEnabled = + Environment.GetEnvironmentVariable("SHARPEMU_REGISTER_SNAPSHOT_REUSE") != "0"; + + private RegisterBanks? _lastSnapshot; + + // Raised by every writer hook below; lowered when a snapshot is taken. + private bool _writtenSinceSnapshot = true; + // A read-only copy that shares every part with the live banks until they next write it: a // deferred draw takes one per draw, and between draws only a few parts change. public RegisterBanks Snapshot() { + if (!_writtenSinceSnapshot && _lastSnapshot is { } last && + last.CompositeDepthSizeXy == CompositeDepthSizeXy && + last.UserDataMarker == UserDataMarker && + last.IndexTypeAndSize == IndexTypeAndSize) + { + return last; + } + var snapshot = new RegisterBanks(Fatal, Context, Shader.ShareStages(), UserConfig) { CompositeDepthSizeXy = CompositeDepthSizeXy, UserDataMarker = UserDataMarker, IndexTypeAndSize = IndexTypeAndSize, }; + _lastSnapshot = SnapshotReuseEnabled ? snapshot : null; + _writtenSinceSnapshot = false; _contextShared = _vertexShared = _pixelShared = _computeShared = _userConfigShared = true; _pixelScalarsShared = _hullScalarsShared = _geometryScalarsShared = _legacyVertexScalarsShared = _exportScalarsShared = _computeScalarsShared = true; return snapshot; @@ -83,6 +106,7 @@ public RegisterBanks Snapshot() // Called by the register writers before they store into a part. public void PrepareContextWrite() { + _writtenSinceSnapshot = true; if (_contextShared) { Context = Context.ShallowCopy(); @@ -109,6 +133,7 @@ private void SetContextPartsShared(bool shared) => private void OwnContext(ContextRegisters context) { + _writtenSinceSnapshot = true; Context = context; _contextShared = false; SetContextPartsShared(false); @@ -251,6 +276,7 @@ public Dictionary UnmodeledTableRegistersForWrite public void PrepareUserConfigWrite() { + _writtenSinceSnapshot = true; if (_userConfigShared) { UserConfig = UserConfig.Copy(); @@ -278,6 +304,7 @@ private static bool Overlaps(uint first, uint last, uint windowStart) => // compute range, compute from there. public void PrepareShaderWrite(uint offset, uint count) { + _writtenSinceSnapshot = true; var last = offset + Math.Max(count, 1) - 1; if (offset < ShaderRegisterOffset.SpiShaderPgmLoVs) { @@ -346,6 +373,7 @@ public void PrepareShaderWrite(uint offset, uint count) public void Reset() { + _writtenSinceSnapshot = true; _contextShared = _vertexShared = _pixelShared = _computeShared = _userConfigShared = false; _pixelScalarsShared = _hullScalarsShared = _geometryScalarsShared = _legacyVertexScalarsShared = _exportScalarsShared = _computeScalarsShared = false; OwnContext(new ContextRegisters()); diff --git a/tests/SharpEmu.Libs.Tests/Gpu/GpuCommands/Registers/RegisterBanksSnapshotTests.cs b/tests/SharpEmu.Libs.Tests/Gpu/GpuCommands/Registers/RegisterBanksSnapshotTests.cs index 18ccd44b9..50f5a6818 100644 --- a/tests/SharpEmu.Libs.Tests/Gpu/GpuCommands/Registers/RegisterBanksSnapshotTests.cs +++ b/tests/SharpEmu.Libs.Tests/Gpu/GpuCommands/Registers/RegisterBanksSnapshotTests.cs @@ -233,4 +233,67 @@ public void ASecondSnapshotSharesAgainUntilTheNextWrite() Assert.True(second.Context.DepthTarget.DepthTestEnabled); Assert.False(banks.Context.DepthTarget.DepthTestEnabled); } + + [Fact] + public void ASnapshotWithNoWriteSinceIsHandedOutAgain() + { + var banks = NewBanks(); + WriteContext(banks, DbDepthControl, 0x0000_0006); + var first = banks.Snapshot(); + + Assert.Same(first, banks.Snapshot()); + Assert.Same(first, banks.Snapshot()); + } + + [Fact] + public void EveryKindOfWriteEndsTheReuseOfTheLastSnapshot() + { + var writes = new Action[] + { + banks => WriteContext(banks, DbDepthControl, 0), + banks => WriteShader(banks, SpiShaderPgmLoPs, 0x4000), + banks => WriteShader(banks, SpiShaderUserDataPs0, 5), + banks => WriteShader(banks, ComputePgmLo, 0x4000), + banks => WriteUserConfig(banks, VgtPrimitiveType, 6), + banks => banks.ApplyContextState(ContextStateOperation.Clear), + banks => banks.Reset(), + banks => banks.IndexTypeAndSize = 2, + banks => banks.CompositeDepthSizeXy = 0x1234, + banks => banks.UserDataMarker = UserScalarKind.BufferResource, + }; + foreach (var write in writes) + { + var banks = NewBanks(); + WriteContext(banks, DbDepthControl, 0x0000_0006); + var before = banks.Snapshot(); + + write(banks); + var after = banks.Snapshot(); + + Assert.NotSame(before, after); + Assert.Same(after, banks.Snapshot()); + } + } + + [Fact] + public void AWriteAfterAReusedSnapshotLeavesEveryHolderOfItUnchanged() + { + var banks = NewBanks(); + WriteContext(banks, DbDepthControl, 0x0000_0006); + WriteShader(banks, SpiShaderUserDataPs0, 11, 12); + var first = banks.Snapshot(); + var second = banks.Snapshot(); + Assert.Same(first, second); + + WriteContext(banks, DbDepthControl, 0x0000_0000); + WriteShader(banks, SpiShaderUserDataPs0, 21, 22); + var third = banks.Snapshot(); + + Assert.NotSame(second, third); + Assert.True(first.Context.DepthTarget.DepthTestEnabled); + Assert.True(second.Context.DepthTarget.DepthTestEnabled); + Assert.Equal([11u, 12u], first.Shader.Pixel.UserScalars.Values[..2]); + Assert.False(third.Context.DepthTarget.DepthTestEnabled); + Assert.Equal([21u, 22u], third.Shader.Pixel.UserScalars.Values[..2]); + } } From ae796903e51792f4876ef7786630f2551692df5e Mon Sep 17 00:00:00 2001 From: kuba Date: Mon, 26 Oct 2026 05:00:58 +0100 Subject: [PATCH 04/82] [GPU] The command queue pulses its gate only when a thread waits and polls watched labels from 50 us SHARPEMU_CP_QUEUE_PULSE=0 and SHARPEMU_CP_WATCH_ADAPTIVE=0 restore the old behaviour. --- .../Gpu/GpuCommands/CommandStreamQueue.cs | 122 +++++++++++++--- .../CommandStreamQueueWakeTests.cs | 137 ++++++++++++++++++ 2 files changed, 237 insertions(+), 22 deletions(-) create mode 100644 tests/SharpEmu.Libs.Tests/Gpu/GpuCommands/CommandStreamQueueWakeTests.cs diff --git a/src/SharpEmu.Libs/Gpu/GpuCommands/CommandStreamQueue.cs b/src/SharpEmu.Libs/Gpu/GpuCommands/CommandStreamQueue.cs index 066acf461..195317376 100644 --- a/src/SharpEmu.Libs/Gpu/GpuCommands/CommandStreamQueue.cs +++ b/src/SharpEmu.Libs/Gpu/GpuCommands/CommandStreamQueue.cs @@ -297,7 +297,7 @@ internal GuestSubmissionPreparationResult WaitForPreparation(CommandSubmissionPr var remaining = deadline - Environment.TickCount64; if (remaining <= 0) { failure = new TimeoutException("Accepted command preparation did not reach a runnable-prefix boundary."); break; } - Monitor.Wait(_gate, (int)Math.Min(remaining, 1000)); + WaitLocked((int)Math.Min(remaining, 1000)); } failure ??= receipt.Failure ?? _preparationFailure; if (failure is null && receipt.Status == CommandSubmissionPreparation.State.Failed) @@ -485,7 +485,7 @@ submission.Kind is CommandSubmissionKind.Graphics or CommandSubmissionKind.Compu (ulong)submission.Kind, submission.DwordCount); } - Monitor.PulseAll(_gate); + PulseWaitersLocked(); } public bool TryEnqueueControlBarrier(Action work, Action cancel) @@ -500,7 +500,7 @@ public bool TryEnqueueControlBarrier(Action work, Action cancel) } _controlBarriers.Enqueue(new ControlBarrier(_admissionOrdinal, work, cancel)); - Monitor.PulseAll(_gate); + PulseWaitersLocked(); return true; } } @@ -531,7 +531,7 @@ public IdleOutcome Done() // Shutdown: no slice will run a queued boundary, so drain as before. while (_outcome == IdleOutcome.Completed && (_processing || _submissionCount != 0)) { - Monitor.Wait(_gate); + WaitLocked(Timeout.Infinite); } _doneCount++; @@ -545,7 +545,7 @@ public IdleOutcome Done() var dsStarted = dsSuspend ? System.Diagnostics.Stopwatch.GetTimestamp() : 0L; while (_outcome == IdleOutcome.Completed && _pendingBoundaries > _boundariesInFlight) { - Monitor.Wait(_gate); + WaitLocked(Timeout.Infinite); } if (dsSuspend) @@ -569,7 +569,7 @@ public IdleOutcome WaitForIdle() while (_outcome == IdleOutcome.Completed && (_processing || _submissionCount != 0 || _controlBarriers.Count != 0)) { - Monitor.Wait(_gate); + WaitLocked(Timeout.Infinite); } return _outcome; @@ -580,7 +580,7 @@ public void Wake() { lock (_gate) { - Monitor.PulseAll(_gate); + PulseWaitersLocked(); } } @@ -605,13 +605,90 @@ public bool WaitForWork(int timeoutMilliseconds) } } + // Threads inside a wait on the gate (counted under it). Every waiter shares the one monitor: the + // worker for work, a frame boundary or a drain for completion, a preparation for its prefix. + private int _waiters; + + // SHARPEMU_CP_QUEUE_PULSE=0 pulses every waiter on every state change again. + private static readonly bool ElidePulses = Environment.GetEnvironmentVariable("SHARPEMU_CP_QUEUE_PULSE") != "0"; + + // Caller holds the gate. + private void WaitLocked(int timeoutMilliseconds) + { + _waiters++; + try + { + Monitor.Wait(_gate, timeoutMilliseconds); + } + finally + { + _waiters--; + } + } + + // Caller holds the gate. Wakes the threads parked on it: none when nobody waits (a busy worker, + // a producer with no frame boundary pending), a single pulse when one does, a broadcast only + // when several kinds of waiter could be parked. A waiter counted but already pulsed misses a + // second pulse and does not need it: it re-reads the state when it takes the gate. + private void PulseWaitersLocked() + { + if (!ElidePulses) + { + Monitor.PulseAll(_gate); + } + else if (_waiters == 1) + { + Monitor.Pulse(_gate); + } + else if (_waiters != 0) + { + Monitor.PulseAll(_gate); + } + } + + // SHARPEMU_CP_WATCH_ADAPTIVE=0 keeps the fixed watch-interval poll. + private static readonly bool AdaptiveWatch = Environment.GetEnvironmentVariable("SHARPEMU_CP_WATCH_ADAPTIVE") != "0"; + + private const int WatchPollStartMicroseconds = 50; + + // The current poll of a watched wait: starts short, so a store the CP waits for is noticed in + // microseconds, and doubles up to the watch interval while nothing changes. Reset when a re-read + // sees a change. + private int _watchPollMicroseconds = WatchPollStartMicroseconds; + + // Caller holds the gate (once). One poll of a watched wait. Below the watch interval it releases + // the gate for a precise sleep, as Monitor.Wait cannot wait less than a millisecond: a submission + // enqueued meanwhile waits out the sleep (at most the poll, under the interval). At the interval + // it is a gate wait, which an enqueue cuts short. + private void WaitWatchIntervalLocked(long remainingMilliseconds) + { + var intervalMicroseconds = WatchIntervalMilliseconds * 1000; + var poll = _watchPollMicroseconds; + if (!AdaptiveWatch || poll >= intervalMicroseconds) + { + WaitLocked((int)Math.Min(remainingMilliseconds, WatchIntervalMilliseconds)); + return; + } + + _watchPollMicroseconds = Math.Min(poll * 2, intervalMicroseconds); + Monitor.Exit(_gate); + try + { + HostTiming.SleepMicroseconds(Math.Min(poll, remainingMilliseconds * 1000)); + } + finally + { + Monitor.Enter(_gate); + } + } + // Caller holds the gate. Sleeps up to the timeout, cut into watch intervals while a // parked head waits on a watched label, and returns early once a re-read woke one. private void WaitWatchingLocked(int timeoutMilliseconds) { if (WatchIntervalMilliseconds == 0 || !HasWatchedHeadLocked()) { - Monitor.Wait(_gate, timeoutMilliseconds); + WaitLocked(timeoutMilliseconds); return; } @@ -624,7 +701,7 @@ private void WaitWatchingLocked(int timeoutMilliseconds) return; } - Monitor.Wait(_gate, (int)Math.Min(remaining, WatchIntervalMilliseconds)); + WaitWatchIntervalLocked(remaining); if (RecheckWatchedLocked()) { return; @@ -685,6 +762,7 @@ private bool RecheckWatchedLocked() } head.WatchedValue = value; + _watchPollMicroseconds = WatchPollStartMicroseconds; if (wait.Compares && (!Packets.WaitOperation.TryCompare(value, wait.Reference, wait.Mask, wait.Function, out var satisfied) || !satisfied)) { @@ -699,7 +777,7 @@ private bool RecheckWatchedLocked() if (woke) { _wakeGeneration++; - Monitor.PulseAll(_gate); + PulseWaitersLocked(); } return woke; @@ -721,12 +799,12 @@ public bool WaitForRetryInterval(int intervalMilliseconds) if (WatchIntervalMilliseconds != 0 && HasWatchedHeadLocked()) { - Monitor.Wait(_gate, (int)Math.Min(remaining, WatchIntervalMilliseconds)); + WaitWatchIntervalLocked(remaining); _ = RecheckWatchedLocked(); } else { - Monitor.Wait(_gate, (int)remaining); + WaitLocked((int)remaining); } } @@ -777,7 +855,7 @@ private void FailLocked() _activePreparation?.Resolve(CommandSubmissionPreparation.State.Failed, _preparationFailure); _activePreparation = null; _processing = false; - Monitor.PulseAll(_gate); + PulseWaitersLocked(); } // A completed GPU tick can make a blocked memory wait ready. A flip slot or a @@ -1051,7 +1129,7 @@ public void NotifyPublicationCompleted() BlockedRetries++; } - Monitor.PulseAll(_gate); + PulseWaitersLocked(); } } @@ -1095,7 +1173,7 @@ public void RetryBlocked() PacketProfile.RecordRetryAll(); } - Monitor.PulseAll(_gate); + PulseWaitersLocked(); } } @@ -1196,7 +1274,7 @@ private void WakeMatchingLocked(TState state, Func StreamRunner.Packet(PacketOpcode.NumInstances, count); + + private static void Enqueue(RecordingCommandStreamHost host, CommandStreamQueue queue, ulong submissionId) + { + var words = Instances(3); + host.WriteWords(Graphics, words); + queue.EnqueueGraphics(Graphics, (uint)words.Length, submissionId, null); + } + + // Waits until the thread has parked inside the queue's wait (it cannot be observed, so it is + // given a moment after it announces itself; a wake that came too early is still found by the + // thread's own check, which is the case the repeated rounds cover). + private static void LetItPark(ManualResetEventSlim announced) + { + Assert.True(announced.Wait(TimeSpan.FromSeconds(30))); + Thread.Sleep(2); + } + + [Fact] + public void AnEnqueueWakesAParkedWorkerEveryTime() + { + var host = new RecordingCommandStreamHost(); + var queue = new CommandStreamQueue(host); + for (var round = 0; round < 300; round++) + { + using var announced = new ManualResetEventSlim(); + var worker = Task.Run(() => + { + announced.Set(); + // A lost wakeup shows as the timeout passing with nothing runnable. + return queue.WaitForWork(20_000); + }); + LetItPark(announced); + + Enqueue(host, queue, (ulong)round + 1); + + Assert.True(worker.Wait(TimeSpan.FromSeconds(60))); + Assert.True(worker.Result, $"round {round}: the worker slept through an enqueue"); + Assert.Equal(SliceResult.Completed, queue.ProcessOne()); + } + } + + [Fact] + public void AnEnqueueWakesTheWorkerWhileAFrameBoundaryWaitsToo() + { + for (var round = 0; round < 25; round++) + { + var host = new RecordingCommandStreamHost(); + var queue = new CommandStreamQueue(host, boundariesInFlight: 0); + // A head that parks with no label to watch: the frame boundary queued behind it cannot run. + var blocker = StreamRunner.Packet(PacketOpcode.WaitOnDeCounterDiff, 0); + host.WriteWords(Graphics, blocker); + queue.EnqueueGraphics(Graphics, (uint)blocker.Length, 1, null); + Assert.Equal(SliceResult.BlockedWithoutProgress, queue.ProcessOne()); + + // The boundary's guest thread now waits on the gate for the worker to run the boundary. + var guest = Task.Run(queue.Done); + Assert.True(SpinWait.SpinUntil(() => queue.PendingSubmissionCount == 2, TimeSpan.FromSeconds(30))); + + using var announced = new ManualResetEventSlim(); + var worker = Task.Run(() => + { + announced.Set(); + return queue.WaitForWork(20_000); + }); + LetItPark(announced); + + // Two kinds of waiter are parked: a compute submission wakes the worker, not just the boundary. + var compute = Instances(3); + host.WriteWords(StreamRunner.DataAddress, compute); + queue.EnqueueCompute(GpuCommandInterpreter.ComputeQueueBase, StreamRunner.DataAddress, (uint)compute.Length, 2, null); + Assert.True(worker.Wait(TimeSpan.FromSeconds(60))); + Assert.True(worker.Result, $"round {round}: the worker slept through an enqueue"); + + // Ending the queue reaches the boundary's waiter as well. + queue.Fail(); + Assert.Equal(IdleOutcome.Failed, guest.WaitAsync(TimeSpan.FromSeconds(60)).GetAwaiter().GetResult()); + } + } + + [Fact] + public void ARunSliceWakesADrainWaitingForIdleEveryTime() + { + var host = new RecordingCommandStreamHost(); + var queue = new CommandStreamQueue(host); + for (var round = 0; round < 200; round++) + { + Enqueue(host, queue, (ulong)round + 1); + var drain = Task.Run(queue.WaitForIdle); + Thread.Sleep(1); + + while (queue.HasPending) + { + Assert.NotEqual(SliceResult.NoWork, queue.ProcessOne()); + } + + Assert.Equal(IdleOutcome.Completed, drain.WaitAsync(TimeSpan.FromSeconds(60)).GetAwaiter().GetResult()); + } + } + + [Fact] + public void WakeReleasesAParkedWorkerWhoseStateChangedOutsideTheQueue() + { + var host = new RecordingCommandStreamHost(); + var queue = new CommandStreamQueue(host); + using var announced = new ManualResetEventSlim(); + var worker = Task.Run(() => + { + announced.Set(); + // Nothing is runnable and no head is watched: only a timeout or a pulse ends the wait. + var stopwatch = System.Diagnostics.Stopwatch.StartNew(); + queue.WaitForWork(20_000); + return stopwatch.Elapsed; + }); + LetItPark(announced); + + queue.Wake(); + + Assert.True(worker.Wait(TimeSpan.FromSeconds(60))); + Assert.True(worker.Result < TimeSpan.FromSeconds(15), $"the worker waited {worker.Result.TotalSeconds:F1} s for a wake"); + } +} From cdf32422887de11a765c4061d536cbc02335de2b Mon Sep 17 00:00:00 2001 From: kuba Date: Mon, 26 Oct 2026 05:01:00 +0100 Subject: [PATCH 05/82] [Split] The mirror translator counts cache hits and misses only under SHARPEMU_SPLIT_STATS Every guest address translation from the worker, the program resolvers and the ring threads did a plain shared _hits++ or _misses++, one cache line bouncing between cores per descriptor word, shader read and staged range. The counters are now off unless SHARPEMU_SPLIT_STATS=1 (CountCacheAccesses); the existing hit/miss test turns them on. --- .../GuestMemory/GuestMirrorTranslator.cs | 12 ++++++++++-- .../Memory/GuestMemory/GuestViewTableTests.cs | 5 ++++- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/src/SharpEmu.HLE/GuestMemory/GuestMirrorTranslator.cs b/src/SharpEmu.HLE/GuestMemory/GuestMirrorTranslator.cs index 63d9fe7eb..c2e085899 100644 --- a/src/SharpEmu.HLE/GuestMemory/GuestMirrorTranslator.cs +++ b/src/SharpEmu.HLE/GuestMemory/GuestMirrorTranslator.cs @@ -80,6 +80,14 @@ public GuestMirrorTranslator(HostSharedRegion table, HostBackingObject mirror) public ulong MappingGeneration => Volatile.Read(ref _header->MappingGeneration); + /// + /// Whether translations count their cache hits and misses (, + /// ). Off unless SHARPEMU_SPLIT_STATS=1: the worker, the program + /// resolvers and the ring threads all translate, and a counter they share is a cache line + /// bouncing between cores on every address of every descriptor, shader read and staged range. + /// + public bool CountCacheAccesses { get; set; } = GpuMemory.SplitCrossingStats.Enabled; + // Diagnostics only: plain increments, so counts from racing threads may be lost. public long CacheHits => Volatile.Read(ref _hits); @@ -153,12 +161,12 @@ private bool TryGetRange(ulong address, out CachedRange range) var cached = Volatile.Read(ref slot); if (cached is not null && cached.Generation == generation && address >= cached.Start && address < cached.End) { - _hits++; + if (CountCacheAccesses) _hits++; range = cached; return true; } - _misses++; + if (CountCacheAccesses) _misses++; range = null!; if (!GuestViewTableReader.TryFind(_header, _entries, _capacity, address, joinRun: true, out var entry, out var readGeneration)) return false; diff --git a/tests/SharpEmu.Libs.Tests/Memory/GuestMemory/GuestViewTableTests.cs b/tests/SharpEmu.Libs.Tests/Memory/GuestMemory/GuestViewTableTests.cs index 567228ca8..61145a2ce 100644 --- a/tests/SharpEmu.Libs.Tests/Memory/GuestMemory/GuestViewTableTests.cs +++ b/tests/SharpEmu.Libs.Tests/Memory/GuestMemory/GuestViewTableTests.cs @@ -182,7 +182,10 @@ public void TranslatorCacheSurvivesProtectionChangesButNotUnmaps() using var table = NewTable(); table.Map(Base, 4 * Block, 3 * Block, HostPageProtection.ReadWrite); table.Map(Base + 4 * Block, Block, 40 * Block, HostPageProtection.ReadWrite); - var translator = new GuestMirrorTranslator(table.Region.Address, table.Region.Size, MirrorBase, MirrorBytes); + var translator = new GuestMirrorTranslator(table.Region.Address, table.Region.Size, MirrorBase, MirrorBytes) + { + CountCacheAccesses = true, + }; Assert.True(translator.TryTranslate(Base + 0x100, 0x100, out var mirror)); Assert.Equal(MirrorBase + 3 * Block + 0x100, mirror); From 2a9b016e820eb9866d16d66e22b86814c0da089e Mon Sep 17 00:00:00 2001 From: kuba Date: Mon, 26 Oct 2026 05:01:00 +0100 Subject: [PATCH 06/82] [Split] A posted call wakes one sleeping server thread instead of all of them The call tables' service bell woke every sleeper (8 threads in the GPU process, 4 in the guest process); all but one found nothing, spun 50/20 us and slept again. Doorbell.RingOne wakes one waiter; the item is published before the ring, waiters re-check after registering and a woken server drains the table, so no wake is lost. Test: 400 sequential calls with 8 sleeping servers caused 2691 empty wake-ups before and fewer than one per call after; a 6-caller x 3000 call contention test never loses a call. --- src/SharpEmu.GpuRemote/Doorbell.cs | 17 +++ src/SharpEmu.GpuRemote/RemoteWait.cs | 14 +++ src/SharpEmu.GpuRemote/SyncCallTable.cs | 3 +- .../SyncCallTableTests.cs | 113 ++++++++++++++++++ 4 files changed, 146 insertions(+), 1 deletion(-) diff --git a/src/SharpEmu.GpuRemote/Doorbell.cs b/src/SharpEmu.GpuRemote/Doorbell.cs index 112d568c0..9c1f0cae6 100644 --- a/src/SharpEmu.GpuRemote/Doorbell.cs +++ b/src/SharpEmu.GpuRemote/Doorbell.cs @@ -56,6 +56,23 @@ public void Ring() } } + /// + /// Publishes a signal that one waiter can act on: wakes one registered sleeper, if any. For a + /// bell whose waiters are interchangeable servers of a shared queue, where each signal + /// announces one item. No wake is lost: the item is published before the ring, every waiter + /// re-checks the queue after registering, and a woken waiter drains the queue before it + /// sleeps again, so an item posted while the only sleeper is on its way up is still found. + /// + [MethodImpl(MethodImplOptions.AggressiveOptimization)] + public void RingOne() + { + Interlocked.Increment(ref _state->Sequence); + if (Volatile.Read(ref _state->Sleepers) != 0) + { + RemoteWait.WakeOne(&_state->Sequence); + } + } + /// /// Bumps the sequence and wakes every waiter regardless of registration; used when one side /// is going away and every wait must notice. diff --git a/src/SharpEmu.GpuRemote/RemoteWait.cs b/src/SharpEmu.GpuRemote/RemoteWait.cs index 6d4f301fc..cb72962ca 100644 --- a/src/SharpEmu.GpuRemote/RemoteWait.cs +++ b/src/SharpEmu.GpuRemote/RemoteWait.cs @@ -95,6 +95,20 @@ public static void WakeAll(uint* word) } } + /// + /// Wakes one waiter blocked on , in any process. For a word several + /// threads of one process serve alike (the call tables' service bell): one request needs one + /// thread, and waking the rest only makes each of them spin and go back to sleep. + /// + [MethodImpl(MethodImplOptions.AggressiveOptimization)] + public static void WakeOne(uint* word) + { + if (RemoteNative.UlockWake != null) + { + _ = RemoteNative.UlockWake(UL_COMPARE_AND_WAIT_SHARED | ULF_NO_ERRNO, word, 0); + } + } + /// One spin-loop pause instruction (PAUSE on x86-64, translated by Rosetta; YIELD on arm64). [MethodImpl(MethodImplOptions.AggressiveInlining)] public static void Pause() diff --git a/src/SharpEmu.GpuRemote/SyncCallTable.cs b/src/SharpEmu.GpuRemote/SyncCallTable.cs index 10b5f3e49..e40a9a678 100644 --- a/src/SharpEmu.GpuRemote/SyncCallTable.cs +++ b/src/SharpEmu.GpuRemote/SyncCallTable.cs @@ -210,7 +210,8 @@ public SyncCallStatus Call( Volatile.Write(ref slot->State, SyncSlotState.Posted); Interlocked.Or(ref _control->Pending, 1UL << index); - new Doorbell(&_control->ServiceBell).Ring(); + // One request, one server thread: the others stay asleep instead of waking to spin. + new Doorbell(&_control->ServiceBell).RingOne(); var outcome = AwaitAnswer(slot, start, timeoutNanoseconds, spinNanoseconds); if (outcome != SyncCallStatus.Completed) diff --git a/tests/SharpEmu.GpuRemote.Tests/SyncCallTableTests.cs b/tests/SharpEmu.GpuRemote.Tests/SyncCallTableTests.cs index 10c533cc6..98729b7dc 100644 --- a/tests/SharpEmu.GpuRemote.Tests/SyncCallTableTests.cs +++ b/tests/SharpEmu.GpuRemote.Tests/SyncCallTableTests.cs @@ -163,6 +163,58 @@ public void CallsAllocateNothingOnceWarm() } } + [MacOSFact] + public void SleepingServerPoolNeverLosesACallUnderContention() + { + using var table = new TableUnderTest(slots: 16); + using var pool = new ServerPool(table.Calls, threads: 8); + const int Callers = 6; + const int CallsEach = 3000; + var failures = 0; + var callers = Enumerable.Range(0, Callers).Select(caller => new Thread(() => + { + Span request = stackalloc byte[8]; + Span response = stackalloc byte[8]; + for (var i = 0; i < CallsEach; i++) + { + var value = ((long)caller << 32) | (uint)i; + BinaryPrimitives.WriteInt64LittleEndian(request, value); + // Servers never spin and never time out, so a wake-up that is lost is a call that + // is not answered; the call's own limit turns that into a failure, not a hang. + var result = table.Calls.Call(1, request, response, out var length, out _, 30 * Second, i % 3 == 0 ? 0 : 20_000); + if (result != SyncCallStatus.Completed || length != 8 || BinaryPrimitives.ReadInt64LittleEndian(response) != value) + { + Interlocked.Increment(ref failures); + } + } + })).ToList(); + callers.ForEach(thread => thread.Start()); + callers.ForEach(thread => thread.Join()); + Assert.Equal(0, failures); + Assert.Equal(Callers * CallsEach, pool.Served); + } + + [MacOSFact] + public void OneCallWakesOneSleepingServerNotTheWholePool() + { + using var table = new TableUnderTest(slots: 4); + using var pool = new ServerPool(table.Calls, threads: 8); + // Let every server go to sleep. + Thread.Sleep(200); + const int Calls = 400; + var response = new byte[8]; + for (var i = 0; i < Calls; i++) + { + Assert.Equal(SyncCallStatus.Completed, table.Calls.Call(1, [1], response, out _, out _, 30 * Second, 0)); + // Let the woken server go back to sleep, as it does between a game's calls. + Thread.Sleep(1); + } + + Assert.Equal(Calls, pool.Served); + // Waking every sleeper would make about seven of the eight find nothing, per call. + Assert.True(pool.EmptyWakes < Calls, $"{pool.EmptyWakes} wake-ups found nothing to serve for {Calls} calls."); + } + private static int Echo(uint opcode, ReadOnlySpan request, Span response, out int status) { status = 0; @@ -202,6 +254,67 @@ public TableUnderTest(int slots) public void Dispose() => Block.Dispose(); } + // The serving pool of the split (SplitGpuSession.ServeReverseCalls, GpuProcessClient.ServeSyncCalls): + // several threads asleep on one table, each serving one call at a time. + private sealed class ServerPool : IDisposable + { + private readonly SyncCallTable _calls; + private readonly List _threads = new(); + private volatile bool _stop; + private long _served; + private long _emptyWakes; + + public ServerPool(SyncCallTable calls, int threads) + { + _calls = calls; + for (var i = 0; i < threads; i++) + { + var thread = new Thread(Serve) { IsBackground = true }; + _threads.Add(thread); + thread.Start(); + } + } + + public long Served => Interlocked.Read(ref _served); + + public long EmptyWakes => Interlocked.Read(ref _emptyWakes); + + private void Serve() + { + SyncCallHandler handler = (uint opcode, ReadOnlySpan request, Span response, out int status) => + { + Interlocked.Increment(ref _served); + return Echo(opcode, request, response, out status); + }; + while (!_stop) + { + if (_calls.WaitForRequests(-1, 0)) + { + while (!_stop && _calls.ServeOne(handler)) + { + } + } + else if (!_stop) + { + Interlocked.Increment(ref _emptyWakes); + } + } + } + + public void Dispose() + { + _stop = true; + // A server between its stop check and its sleep misses one wake-up: ring until all are gone. + foreach (var thread in _threads) + { + while (!thread.Join(20)) + { + _calls.WakeAll(); + } + } + } + } + private sealed class Server : IDisposable { private readonly SyncCallTable _calls; From c99e502e49ec651557a35f20191e46631a09bb75 Mon Sep 17 00:00:00 2001 From: kuba Date: Mon, 26 Oct 2026 05:01:07 +0100 Subject: [PATCH 07/82] [Split] Managed-write interest is per 4 KiB page, and one watch change waits once for its protections Interest: the summary in shared memory had one bit per 64 KiB, so any managed write next to a watched page (command buffers sit beside the images they draw to) made a blocking CpuWrite call of 4-25 us plus a ring record. It is now one bit per 4 KiB page, set and cleared by B's page watch transitions (32 MiB of sparse address space). SHARPEMU_SPLIT_STATS=1 adds ManagedWriteInterest, ManagedWriteCall (timed) and ProtectWait (timed) to the [SPLIT-X] line. Protections: every watch protection that takes rights away waited for A's acknowledgement once per contiguous run per 4 MiB block, with the page guard's block lock held. PageGuard now brackets a watch change with IProtectionBatchSpace; the GPU process's address space defers the waits to the end, outside the locks, and waits once for the last request (the counter is cumulative). A change that found its pages already watched by another thread waits for that thread's pending number too. SHARPEMU_SPLIT_BATCH_PROTECT=0 restores a wait per protection. Tests: page-granular summary (a write to an unwatched page in a watched 64 KiB group is not of interest), batching (one wait for a nested batch, for a sparse 3-run mask, and for another thread's pending protection). --- docs/macos/split-protocol.md | 25 ++- .../RemoteGuestAddressSpace.cs | 116 +++++++++++--- src/SharpEmu.GpuSplit/RemoteImageStore.cs | 12 +- src/SharpEmu.GpuSplit/SplitSharedState.cs | 52 ++++--- .../GpuMemory/IProtectionBatchSpace.cs | 23 +++ src/SharpEmu.HLE/GpuMemory/PageGuard.cs | 42 +++++- .../GpuMemory/SplitCrossingStats.cs | 25 +++ .../RemoteGuestAddressSpaceTests.cs | 142 ++++++++++++++++-- 8 files changed, 366 insertions(+), 71 deletions(-) create mode 100644 src/SharpEmu.HLE/GpuMemory/IProtectionBatchSpace.cs diff --git a/docs/macos/split-protocol.md b/docs/macos/split-protocol.md index 29e751906..efdf8e1cd 100644 --- a/docs/macos/split-protocol.md +++ b/docs/macos/split-protocol.md @@ -130,9 +130,13 @@ bridge refuses (and reports) a call from the ring thread. - **Protection acknowledgement** (offset 0, a `SharedCounter`): A raises it to a `ProtectRequest`'s sequence once that request and every one before it is applied; B waits on it. (The first contract had a `ProtectAck` ring record; a shared counter means B's waits never depend on A's ring writers, which may be blocked on a full ring.) -- **GPU-interest summary** (offset 4096, 2 MiB): one bit per 64 KiB below 1 TiB, set while any page in it carries a - `PageGuard` watch in B (`IPageWatchObserver`, under the guard's block lock; one word per 4 MiB block). B sets a bit - before the watch's protection is requested, with a full barrier; A filters managed writes with it. +- **GPU-interest summary** (offset 4096, 32 MiB of address space, touched only where watches are): one bit per 4 KiB + page below 1 TiB, set while the page carries a `PageGuard` watch in B (`IPageWatchObserver`, under the guard's block + lock; 16 words per 4 MiB block). B sets a bit before the watch's protection is requested, with a full barrier; A + filters managed writes with it, so a write next to a watched page (command buffers sit beside the images they draw + to) does not block. `SHARPEMU_SPLIT_STATS=1` adds `ManagedWriteInterest` (writes that hit a set bit), + `ManagedWriteCall` (the blocking `CpuWrite` calls, with their average time) and `ProtectWait` (confirmation waits, + average time) to the `[SPLIT-X]` line. ## Teardown and failure @@ -148,15 +152,22 @@ bridge refuses (and reports) a call from the ring thread. - B's console is A's console (inherited); B tees it into `SHARPEMU_GPU_HOST_LOG`, which the CLI sets to `.gpuhost.log` when it runs with `--log-file`. +## Watch protections wait once + +A watch change (`AddWatch`, `AddWatchMask` and their removals) sends every protection it needs and waits for A's +acknowledgement once, after the page guard released its block locks (`IProtectionBatchSpace`; the shared counter is +cumulative, so the last request's number covers the rest). A change that found its pages already watched by another +thread, whose protection that thread has not confirmed yet, waits for that thread's number too. Protections made +outside a watch change (ledger changes, `TryProtect`) still wait where they are made. `SHARPEMU_SPLIT_BATCH_PROTECT=0` +waits at each protection, as before. + ## Not carried yet - Host movies (`MoviePresentationServices`: frames A's decoder hands the presenter) do not reach B's presenter. - `TraceGpuFlip` (diagnostics) does nothing in B. - Private guest memory (above). -- Fast paths: restricting watches wait one round trip each (batch them and wait once before the bytes are read); - `CanPresentFlip`/`NextFlipPresentTimestamp` cost a round trip per presenter pass (mirror the pacing state); managed +- Fast paths: `CanPresentFlip`/`NextFlipPresentTimestamp` cost a round trip per presenter pass (mirror the pacing state); managed writes under a watch are a call each. -- Known windows, not closed yet: B's page-guard block lock (a spin) is held while a protection that takes rights away - waits for A's acknowledgement; managed writes made inside a bracket reach B as records and may be handled after the +- Known windows, not closed yet: managed writes made inside a bracket reach B as records and may be handled after the worker resumes; the shutdown no longer orders B's watch teardown before A releases its page guard (A's guard holds no watches, and A acknowledges late requests without applying them). diff --git a/src/SharpEmu.GpuSplit/RemoteGuestAddressSpace.cs b/src/SharpEmu.GpuSplit/RemoteGuestAddressSpace.cs index 7da24c662..ee2068ae7 100644 --- a/src/SharpEmu.GpuSplit/RemoteGuestAddressSpace.cs +++ b/src/SharpEmu.GpuSplit/RemoteGuestAddressSpace.cs @@ -1,6 +1,7 @@ // Copyright (C) 2026 SharpEmu Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later +using System.Diagnostics; using SharpEmu.GpuRemote; using SharpEmu.HLE; using SharpEmu.HLE.GpuMemory; @@ -25,9 +26,9 @@ internal interface IProtectionRequests /// The GPU-interest page summary as the GPU process writes it (). internal interface IInterestSummary { - void SetGroup(ulong address); + void SetPage(ulong address); - void ClearGroup(ulong address); + void ClearPage(ulong address); } /// @@ -54,7 +55,8 @@ internal interface IInterestSummary /// The GPU side never allocates or maps guest memory: those members refuse. /// internal sealed unsafe class RemoteGuestAddressSpace : - ICpuMemory, IGuestBackedSpace, IGuestAddressSpace, IGuestDerivedProtectionAddressSpace, IPageWatchObserver + ICpuMemory, IGuestBackedSpace, IGuestAddressSpace, IGuestDerivedProtectionAddressSpace, IPageWatchObserver, + IProtectionBatchSpace { private const int MaxMissReports = 16; @@ -64,7 +66,6 @@ internal sealed unsafe class RemoteGuestAddressSpace : private readonly GuestReplica? _replica; private readonly IProtectionRequests _protections; private readonly IInterestSummary _summary; - private readonly byte[]?[] _groupWatches = new byte[]?[TrackerLayout.BlockCount]; private int _missReports; private long _misses; @@ -309,7 +310,7 @@ public bool TryProtect(ulong address, ulong size, GuestPageProtection protection var sequence = _protections.Request(address, size, protection, protection, ProtectRequestFlags.None); if (sequence != 0) { - _ = _protections.WaitApplied(sequence); + _ = WaitAppliedTimed(sequence); } return true; @@ -330,16 +331,102 @@ public bool TryApplyDerivedProtection(ulong address, ulong size, GuestPageProtec // here would only make the page guard fail the whole process. if (sequence != 0 && derivedAccess != guestAccess) { - _ = _protections.WaitApplied(sequence); + if (t_batchDepth != 0 && BatchingEnabled) + { + // Inside a watch change: the guard waits once, after its block locks are released. + // (Requests are applied in order, so the last one's number covers the rest.) + if (sequence > t_batchSequence) t_batchSequence = sequence; + RaisePending(sequence); + } + else + { + _ = WaitAppliedTimed(sequence); + } } return true; } + // ---- IProtectionBatchSpace ---- + + // SHARPEMU_SPLIT_BATCH_PROTECT=0 waits for every protection where it is made, as before. + private static readonly bool BatchingEnabled = !string.Equals( + Environment.GetEnvironmentVariable("SHARPEMU_SPLIT_BATCH_PROTECT"), "0", StringComparison.Ordinal); + + // The depth of the watch changes this thread is inside, and the highest protection number it + // left unconfirmed in them. + [ThreadStatic] + private static int t_batchDepth; + + [ThreadStatic] + private static ulong t_batchSequence; + + // The highest number any thread left unconfirmed in a batch. A thread whose watch change found + // the pages already watched by another (whose protection that thread has not confirmed yet) sent + // nothing of its own, but must not return before that protection is applied: the old wait under + // the block lock held it back, so it waits for this number as well. + private long _pendingSequence; + + public void BeginProtectionBatch() => t_batchDepth++; + + public void EndProtectionBatch() + { + if (--t_batchDepth != 0) + { + return; + } + + var own = t_batchSequence; + t_batchSequence = 0; + var target = Math.Max(own, (ulong)Interlocked.Read(ref _pendingSequence)); + if (target == 0) + { + return; + } + + _ = WaitAppliedTimed(target); + // Applied (or the wait failed and was reported): nothing is pending up to target any more. + var pending = Interlocked.Read(ref _pendingSequence); + while ((ulong)pending <= target && pending != 0 && + Interlocked.CompareExchange(ref _pendingSequence, 0, pending) != pending) + { + pending = Interlocked.Read(ref _pendingSequence); + } + } + + private void RaisePending(ulong sequence) + { + var current = Interlocked.Read(ref _pendingSequence); + while ((ulong)current < sequence) + { + var observed = Interlocked.CompareExchange(ref _pendingSequence, (long)sequence, current); + if (observed == current) + { + break; + } + + current = observed; + } + } + + private bool WaitAppliedTimed(ulong sequence) + { + if (!SplitCrossingStats.Enabled) + { + return _protections.WaitApplied(sequence); + } + + var start = Stopwatch.GetTimestamp(); + var applied = _protections.WaitApplied(sequence); + SplitCrossingStats.Count(SplitCrossing.ProtectWait); + SplitCrossingStats.Time(SplitCrossing.ProtectWait, Stopwatch.GetTimestamp() - start); + return applied; + } + // ---- IPageWatchObserver ---- - // Under the page guard's block lock: every group of a block (and every bit of its summary - // word) changes under that one lock. + // Called once per page whose watched state changes, under the page guard's block lock: every + // page of a block (and every bit of its summary words) changes under that one lock. public void OnPageWatchChanged(ulong page, bool watched) { if (page >= TrackerLayout.SpaceBytes) @@ -347,20 +434,13 @@ public void OnPageWatchChanged(ulong page, bool watched) return; } - var block = (int)(page / TrackerLayout.BlockBytes); - var group = (int)((page >> SplitSharedState.GroupShift) & 63); - var counts = _groupWatches[block]; if (watched) { - counts ??= _groupWatches[block] = new byte[(int)(TrackerLayout.BlockBytes >> SplitSharedState.GroupShift)]; - if (counts[group]++ == 0) - { - _summary.SetGroup(page); - } + _summary.SetPage(page); } - else if (counts is not null && counts[group] != 0 && --counts[group] == 0) + else { - _summary.ClearGroup(page); + _summary.ClearPage(page); } } diff --git a/src/SharpEmu.GpuSplit/RemoteImageStore.cs b/src/SharpEmu.GpuSplit/RemoteImageStore.cs index bf19d77f7..5bb9049d3 100644 --- a/src/SharpEmu.GpuSplit/RemoteImageStore.cs +++ b/src/SharpEmu.GpuSplit/RemoteImageStore.cs @@ -1,6 +1,7 @@ // Copyright (C) 2026 SharpEmu Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later +using System.Diagnostics; using System.Runtime.CompilerServices; using System.Runtime.Versioning; using SharpEmu.GpuRemote; @@ -110,6 +111,7 @@ public bool MarkPreciseCpuWrite(ulong address, ulong size) return false; } + SplitCrossingStats.Count(SplitCrossing.ManagedWriteInterest); if (RemoteGpuQueueRelay.CurrentBracket != 0 || SplitGpuSession.OnReverseServerThread) { return SendLanded(address, size); @@ -119,8 +121,16 @@ public bool MarkPreciseCpuWrite(ulong address, ulong size) // the worker as the write is published, so the GPU caches need nothing more from the ring // here, and a managed write does not wait for a ring thread held by the shader prewarm. var request = new CpuWriteRequest { Barrier = 0, Address = address, Size = size }; - return _session.Call(GpuRemoteSyncOpcode.CpuWrite, request, out HandledResponse response, SplitGpuSession.CallNanoseconds) && + var start = SplitCrossingStats.Enabled ? Stopwatch.GetTimestamp() : 0; + var handled = _session.Call(GpuRemoteSyncOpcode.CpuWrite, request, out HandledResponse response, SplitGpuSession.CallNanoseconds) && response.Handled != 0; + if (SplitCrossingStats.Enabled) + { + SplitCrossingStats.Count(SplitCrossing.ManagedWriteCall); + SplitCrossingStats.Time(SplitCrossing.ManagedWriteCall, Stopwatch.GetTimestamp() - start); + } + + return handled; } public bool MarkCpuWriteLanded(ulong address, ulong size) => diff --git a/src/SharpEmu.GpuSplit/SplitSharedState.cs b/src/SharpEmu.GpuSplit/SplitSharedState.cs index 29b26bcb4..d631ceb73 100644 --- a/src/SharpEmu.GpuSplit/SplitSharedState.cs +++ b/src/SharpEmu.GpuSplit/SplitSharedState.cs @@ -16,24 +16,26 @@ namespace SharpEmu.GpuSplit; /// to a request's number once that request and every one before it is applied. B waits on it /// before it relies on a watch it armed. /// -/// GPU-interest summary. One bit per 64 KiB of guest address space below -/// , set while any page in it carries a page watch in B +/// GPU-interest summary. One bit per 4 KiB page of guest address space below +/// , set while the page carries a page watch in B /// (). A reports a managed write to B only when a bit under it is /// set: GTA V makes ~1.2 million managed writes a second, and the GPU caches care about a few of -/// them. One word covers one 4 MiB block of the page guard, so every change of a word happens -/// under that block's lock in B. B sets a bit before the watch's protection is requested, and the -/// set is a full barrier: a managed write in A either sees the bit (and is reported) or lands -/// before B reads the bytes the watch covers. +/// them. (The first version had one bit per 64 KiB, so a write to a page nobody watches, next to +/// one somebody does, still cost a blocking call; command buffers sit next to the images they +/// draw to.) A page's watch changes under its 4 MiB block's lock in B, and one block is 16 words +/// of the summary. B sets a bit before the watch's protection is requested, and the set is a full +/// barrier: a managed write in A either sees the bit (and is reported) or lands before B reads +/// the bytes the watch covers. /// /// Layout: the counter at 0 ( bytes), the summary at -/// , words (2 MiB, touched only where -/// watches are). +/// , words (32 MiB of address space in the +/// region, touched only where watches are). /// public sealed unsafe class SplitSharedState : IInterestSummary { - public const int GroupShift = 16; + public const int PageShift = 12; public const int SummaryOffset = 4096; - public const int SummaryWords = (int)(TrackerLayout.SpaceBytes >> GroupShift >> 6); + public const int SummaryWords = (int)(TrackerLayout.SpaceBytes >> PageShift >> 6); public const int Bytes = SummaryOffset + SummaryWords * sizeof(ulong); private readonly ulong* _summary; @@ -57,7 +59,7 @@ public SplitSharedState(GpuRemoteChannel channel) /// The number of the last page protection A applied (with every one before it). public SharedCounter ProtectApplied { get; } - /// A: whether any 64 KiB group under [address, address + size) holds a watched page. + /// A: whether any 4 KiB page under [address, address + size) carries a watch. public bool MayInterest(ulong address, ulong size) { if (size == 0 || address >= TrackerLayout.SpaceBytes) @@ -66,16 +68,16 @@ public bool MayInterest(ulong address, ulong size) } var end = size > TrackerLayout.SpaceBytes - address ? TrackerLayout.SpaceBytes : address + size; - var group = address >> GroupShift; - var last = (end - 1) >> GroupShift; - while (group <= last) + var page = address >> PageShift; + var last = (end - 1) >> PageShift; + while (page <= last) { - var word = group >> 6; + var word = page >> 6; var bits = Volatile.Read(ref _summary[word]); var wordLast = Math.Min(last, (word << 6) | 63); if (bits != 0) { - var low = (int)(group & 63); + var low = (int)(page & 63); var high = (int)(wordLast & 63); var mask = (high == 63 ? ulong.MaxValue : (1UL << (high + 1)) - 1) & ~((1UL << low) - 1); if ((bits & mask) != 0) @@ -84,23 +86,23 @@ public bool MayInterest(ulong address, ulong size) } } - group = wordLast + 1; + page = wordLast + 1; } return false; } - /// B: marks the 64 KiB group holding as interesting (a full barrier). - public void SetGroup(ulong address) + /// B: marks the page holding as watched (a full barrier). + public void SetPage(ulong address) { - var group = address >> GroupShift; - Interlocked.Or(ref _summary[group >> 6], 1UL << (int)(group & 63)); + var page = address >> PageShift; + Interlocked.Or(ref _summary[page >> 6], 1UL << (int)(page & 63)); } - /// B: the group holding has no watched page any more. - public void ClearGroup(ulong address) + /// B: the page holding has no watch any more. + public void ClearPage(ulong address) { - var group = address >> GroupShift; - Interlocked.And(ref _summary[group >> 6], ~(1UL << (int)(group & 63))); + var page = address >> PageShift; + Interlocked.And(ref _summary[page >> 6], ~(1UL << (int)(page & 63))); } } diff --git a/src/SharpEmu.HLE/GpuMemory/IProtectionBatchSpace.cs b/src/SharpEmu.HLE/GpuMemory/IProtectionBatchSpace.cs new file mode 100644 index 000000000..5ac6e616f --- /dev/null +++ b/src/SharpEmu.HLE/GpuMemory/IProtectionBatchSpace.cs @@ -0,0 +1,23 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +namespace SharpEmu.HLE.GpuMemory; + +/// +/// An address space whose protections that take rights away are slow to confirm (the two-process +/// split's GPU process asks the guest process and waits for its acknowledgement). A +/// watch change brackets all of its protections with these calls, so the +/// space can send them all and wait once, after the guard released its block locks, instead of +/// once per protection under a lock. +/// +public interface IProtectionBatchSpace +{ + /// A watch change starts on this thread. Calls nest; only the outermost end waits. + void BeginProtectionBatch(); + + /// + /// The watch change is over: every protection it asked for that takes rights away must be + /// confirmed before this returns. + /// + void EndProtectionBatch(); +} diff --git a/src/SharpEmu.HLE/GpuMemory/PageGuard.cs b/src/SharpEmu.HLE/GpuMemory/PageGuard.cs index 7d9d76947..eb792b65c 100644 --- a/src/SharpEmu.HLE/GpuMemory/PageGuard.cs +++ b/src/SharpEmu.HLE/GpuMemory/PageGuard.cs @@ -122,6 +122,7 @@ public BlockLock(PageBlock block) private readonly IGuestAddressSpace _addressSpace; private readonly IPageWatchObserver? _watchObserver; + private readonly IProtectionBatchSpace? _batchSpace; private readonly PageBlock?[] _blocks = new PageBlock?[TrackerLayout.BlockCount]; private readonly object _blockGate = new(); @@ -135,6 +136,7 @@ public PageGuard(IGuestAddressSpace addressSpace) _addressSpace = addressSpace; _watchObserver = addressSpace as IPageWatchObserver; + _batchSpace = addressSpace as IProtectionBatchSpace; } public GuestPermissionLedger Permissions { get; } = new(); @@ -180,25 +182,57 @@ public void Dispose() public void AddWatch(ulong address, ulong size, bool blockReads) { SplitCrossingStats.Count(SplitCrossing.WatchAdd); - UpdatePages(address, size, true, blockReads); + _batchSpace?.BeginProtectionBatch(); + try + { + UpdatePages(address, size, true, blockReads); + } + finally + { + _batchSpace?.EndProtectionBatch(); + } } public void RemoveWatch(ulong address, ulong size, bool blockReads) { SplitCrossingStats.Count(SplitCrossing.WatchRemove); - UpdatePages(address, size, false, blockReads); + _batchSpace?.BeginProtectionBatch(); + try + { + UpdatePages(address, size, false, blockReads); + } + finally + { + _batchSpace?.EndProtectionBatch(); + } } public void AddWatchMask(ulong blockBase, in PageMask pages, bool blockReads) { SplitCrossingStats.Count(SplitCrossing.WatchAdd); - UpdateMask(blockBase, pages, true, blockReads); + _batchSpace?.BeginProtectionBatch(); + try + { + UpdateMask(blockBase, pages, true, blockReads); + } + finally + { + _batchSpace?.EndProtectionBatch(); + } } public void RemoveWatchMask(ulong blockBase, in PageMask pages, bool blockReads) { SplitCrossingStats.Count(SplitCrossing.WatchRemove); - UpdateMask(blockBase, pages, false, blockReads); + _batchSpace?.BeginProtectionBatch(); + try + { + UpdateMask(blockBase, pages, false, blockReads); + } + finally + { + _batchSpace?.EndProtectionBatch(); + } } // Watcher state only; the guest's own permissions are the ledger's. diff --git a/src/SharpEmu.HLE/GpuMemory/SplitCrossingStats.cs b/src/SharpEmu.HLE/GpuMemory/SplitCrossingStats.cs index 27f51a08a..c68abd584 100644 --- a/src/SharpEmu.HLE/GpuMemory/SplitCrossingStats.cs +++ b/src/SharpEmu.HLE/GpuMemory/SplitCrossingStats.cs @@ -38,6 +38,12 @@ public enum SplitCrossing FlipComplete, // Guest state the GPU side reads (AGC shader headers). ShaderHeaderLookup, + // Split only. Managed writes under a page the GPU side watches (the interest summary says so), + // the blocking CpuWrite calls those made (timed), and the waits for the guest process to + // confirm a watch's protection (timed). + ManagedWriteInterest, + ManagedWriteCall, + ProtectWait, } public static class SplitCrossingStats @@ -50,6 +56,8 @@ public static class SplitCrossingStats private static readonly int KindCount = Enum.GetValues().Length; private static readonly long[] Counts = new long[KindCount]; private static readonly long[] Reported = new long[KindCount]; + private static readonly long[] TimeTicks = new long[KindCount]; + private static readonly long[] ReportedTimeTicks = new long[KindCount]; private static long _storeFaultTicks; private static long _slowStoreFaults; private static long _maxStoreFaultTicks; @@ -70,6 +78,15 @@ public static void Count(SplitCrossing kind) EnsureTimer(); } + /// Adds the duration of one counted crossing of ; reported as its average. + public static void Time(SplitCrossing kind, long ticks) + { + if (Enabled) + { + Interlocked.Add(ref TimeTicks[(int)kind], ticks); + } + } + /// Records how long the GPU-side stores held one fault. public static void StoreFaultTime(long ticks) { @@ -116,6 +133,14 @@ private static void Report() var delta = total - Reported[kind]; Reported[kind] = total; line.Append(CultureInfo.InvariantCulture, $" {(SplitCrossing)kind}={delta / seconds:F0}/s"); + var timed = Interlocked.Read(ref TimeTicks[kind]); + var timedDelta = timed - ReportedTimeTicks[kind]; + ReportedTimeTicks[kind] = timed; + if (timed != 0) + { + line.Append(CultureInfo.InvariantCulture, + $"(avg {timedDelta * 1e6 / Stopwatch.Frequency / Math.Max(delta, 1):F1}us)"); + } } var faults = Math.Max(Interlocked.Read(ref Reported[(int)SplitCrossing.FaultToStores]), 1); diff --git a/tests/SharpEmu.GpuSplit.Tests/RemoteGuestAddressSpaceTests.cs b/tests/SharpEmu.GpuSplit.Tests/RemoteGuestAddressSpaceTests.cs index 53a0fde1f..5ef092e28 100644 --- a/tests/SharpEmu.GpuSplit.Tests/RemoteGuestAddressSpaceTests.cs +++ b/tests/SharpEmu.GpuSplit.Tests/RemoteGuestAddressSpaceTests.cs @@ -14,31 +14,60 @@ namespace SharpEmu.GpuSplit.Tests; public sealed class SplitSharedStateTests { [MacOSFact] - public void TheSummaryAnswersForEveryGroupARangeTouches() + public void TheSummaryAnswersForEveryPageARangeTouches() { using var channel = GpuRemoteChannel.Create(SharedMemoryRegion.CreateUniqueName("sesplit"), SplitSharedState.Layout); var state = new SplitSharedState(channel); - const ulong Group = 1UL << SplitSharedState.GroupShift; + const ulong Page = 1UL << SplitSharedState.PageShift; + Assert.Equal(TrackerLayout.PageBytes, Page); Assert.False(state.MayInterest(0, TrackerLayout.SpaceBytes)); - state.SetGroup(0x40_0000 + 3 * Group + 0x123); - Assert.True(state.MayInterest(0x40_0000 + 3 * Group, 8)); - Assert.True(state.MayInterest(0x40_0000 + 4 * Group - 1, 1)); - Assert.False(state.MayInterest(0x40_0000 + 4 * Group, Group)); - Assert.False(state.MayInterest(0x40_0000, 3 * Group)); - // Across words: one 4 MiB block per word. + state.SetPage(0x40_0000 + 3 * Page + 0x123); + Assert.True(state.MayInterest(0x40_0000 + 3 * Page, 8)); + Assert.True(state.MayInterest(0x40_0000 + 4 * Page - 1, 1)); + Assert.False(state.MayInterest(0x40_0000 + 4 * Page, Page)); + Assert.False(state.MayInterest(0x40_0000, 3 * Page)); + // Across words: 64 pages each. Assert.True(state.MayInterest(0x10_0000, 0x40_0000)); Assert.True(state.MayInterest(0, TrackerLayout.SpaceBytes)); Assert.False(state.MayInterest(TrackerLayout.SpaceBytes, 8)); - Assert.False(state.MayInterest(0x40_0000 + 3 * Group, 0)); + Assert.False(state.MayInterest(0x40_0000 + 3 * Page, 0)); - state.SetGroup(TrackerLayout.SpaceBytes - 1); - Assert.True(state.MayInterest(TrackerLayout.SpaceBytes - Group, ulong.MaxValue / 2)); + state.SetPage(TrackerLayout.SpaceBytes - 1); + Assert.True(state.MayInterest(TrackerLayout.SpaceBytes - Page, ulong.MaxValue / 2)); - state.ClearGroup(0x40_0000 + 3 * Group); + state.ClearPage(0x40_0000 + 3 * Page); Assert.False(state.MayInterest(0x40_0000, 0x40_0000)); } + [MacOSFact] + public void AWriteToAnUnwatchedPageNextToAWatchedOneIsNotOfInterest() + { + using var channel = GpuRemoteChannel.Create(SharedMemoryRegion.CreateUniqueName("sesplit"), SplitSharedState.Layout); + var state = new SplitSharedState(channel); + const ulong Group = 0x10_0000; + // One watched page in the middle of a 64 KiB group (the old summary's granularity). + state.SetPage(Group + 5 * 0x1000); + + Assert.True(state.MayInterest(Group + 5 * 0x1000, 1)); + Assert.True(state.MayInterest(Group + 5 * 0x1000 + 0xFFF, 1)); + // The same 64 KiB group, every other page. + for (ulong page = 0; page < 16; page++) + { + Assert.Equal(page == 5, state.MayInterest(Group + page * 0x1000, 8)); + Assert.Equal(page == 5, state.MayInterest(Group + page * 0x1000 + 0xFF8, 8)); + } + + // A write that straddles into the watched page is of interest, one that stops short is not. + Assert.True(state.MayInterest(Group + 5 * 0x1000 - 4, 8)); + Assert.False(state.MayInterest(Group + 5 * 0x1000 - 8, 8)); + Assert.False(state.MayInterest(Group + 6 * 0x1000, 0x1000)); + Assert.False(state.MayInterest(Group, 5 * 0x1000)); + + state.ClearPage(Group + 5 * 0x1000); + Assert.False(state.MayInterest(Group, 16 * 0x1000)); + } + [MacOSFact] public void AProcessWithoutTheBlockIsRefused() { @@ -178,19 +207,100 @@ public void WatchesOfTheGpuMemoryManagerKeepTheInterestSummary() memory.Pages.AddWatch(_guest.Address, 0x1000, blockReads: false); memory.Pages.AddWatch(_guest.Address + 0x1000, 0x1000, blockReads: false); memory.Pages.AddWatch(_guest.Address + SharedGuestMemory.Chunk, 0x1000, blockReads: false); - Assert.Equal(new[] { _guest.Address, _guest.Address + SharedGuestMemory.Chunk }, _summary.Set); + // One bit per watched page, not per 64 KiB group around it. + Assert.Equal(new[] { _guest.Address, _guest.Address + 0x1000, _guest.Address + SharedGuestMemory.Chunk }, _summary.Set); // Each watch takes write access from the guest, and waits for the guest to apply it. Assert.Equal(3, _protections.Requests.Count(request => request.Access == GuestPageProtection.Read)); Assert.Equal(3, _protections.Waits.Count); + // A second watcher of a page keeps its bit until the last one goes. + memory.Pages.AddWatch(_guest.Address, 0x1000, blockReads: false); memory.Pages.RemoveWatch(_guest.Address, 0x1000, blockReads: false); Assert.Empty(_summary.Cleared); + memory.Pages.RemoveWatch(_guest.Address, 0x1000, blockReads: false); memory.Pages.RemoveWatch(_guest.Address + 0x1000, 0x1000, blockReads: false); memory.Pages.RemoveWatch(_guest.Address + SharedGuestMemory.Chunk, 0x1000, blockReads: false); - Assert.Equal(new[] { _guest.Address + 0x1000, _guest.Address + SharedGuestMemory.Chunk }, _summary.Cleared); + Assert.Equal(new[] { _guest.Address, _guest.Address + 0x1000, _guest.Address + SharedGuestMemory.Chunk }, _summary.Cleared); Assert.Equal(3, _protections.Waits.Count); } + [MacOSFact] + public void ProtectionsOfOneWatchChangeAreConfirmedOnceAtItsEnd() + { + _space.BeginProtectionBatch(); + _space.BeginProtectionBatch(); + Assert.True(_space.TryApplyDerivedProtection(_guest.Address, 0x1000, ReadWrite, GuestPageProtection.Read, ordinaryEnrollment: false)); + Assert.True(_space.TryApplyDerivedProtection(_guest.Address + 0x4000, 0x1000, ReadWrite, GuestPageProtection.Read, ordinaryEnrollment: false)); + // Giving rights back never waits, batched or not. + Assert.True(_space.TryApplyDerivedProtection(_guest.Address + 0x8000, 0x1000, ReadWrite, ReadWrite, ordinaryEnrollment: false)); + _space.EndProtectionBatch(); + Assert.Empty(_protections.Waits); + Assert.Equal(3, _protections.Requests.Count); + _space.EndProtectionBatch(); + // One wait for the last of them: the guest process applies them in order. + Assert.Equal(new ulong[] { 2 }, _protections.Waits); + + // Nothing stays pending for the next change. + _space.BeginProtectionBatch(); + _space.EndProtectionBatch(); + Assert.Single(_protections.Waits); + + // Outside a batch every protection waits at once. + Assert.True(_space.TryApplyDerivedProtection(_guest.Address, 0x1000, ReadWrite, GuestPageProtection.Read, ordinaryEnrollment: false)); + Assert.Equal(new ulong[] { 2, 4 }, _protections.Waits); + } + + [MacOSFact] + public void AWatchChangeThatSentNothingStillWaitsForAnotherThreadsProtection() + { + using var sent = new ManualResetEventSlim(false); + using var finish = new ManualResetEventSlim(false); + var other = new Thread(() => + { + _space.BeginProtectionBatch(); + _ = _space.TryApplyDerivedProtection(_guest.Address, 0x1000, ReadWrite, GuestPageProtection.Read, ordinaryEnrollment: false); + sent.Set(); + finish.Wait(10_000); + _space.EndProtectionBatch(); + }); + other.Start(); + Assert.True(sent.Wait(10_000)); + + // This thread's watch found the page already watched by the other, so it sent nothing; the + // other's protection may not be applied yet, and this thread must not return before it is. + _space.BeginProtectionBatch(); + _space.EndProtectionBatch(); + Assert.Equal(new ulong[] { 1 }, _protections.Waits); + + finish.Set(); + other.Join(); + } + + [MacOSFact] + public void ASparseWatchMaskWaitsOnceForAllItsRuns() + { + using var memory = new GuestGpuMemory(_space); + memory.Register(_guest.Address, SharedGuestMemory.ViewBytes, ReadWrite); + _protections.Requests.Clear(); + _protections.Waits.Clear(); + + var blockBase = _guest.Address & ~(TrackerLayout.BlockBytes - 1); + var first = (int)((_guest.Address - blockBase) / TrackerLayout.PageBytes); + var mask = new PageMask(); + mask.Set(first); + mask.Set(first + 2); + mask.Set(first + 4); + memory.Pages.AddWatchMask(blockBase, mask, blockReads: false); + + // Three runs, three requests, one wait for the last. + Assert.Equal(3, _protections.Requests.Count(request => request.Access == GuestPageProtection.Read)); + Assert.Equal(new[] { _protections.LastRequested }, _protections.Waits); + + _protections.Waits.Clear(); + memory.Pages.RemoveWatchMask(blockBase, mask, blockReads: false); + Assert.Empty(_protections.Waits); + } + [MacOSFact] public void TheGpuSideNeverAllocatesOrMaps() { @@ -230,8 +340,8 @@ private sealed class RecordingSummary : IInterestSummary public List Cleared { get; } = new(); - public void SetGroup(ulong address) => Set.Add(address); + public void SetPage(ulong address) => Set.Add(address); - public void ClearGroup(ulong address) => Cleared.Add(address); + public void ClearPage(ulong address) => Cleared.Add(address); } } From d30b0a525ab20e85a69b10b461a2e8253b6130f7 Mon Sep 17 00:00:00 2001 From: kuba Date: Mon, 26 Oct 2026 05:00:52 +0100 Subject: [PATCH 08/82] [GPU] Per-draw bindings reuse the ring ranges, window placement and encoder residency they already resolved Argument buffers, push data and the uploaded flattened/shader-data tables are filled in host scratch and found again by content in a per-submission memo (MetalRegionMemo); a range is only reused by the submission that wrote it, so a ring still in flight is never overwritten or handed to a later submission. Guest window resolves answer from the window the submission last placed a range in (keyed on serial, partition version and trap version; single-copy releases, holes and stale windows take the full path). The submission resident list is named once per encoder and list version instead of walked every draw, and a stage's memoised textures are confirmed under one image cache hold instead of one per texture. Measured (e2e 8a-8f, x64 and arm64, machine load 20-140 so only the quietest run is trusted): x64 8e 7486 -> 7075 ns/draw, 8d 4399 -> 4358, 8c 7535 -> 7429, 8f/8a/8b within noise; B/op unchanged (1393 / 3720). arm64 runs were not separable from noise. Switches (default on, =0 turns off): SHARPEMU_METAL_V2_ARGUMENT_REUSE, SHARPEMU_METAL_V2_WINDOW_MEMO, SHARPEMU_METAL_V2_RESIDENT_WATERMARK. Verify mode via SHARPEMU_PERF_VERIFY=1. --- .../SharpEmu.Gpu.Metal/V2/MetalGuestMemory.cs | 141 +++++++++++ .../SharpEmu.Gpu.Metal/V2/MetalImageCache.cs | 14 ++ .../SharpEmu.Gpu.Metal/V2/MetalRegionMemo.cs | 91 +++++++ .../V2/MetalRenderHost.Descriptors.cs | 204 ++++++++++++++- .../MetalBindingReuseTests.cs | 234 ++++++++++++++++++ .../Scenes/BindingReuseSceneTests.cs | 109 ++++++++ 6 files changed, 781 insertions(+), 12 deletions(-) create mode 100644 src/graphics/SharpEmu.Gpu.Metal/V2/MetalRegionMemo.cs create mode 100644 tests/SharpEmu.Gpu.Metal.Tests/MetalBindingReuseTests.cs create mode 100644 tests/SharpEmu.Gpu.Metal.Tests/Scenes/BindingReuseSceneTests.cs diff --git a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalGuestMemory.cs b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalGuestMemory.cs index 87805ff0e..dbcc2a744 100644 --- a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalGuestMemory.cs +++ b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalGuestMemory.cs @@ -1032,7 +1032,125 @@ private void ReportUnresolved(ulong gpuVa, ulong size) $"alias_contiguous={contiguous} start_alias=0x{startAlias:X16} end_alias=0x{endAlias:X16}."); } + // SHARPEMU_METAL_V2_WINDOW_MEMO=0 takes the window lock and places the range on every resolve. + private static readonly bool WindowMemoEnabled = + SharpEmu.ShaderCompiler.PerfSwitches.Enabled("SHARPEMU_METAL_V2_WINDOW_MEMO"); + + // Verify mode (SHARPEMU_METAL_V2_WINDOW_MEMO=1 or SHARPEMU_PERF_VERIFY=1): every resolve also takes + // the full path and the two answers must agree. + private static readonly bool WindowMemoVerify = + SharpEmu.ShaderCompiler.PerfSwitches.Verify("SHARPEMU_METAL_V2_WINDOW_MEMO"); + + /// + /// A resolve is answered from the window it last placed a range in. Within one submission a + /// window a resolve touched is pinned: its cuts do not move, so every later range inside it + /// places into the same window, and the single-copy bookkeeping the full path does (restoring + /// released chunks, filling holes, refusing a stale window) has nothing left to do for it while + /// the trap version holds. Reads of the backing stay per call, so a remapped range never + /// resolves from here. + /// + private sealed class WindowMemoEntry( + MetalWindowPartition.Window window, ulong coverStart, ulong coverEnd, ulong serial, long partitionVersion, + long trapVersion) + { + public readonly MetalWindowPartition.Window Window = window; + + // The backing offsets the entry answers for: the whole window when no hole or stale + // window could be under any of it, else only the range that was placed. + public readonly ulong CoverStart = coverStart; + public readonly ulong CoverEnd = coverEnd; + public readonly ulong Serial = serial; + public readonly long PartitionVersion = partitionVersion; + public readonly long TrapVersion = trapVersion; + public readonly nint Buffer = window.Buffer; + } + + private readonly WindowMemoEntry?[] _windowMemo = new WindowMemoEntry?[8]; + private int _windowMemoNext; + private long _windowMemoHits; + private long _windowMemoMisses; + + /// Resolves answered from the window memo, and those that took the full path. + internal (long Hits, long Misses) WindowMemoCounts => + (Volatile.Read(ref _windowMemoHits), Volatile.Read(ref _windowMemoMisses)); + + private bool TryResolveWindowFromMemo(ulong offset, ulong size, out MetalGuestRange resolved) + { + resolved = default; + if (Trap is not null || (_singleCopy is { } singleCopy && singleCopy.AnyReleased(offset, size))) + { + return false; + } + + var serial = Volatile.Read(ref _submissionSerial); + var partitionVersion = _partition.Version; + var trapVersion = Volatile.Read(ref _trapVersion); + var memo = _windowMemo; + for (var index = 0; index < memo.Length; index++) + { + if (Volatile.Read(ref memo[index]) is not { } entry || + entry.Serial != serial || offset < entry.CoverStart || offset > entry.CoverEnd || + size > entry.CoverEnd - offset || entry.PartitionVersion != partitionVersion || + entry.TrapVersion != trapVersion || entry.Window.Buffer != entry.Buffer) + { + continue; + } + + var windowOffset = offset - entry.Window.Start; + resolved = new MetalGuestRange(entry.Buffer, windowOffset, size, entry.Window.GpuAddress + windowOffset); + Interlocked.Increment(ref _windowMemoHits); + return true; + } + + return false; + } + + // Caller holds _windowLock, right after the full path placed the range and pinned its window. + private void PublishWindowMemoLocked(MetalWindowPartition.Window window, ulong offset, ulong size) + { + if (Trap is not null || window.Buffer == 0) + { + return; + } + + var whole = (_holes is null || _holeCount == 0) && _staleWindows.Count == 0; + var coverStart = whole ? window.Start : offset; + var coverEnd = whole ? window.End : offset + size; + var version = _partition.Version; + foreach (var held in _windowMemo) + { + if (held is not null && ReferenceEquals(held.Window, window) && held.Serial == _submissionSerial && + held.PartitionVersion == version && held.TrapVersion == _trapVersion && held.Buffer == window.Buffer && + held.CoverStart <= coverStart && held.CoverEnd >= coverEnd) + { + return; + } + } + + var slot = (int)((uint)Interlocked.Increment(ref _windowMemoNext) % (uint)_windowMemo.Length); + Volatile.Write( + ref _windowMemo[slot], + new WindowMemoEntry(window, coverStart, coverEnd, _submissionSerial, version, _trapVersion)); + } + private bool TryResolveWindow(ulong gpuVa, ulong size, out MetalGuestRange resolved, bool warn = true) + { + var ok = TryResolveWindowCore(gpuVa, size, out resolved, warn, WindowMemoEnabled); + if (WindowMemoVerify && WindowMemoEnabled) + { + var fullOk = TryResolveWindowCore(gpuVa, size, out var full, warn: false, useMemo: false); + if (ok != fullOk || (ok && resolved != full)) + { + throw GpuFatal.Fatal( + $"A memoised window resolve disagrees with the full path: address=0x{gpuVa:X16} size=0x{size:X} " + + $"memo={ok}:{resolved} full={fullOk}:{full}."); + } + } + + return ok; + } + + private bool TryResolveWindowCore(ulong gpuVa, ulong size, out MetalGuestRange resolved, bool warn, bool useMemo) { using var phaseScope = SharpEmu.Libs.VideoOut.RenderPhaseProfile.MeasureDetail(SharpEmu.Libs.VideoOut.RenderPhaseProfile.Phase.GuestWindowResolve); resolved = default; @@ -1055,6 +1173,16 @@ private bool TryResolveWindow(ulong gpuVa, ulong size, out MetalGuestRange resol } var offset = alias - _aliasBase; + if (useMemo) + { + if (TryResolveWindowFromMemo(offset, size, out resolved)) + { + return true; + } + + Interlocked.Increment(ref _windowMemoMisses); + } + // A released chunk gets its bytes back before any window names it (single-copy textures). RestoreSingleCopyRange(offset, size); MetalWindowPartition.Window window; @@ -1103,6 +1231,10 @@ private bool TryResolveWindow(ulong gpuVa, ulong size, out MetalGuestRange resol EnsureBuffer(window); window.LastUsedSerial = _submissionSerial; _pinnedWindows.Add(window.Id); + if (useMemo) + { + PublishWindowMemoLocked(window, offset, size); + } } if (_touchedWindows.Add(window.Id)) @@ -1659,6 +1791,7 @@ public ReadOnlySpan ResidentBuffers if (_residentDirty) { _residentDirty = false; + _residentVersion++; _submissionResidentCount = 0; lock (_windowLock) { @@ -1687,6 +1820,14 @@ public ReadOnlySpan ResidentBuffers } } + /// + /// Changes whenever is rebuilt: the list is the same set while + /// it holds. Read after the list, which is what rebuilds it. + /// + public long ResidentVersion => _residentVersion; + + private long _residentVersion; + private static void Append(ref nint[] buffers, ref int count, nint buffer) { if (count == buffers.Length) diff --git a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalImageCache.cs b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalImageCache.cs index 8b1e4a9e1..2c7b4cee5 100644 --- a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalImageCache.cs +++ b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalImageCache.cs @@ -1221,6 +1221,20 @@ private nint AcquireTextureViewLocked(ResourceSlotIdentifier imageIdentifier, in public bool TryReacquireTextureView(ResourceSlotIdentifier imageIdentifier, in ImageRequest request, in TextureMemoStamp stamp) { using var held = _lock.Hold(); + return TryReacquireTextureViewHeld(imageIdentifier, in request, in stamp); + } + + /// + /// One hold of the cache lock for a run of calls: a stage + /// that binds eight memoised textures takes the lock once instead of eight times, which is + /// eight fewer chances to queue behind an upload, a compile or a fault handler. Nothing between + /// the hold and its end may take the cache lock again (the lock is not reentrant). + /// + internal SharpEmu.HLE.GpuMemory.RegionLock.Held HoldForReacquire() => _lock.Hold(); + + /// for a caller that already holds the lock (). + internal bool TryReacquireTextureViewHeld(ResourceSlotIdentifier imageIdentifier, in ImageRequest request, in TextureMemoStamp stamp) + { var image = stamp.Image; if (_deferredRefreshes.Count != 0 || !ReferenceEquals(_slots.TryGet(imageIdentifier), image) || image.BindingVersion != stamp.ImageVersion || _stencilOwnersVersion != stamp.StencilOwners || diff --git a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRegionMemo.cs b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRegionMemo.cs new file mode 100644 index 000000000..cb6c085a2 --- /dev/null +++ b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRegionMemo.cs @@ -0,0 +1,91 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +namespace SharpEmu.Gpu.Metal.V2; + +/// +/// The ring ranges one submission already filled, found again by content: an argument buffer or +/// an uploaded table that comes out byte for byte the same as one an earlier draw of the same +/// command buffer wrote is the same range, so the draw binds it instead of reserving, clearing +/// and filling a new one. +/// +/// A range is only reused by the submission that wrote it. The ring a range lives in belongs to +/// that submission until it completes ( stamps it), and a +/// later submission may find the ring recycled and its bytes overwritten, so a range an earlier +/// serial wrote is never handed out again. Within one submission nothing rewrites a range: the +/// bytes the GPU reads when the command buffer runs are the bytes the draw recorded. +/// +/// One thread uses an instance: the one recording draws. +/// +internal sealed class MetalRegionMemo +{ + private struct Slot + { + public ulong Serial; + public MetalGuestRange Range; + public byte[]? Bytes; + public int Length; + } + + private readonly Slot[] _slots; + private int _next; + + public MetalRegionMemo(int slots) + { + ArgumentOutOfRangeException.ThrowIfLessThan(slots, 1); + _slots = new Slot[slots]; + } + + /// Ranges handed back instead of a new one. + public long Hits { get; private set; } + + /// Lookups that found nothing for the submission. + public long Misses { get; private set; } + + /// The range this submission already holds these bytes in, if it does. + public bool TryFind(ReadOnlySpan bytes, ulong serial, out MetalGuestRange range) + { + var slots = _slots; + for (var index = 0; index < slots.Length; index++) + { + ref var slot = ref slots[index]; + if (slot.Bytes is { } held && slot.Serial == serial && slot.Length == bytes.Length && + held.AsSpan(0, slot.Length).SequenceEqual(bytes)) + { + range = slot.Range; + Hits++; + return true; + } + } + + range = default; + Misses++; + return false; + } + + /// Remembers a range this submission just filled with these bytes. + public void Add(ReadOnlySpan bytes, ulong serial, in MetalGuestRange range) + { + ref var slot = ref _slots[_next]; + _next = (_next + 1) % _slots.Length; + if (slot.Bytes is null || slot.Bytes.Length < bytes.Length) + { + slot.Bytes = new byte[bytes.Length]; + } + + bytes.CopyTo(slot.Bytes); + slot.Length = bytes.Length; + slot.Serial = serial; + slot.Range = range; + } + + /// Forgets everything (a new device or a disposed ring pool). + public void Clear() + { + for (var index = 0; index < _slots.Length; index++) + { + _slots[index].Bytes = null; + _slots[index].Length = 0; + } + } +} diff --git a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.Descriptors.cs b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.Descriptors.cs index e45e8609b..d8084b952 100644 --- a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.Descriptors.cs +++ b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.Descriptors.cs @@ -673,6 +673,43 @@ private void TakeSingleCopyTrapHits() } } + // The memoised sampled bindings from `first` on that still hold, confirmed under one hold of the + // image cache lock (MetalImageCache.HoldForReacquire) rather than one hold each. Stops at the + // first binding that is a movie plane, has no memo entry or whose stamp no longer holds, so every + // binding is still looked at in order, after the ones before it have taken effect. Returns how + // many it bound; zero means the binding at `first` is stale. + private int ReacquireMemoRun(MetalStageBindings stage, int first) + { + var count = 0; + using (var held = _imageCache.HoldForReacquire()) + { + for (var index = first; index < stage.ImageCount; index++) + { + if (index == stage.MovieLuma || index == stage.MovieChroma || + stage.MemoHits[index] is not { } hit || + !_imageCache.TryReacquireTextureViewHeld(hit.Image, in hit.Request, in hit.Stamp)) + { + break; + } + + stage.Textures[index] = hit.Texture; + stage.TextureIds[index] = hit.ResourceId; + stage.MipTextures[index] = null; + stage.MipTextureIds[index] = null; + count++; + } + } + + for (var index = first; index < first + count; index++) + { + var hit = stage.MemoHits[index]!; + NotePendingInputDump(hit.Texture, hit.Image, index, ImageRole.Texture); + } + + _textureMemoHits += count; + return count; + } + public void BindResources(IPreparedBindings prepared) { using var phaseScope = SharpEmu.Libs.VideoOut.RenderPhaseProfile.MeasureDetail(SharpEmu.Libs.VideoOut.RenderPhaseProfile.Phase.ResourceBinding); @@ -690,14 +727,10 @@ public void BindResources(IPreparedBindings prepared) if (stage.MemoHits[index] is { } hit) { - if (_imageCache.TryReacquireTextureView(hit.Image, in hit.Request, in hit.Stamp)) + var reacquired = ReacquireMemoRun(stage, index); + if (reacquired > 0) { - stage.Textures[index] = hit.Texture; - stage.TextureIds[index] = hit.ResourceId; - NotePendingInputDump(hit.Texture, hit.Image, index, ImageRole.Texture); - stage.MipTextures[index] = null; - stage.MipTextureIds[index] = null; - _textureMemoHits++; + index += reacquired - 1; continue; } @@ -1132,8 +1165,22 @@ public void CommitBindings(GpuBindPoint bindPoint, in PipelineHandle pipeline, R if (hasPushData) { - var range = _guestMemory.Allocate(PushData.ByteSize, 256, out var span); - System.Runtime.InteropServices.MemoryMarshal.AsBytes(pushData).CopyTo(span); + var pushBytes = System.Runtime.InteropServices.MemoryMarshal.AsBytes(pushData); + MetalGuestRange range; + if (ArgumentBufferReuse && _pushRegions.TryFind(pushBytes, _submissionSerial, out var found)) + { + range = found; + } + else + { + range = _guestMemory.Allocate(PushData.ByteSize, 256, out var span); + pushBytes.CopyTo(span); + if (ArgumentBufferReuse) + { + _pushRegions.Add(pushBytes, _submissionSerial, in range); + } + } + _pushBuffer = range.Buffer; _pushOffset = range.Offset; _hasPushData = true; @@ -1153,7 +1200,17 @@ private void WriteArgumentBuffer(MetalStageBindings stage, int stageIndex) return; } - var range = _guestMemory.Allocate(Math.Max(argumentLayout.ByteSize, 16u), 256, out var argument); + // Filled in host scratch first: a draw whose argument buffer comes out the same as one this + // command buffer already wrote binds that range (ArgumentBufferReuse) instead of reserving, + // clearing and filling a new one. Every side effect of the fill (the textures it names, the + // tables it uploads, the device-address use) still happens on every draw. + var argumentBytes = (int)Math.Max(argumentLayout.ByteSize, 16u); + if (_argumentScratch.Length < argumentBytes) + { + _argumentScratch = new byte[Math.Max(argumentBytes, _argumentScratch.Length * 2)]; + } + + var argument = _argumentScratch.AsSpan(0, argumentBytes); argument.Clear(); var imageCursor = 0; var fields = argumentLayout.Fields; @@ -1240,11 +1297,73 @@ private void WriteArgumentBuffer(MetalStageBindings stage, int stageIndex) } } + var range = ArgumentBufferReuse ? CommitArgumentBytes(stageIndex, argument) : WriteArgumentRange(argument); _stageArgumentBuffers[stageIndex] = range.Buffer; _stageArgumentOffsets[stageIndex] = range.Offset; _stageArgumentValid[stageIndex] = true; } + // SHARPEMU_METAL_V2_ARGUMENT_REUSE=0 reserves and fills a ring range for every stage of every draw. + private static readonly bool ArgumentBufferReuse = + SharpEmu.ShaderCompiler.PerfSwitches.Enabled("SHARPEMU_METAL_V2_ARGUMENT_REUSE"); + + private static readonly bool ArgumentReuseVerify = + SharpEmu.ShaderCompiler.PerfSwitches.Verify("SHARPEMU_METAL_V2_ARGUMENT_REUSE"); + + private byte[] _argumentScratch = new byte[512]; + + // Four recent argument buffers per stage: draws alternate between a few materials. + private readonly MetalRegionMemo[] _argumentRegions = CreateRegionMemos(StageCount, slots: 4); + + // The push-data block of a draw: the shader data of its stages, as one buffer. + private readonly MetalRegionMemo _pushRegions = new(slots: 4); + + // The uploaded tables (flattened resource table, shader data) of every stage. + private readonly MetalRegionMemo _uploadedTables = new(slots: 8); + + private static MetalRegionMemo[] CreateRegionMemos(int count, int slots) + { + var memos = new MetalRegionMemo[count]; + for (var index = 0; index < memos.Length; index++) + { + memos[index] = new MetalRegionMemo(slots); + } + + return memos; + } + + /// Argument buffers a draw bound from an earlier draw of its command buffer. + internal long ArgumentBufferReuses => _argumentRegions.Sum(memo => memo.Hits) + _pushRegions.Hits; + + /// Argument buffers written to a new ring range. + internal long ArgumentBufferWrites => _argumentRegions.Sum(memo => memo.Misses) + _pushRegions.Misses; + + private MetalGuestRange WriteArgumentRange(ReadOnlySpan bytes) + { + var range = _guestMemory.Allocate((ulong)bytes.Length, 256, out var span); + span.Clear(); + bytes.CopyTo(span); + return range; + } + + private MetalGuestRange CommitArgumentBytes(int stageIndex, ReadOnlySpan bytes) + { + var memo = _argumentRegions[stageIndex]; + if (memo.TryFind(bytes, _submissionSerial, out var found)) + { + if (ArgumentReuseVerify && !_guestMemory.RingSpan(in found).SequenceEqual(bytes)) + { + throw Fatal($"A reused argument buffer no longer holds what the draw wrote: stage={stageIndex}."); + } + + return found; + } + + var range = WriteArgumentRange(bytes); + memo.Add(bytes, _submissionSerial, in range); + return range; + } + // A binding lists one resource index per declared element; a dynamic-mip storage // image lists the same index once per mip, and each occurrence takes its own view. private void WriteTextureField( @@ -1443,9 +1562,19 @@ internal static void AddDeviceAddressEntry( private ulong UploadDwords(ReadOnlySpan words) { var bytes = System.Runtime.InteropServices.MemoryMarshal.AsBytes(words); + if (ArgumentBufferReuse && _uploadedTables.TryFind(bytes, _submissionSerial, out var found)) + { + return found.GpuAddress; + } + var range = _guestMemory.Allocate((ulong)Math.Max(bytes.Length, 16), 256, out var span); span.Clear(); bytes.CopyTo(span); + if (ArgumentBufferReuse) + { + _uploadedTables.Add(bytes, _submissionSerial, in range); + } + return range.GpuAddress; } @@ -1546,7 +1675,7 @@ private void ApplyStageBindings(bool graphics) : Mtl.RenderStageVertex | Mtl.RenderStageFragment; BindStageBuffers(_renderEncoder, StageVertex, meshStage ? SelSetMeshBuffer : SelSetVertexBuffer); BindStageBuffers(_renderEncoder, StageFragment, SelSetFragmentBuffer); - NameResidentBuffers(_renderEncoder, _guestMemory.ResidentBuffers, stages); + NameSubmissionResidents(_renderEncoder, stages, ResidentMarkGraphics); for (var index = 0; index < _usedResourceCount; index++) { @@ -1567,7 +1696,7 @@ private void ApplyStageBindings(bool graphics) } BindStageBuffers(_computeEncoder, StageCompute, SelSetComputeBuffer); - NameResidentBuffers(_computeEncoder, _guestMemory.ResidentBuffers, stages: 0); + NameSubmissionResidents(_computeEncoder, stages: 0, ResidentMarkCompute); for (var index = 0; index < _usedResourceCount; index++) { @@ -1635,7 +1764,15 @@ private void NameDeviceAddressWindows(nint encoder, nuint stages) else ResolveDeviceAddressBuffers(); + ref var mark = ref _residentMarks[ResidentMarkDeviceAddress]; + if (ResidentWatermark && mark.Matches(_encoderEpoch, _deviceAddressBuffersVersion, stages)) + { + _residentWalksSkipped++; + return; + } + NameResidentBuffers(encoder, System.Runtime.InteropServices.CollectionsMarshal.AsSpan(_deviceAddressBuffers), stages); + mark = new ResidentMark(_encoderEpoch, _deviceAddressBuffersVersion, stages); } // BeginDeviceAddressReach pins the table's window cuts for this submission. @@ -1650,6 +1787,7 @@ private void NameDeviceAddressWindows(nint encoder, nuint stages) private void ResolveDeviceAddressBuffers() { _deviceAddressBufferBuilds++; + _deviceAddressBuffersVersion++; _deviceAddressBuffers.Clear(); foreach (var id in DeviceAddressNamedWindows()) { @@ -1660,6 +1798,48 @@ private void ResolveDeviceAddressBuffers() } } + private long _deviceAddressBuffersVersion; + + // What an encoder was last told about a list of buffers: the list is the same set while its + // version holds, and a name sticks to the encoder for its life, so a draw that finds the + // encoder, the version and the stages unchanged has nothing to name and need not walk the + // list (one dictionary probe per window per draw otherwise). SHARPEMU_METAL_V2_RESIDENT_WATERMARK=0 + // walks it every draw. + private static readonly bool ResidentWatermark = + SharpEmu.ShaderCompiler.PerfSwitches.Enabled("SHARPEMU_METAL_V2_RESIDENT_WATERMARK"); + + private const int ResidentMarkGraphics = 0; + private const int ResidentMarkCompute = 1; + private const int ResidentMarkDeviceAddress = 2; + + private readonly record struct ResidentMark(long Epoch, long Version, nuint Stages) + { + public bool Matches(long epoch, long version, nuint stages) => + Epoch == epoch && Version == version && Stages == stages; + } + + private readonly ResidentMark[] _residentMarks = new ResidentMark[3] { new(-1, -1, 0), new(-1, -1, 0), new(-1, -1, 0) }; + private long _residentWalksSkipped; + + /// Per-draw walks of a resident list that the encoder watermark skipped. + internal long ResidentWalksSkipped => Volatile.Read(ref _residentWalksSkipped); + + // The submission's window and ring buffers, named on the encoder once per change of the list. + private void NameSubmissionResidents(nint encoder, nuint stages, int markIndex) + { + var buffers = _guestMemory.ResidentBuffers; + var version = _guestMemory.ResidentVersion; + ref var mark = ref _residentMarks[markIndex]; + if (ResidentWatermark && mark.Matches(_encoderEpoch, version, stages)) + { + _residentWalksSkipped++; + return; + } + + NameResidentBuffers(encoder, buffers, stages); + mark = new ResidentMark(_encoderEpoch, version, stages); + } + private nint[] _useResourcesScratch = new nint[64]; // Names guest windows and resident shared buffers read-write on an encoder, those it has diff --git a/tests/SharpEmu.Gpu.Metal.Tests/MetalBindingReuseTests.cs b/tests/SharpEmu.Gpu.Metal.Tests/MetalBindingReuseTests.cs new file mode 100644 index 000000000..77691c2bc --- /dev/null +++ b/tests/SharpEmu.Gpu.Metal.Tests/MetalBindingReuseTests.cs @@ -0,0 +1,234 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using System.Runtime.InteropServices; +using SharpEmu.Gpu.Metal.V2; +using SharpEmu.HLE; +using SharpEmu.HLE.Host; +using Xunit; +using Xunit.Abstractions; + +namespace SharpEmu.Gpu.Metal.Tests; + +/// +/// The per-draw binding memos: argument-buffer ranges found again by content, only within the +/// submission that wrote them, and the guest-window resolve memo, which has to follow the +/// submission and the window layout. +/// +public sealed unsafe class MetalBindingReuseTests(ITestOutputHelper output) +{ + private const ulong Mib = 1024 * 1024; + private const ulong GuestBase = 0x3_0000_0000; + private const ulong BackingSize = 64 * Mib; + private const ulong Granule = Mib; + + private static readonly Lazy Device = new(MetalNative.MTLCreateSystemDefaultDevice); + + private static byte[] Bytes(byte fill, int length = 64) => Enumerable.Repeat(fill, length).ToArray(); + + private static MetalGuestRange Range(nint buffer, ulong offset) => new(buffer, offset, 64, 0x1000 + offset); + + [Fact] + public void ARangeIsFoundOnlyBySameBytesInTheSameSubmission() + { + var memo = new MetalRegionMemo(4); + var range = Range(1, 0); + memo.Add(Bytes(1), serial: 7, in range); + + Assert.True(memo.TryFind(Bytes(1), 7, out var found)); + Assert.Equal(range, found); + // A changed binding is a different range, and a shorter or longer one is too. + Assert.False(memo.TryFind(Bytes(2), 7, out _)); + Assert.False(memo.TryFind(Bytes(1, 32), 7, out _)); + Assert.False(memo.TryFind(Bytes(1, 128), 7, out _)); + // The next submission never gets the range back: the ring may be recycled under it. + Assert.False(memo.TryFind(Bytes(1), 8, out _)); + Assert.False(memo.TryFind(Bytes(1), 6, out _)); + Assert.Equal(1, memo.Hits); + } + + [Fact] + public void TheOldestRangeIsReplacedWhenTheSlotsAreFull() + { + var memo = new MetalRegionMemo(2); + memo.Add(Bytes(1), 1, Range(1, 0)); + memo.Add(Bytes(2), 1, Range(1, 256)); + memo.Add(Bytes(3), 1, Range(1, 512)); + Assert.False(memo.TryFind(Bytes(1), 1, out _)); + Assert.True(memo.TryFind(Bytes(2), 1, out var second)); + Assert.Equal(256UL, second.Offset); + Assert.True(memo.TryFind(Bytes(3), 1, out var third)); + Assert.Equal(512UL, third.Offset); + memo.Clear(); + Assert.False(memo.TryFind(Bytes(3), 1, out _)); + } + + // A range written for a command buffer that has not completed is never overwritten by the next + // one, and the memo does not hand it to the next one either: the new bytes get a new range, + // the old range keeps the old bytes, and a recycled ring is never mistaken for the old range. + [Fact] + public void ARangeStillInFlightIsNeitherReusedNorOverwritten() + { + if (!RequireDevice()) + { + return; + } + + using var backing = new MappedBacking(); + using var memory = new MetalGuestMemory(Device.Value, backing, new NoCpu(), Granule, 64 * Mib); + var memo = new MetalRegionMemo(4); + var first = Bytes(0xA1); + var second = Bytes(0xB2); + + memory.BeginSubmission(1, 0); + var range = memory.Allocate(64, 256, out var span); + first.CopyTo(span); + memo.Add(first, 1, in range); + Assert.True(memo.TryFind(first, 1, out var again)); + Assert.Equal(range, again); + + // Submission 1 has not completed (completed = 0): a lookup in submission 2 finds nothing, + // and what it writes lands elsewhere in the ring. + memory.BeginSubmission(2, 0); + Assert.False(memo.TryFind(first, 2, out _)); + var next = memory.Allocate(64, 256, out var nextSpan); + second.CopyTo(nextSpan); + Assert.False(next.Buffer == range.Buffer && next.Offset == range.Offset); + Assert.True(memory.RingSpan(in range).SequenceEqual(first), "the in-flight range was overwritten"); + memo.Add(second, 2, in next); + + // Submissions 1 and 2 complete: the ring may be recycled and rewritten, and no old range + // is offered for the same bytes in submission 3. + memory.BeginSubmission(3, 2); + Assert.False(memo.TryFind(first, 3, out _)); + Assert.False(memo.TryFind(second, 3, out _)); + output.WriteLine($"in-flight range 0x{range.Offset:X}, next 0x{next.Offset:X}"); + } + + // Two resolves of ranges in one window take the memo; a new submission, a moved cut and a + // range in another window each take the full path, and every answer is the window's address + // plus the offset in it. + [Fact] + public void TheWindowMemoFollowsTheSubmissionAndTheLayout() + { + if (!RequireDevice()) + { + return; + } + + using var backing = new MappedBacking(); + using var memory = new MetalGuestMemory(Device.Value, backing, new NoCpu(), Granule, 64 * Mib); + + MetalGuestRange Resolve(ulong offset, ulong size) + { + Assert.True(memory.TryResolveInPlace(GuestBase + offset, size, out var range)); + Assert.Equal(size, range.Size); + Assert.NotEqual(0, range.Buffer); + return range; + } + + memory.BeginSubmission(1, 0); + var (hits0, misses0) = memory.WindowMemoCounts; + var a = Resolve(0x100, 0x40); + Assert.Equal((hits0, misses0 + 1), memory.WindowMemoCounts); + + // The same range, and another inside the window: no lock, no placement. + Assert.Equal(a, Resolve(0x100, 0x40)); + var b = Resolve(0x2000, 0x80); + Assert.Equal(a.Buffer, b.Buffer); + Assert.Equal(0x2000UL - 0x100, b.GpuAddress - a.GpuAddress); + var (hits1, misses1) = memory.WindowMemoCounts; + Assert.Equal((hits0 + 2, misses0 + 1), (hits1, misses1)); + + // A range across an edge no resolve used moves the cut: the layout version changed, so + // the next resolve of the first window's range is placed again (and still right). + var versionBefore = memory.PartitionVersion; + var across = Resolve(5 * Granule - 0x100, 0x200); + Assert.NotEqual(versionBefore, memory.PartitionVersion); + var (hits2, misses2) = memory.WindowMemoCounts; + Assert.Equal((hits1, misses1 + 1), (hits2, misses2)); + Assert.Equal(a, Resolve(0x100, 0x40)); + Assert.Equal((hits2, misses2 + 1), memory.WindowMemoCounts); + Assert.Equal(a, Resolve(0x100, 0x40)); + Assert.Equal((hits2 + 1, misses2 + 1), memory.WindowMemoCounts); + Assert.True(across.IsValid); + + // The next submission starts over, and a range in another window is not the first's. + memory.BeginSubmission(2, 1); + var (hits3, misses3) = memory.WindowMemoCounts; + Assert.Equal(a, Resolve(0x100, 0x40)); + Assert.Equal((hits3, misses3 + 1), memory.WindowMemoCounts); + var other = Resolve(9 * Granule + 0x10, 0x20); + Assert.NotEqual(a.Buffer, other.Buffer); + Assert.Equal((hits3, misses3 + 2), memory.WindowMemoCounts); + Assert.Equal(other, Resolve(9 * Granule + 0x10, 0x20)); + Assert.Equal((hits3 + 1, misses3 + 2), memory.WindowMemoCounts); + } + + private bool RequireDevice() + { + if (Environment.GetEnvironmentVariable("SHARPEMU_TEST_REQUIRE_DEVICE") != "1") + { + output.WriteLine("Runs on the device only behind the exclusive device gate."); + return false; + } + + Assert.True(OperatingSystem.IsMacOS()); + Assert.NotEqual(0, Device.Value); + return true; + } + + private sealed class NoCpu : ICpuMemory + { + public bool TryRead(ulong virtualAddress, Span destination) => false; + public bool TryWrite(ulong virtualAddress, ReadOnlySpan source) => false; + public bool CanRead(ulong address, ulong size) => false; + } + + // An anonymous shared mapping standing in for the guest backing; GuestBase is its first byte. + private sealed class MappedBacking : IGuestBackedSpace, IDisposable + { + public MappedBacking() + { + Base = Mmap(0, (nuint)BackingSize, 3, 0x0001 | 0x1000, -1, 0); + Assert.NotEqual(-1, Base); + } + + public nint Base { get; } + + public void Dispose() => Munmap(Base, (nuint)BackingSize); + + private static bool Contains(ulong address, ulong length) => + address >= GuestBase && address - GuestBase + length <= BackingSize; + + public bool TryHoldRange(ulong address, ulong size) => false; + public bool TryHoldRangeAtOrAbove(ulong searchStart, ulong size, ulong alignment, out ulong address) { address = 0; return false; } + public bool TryMapBacked(ulong address, ulong size, ulong backingOffset, GuestPageProtection protection, out HostViewFailure failure) { failure = HostViewFailure.None; return false; } + public bool TryUnmapBacked(ulong address, ulong size) => false; + public bool TryClearBacking(ulong offset, ulong size) => false; + public bool IsBackedView(ulong address) => Contains(address, 1); + public bool IsBackedRange(ulong address, ulong size) => Contains(address, size); + public (ulong Base, ulong Size) BackingAliasRange => ((ulong)Base, BackingSize); + + public bool TryResolveBackingAlias(ulong address, ulong size, out ulong alias) + { + alias = Contains(address, size) ? (ulong)Base + (address - GuestBase) : 0; + return alias != 0; + } + + public bool TryWriteBacking(ulong address, ReadOnlySpan data) => false; + + public bool TryReadBacking(ulong address, Span data) + { + if (!Contains(address, (ulong)data.Length)) return false; + new ReadOnlySpan((byte*)Base + (address - GuestBase), data.Length).CopyTo(data); + return true; + } + + [DllImport("libc", EntryPoint = "mmap")] + private static extern nint Mmap(nint address, nuint length, int protection, int flags, int fd, long offset); + + [DllImport("libc", EntryPoint = "munmap")] + private static extern int Munmap(nint address, nuint length); + } +} diff --git a/tests/SharpEmu.Gpu.Metal.Tests/Scenes/BindingReuseSceneTests.cs b/tests/SharpEmu.Gpu.Metal.Tests/Scenes/BindingReuseSceneTests.cs new file mode 100644 index 000000000..c9ae1f023 --- /dev/null +++ b/tests/SharpEmu.Gpu.Metal.Tests/Scenes/BindingReuseSceneTests.cs @@ -0,0 +1,109 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using System.Runtime.InteropServices; +using Xunit; +using Xunit.Abstractions; + +namespace SharpEmu.Gpu.Metal.Tests.Scenes; + +/// +/// Draws that bind the same textures in one command buffer reuse the ring range their +/// argument buffer was written to; a draw whose binding changed does not, and renders what it +/// binds. Every colour is read back from the target the draw rendered to. +/// +public sealed class BindingReuseSceneTests(ITestOutputHelper output) : SceneTestBase(output) +{ + private const uint Rgba8 = 56; + private const uint Size = 8; + + private static readonly uint[] Sample = + [ + 0xBEFC0310, // s_mov_b32 m0, s16 + 0xC8080000, // v_interp_p1_f32_e32 v2, v0, attr0.x + 0xC8090001, // v_interp_p2_f32_e32 v2, v1, attr0.x + 0xC80C0100, // v_interp_p1_f32_e32 v3, v0, attr0.y + 0xC80D0101, // v_interp_p2_f32_e32 v3, v1, attr0.y + 0xC8100200, // v_interp_p1_f32_e32 v4, v0, attr0.z + 0xC8110201, // v_interp_p2_f32_e32 v4, v1, attr0.z + 0xF0800F08, 0x00400802, // image_sample v[8:11], v[2:3], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D + 0xBF8C3F70, // s_waitcnt vmcnt(0) + 0xF800180F, 0x0B0A0908, // exp mrt0 v8, v9, v10, v11 done vm + 0xBF810000, // s_endpgm + ]; + + private static ulong SolidTexture(SceneRig rig, uint abgr) + { + var texels = new uint[Size * 64]; + Array.Fill(texels, abgr); + var address = rig.Memory.Allocate(Size * 64 * 4, 0x10000); + rig.Memory.Write(address, MemoryMarshal.AsBytes(texels.AsSpan())); + return address; + } + + private static (ulong Target, uint Pitch) DrawTextured(SceneRig rig, ScenePm4 pm4, ulong texture) + { + const uint width = 16; + var target = rig.ColorTarget(width, width, 16, 0xFF); + var state = new SceneState(width, width).ColorTarget(0, target, width, width, format: 14, numberType: 7); + static SceneVertex V(float x, float y, float u, float v) => SceneVertex.At(x, y, 0.5f, u, v, 0, 1); + SceneVertex[] quad = + [ + V(-1, -1, 0, 1), V(1, -1, 1, 1), V(-1, 1, 0, 0), + V(-1, 1, 0, 0), V(1, -1, 1, 1), V(1, 1, 1, 0), + ]; + uint[] userData = + [ + .. Descriptors.Texture(texture, Rgba8, Size, Size), .. Descriptors.Sampler(), + SceneState.F(0), SceneState.F(0), SceneState.F(0), SceneState.F(0), + ]; + rig.DrawMesh(pm4, state, quad, Sample, userData); + return (target, width); + } + + private static float[] Centre(SceneRig rig, (ulong Target, uint Pitch) draw) + { + var pixels = MemoryMarshal.Cast(rig.ReadImage(draw.Target, draw.Pitch, draw.Pitch, 16)); + return pixels.Slice((int)(((8 * draw.Pitch) + 8) * 4), 4).ToArray(); + } + + private static void AssertColour(string what, float[] actual, uint abgr) + { + float[] expected = + [(abgr & 0xFF) / 255f, ((abgr >> 8) & 0xFF) / 255f, ((abgr >> 16) & 0xFF) / 255f, ((abgr >> 24) & 0xFF) / 255f]; + for (var channel = 0; channel < 4; channel++) + { + Assert.True( + MathF.Abs(actual[channel] - expected[channel]) <= 0.01f, + $"{what}: channel {channel} is {actual[channel]}, expected {expected[channel]}"); + } + } + + [Fact] + public void DrawsWithTheSameBindingsShareArgumentBuffersAndAChangedBindingIsSeen() + { + using var rig = CreateRig(); + if (rig is null) return; + const uint red = 0xFF0000FF; + const uint green = 0xFF00FF00; + var textures = new Dictionary { [red] = SolidTexture(rig, red), [green] = SolidTexture(rig, green) }; + + // The binding changes after a reuse (red, red, green) and changes back (green, red). + uint[] sequence = [red, red, red, green, green, red, green]; + var pm4 = new ScenePm4(); + var draws = sequence.Select(colour => DrawTextured(rig, pm4, textures[colour])).ToArray(); + var reusesBefore = rig.Host.ArgumentBufferReuses; + rig.Submit(pm4); + + for (var index = 0; index < draws.Length; index++) + { + AssertColour($"draw {index}", Centre(rig, draws[index]), sequence[index]); + } + + var reuses = rig.Host.ArgumentBufferReuses - reusesBefore; + var writes = rig.Host.ArgumentBufferWrites; + output.WriteLine($"{sequence.Length} draws: {reuses} argument/push ranges reused, {writes} written"); + Assert.True(reuses > 0, "no draw reused an earlier draw's argument buffer"); + Assert.True(writes > 0, "no argument buffer was written"); + } +} From 983fbb49a59866e4ff107ffd9c3be116214b3c0e Mon Sep 17 00:00:00 2001 From: kuba Date: Mon, 26 Oct 2026 05:34:44 +0100 Subject: [PATCH 09/82] [GPU] Draws through moving descriptor tables find their materialization by the descriptors they point at ResourceMaterializationCache keyed a draw on every user data word its plan reads, so a descriptor-table pointer that moves every frame (a per-frame descriptor ring) put the same object under a new key every frame. The address rebase could take the pointer out of the key, but it learned the pointer only by comparing a draw with the latest draw of the same plan, which is another object whenever a shader draws several materials: it never confirmed, and every draw was a key-not-found miss and a full walk (the game's 62% key-not-found). The rebase now keeps, per plan, the latest entry per content print as well: a hash of the descriptor dwords the plan loads through each raw pointer candidate (flattened table slots resolved, buffer base dwords left out because memory-loaded bases move every draw). Learning compares a draw with the same object's previous draw through another pointer, so the pointer confirms after a few draws. Once it is rebased, the key holds the print of the descriptors the pointer points at now, read from guest memory, so objects drawn through the same moving pointer keep separate keys and the same object finds its entry wherever its table lies. The print only chooses the key; Entry.Matches and Validate (every recorded range re-read at its moved address and compared) still decide every hit, so changed descriptor words miss or refresh as before. A hit through rebased or differing user data built a new ResourceSnapshot (and patched buffer copies) unless it repeated the entry's single last hit. Each entry now keeps its last 8 hit snapshots, immutable and shared, so draws cycling through a few pointers allocate nothing; a snapshot handed to a draw is never written afterwards. Switch: SHARPEMU_RESOURCE_CONTENT_KEY=0 turns the print (key and learning partner) off. Microbench (run-bench.sh, machine heavily loaded: load average 100-240, so ns are noisy; hit rates and B/op are exact): - new 3g, 32 materials/frame through a drifting descriptor ring, same contents each frame: hit rate 0% -> 100% steady (74% over the first 8 frames); 9,181 -> 856 B/op (688 of it is the bench's WriteTable); arm64 6,588 ns (quiet base) -> 1,069-1,481 ns (loaded); x64 20,364-30,249 -> 1,158-2,060 ns. - new 3f, 50 draws over 8 tables with identical contents: hit rate 92%/100% before and after; 127 -> 64 B/op (64 B is the bench's delegate, the cache now allocates 0). - 3a-3e and 8a-8f: B/op unchanged (3a 3,152, 3b 64, 3c ~8.2 K, 3d ~8.4 K, 8a-c 3,720, 8d-f 1,393 producer-side); 8e sampled allocations by type lose ResourceSnapshot (320 B/draw sampled on base, none after). Timings within the load noise. --- .../Resources/ResourceMaterializationCache.cs | 261 ++++++++++++++-- .../ResourceMaterializationContentKeyTests.cs | 283 ++++++++++++++++++ 2 files changed, 526 insertions(+), 18 deletions(-) create mode 100644 tests/SharpEmu.ShaderCompiler.Tests/Resources/ResourceMaterializationContentKeyTests.cs diff --git a/src/SharpEmu.ShaderCompiler/Resources/ResourceMaterializationCache.cs b/src/SharpEmu.ShaderCompiler/Resources/ResourceMaterializationCache.cs index 9d5dd5372..33113dbdf 100644 --- a/src/SharpEmu.ShaderCompiler/Resources/ResourceMaterializationCache.cs +++ b/src/SharpEmu.ShaderCompiler/Resources/ResourceMaterializationCache.cs @@ -101,6 +101,10 @@ private readonly record struct Candidate(int UserData, int Buffer, int High = -1 private readonly Dictionary _confirmed = new(); private const int RecentCapacity = 512; private readonly Dictionary _recent = new(); + // The latest entry per structure and content print (EntryPrint): the same object drawn + // before, which _recent misses when other objects are drawn in between. + private readonly Dictionary _recentByPrint = new(); + private long _recentByPrintBytes; // The estimated bytes the recent entries hold (Entry.EstimatedBytes). public long RecentBytes { get; private set; } @@ -108,9 +112,50 @@ private readonly record struct Candidate(int UserData, int Buffer, int High = -1 public void ForgetRecent() { _recent.Clear(); + _recentByPrint.Clear(); + _recentByPrintBytes = 0; RecentBytes = 0; } + // Keeps built as the latest entry of its structure and print; the one it replaces, if any. + private Entry? RememberByPrint(ulong structure, Entry built) + { + if (!_hasPrint || EntryPrint(built) is not { } print) + return null; + var key = MixPrint(structure, print); + _recentByPrint.TryGetValue(key, out var previous); + if (_recentByPrint.Count >= RecentCapacity) + { + _recentByPrint.Clear(); + RecentBytes -= _recentByPrintBytes; + _recentByPrintBytes = 0; + } + else if (previous is not null) + { + RecentBytes -= previous.EstimatedBytes; + _recentByPrintBytes -= previous.EstimatedBytes; + } + + _recentByPrint[key] = built; + RecentBytes += built.EstimatedBytes; + _recentByPrintBytes += built.EstimatedBytes; + return previous; + } + + // Some candidate word the entries read differs: they are drawn through other pointers. + private bool CandidateWordsDiffer(Entry left, Entry right) + { + foreach (var candidate in _candidates) + { + var index = candidate.UserData; + if (index < left.UserData.Length && index < right.UserData.Length && IsUsed(left.UsedUserData, index) && + left.UserData[index] != right.UserData[index]) + return true; + } + + return false; + } + private ulong StructureOf(Entry entry) { var hash = new HashCode(); @@ -148,6 +193,8 @@ public AddressRebase(ShaderResourcePlan plan) _candidates = candidates.ToArray(); _candidateMask = MaskOf(_candidates.Select(candidate => candidate.UserData)); + _printOffsets = PrintOffsetsOf(plan); + _hasPrint = _printOffsets.Any(offsets => offsets.Length != 0); var memoryBases = MemoryBaseCandidates(plan); MemoryBases = memoryBases.Select(candidate => candidate.Buffer).ToArray(); _memoryState = new int[plan.Info.Buffers.Count]; @@ -427,10 +474,7 @@ private IEnumerable RawCandidates(ShaderResourcePlan plan, HashSet RawCandidates(ShaderResourcePlan plan, HashSet !disqualified.Contains(candidate.UserData)); } + // The byte offset from its handle's base of the dword a raw scalar load reads. + private static long RelativeOffset(ShaderResourcePlan plan, ScalarValue value) + { + var immediate = (long)plan.Memory[value.MemoryIndex].Offset; + var offset = (uint)value.Operands[1].Payload; + return value.Kind == ScalarValueKind.ScalarBufferWord + ? (immediate + (long)offset) & ~3L + : (immediate & ~3L) + (long)(offset & ~3u); + } + + // The content print. A raw candidate is a pointer to a descriptor table. Once it is rebased the + // key leaves it out, so every object a shader draws through such a pointer would share one key + // and the few variants chained behind it; and before it is, an object whose table moves every + // frame is under a new key every frame. The print puts back what tells objects apart: the + // descriptor dwords the plan loads through the pointer, read where it points now. It only + // chooses the key: Entry.Matches and Validate still decide a hit on the entry it finds. + // + // Per candidate, the byte offsets from its base of up to PrintWords such dwords, spread over + // the ones within PrintSpanBytes of the first so one read covers them; empty for a buffer + // candidate and for a pointer the plan loads no descriptor dword through. A buffer's base dword + // is left out: a memory-loaded base moves on every draw and TryRebaseRefresh patches it. + private readonly long[][] _printOffsets; + private readonly bool _hasPrint; + private const int PrintWords = 32; + private const int PrintSpanBytes = 4096; + private const ulong PrintSeed = 0x243F_6A88_85A3_08D3UL; + // A word that cannot be found or read: outside the range of a word. + private const ulong MissingWord = 1UL << 32; + + private long[][] PrintOffsetsOf(ShaderResourcePlan plan) + { + var offsets = new long[_candidates.Length][]; + offsets.AsSpan().Fill([]); + if (!ContentKey.Enabled) + return offsets; + var bufferSources = new HashSet(); + foreach (var buffer in plan.Info.Buffers) + bufferSources.Add((int)buffer.Source); + var found = new SortedSet?[_candidates.Length]; + for (var source = 0; source < plan.DescriptorSources.Count; source++) + { + var dwords = plan.DescriptorSources[source].Dwords; + for (var dword = bufferSources.Contains(source) ? 1 : 0; dword < dwords.Length; dword++) + { + // A descriptor dword the plan flattened into its table is that slot's load. + var value = dwords[dword]; + if (value.Kind == ScalarValueKind.ResourceTableWord && value.Payload < (ulong)plan.TableReads.Count) + value = plan.TableReads[(int)value.Payload].Value; + if (value.Kind is not (ScalarValueKind.ScalarAddressWord or ScalarValueKind.ScalarBufferWord) || + value.Operands.Length < 2 || value.Operands[0].Operands.Length < 2 || + value.Operands[1].Kind != ScalarValueKind.Constant || (uint)value.MemoryIndex >= (uint)plan.Memory.Count || + UserDataIndex(value.Operands[0].Operands[0]) is not { } index) + continue; + var candidate = Array.FindIndex(_candidates, candidate => candidate.UserData == index && candidate.Buffer < 0); + if (candidate >= 0) + (found[candidate] ??= []).Add(RelativeOffset(plan, value)); + } + } + + for (var candidate = 0; candidate < found.Length; candidate++) + { + if (found[candidate] is not { Count: > 0 } set) + continue; + var first = set.Min; + var all = set.Where(offset => offset - first <= PrintSpanBytes - sizeof(uint)).ToArray(); + if (all.Length > PrintWords) + all = Enumerable.Range(0, PrintWords).Select(index => all[(int)((long)index * all.Length / PrintWords)]).ToArray(); + offsets[candidate] = all; + } + + return offsets; + } + + private static ulong MixPrint(ulong print, ulong value) => + System.Numerics.BitOperations.RotateLeft((print ^ value) * 0x9E37_79B9_7F4A_7C15UL, 29); + + // The print of the descriptors the entry read through every candidate not rejected: what the + // print of the same object drawn again is, wherever its table then lies. Null when none has one. + private ulong? EntryPrint(Entry entry) + { + var print = PrintSeed; + var any = false; + for (var index = 0; index < _candidates.Length; index++) + { + var offsets = _printOffsets[index]; + var candidate = _candidates[index]; + if (offsets.Length == 0 || IsUsed(_rejected, candidate.UserData)) + continue; + if (candidate.UserData >= entry.UserData.Length || candidate.High >= entry.UserData.Length) + return null; + var addressBase = RawBase(entry.UserData, candidate, entry.UserData[candidate.UserData]); + any = true; + print = MixPrint(print, (ulong)index); + foreach (var offset in offsets) + { + print = TryFindWord(entry, unchecked(addressBase + (ulong)offset), out var at) + ? MixPrint(print, System.Buffers.Binary.BinaryPrimitives.ReadUInt32LittleEndian(entry.Bytes.AsSpan(at, sizeof(uint)))) + : MixPrint(print, MissingWord); + } + } + + return any ? print : null; + } + + // The print of the descriptors behind this draw's rebased pointers, read where they lie now; + // zero when no rebased candidate has one. Words that cannot be read print as missing, the + // same for the lookup and for the entry stored after it. + public ulong MemoryPrint(IReadOnlyList userData, ResidentGuestBytesReader reader, ref byte[] scratch) + { + if (!_hasPrint || Active.Length == 0) + return 0; + var print = PrintSeed; + var any = false; + for (var index = 0; index < _candidates.Length; index++) + { + var offsets = _printOffsets[index]; + var candidate = _candidates[index]; + if (offsets.Length == 0 || !IsUsed(Active, candidate.UserData)) + continue; + if (candidate.UserData >= userData.Count || candidate.High >= userData.Count) + return 0; + var addressBase = RawBase(userData, candidate, userData[candidate.UserData]); + any = true; + print = MixPrint(print, (ulong)index); + var length = (int)(offsets[^1] - offsets[0]) + sizeof(uint); + if (scratch.Length < length) + scratch = new byte[Math.Max(length, scratch.Length * 2)]; + var bytes = scratch.AsSpan(0, length); + var start = (long)addressBase + offsets[0]; + var read = start >= 0 && reader((ulong)start, bytes, false); + foreach (var offset in offsets) + { + print = read + ? MixPrint(print, System.Buffers.Binary.BinaryPrimitives.ReadUInt32LittleEndian(bytes[(int)(offset - offsets[0])..])) + : MixPrint(print, MissingWord); + } + } + + return any ? print : 0; + } + // The window [start, start + size) of reads a candidate's base covers in this entry. private bool TryWindow(Entry entry, in Candidate candidate, out ulong start, out ulong size, out ulong addressBase) { @@ -561,7 +746,7 @@ public void Learn(Entry built, if (_recent.Count >= RecentCapacity) { _recent.Clear(); - RecentBytes = 0; + RecentBytes = _recentByPrintBytes; } else if (previous is not null) { @@ -570,6 +755,11 @@ public void Learn(Entry built, _recent[structure] = built; RecentBytes += built.EstimatedBytes; + // The same object's previous draw through another pointer is the comparison that can + // confirm the pointer; the latest draw is often another object, which never does. + if (RememberByPrint(structure, built) is { } same && !ReferenceEquals(same, previous) && + CandidateWordsDiffer(same, built)) + previous = same; if (previous is null || previous.UserData.Length != built.UserData.Length || !previous.UsedUserData.AsSpan().SequenceEqual(built.UsedUserData)) return; @@ -1104,7 +1294,7 @@ private bool MaterializeCore( drawKey = DrawKeyOf(plan, inputs); _learnedUserData.TryGetValue(drawKey, out learned); rebase = RebaseOf(drawKey, plan); - key = KeyOf(drawKey, inputs.UserData, learned, rebase.Active); + key = KeyFor(drawKey, inputs, learned, rebase, residentReader); } bool found; @@ -1354,7 +1544,7 @@ private bool MaterializeCore( if (!ReferenceEquals(widened, learned) || !ReferenceEquals(activeBefore, rebase.Active)) { _learnedUserData[drawKey] = widened; - key = KeyOf(drawKey, inputs.UserData, widened, rebase.Active); + key = KeyFor(drawKey, inputs, widened, rebase, residentReader); found = false; } @@ -1595,6 +1785,26 @@ private static ulong KeyOf(ulong drawKey, IReadOnlyList userData, ulong[]? return ((ulong)high << 32) | low; } + private byte[] _printScratch = new byte[256]; + + // KeyOf, and for a rebased descriptor-table pointer the print of the descriptors it points at + // now (AddressRebase.MemoryPrint): objects drawn through moving tables keep apart, and the same + // object finds its entry wherever its table lies this time. + private ulong KeyFor(ulong drawKey, ResourceRuntimeInputs inputs, ulong[]? learned, AddressRebase rebase, + ResidentGuestBytesReader residentReader) + { + var key = KeyOf(drawKey, inputs.UserData, learned, rebase.Active); + var print = rebase.MemoryPrint(inputs.UserData, residentReader, ref _printScratch); + return print == 0 ? key : key ^ (print * 0xC2B2_AE3D_27D4_EB4FUL); + } + + // SHARPEMU_RESOURCE_CONTENT_KEY=0: no content print, so a rebased pointer's key holds every + // object drawn through it, and a pointer is learned only against the plan's latest draw. + private static class ContentKey + { + public static readonly bool Enabled = PerfSwitches.Enabled("SHARPEMU_RESOURCE_CONTENT_KEY"); + } + private static bool IsUsed(ulong[] mask, int index) => (uint)(index >> 6) < (uint)mask.Length && (mask[index >> 6] & (1UL << (index & 63))) != 0; @@ -1647,21 +1857,33 @@ private static bool SameUserData(uint[] words, IReadOnlyList userData) // The snapshot a hit on the entry gives this draw (AddressRebase.Apply). A draw that differs from // the entry's in user data words its plan never reads differs on every hit, and so does a rebased - // one: the entry keeps the last snapshot it made for a draw, which an identical draw shares. + // one: the entry keeps the last HitSnapshotCapacity snapshots it made, which identical draws share. + // Draws that cycle through a few pointers (meshes, descriptor tables) then allocate nothing. + // A kept snapshot is never written: it is handed out as is, so a draw recorded earlier and + // encoded later still holds exactly the snapshot it was given. private static ResourceSnapshot HitSnapshot(Entry entry, IReadOnlyList userData, AddressRebase rebase) { - if (entry.LastHit is { } last && ReferenceEquals(entry.LastHitActive, rebase.Active) && SameUserData(last.UserData, userData)) - return last; + var kept = entry.HitSnapshots; + if (kept is not null) + { + foreach (var (last, active) in kept) + if (last is not null && ReferenceEquals(active, rebase.Active) && SameUserData(last.UserData, userData)) + return last; + } + var snapshot = rebase.Apply(entry.Snapshot, userData); if (!ReferenceEquals(snapshot, entry.Snapshot)) { - entry.LastHit = snapshot; - entry.LastHitActive = rebase.Active; + kept ??= entry.HitSnapshots = new (ResourceSnapshot?, ulong[]?)[HitSnapshotCapacity]; + kept[entry.NextHitSnapshot] = (snapshot, rebase.Active); + entry.NextHitSnapshot = (entry.NextHitSnapshot + 1) % HitSnapshotCapacity; } return snapshot; } + private const int HitSnapshotCapacity = 8; + private bool TryFind(ulong key, ShaderResourcePlan plan, ResourceRuntimeInputs inputs, ulong[] rebased, out Entry entry) { if (_young.TryGetValue(key, out entry!)) @@ -2227,9 +2449,10 @@ private sealed class Entry // its arrays that come out equal are shared when this entry's are made, then dropped. public JoinedReads? JoinedHint { get; set; } public SlotModel? SlotsHint { get; set; } - // The last snapshot a hit made for a draw (HitSnapshot), with the rebased words it applied. - public ResourceSnapshot? LastHit { get; set; } - public ulong[]? LastHitActive { get; set; } + // The last snapshots hits made for draws (HitSnapshot), each with the rebased words it applied; + // NextHitSnapshot is the slot the next one replaces. + public (ResourceSnapshot? Snapshot, ulong[]? Active)[]? HitSnapshots { get; set; } + public int NextHitSnapshot { get; set; } public Entry? Next { get; set; } // The chain bytes (ChainBytes) this entry was stored with as a key's head. @@ -2239,7 +2462,7 @@ private sealed class Entry // An upper bound on what the entry keeps alive, from its immutable arrays. Arrays it shares // with the entry it was rebuilt from are counted in both. The derived data filled in later - // (RangeWindows, BaseWordProbes, TableCopySources, Joined, Slots, LastHit) is counted up + // (RangeWindows, BaseWordProbes, TableCopySources, Joined, Slots, HitSnapshots) is counted up // front from the sizes it is made of, so the estimate never changes. public long EstimatedBytes { @@ -2261,8 +2484,10 @@ public long EstimatedBytes ObjectBytes + ArrayBytes(ranges, sizeof(ulong)) + 4 * ArrayBytes(ranges, sizeof(int)) + ArrayBytes(ranges, sizeof(bool)) + // SlotModel: at most one slot per table word. ObjectBytes + 2 * ArrayBytes(table, sizeof(ulong)) + 2 * ArrayBytes(table, sizeof(int)) + - // LastHit: a snapshot shell with its patched buffers. - ObjectBytes + JaggedBytes(Snapshot.Buffers) + ArrayBytes(UsedUserData.Length, sizeof(ulong)); + // HitSnapshots: the slot array, and per slot a snapshot shell with its patched buffers + // and the user data array it keeps alive. + ArrayBytes(HitSnapshotCapacity, 2 * IntPtr.Size) + HitSnapshotCapacity * + (ObjectBytes + JaggedBytes(Snapshot.Buffers) + ArrayBytes(UserData.Length, sizeof(uint))); _estimatedBytes = size; return size; } diff --git a/tests/SharpEmu.ShaderCompiler.Tests/Resources/ResourceMaterializationContentKeyTests.cs b/tests/SharpEmu.ShaderCompiler.Tests/Resources/ResourceMaterializationContentKeyTests.cs new file mode 100644 index 000000000..c29476b34 --- /dev/null +++ b/tests/SharpEmu.ShaderCompiler.Tests/Resources/ResourceMaterializationContentKeyTests.cs @@ -0,0 +1,283 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using SharpEmu.ShaderCompiler.Resources; +using Xunit; +using static SharpEmu.ShaderCompiler.Tests.Resources.ResourceTestProgram; + +namespace SharpEmu.ShaderCompiler.Tests.Resources; + +// Draws that bind their descriptors through a table pointer in user data: the pointer moves from +// draw to draw (a per-frame descriptor ring, or several copies of one table), the descriptors it +// points at often do not. Every result must equal a full walk of the same draw. +public sealed class ResourceMaterializationContentKeyTests +{ + private const ulong MemoryBase = 0x10_0000; + private const int MemoryWords = 0x40000; + + // textures T# and buffers V#, all loaded through the table pointer s[0:1]; the shader samples + // every texture with one sampler (also from the table) and loads from every buffer. + private static ShaderResourcePlan TablePlan(int textures, int buffers, int[]? textureOffsets = null, int samplerOffset = -1, + int[]? bufferOffsets = null) + { + textureOffsets ??= Enumerable.Range(0, textures).Select(index => index * 32).ToArray(); + samplerOffset = samplerOffset >= 0 ? samplerOffset : textures * 32; + bufferOffsets ??= Enumerable.Range(0, buffers).Select(index => samplerOffset + 16 + index * 16).ToArray(); + var program = new List(); + uint pc = 0; + for (var index = 0; index < textures; index++, pc += 8) + program.Add(ScalarLoad(pc, 0, 8 + 8 * (uint)index, 8, textureOffsets[index])); + program.Add(ScalarLoad(pc, 0, 72, 4, samplerOffset)); + pc += 8; + for (var index = 0; index < buffers; index++, pc += 8) + program.Add(ScalarLoad(pc, 0, 80 + 4 * (uint)index, 4, bufferOffsets[index])); + for (var index = 0; index < textures; index++, pc += 8) + program.Add(Image(pc, "ImageSample", 8 + 8 * (uint)index, 72)); + for (var index = 0; index < buffers; index++, pc += 8) + program.Add(BufferLoad(pc, 80 + 4 * (uint)index)); + program.Add(EndProgram(pc)); + return Extract(Program([.. program]), userDataCount: 2, stage: ShaderStage.Pixel); + } + + private static uint[] Texture(uint address) => [address >> 8, 20u << 20, 0, 0xFACu | (9u << 28), 0, 0, 0, 0]; + + private static uint[] Buffer(uint address, uint size = 256) => [address, 0, size, 0x2_0000]; + + // Material m's descriptors, written at table. + private static void WriteTable(TestWordMemory memory, ulong table, int material, int textures, int buffers, int[]? textureOffsets = null, + int samplerOffset = -1, int[]? bufferOffsets = null) + { + textureOffsets ??= Enumerable.Range(0, textures).Select(index => index * 32).ToArray(); + samplerOffset = samplerOffset >= 0 ? samplerOffset : textures * 32; + bufferOffsets ??= Enumerable.Range(0, buffers).Select(index => samplerOffset + 16 + index * 16).ToArray(); + for (var index = 0; index < textures; index++) + Write(memory, table + (ulong)textureOffsets[index], Texture(0x0100_0000u + (uint)(material * 16 + index) * 0x1_0000u)); + Write(memory, table + (ulong)samplerOffset, [0, 0, 0, 0]); + for (var index = 0; index < buffers; index++) + Write(memory, table + (ulong)bufferOffsets[index], Buffer(0x0200_0000u + (uint)(material * 8 + index) * 0x1000u)); + } + + private static void Write(TestWordMemory memory, ulong address, uint[] words) + { + for (var index = 0; index < words.Length; index++) + memory.At(address + (ulong)index * 4) = words[index]; + } + + private static TestWordMemory NewMemory() => new() { Base = MemoryBase, Words = new uint[MemoryWords] }; + + private static uint[] Pointer(ulong table) => [(uint)table, (uint)(table >> 32)]; + + private static bool Run(ResourceMaterializationCache cache, ShaderResourcePlan plan, TestWordMemory memory, uint[] userData, + out ResourceSnapshot snapshot, out ResourceSpecialization specialization) + { + snapshot = new ResourceSnapshot(); + specialization = new ResourceSpecialization(); + return cache.Materialize(plan, Inputs(userData, memory.Read, memory.Read), (address, destination, _) => + { + for (var offset = 0; offset < destination.Length; offset += 4) + { + if (!memory.Read(address + (ulong)offset, out var word)) + return false; + BitConverter.TryWriteBytes(destination[offset..], word); + } + + return true; + }, ref snapshot, ref specialization, out _); + } + + private static void AssertFullWalk(ShaderResourcePlan plan, TestWordMemory memory, uint[] userData, ResourceSnapshot snapshot, + ResourceSpecialization specialization) + { + var expected = new ResourceSnapshot(); + var expectedSpecialization = new ResourceSpecialization(); + Assert.True(ResourceMaterializer.Materialize(plan, Inputs(userData, memory.Read, memory.Read), ref expected, ref expectedSpecialization)); + Assert.Equal(expected.Buffers.Select(words => words.ToArray()), snapshot.Buffers.Select(words => words.ToArray())); + Assert.Equal(expected.Images.Select(words => words.ToArray()), snapshot.Images.Select(words => words.ToArray())); + Assert.Equal(expected.Samplers.Select(words => words.ToArray()), snapshot.Samplers.Select(words => words.ToArray())); + Assert.Equal(expected.FlattenedResourceTable, snapshot.FlattenedResourceTable); + Assert.Equal(expected.DeviceAddressRanges, snapshot.DeviceAddressRanges); + Assert.Equal(userData, snapshot.UserData); + Assert.Equal(expectedSpecialization, specialization); + } + + [Fact] + public void IdenticalDescriptorsAtDifferentTableAddressesHit() + { + var plan = TablePlan(textures: 4, buffers: 2); + var memory = NewMemory(); + var tables = Enumerable.Range(0, 8).Select(index => MemoryBase + 0x1000 + (ulong)index * 0x300).ToArray(); + foreach (var table in tables) + WriteTable(memory, table, material: 3, textures: 4, buffers: 2); + var cache = new ResourceMaterializationCache(); + for (var draw = 0; draw < 50; draw++) + { + var userData = Pointer(tables[draw % tables.Length]); + Assert.True(Run(cache, plan, memory, userData, out var snapshot, out var specialization)); + AssertFullWalk(plan, memory, userData, snapshot, specialization); + } + + // One miss to build, three to confirm the pointer; every later draw is a hit. + Assert.Equal((46, 4), (cache.Hits, cache.Misses)); + } + + [Fact] + public void ObjectsWhoseTablesMoveEveryFrameHitOnceThePointerIsConfirmed() + { + const int Materials = 6; + var plan = TablePlan(textures: 4, buffers: 2); + var memory = NewMemory(); + var cache = new ResourceMaterializationCache(); + for (var frame = 0; frame < 12; frame++) + { + for (var material = 0; material < Materials; material++) + { + // A ring that drifts, so no material lies where it lay before. + var table = MemoryBase + 0x1000 + (ulong)((frame * 7 + material) % 300) * 0x200; + WriteTable(memory, table, material, textures: 4, buffers: 2); + var userData = Pointer(table); + Assert.True(Run(cache, plan, memory, userData, out var snapshot, out var specialization)); + AssertFullWalk(plan, memory, userData, snapshot, specialization); + } + } + + // The first frame builds, the second confirms the pointer, and from the third every object + // finds its own entry wherever its table lies. + Assert.True(cache.Hits >= 9 * Materials, $"hits={cache.Hits} misses={cache.Misses}"); + } + + [Fact] + public void AChangedDescriptorWordIsNeverReused() + { + var plan = TablePlan(textures: 4, buffers: 2); + var memory = NewMemory(); + var tables = Enumerable.Range(0, 8).Select(index => MemoryBase + 0x1000 + (ulong)index * 0x300).ToArray(); + foreach (var table in tables) + WriteTable(memory, table, material: 3, textures: 4, buffers: 2); + var cache = new ResourceMaterializationCache(); + for (var draw = 0; draw < 24; draw++) + Assert.True(Run(cache, plan, memory, Pointer(tables[draw % tables.Length]), out _, out _)); + var hits = cache.Hits; + + // A texture rewritten in place at an address the cache has seen. + Write(memory, tables[5] + 32, Texture(0x0700_0000)); + Assert.True(Run(cache, plan, memory, Pointer(tables[5]), out var rewritten, out var rewrittenSpecialization)); + AssertFullWalk(plan, memory, Pointer(tables[5]), rewritten, rewrittenSpecialization); + Assert.Equal(hits, cache.Hits); + + // A buffer resized behind a pointer the cache has never seen, whose other words match. + var moved = MemoryBase + 0x8000; + WriteTable(memory, moved, material: 3, textures: 4, buffers: 2); + Write(memory, moved + 4 * 32 + 16 + 8, [512]); + Assert.True(Run(cache, plan, memory, Pointer(moved), out var resized, out var resizedSpecialization)); + AssertFullWalk(plan, memory, Pointer(moved), resized, resizedSpecialization); + Assert.Equal(512u, resized.Buffers[0][2]); + + // The untouched tables still hit, and still give their own descriptors. + Assert.True(Run(cache, plan, memory, Pointer(tables[2]), out var untouched, out var untouchedSpecialization)); + AssertFullWalk(plan, memory, Pointer(tables[2]), untouched, untouchedSpecialization); + Assert.Equal(hits + 1, cache.Hits); + } + + // A hit through a moved pointer gives each pointer its own snapshot; draws that come back to a + // pointer share the one made for it, and nothing a later draw does changes an earlier one's. + [Fact] + public void HitSnapshotsAreSharedPerPointerAndNeverWritten() + { + var plan = TablePlan(textures: 2, buffers: 1); + var memory = NewMemory(); + var tables = Enumerable.Range(0, 8).Select(index => MemoryBase + 0x1000 + (ulong)index * 0x300).ToArray(); + foreach (var table in tables) + WriteTable(memory, table, material: 1, textures: 2, buffers: 1); + var userData = tables.Select(Pointer).ToArray(); + var cache = new ResourceMaterializationCache(); + ResidentGuestBytesReader resident = (address, destination, _) => + { + for (var offset = 0; offset < destination.Length; offset += 4) + { + if (!memory.Read(address + (ulong)offset, out var word)) + return false; + BitConverter.TryWriteBytes(destination[offset..], word); + } + + return true; + }; + var inputs = userData.Select(words => Inputs(words, memory.Read, memory.Read)).ToArray(); + var snapshot = new ResourceSnapshot(); + var specialization = new ResourceSpecialization(); + var first = new ResourceSnapshot[tables.Length]; + for (var draw = 0; draw < 4 * tables.Length; draw++) + { + Assert.True(cache.Materialize(plan, inputs[draw % tables.Length], resident, ref snapshot, ref specialization, out _)); + if (draw >= 2 * tables.Length && draw < 3 * tables.Length) + first[draw % tables.Length] = snapshot; + if (draw >= 3 * tables.Length) + Assert.Same(first[draw % tables.Length], snapshot); + } + + for (var index = 0; index < tables.Length; index++) + { + Assert.Equal(userData[index], first[index].UserData); + AssertFullWalk(plan, memory, userData[index], first[index], specialization); + } + + // Steady state: a cycle of hits allocates nothing. + var before = GC.GetAllocatedBytesForCurrentThread(); + for (var draw = 0; draw < tables.Length; draw++) + Assert.True(cache.Materialize(plan, inputs[draw], resident, ref snapshot, ref specialization, out _)); + Assert.Equal(0, GC.GetAllocatedBytesForCurrentThread() - before); + } + + // Random layouts, random materials at random and reused table addresses, in-place rewrites and + // tables that move: whatever the cache answers, hit, refresh or miss, is what a full walk gives. + [Fact] + public void EveryAnswerEqualsAFullWalkOverRandomLayouts() + { + var random = new Random(0x5EED); + for (var layout = 0; layout < 24; layout++) + { + var textures = random.Next(1, 7); + var buffers = random.Next(0, 4); + // Descriptor slots in a shuffled order with gaps; textures need 32 bytes, the rest 16. + var slots = Enumerable.Range(0, textures + 1 + buffers + random.Next(0, 4)).OrderBy(_ => random.Next()).ToArray(); + var textureOffsets = slots.Take(textures).Select(slot => slot * 32).ToArray(); + var samplerOffset = slots[textures] * 32; + var bufferOffsets = slots.Skip(textures + 1).Take(buffers).Select(slot => slot * 32 + 16 * random.Next(0, 2)).ToArray(); + var plan = TablePlan(textures, buffers, textureOffsets, samplerOffset, bufferOffsets); + var memory = NewMemory(); + var cache = new ResourceMaterializationCache(); + var materials = random.Next(1, 6); + var addresses = Enumerable.Range(0, 12).Select(index => MemoryBase + 0x1000 + (ulong)index * 0x400).ToArray(); + var tableOf = new int[addresses.Length]; + Array.Fill(tableOf, -1); + for (var draw = 0; draw < 160; draw++) + { + var slot = random.Next(addresses.Length); + var table = addresses[slot]; + var action = random.Next(10); + if (tableOf[slot] < 0 || action < 3) + { + tableOf[slot] = random.Next(materials); + WriteTable(memory, table, tableOf[slot], textures, buffers, textureOffsets, samplerOffset, bufferOffsets); + } + else if (action == 3) + { + // One descriptor taken from another material, in place. + var texture = random.Next(textures); + Write(memory, table + (ulong)textureOffsets[texture], Texture(0x0300_0000u + (uint)random.Next(64) * 0x1_0000u)); + tableOf[slot] = materials + draw; + } + else if (action == 4 && buffers != 0) + { + Write(memory, table + (ulong)bufferOffsets[random.Next(buffers)] + 8, [(uint)random.Next(1, 4) * 256]); + tableOf[slot] = materials + draw; + } + + var userData = Pointer(table); + Assert.True(Run(cache, plan, memory, userData, out var snapshot, out var specialization)); + AssertFullWalk(plan, memory, userData, snapshot, specialization); + } + + Assert.True(cache.Hits > 0, $"layout {layout}: hits={cache.Hits} misses={cache.Misses}"); + } + } +} From 8601ce1d72ea224c7c856ecf86ed69abecdcbb46 Mon Sep 17 00:00:00 2001 From: kuba Date: Mon, 26 Oct 2026 05:36:27 +0100 Subject: [PATCH 10/82] [GPU] A guest store into a large staged entry re-copies only its pages; single-copy retile reuses its staging buffer Staged range cache: one CPU store used to drop the watch on the whole entry, so the next bind copied the whole entry (up to 64 MB) and re-added a watch on every page. For entries of 1 MiB or more, a fault or a precise managed write now marks only the 4 KiB pages it touches (bitmap per entry), drops the watch on those pages, and leaves the rest current and watched. The next bind rewatches the dirty pages and copies just them (and their pristine twin) in place. Falls back to the old whole-entry stale for: GPU/host writes, shader stores never written back, a writeback with unwindowed bytes, an entry a submission still reads, an entry with no pristine twin when a written binding needs one, and more than 32 dirty pages (then the next 8 stagings drop the whole watch on the first store, so a wholesale writer pays at most 32 faults once per 9 cycles). Switch: SHARPEMU_METAL_V2_STAGED_CACHE_PARTIAL=0 restores the whole-entry behaviour. Single-copy restore: the retile staging buffer is taken from a small pool (4 buffers, 256 MiB, 1 MiB size classes) instead of a fresh newBuffer per restore on the faulting guest thread. The zero fill and the wait are kept. Switch: SHARPEMU_SINGLE_COPY_RETILE_POOL=0. Synthetic bench (64 MB entry, one 4-byte store, rebind; noisy shared machine, load 100+): read-only, in place: 6.7-13 ms and 64 MiB copied -> 40-53 us and 4 KiB copied; written (with pristine): 15-20 ms -> 36-49 us. Watch churn per rebind 16384 pages added and removed -> 1 page. Not changed: PageGuard block lock across mprotect, and the written-binding double copy. --- .../V2/MetalRenderHost.StagedStorage.cs | 15 + .../SharpEmu.Gpu.Metal/V2/MetalSingleCopy.cs | 95 ++- .../V2/MetalStagedRangeCache.cs | 283 ++++++++- .../V2/MetalStagedStorage.cs | 4 + .../MetalSingleCopyTests.cs | 22 + .../MetalStagedRangePartialTests.cs | 548 ++++++++++++++++++ 6 files changed, 959 insertions(+), 8 deletions(-) create mode 100644 tests/SharpEmu.Gpu.Metal.Tests/MetalStagedRangePartialTests.cs diff --git a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.StagedStorage.cs b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.StagedStorage.cs index cafd40090..51b2d36b2 100644 --- a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.StagedStorage.cs +++ b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.StagedStorage.cs @@ -226,8 +226,23 @@ private MetalGuestRange BindStagedRange(ulong address, ulong size, bool written) return entry.Staging; } + private MetalStagedRangeCache.GuestReader? _stagingReader; + private unsafe void FillStagedRange(MetalStagedRangeCache.Entry entry, MetalStagedStorage staged) { + if (entry.Plan is not null) + { + // Stores touched a few pages of the entry: those are copied, unless the GPU still has + // bytes to write there, which the gather below picks up for the whole entry. + if (_gpuWrites.PendingSerial(entry.Address, entry.Size, CompletedSerial) == 0 && + _stagedRanges!.RefillDirty(entry, _stagingReader ??= _guestMemory.ReadForStaging)) + { + return; + } + + entry.Plan = null; + } + var size = entry.Size; var bytes = new Span((void*)entry.StagingBuffer!.Contents, checked((int)size)); _guestMemory.ReadForStaging(entry.Address, bytes); diff --git a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalSingleCopy.cs b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalSingleCopy.cs index 35a2206de..cd6e5b7ba 100644 --- a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalSingleCopy.cs +++ b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalSingleCopy.cs @@ -121,6 +121,7 @@ internal sealed unsafe class MetalSingleCopy : IGuestSingleCopyHandler, IDisposa private static readonly nint SelRetain = MetalNative.Selector("retain"); private static readonly nint SelRelease = MetalNative.Selector("release"); private static readonly nint SelContents = MetalNative.Selector("contents"); + private static readonly nint SelLength = MetalNative.Selector("length"); private static readonly nint SelNewBuffer = MetalNative.Selector("newBufferWithLength:options:"); /// Timings and caps; the defaults are the protocol's, tests shorten them. @@ -237,6 +238,81 @@ private sealed class RestoreContext(nint device) : IMetalSubmissionContext private readonly ConcurrentQueue _unwatch = new(); private readonly object _gpuLock = new(); + + // SHARPEMU_SINGLE_COPY_RETILE_POOL=0 allocates and releases a staging buffer per retile, as before. + private static readonly bool PoolRetileStaging = + Environment.GetEnvironmentVariable("SHARPEMU_SINGLE_COPY_RETILE_POOL") != "0"; + + // Retile staging buffers kept for the next retile: a restore runs on the faulting guest thread, and a + // fresh shared buffer costs the allocation and its first-touch page faults every time. A retile only + // needs the bytes it fills and the texture writes, so any buffer at least as large will do. + private const int RetilePoolMaxBuffers = 4; + private const ulong RetilePoolMaxBytes = 256UL * 1024 * 1024; + private const ulong RetileClassBytes = 1024 * 1024; + private readonly List<(nint Buffer, ulong Capacity)> _retilePool = new(); + private ulong _retilePoolBytes; + private long _retilePoolHits; + private long _retilePoolMisses; + + /// Retile staging buffers taken from the pool, and those created, since the start. + internal (long Hits, long Misses) RetilePoolCounts => + (Interlocked.Read(ref _retilePoolHits), Interlocked.Read(ref _retilePoolMisses)); + + // The smallest pooled buffer that holds the run, or a new one of the run's size class. + private nint TakeRetileStaging(ulong bytes) + { + if (PoolRetileStaging) + { + lock (_retilePool) + { + var best = -1; + for (var index = 0; index < _retilePool.Count; index++) + { + if (_retilePool[index].Capacity >= bytes && + (best < 0 || _retilePool[index].Capacity < _retilePool[best].Capacity)) + { + best = index; + } + } + + if (best >= 0) + { + var taken = _retilePool[best].Buffer; + _retilePoolBytes -= _retilePool[best].Capacity; + _retilePool.RemoveAt(best); + Interlocked.Increment(ref _retilePoolHits); + return taken; + } + } + } + + Interlocked.Increment(ref _retilePoolMisses); + var capacity = PoolRetileStaging + ? (bytes + RetileClassBytes - 1) / RetileClassBytes * RetileClassBytes + : bytes; + return MetalNative.SendNewBuffer(_device, SelNewBuffer, (nuint)capacity, 0); + } + + // Keeps the buffer for the next retile, or releases it when the pool is full. + private void ReleaseRetileStaging(nint staging) + { + if (PoolRetileStaging && staging != 0) + { + var capacity = (ulong)MetalV2Native.SendValue(staging, SelLength); + lock (_retilePool) + { + if (!_disposed && _retilePool.Count < RetilePoolMaxBuffers && + _retilePoolBytes + capacity <= RetilePoolMaxBytes) + { + _retilePool.Add((staging, capacity)); + _retilePoolBytes += capacity; + return; + } + } + } + + MetalNative.SendVoid(staging, SelRelease); + } private long _lastScan; private bool _stopped; private bool _disposed; @@ -871,7 +947,7 @@ private void EmptyRecord(Record record, long now, long grace) } finally { - MetalNative.SendVoid(staging, SelRelease); + ReleaseRetileStaging(staging); } } finally @@ -993,7 +1069,7 @@ private nint Retile(Record record, out nint contents) lock (_gpuLock) { using var pool = MetalAutoreleasePool.Enter(); - var staging = MetalNative.SendNewBuffer(_device, SelNewBuffer, (nuint)((runSize + 3) & ~3UL), 0); + var staging = TakeRetileStaging((runSize + 3) & ~3UL); if (staging == 0) { throw GpuFatal.Fatal($"Single-copy textures could not allocate 0x{runSize:X} bytes of retile staging."); @@ -1508,7 +1584,7 @@ private void RestoreRecord(Record record, GuestSingleCopyAccess access, bool pen } finally { - MetalNative.SendVoid(staging, SelRelease); + ReleaseRetileStaging(staging); } } } @@ -1750,7 +1826,7 @@ public void EndOverwrite(object token, ulong offset, ulong written) } finally { - MetalNative.SendVoid(staging, SelRelease); + ReleaseRetileStaging(staging); } } } @@ -2042,6 +2118,17 @@ public void Dispose() ReleaseTexture(record); } + lock (_retilePool) + { + foreach (var (buffer, _) in _retilePool) + { + MetalNative.SendVoid(buffer, SelRelease); + } + + _retilePool.Clear(); + _retilePoolBytes = 0; + } + lock (_gpuLock) { _transfer.Dispose(); diff --git a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalStagedRangeCache.cs b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalStagedRangeCache.cs index 74ce1827d..c84afda52 100644 --- a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalStagedRangeCache.cs +++ b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalStagedRangeCache.cs @@ -51,6 +51,19 @@ internal sealed class MetalStagedRangeCache internal static readonly bool Enabled = Environment.GetEnvironmentVariable("SHARPEMU_METAL_V2_STAGED_CACHE") != "0"; + // SHARPEMU_METAL_V2_STAGED_CACHE_PARTIAL=0 stales a whole entry on any guest store, as before. + // Otherwise a store into a large entry marks only its own pages, and the next bind copies those. + internal static readonly bool PartialDefault = + Environment.GetEnvironmentVariable("SHARPEMU_METAL_V2_STAGED_CACHE_PARTIAL") != "0"; + + // Below this an entry is cheaper to copy whole than to fault page by page. + public const ulong PartialMinBytes = 1024 * 1024; + + // More dirty pages than this and the entry is being rewritten wholesale: it is staled whole, and + // for the next PartialHoldStagings stagings its first store drops the watch over all of it. + public const int MaxDirtyPages = 32; + public const int PartialHoldStagings = 8; + public enum VerifyKind { Off, @@ -204,10 +217,34 @@ internal sealed class Entry public bool Removed; internal uint QueryEpoch; + // Guest stores into single pages of a large entry (see MetalStagedRangeCache.TryMarkPages): the pages + // are dirty, and their watch is gone, while the rest of the entry stays current and watched. + // A bit per page counted from the page holding Address. Under the cache lock. + internal ulong[]? DirtyPages; + internal int DirtyPageCount; + + // The bytes of the staging may not be what guest memory holds on the pages no store touched: + // the entry is staged whole next time. True until its first staging completes a watch. + public bool NeedsWholeStage = true; + + // Stagings left in which a store drops the whole watch rather than one page of it. + internal int PartialHold; + + // What the worker copies between Acquire and the bind: only the pages that were stored into. + internal RefillPlan? Plan; + public bool Overlaps(ulong address, ulong size) => address < Address + Size && Address < address + size; } + /// The part of an entry copies. + internal sealed class RefillPlan + { + // Byte ranges of the entry, ascending and disjoint. + public readonly List<(ulong Offset, ulong Length)> DirtyRuns = new(); + public ulong DirtyBytes; + } + private readonly IStagedRangeBackend _backend; private readonly ulong _budget; private readonly object _gate = new(); @@ -228,12 +265,27 @@ public bool Overlaps(ulong address, ulong size) => private long _evictions; private readonly long[] _invalidations = new long[5]; - public MetalStagedRangeCache(IStagedRangeBackend backend, ulong budgetBytes) + private readonly bool _partial; + private long _partialRefills; + private long _partialDirtyBytes; + private long _partialSavedBytes; + private long _partialFallbacks; + + public MetalStagedRangeCache(IStagedRangeBackend backend, ulong budgetBytes, bool? partial = null) { _backend = backend; _budget = budgetBytes; + _partial = partial ?? PartialDefault; } + /// + /// Refills of only the stored-into pages of a large entry, the guest bytes they copied, the bytes + /// of the entry that were not copied, and the stores that staled an entry whole instead. + /// + public (long Refills, long GuestBytes, long SavedBytes, long Fallbacks) PartialCounters => + (Interlocked.Read(ref _partialRefills), Interlocked.Read(ref _partialDirtyBytes), + Interlocked.Read(ref _partialSavedBytes), Interlocked.Read(ref _partialFallbacks)); + public ulong CacheBytes { get @@ -358,12 +410,20 @@ public bool Acquire( // buffers go back to the pool when that submission completes, and this entry // gets new ones. var inFlight = current.LastUseSerial > _completedSerial; + + // Stores that touched a few pages of a large entry left the rest current and watched: only + // those pages are copied again, in place. An in-flight entry is staged whole into new buffers. + var refill = existing is not null && !inFlight && CanRefillPartially(current, needPristine); if (inFlight) { RetireBuffers(current, current.LastUseSerial); } - DropWatch(current); + if (!refill) + { + DropWatch(current); + } + if (!needPristine && current.PristineBuffer is { } stalePristine) { // Its bytes are from before this staging, and nothing reads it unless a written binding does. @@ -413,11 +473,26 @@ public bool Acquire( Index(current); } + Interlocked.Increment(ref _misses); + if (refill) + { + var plan = BeginRefill(current); + current.Plan = plan; + Interlocked.Add(ref _stagedBytes, (long)plan.DirtyBytes); + entry = current; + return false; + } + + current.Plan = null; current.WatchBegin = address; current.WatchEnd = address + size; _backend.AddWatch(address, size); + current.NeedsWholeStage = false; + if (current.PartialHold > 0) + { + current.PartialHold--; + } - Interlocked.Increment(ref _misses); Interlocked.Add(ref _stagedBytes, (long)size); entry = current; return false; @@ -445,6 +520,7 @@ public void OnWrittenBack(Entry entry) if (entry.StaleOnClean) { entry.StaleOnClean = false; + entry.NeedsWholeStage = true; Volatile.Write(ref entry.Stale, 1); DropWatch(entry); } @@ -597,6 +673,18 @@ private bool Invalidate(ulong address, ulong size, Reason reason, bool pageGranu continue; } + if (reason is Reason.Fault or Reason.Precise && TryMarkPages(entry, begin, length)) + { + // Only the pages the store touched are behind guest memory. + if (Interlocked.Exchange(ref entry.Stale, 1) == 0) + { + Interlocked.Increment(ref _invalidations[(int)reason]); + } + + continue; + } + + entry.NeedsWholeStage = true; if (Interlocked.Exchange(ref entry.Stale, 1) == 0) { Interlocked.Increment(ref _invalidations[(int)reason]); @@ -609,6 +697,155 @@ private bool Invalidate(ulong address, ulong size, Reason reason, bool pageGranu return found; } + private static ulong PageBase(Entry entry) => entry.Address & ~(WatchPageBytes - 1); + + private static int PageCount(Entry entry) => + (int)(((entry.Address + entry.Size + WatchPageBytes - 1) & ~(WatchPageBytes - 1)) - PageBase(entry)) / + (int)WatchPageBytes; + + private bool CanRefillPartially(Entry entry, bool needPristine) => + _partial && entry.Size >= PartialMinBytes && !entry.Removed && !entry.NeedsWholeStage && !entry.Dirty && + !entry.StaleOnClean && entry.WatchEnd != 0 && entry.DirtyPageCount > 0 && + Volatile.Read(ref entry.Stale) != 0 && entry.StagingBuffer is not null && + (!needPristine || entry.PristineBuffer is not null); + + // Caller holds the lock. A guest store into [begin, begin + length) of a large entry: its pages are + // marked dirty and lose their watch (the store may land once it is gone), the others stay watched. + // False when the entry is too small, its stores come too many pages at a time (then it is staled + // whole, and for a while a store drops all of it), or this cache does not track pages. + private bool TryMarkPages(Entry entry, ulong begin, ulong length) + { + if (!_partial || entry.Size < PartialMinBytes || entry.NeedsWholeStage || entry.PartialHold > 0 || + entry.WatchEnd == 0 || entry.Removed) + { + return false; + } + + var entryBase = PageBase(entry); + var pageCount = PageCount(entry); + var lo = Math.Max(begin & ~(WatchPageBytes - 1), entryBase); + var hi = Math.Min((begin + length + WatchPageBytes - 1) & ~(WatchPageBytes - 1), entryBase + (ulong)pageCount * WatchPageBytes); + if (hi <= lo) + { + return true; + } + + var first = (int)((lo - entryBase) / WatchPageBytes); + var last = (int)((hi - entryBase) / WatchPageBytes); + var bits = entry.DirtyPages ??= new ulong[(pageCount + 63) / 64]; + var fresh = 0; + for (var page = first; page < last; page++) + { + if ((bits[page / 64] & (1UL << (page % 64))) == 0) + { + fresh++; + } + } + + if (entry.DirtyPageCount + fresh > MaxDirtyPages) + { + Interlocked.Increment(ref _partialFallbacks); + entry.PartialHold = PartialHoldStagings; + return false; + } + + for (var page = first; page < last;) + { + if ((bits[page / 64] & (1UL << (page % 64))) != 0) + { + page++; + continue; + } + + var runStart = page; + while (page < last && (bits[page / 64] & (1UL << (page % 64))) == 0) + { + bits[page / 64] |= 1UL << (page % 64); + page++; + } + + entry.DirtyPageCount += page - runStart; + _backend.RemoveWatch(entryBase + (ulong)runStart * WatchPageBytes, (ulong)(page - runStart) * WatchPageBytes); + } + + return true; + } + + // Caller holds the lock. Rewatches the dirty pages, clears them, and lists their bytes for the copy. + private RefillPlan BeginRefill(Entry entry) + { + var plan = new RefillPlan(); + var entryBase = PageBase(entry); + var pageCount = PageCount(entry); + var bits = entry.DirtyPages!; + for (var page = 0; page < pageCount;) + { + if ((bits[page / 64] & (1UL << (page % 64))) == 0) + { + page++; + continue; + } + + var runStart = page; + while (page < pageCount && (bits[page / 64] & (1UL << (page % 64))) != 0) + { + page++; + } + + // Watched before the copy reads the bytes, like a whole staging: a store that lands from + // here on marks the page again. + var runAddress = entryBase + (ulong)runStart * WatchPageBytes; + var runBytes = (ulong)(page - runStart) * WatchPageBytes; + _backend.AddWatch(runAddress, runBytes); + var from = Math.Max(runAddress, entry.Address) - entry.Address; + var to = Math.Min(runAddress + runBytes, entry.Address + entry.Size) - entry.Address; + plan.DirtyRuns.Add((from, to - from)); + plan.DirtyBytes += to - from; + } + + Array.Clear(bits); + entry.DirtyPageCount = 0; + return plan; + } + + /// + /// Copies what left to copy, from the worker, after the entry was acquired and + /// before its buffer is recorded: guest bytes for the pages stored into. Returns false when there is no partial plan, which means the + /// whole entry is to be read from guest memory. + /// + internal unsafe bool RefillDirty(Entry entry, GuestReader read) + { + var plan = entry.Plan; + if (plan is null) + { + return false; + } + + entry.Plan = null; + var size = checked((int)entry.Size); + var staging = new Span((void*)entry.StagingBuffer!.Contents, size); + var pristine = entry.PristineBuffer is { } p && entry.Pristine.IsValid + ? new Span((void*)p.Contents, size) + : default; + foreach (var (offset, length) in plan.DirtyRuns) + { + var piece = staging.Slice((int)offset, (int)length); + read(entry.Address + offset, piece); + if (!pristine.IsEmpty) + { + piece.CopyTo(pristine.Slice((int)offset, (int)length)); + } + } + + Interlocked.Increment(ref _partialRefills); + Interlocked.Add(ref _partialDirtyBytes, (long)plan.DirtyBytes); + Interlocked.Add(ref _partialSavedBytes, (long)(entry.Size - plan.DirtyBytes)); + return true; + } + + /// Reads guest memory at an absolute address, as staging does. + public delegate void GuestReader(ulong address, Span destination); + // Caller holds the lock. The entries whose bytes overlap [address, address + size) among those on the pages. private List Collect(ulong first, ulong lastExclusive, ulong address, ulong size) { @@ -729,6 +966,7 @@ private void DropWatch(Entry entry) { if (entry.WatchEnd == 0) { + ClearDirtyPages(entry); return; } @@ -736,7 +974,44 @@ private void DropWatch(Entry entry) var length = entry.WatchEnd - begin; entry.WatchBegin = 0; entry.WatchEnd = 0; - _backend.RemoveWatch(begin, length); + if (entry.DirtyPageCount == 0) + { + _backend.RemoveWatch(begin, length); + return; + } + + // Pages that stores marked lost their watch already: only the others still hold one. + var entryBase = PageBase(entry); + var pageCount = PageCount(entry); + var bits = entry.DirtyPages!; + for (var page = 0; page < pageCount;) + { + if ((bits[page / 64] & (1UL << (page % 64))) != 0) + { + page++; + continue; + } + + var runStart = page; + while (page < pageCount && (bits[page / 64] & (1UL << (page % 64))) == 0) + { + page++; + } + + _backend.RemoveWatch(entryBase + (ulong)runStart * WatchPageBytes, (ulong)(page - runStart) * WatchPageBytes); + } + + ClearDirtyPages(entry); + } + + private static void ClearDirtyPages(Entry entry) + { + if (entry.DirtyPages is { } bits) + { + Array.Clear(bits); + } + + entry.DirtyPageCount = 0; } private void RetireBuffers(Entry entry, ulong retireAfter) diff --git a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalStagedStorage.cs b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalStagedStorage.cs index ea6f75fd9..6d1616c9c 100644 --- a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalStagedStorage.cs +++ b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalStagedStorage.cs @@ -279,6 +279,7 @@ public void EndSubmission() if (copy.Cached && copy.Dirty) { copy.Dirty = false; + copy.NeedsWholeStage = true; Volatile.Write(ref copy.Stale, 1); } @@ -330,6 +331,9 @@ public void EndSubmission() var missed = EncodeWriteback(encoder, copy); if (missed != 0) { + // The stores to those bytes stay in the copy and guest memory never has them: the + // copy is not what guest memory holds on pages no CPU store touched. + copy.NeedsWholeStage = true; MetalV2Diagnostics.WarnOnce( $"staged-storage-unwritten:{copy.Size:X}", $"A staged storage buffer has bytes no window covers, so the shader's stores to them " + diff --git a/tests/SharpEmu.Gpu.Metal.Tests/MetalSingleCopyTests.cs b/tests/SharpEmu.Gpu.Metal.Tests/MetalSingleCopyTests.cs index 23c009c61..42980fd8b 100644 --- a/tests/SharpEmu.Gpu.Metal.Tests/MetalSingleCopyTests.cs +++ b/tests/SharpEmu.Gpu.Metal.Tests/MetalSingleCopyTests.cs @@ -335,6 +335,28 @@ public void AHostReadOfAReleasedChunkGetsTheExactBytesBack(GuestTileMode tileMod output.WriteLine(rig.SingleCopy.Describe()); } + // The retile staging buffer is kept for the next restore instead of being allocated and released each + // time, and a restore from a reused (already written) buffer still returns exactly the guest's bytes. + [Fact] + public void RestoresReuseTheRetileStagingBufferAndStillReturnTheExactBytes() + { + if (!RequireDevice()) return; + using var rig = new Rig(Device.Value); + var request = Request(rig.ImageAddress, GuestTileMode.Standard64KB); + var size = request.Description.Data.Size; + var bytes = Pattern(size, 90); + rig.Release(request, bytes, (int)(size / Chunk)); + Assert.True(rig.Raw(rig.ImageAddress, size).All(static value => value == 0)); + Assert.Equal(bytes, rig.Read(rig.ImageAddress, size)); + Assert.Equal(bytes, rig.Raw(rig.ImageAddress, size)); + + // The release verifies through a retile and the host read restores through another: one buffer. + var (hits, misses) = rig.SingleCopy.RetilePoolCounts; + output.WriteLine($"retile pool hits={hits} misses={misses}"); + Assert.Equal(1, misses); + Assert.Equal(1, hits); + } + // A fill that covers a whole chunk (a file read into it) takes it over without a // retile; one that stops short gets the rest of its chunk back from the texture. [Fact] diff --git a/tests/SharpEmu.Gpu.Metal.Tests/MetalStagedRangePartialTests.cs b/tests/SharpEmu.Gpu.Metal.Tests/MetalStagedRangePartialTests.cs new file mode 100644 index 000000000..1686cc3c0 --- /dev/null +++ b/tests/SharpEmu.Gpu.Metal.Tests/MetalStagedRangePartialTests.cs @@ -0,0 +1,548 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later +using System.Runtime.InteropServices; +using SharpEmu.Gpu.Metal.V2; +using Xunit; + +namespace SharpEmu.Gpu.Metal.Tests; + +/// +/// A guest store into a large staged entry marks its own pages, and the next bind copies those pages alone. +/// The tests keep a guest of real bytes next to the cache's buffers, so what they check is the staging itself: +/// after a bind it holds exactly what the guest holds, however few bytes the bind copied. +/// +public sealed unsafe class MetalStagedRangePartialTests : IDisposable +{ + private const ulong Mib = 1024 * 1024; + private const ulong Page = 4096; + private const ulong Range = 0x10_0000_0000; + private const ulong Size = 4 * Mib; + + // Real buffers, and a watch model that counts watchers per page the way the page guard does. + private sealed class Backend : IStagedRangeBackend, IDisposable + { + private readonly Dictionary _buffers = new(); + private nint _next = 0x1000; + public readonly Dictionary WatchesByPage = new(); + public int Released; + + public (nint Handle, nint Contents, ulong GpuAddress) CreateBuffer(ulong capacity) + { + var memory = (nint)NativeMemory.AlignedAlloc((nuint)capacity, 16384); + _next += 0x10; + _buffers[_next] = memory; + return (_next, memory, (ulong)_next); + } + + public void ReleaseBuffer(nint handle) => Released++; + + public void AddWatch(ulong address, ulong size) + { + for (var page = address & ~(Page - 1); page < address + size; page += Page) + { + WatchesByPage[page] = WatchesByPage.GetValueOrDefault(page) + 1; + } + } + + public void RemoveWatch(ulong address, ulong size) + { + for (var page = address & ~(Page - 1); page < address + size; page += Page) + { + var count = WatchesByPage.GetValueOrDefault(page); + Assert.True(count > 0, $"watch removed from an unwatched page 0x{page:X}"); + if (count == 1) + { + WatchesByPage.Remove(page); + } + else + { + WatchesByPage[page] = count - 1; + } + } + } + + public void Dispose() + { + foreach (var memory in _buffers.Values) + { + NativeMemory.AlignedFree((void*)memory); + } + } + } + + private readonly Backend _backend = new(); + private readonly byte[] _guest; + private long _guestBytesRead; + + public MetalStagedRangePartialTests() + { + _guest = new byte[Size + 2 * Page]; + new Random(7).NextBytes(_guest); + } + + public void Dispose() => _backend.Dispose(); + + private MetalStagedRangeCache Cache(bool partial = true) => new(_backend, 64 * Mib, partial); + + private void Read(ulong address, Span destination) + { + _guest.AsSpan((int)(address - Range), destination.Length).CopyTo(destination); + _guestBytesRead += destination.Length; + } + + // What MetalRenderHost.BindStagedRange does, minus the GPU: acquire, then copy what Acquire left to copy. + private bool Bind(MetalStagedRangeCache cache, ulong address, ulong size, bool written, ulong serial, + ulong completed, out MetalStagedRangeCache.Entry entry) + { + var hit = cache.Acquire(address, size, written, serial, completed, out var acquired); + entry = acquired!; + if (hit) + { + return true; + } + + if (!cache.RefillDirty(entry, Read)) + { + Read(address, Staging(entry, size)); + if (entry.Pristine.IsValid) + { + Staging(entry, size).CopyTo(new Span((void*)entry.PristineBuffer!.Contents, (int)size)); + } + } + + return false; + } + + private static Span Staging(MetalStagedRangeCache.Entry entry, ulong size) => + new((void*)entry.StagingBuffer!.Contents, (int)size); + + private ReadOnlySpan GuestBytes(ulong address, ulong size) => _guest.AsSpan((int)(address - Range), (int)size); + + private void Store(ulong address, byte value) => _guest[(int)(address - Range)] = value; + + private int WatchedPages(MetalStagedRangeCache.Entry entry) + { + var count = 0; + for (var page = entry.Address & ~(Page - 1); page < entry.Address + entry.Size; page += Page) + { + Assert.InRange(_backend.WatchesByPage.GetValueOrDefault(page), 0, 1); + count += _backend.WatchesByPage.ContainsKey(page) ? 1 : 0; + } + + return count; + } + + private static int PagesOf(ulong address, ulong size) => + (int)(((address + size + Page - 1) & ~(Page - 1)) - (address & ~(Page - 1))) / (int)Page; + + [Fact] + public void AStoreIntoOnePageCopiesThatPageAndNothingElse() + { + var cache = Cache(); + Bind(cache, Range, Size, written: false, 1, 0, out var entry); + Assert.Equal(PagesOf(Range, Size), WatchedPages(entry)); + + // The clean pages are poisoned in the staging: a copy of them would show. + var staging = Staging(entry, Size); + var cleanBefore = staging.ToArray(); + var storeAt = Range + 300 * Page + 40; + Store(storeAt, 0x5A); + Assert.True(cache.OnCpuWrite(storeAt, 4)); + + // The watch over the stored-into page is gone, so the store can land; the others stay. + Assert.Equal(PagesOf(Range, Size) - 1, WatchedPages(entry)); + Assert.False(_backend.WatchesByPage.ContainsKey(Range + 300 * Page)); + Assert.Equal(1, entry.Stale); + + _guestBytesRead = 0; + Assert.False(Bind(cache, Range, Size, written: false, 2, 1, out var again)); + Assert.Same(entry, again); + Assert.Equal((long)Page, _guestBytesRead); + Assert.True(staging.SequenceEqual(GuestBytes(Range, Size))); + Assert.NotEqual(cleanBefore[300 * (int)Page + 40], staging[300 * (int)Page + 40]); + Assert.Equal(PagesOf(Range, Size), WatchedPages(entry)); + Assert.Equal(0, entry.Stale); + Assert.True(Bind(cache, Range, Size, written: false, 3, 2, out _)); + + var counters = cache.PartialCounters; + Assert.Equal((1L, (long)Page, (long)(Size - Page), 0L), counters); + } + + [Fact] + public void AStoreThatLandsAfterTheBindIsNotLost() + { + var cache = Cache(); + Bind(cache, Range, Size, written: false, 1, 0, out var entry); + Store(Range + 5 * Page, 1); + cache.OnCpuWrite(Range + 5 * Page, 8); + Bind(cache, Range, Size, written: false, 2, 1, out _); + + // The page was watched again before it was copied: a second store marks it again. + Store(Range + 5 * Page + 8, 2); + Assert.True(cache.OnCpuWrite(Range + 5 * Page + 8, 8)); + Assert.Equal(1, entry.Stale); + Assert.False(Bind(cache, Range, Size, written: false, 3, 2, out _)); + Assert.True(Staging(entry, Size).SequenceEqual(GuestBytes(Range, Size))); + } + + [Fact] + public void AnUnalignedEntryRefillsTheClippedFirstAndLastPages() + { + var cache = Cache(); + var address = Range + 0x100; + var size = Size - 0x300; + Bind(cache, address, size, written: false, 1, 0, out var entry); + Assert.Equal(PagesOf(address, size), WatchedPages(entry)); + + // A store on the first page and one on the page the entry ends in, past its last byte. + Store(address + 8, 3); + Store(address + size - 8, 4); + cache.OnCpuWrite(address + 8, 4); + cache.OnCpuWrite(address + size - 8, 4); + Assert.Equal(PagesOf(address, size) - 2, WatchedPages(entry)); + + _guestBytesRead = 0; + Bind(cache, address, size, written: false, 2, 1, out _); + // The first page holds Page - 0x100 of the entry; the last holds the rest up to its end. + var lastPageBytes = (address + size) & (Page - 1); + Assert.Equal((long)(Page - 0x100 + lastPageBytes), _guestBytesRead); + Assert.True(Staging(entry, size).SequenceEqual(GuestBytes(address, size))); + Assert.Equal(PagesOf(address, size), WatchedPages(entry)); + } + + [Fact] + public void APreciseWriteMarksOnlyThePagesItsBytesLieOn() + { + var cache = Cache(); + Bind(cache, Range, Size, written: false, 1, 0, out var entry); + for (var offset = 0UL; offset < 8; offset++) + { + Store(Range + 10 * Page - 4 + offset, (byte)(0x40 + offset)); + } + + Assert.True(cache.OnPreciseCpuWrite(Range + 10 * Page - 4, 8, backed: true)); + Assert.Equal(PagesOf(Range, Size) - 2, WatchedPages(entry)); + _guestBytesRead = 0; + Bind(cache, Range, Size, written: false, 2, 1, out _); + Assert.Equal((long)(2 * Page), _guestBytesRead); + Assert.True(Staging(entry, Size).SequenceEqual(GuestBytes(Range, Size))); + } + + [Fact] + public void AnEntryAnInFlightSubmissionReadsKeepsItsBytesAndIsStagedWholeIntoAnotherBuffer() + { + var cache = Cache(); + Bind(cache, Range, Size, written: false, 5, 0, out var entry); + var oldBuffer = entry.StagingBuffer!; + var oldBytes = Staging(entry, Size).ToArray(); + + Store(Range + 700 * Page + 9, 0xEE); + cache.OnCpuWrite(Range + 700 * Page + 9, 4); + + // Submission 5 has not completed: the entry gets another buffer, and the old one is left as it was. + _guestBytesRead = 0; + Bind(cache, Range, Size, written: false, 6, 4, out _); + Assert.NotSame(oldBuffer, entry.StagingBuffer); + Assert.True(new ReadOnlySpan((void*)oldBuffer.Contents, (int)Size).SequenceEqual(oldBytes)); + Assert.Equal(1, cache.RetiredCount); + Assert.Equal((long)Size, _guestBytesRead); + Assert.True(Staging(entry, Size).SequenceEqual(GuestBytes(Range, Size))); + Assert.Equal(0, cache.PartialCounters.Refills); + Assert.Equal(PagesOf(Range, Size), WatchedPages(entry)); + } + + [Fact] + public void AWrittenEntryInFlightIsStagedWhole() + { + var cache = Cache(); + Bind(cache, Range, Size, written: true, 5, 0, out var entry); + + Store(Range + 3 * Page, 1); + cache.OnCpuWrite(Range + 3 * Page, 4); + _guestBytesRead = 0; + Bind(cache, Range, Size, written: true, 6, 4, out _); + Assert.Equal((long)Size, _guestBytesRead); + Assert.True(Staging(entry, Size).SequenceEqual(GuestBytes(Range, Size))); + Assert.Equal(0, cache.PartialCounters.Refills); + } + + [Fact] + public void AWrittenEntryRefillsBothItsStagingAndItsPristineTwin() + { + var cache = Cache(); + Bind(cache, Range, Size, written: true, 1, 0, out var entry); + Store(Range + 40 * Page + 1, 0x77); + Store(Range + 41 * Page + 1, 0x78); + cache.OnCpuWrite(Range + 40 * Page + 1, 1); + cache.OnCpuWrite(Range + 41 * Page + 1, 1); + + _guestBytesRead = 0; + Assert.False(Bind(cache, Range, Size, written: true, 2, 1, out _)); + Assert.Equal((long)(2 * Page), _guestBytesRead); + Assert.True(Staging(entry, Size).SequenceEqual(GuestBytes(Range, Size))); + Assert.True(new ReadOnlySpan((void*)entry.PristineBuffer!.Contents, (int)Size) + .SequenceEqual(GuestBytes(Range, Size))); + } + + [Fact] + public void StoresToManyPagesStaleTheEntryWholeAndASmallerWriterIsTrackedAgainLater() + { + var cache = Cache(); + Bind(cache, Range, Size, written: false, 1, 0, out var entry); + + // A wholesale rewrite: past the limit the entry is staled whole and every watch is dropped. + for (var page = 0; page < MetalStagedRangeCache.MaxDirtyPages + 1; page++) + { + Store(Range + (ulong)page * Page, 9); + cache.OnCpuWrite(Range + (ulong)page * Page, 4); + } + + Assert.Empty(_backend.WatchesByPage); + Assert.Equal(1, cache.PartialCounters.Fallbacks); + _guestBytesRead = 0; + Bind(cache, Range, Size, written: false, 2, 1, out _); + Assert.Equal((long)Size, _guestBytesRead); + Assert.True(Staging(entry, Size).SequenceEqual(GuestBytes(Range, Size))); + + // For the next few stagings (the whole one above was the first) the first store drops the whole + // watch, as it always did. + for (var staging = 0; staging < MetalStagedRangeCache.PartialHoldStagings - 1; staging++) + { + Store(Range + Page, (byte)staging); + cache.OnCpuWrite(Range + Page, 4); + Assert.Empty(_backend.WatchesByPage); + Bind(cache, Range, Size, written: false, (ulong)(3 + staging), (ulong)(2 + staging), out _); + } + + // Then one store marks one page once more. + Store(Range + Page, 0xAB); + cache.OnCpuWrite(Range + Page, 4); + Assert.Equal(PagesOf(Range, Size) - 1, WatchedPages(entry)); + _guestBytesRead = 0; + Bind(cache, Range, Size, written: false, 20, 19, out _); + Assert.Equal((long)Page, _guestBytesRead); + Assert.True(Staging(entry, Size).SequenceEqual(GuestBytes(Range, Size))); + } + + [Theory] + [InlineData(0)] + [InlineData(1)] + [InlineData(2)] + public void AnyOtherWriteAfterAPartialMarkStalesTheEntryWholeAndReleasesExactlyTheWatchesItHolds(int kind) + { + var cache = Cache(); + Bind(cache, Range, Size, written: false, 1, 0, out var entry); + Store(Range + 9 * Page, 1); + cache.OnCpuWrite(Range + 9 * Page, 4); + Assert.Equal(PagesOf(Range, Size) - 1, WatchedPages(entry)); + + // The host or the GPU wrote some other bytes of the entry: nothing says which pages the copy still has right. + var other = Range + 500 * Page; + for (var offset = 0UL; offset < 64; offset++) + { + Store(other + offset, 0x33); + } + + switch (kind) + { + case 0: + cache.OnHostWrite(other, 64); + break; + case 1: + cache.OnGpuWrite(other, 64, MetalGpuWriteOrigin.Fill); + break; + default: + cache.OnUnmap(Range, Size); + Assert.Empty(_backend.WatchesByPage); + return; + } + + Assert.Empty(_backend.WatchesByPage); + _guestBytesRead = 0; + Bind(cache, Range, Size, written: false, 2, 1, out _); + Assert.Equal((long)Size, _guestBytesRead); + Assert.True(Staging(entry, Size).SequenceEqual(GuestBytes(Range, Size))); + Assert.Equal(PagesOf(Range, Size), WatchedPages(entry)); + } + + [Fact] + public void StoresOfAnEntryThatNeverReachedGuestMemoryAreNotKeptByAPartialRefill() + { + // A copy whose shader stores were lost (no writeback) differs from guest memory on pages no CPU store touched. + var cache = Cache(); + Bind(cache, Range, Size, written: true, 1, 0, out var entry); + entry.Dirty = true; + var staged = new MetalStagedStorage(null!, cache); + staged.Bind(entry, written: true); + staged.Arm(); + staged.EndSubmission(); + Assert.False(entry.Dirty); + Assert.Equal(1, entry.Stale); + Assert.True(entry.NeedsWholeStage); + + // A CPU store now must not turn that into a one-page refill. + Store(Range + 2 * Page, 5); + cache.OnCpuWrite(Range + 2 * Page, 4); + _guestBytesRead = 0; + Bind(cache, Range, Size, written: true, 2, 1, out _); + Assert.Equal((long)Size, _guestBytesRead); + } + + [Fact] + public void AStoreWhileTheEntryHasStoresOfItsOwnStalesItWholeOnceTheyAreWrittenBack() + { + var cache = Cache(); + Bind(cache, Range, Size, written: true, 1, 0, out var entry); + entry.Dirty = true; + Store(Range + 2 * Page, 5); + cache.OnCpuWrite(Range + 2 * Page, 4); + cache.OnWrittenBack(entry); + Assert.Equal(1, entry.Stale); + Assert.True(entry.NeedsWholeStage); + _guestBytesRead = 0; + Bind(cache, Range, Size, written: true, 2, 1, out _); + Assert.Equal((long)Size, _guestBytesRead); + Assert.True(Staging(entry, Size).SequenceEqual(GuestBytes(Range, Size))); + } + + [Fact] + public void AnEntryBelowTheThresholdIsStaledWholeAsBefore() + { + var cache = Cache(); + var size = MetalStagedRangeCache.PartialMinBytes - Page; + Bind(cache, Range, size, written: false, 1, 0, out var entry); + cache.OnCpuWrite(Range + 5 * Page, 4); + Assert.Empty(_backend.WatchesByPage); + _guestBytesRead = 0; + Bind(cache, Range, size, written: false, 2, 1, out _); + Assert.Equal((long)size, _guestBytesRead); + Assert.Equal(0, cache.PartialCounters.Refills); + } + + [Fact] + public void WithPartialOffEveryStoreStalesTheWholeEntry() + { + var cache = Cache(partial: false); + Bind(cache, Range, Size, written: false, 1, 0, out _); + cache.OnCpuWrite(Range + 5 * Page, 4); + Assert.Empty(_backend.WatchesByPage); + _guestBytesRead = 0; + Bind(cache, Range, Size, written: false, 2, 1, out _); + Assert.Equal((long)Size, _guestBytesRead); + } + + [Fact] + public void ADisposedOrEvictedEntryReleasesTheWatchesOfItsCleanPagesOnly() + { + var cache = Cache(); + Bind(cache, Range, Size, written: false, 1, 0, out _); + Store(Range + 9 * Page, 1); + cache.OnCpuWrite(Range + 9 * Page, 4); + cache.StopWatching(); + Assert.Empty(_backend.WatchesByPage); + cache.Dispose(); + } + + // Random stores, host and GPU writes and binds, in flight or not, read-only or written, over entries + // with unaligned edges. Whatever the sequence, a bind leaves the staging equal to the guest, and every + // live entry holds exactly one watch on each page that no store has marked. + [Theory] + [InlineData(11, true)] + [InlineData(23, true)] + [InlineData(37, false)] + public void RandomWritesAndBindsKeepTheStagingEqualToGuestMemory(int seed, bool partial) + { + var random = new Random(seed); + var cache = Cache(partial); + var address = Range + 0x240; + var size = Size - 0x980; + ulong serial = 3; + var written = false; + for (var step = 0; step < 600; step++) + { + switch (random.Next(10)) + { + case 0: + case 1: + case 2: + case 3: + { + var at = address + (ulong)random.NextInt64((long)size - 16); + var length = (ulong)random.Next(1, 16); + for (var i = 0UL; i < length; i++) + { + Store(at + i, (byte)random.Next(256)); + } + + if (random.Next(3) == 0) + { + cache.OnPreciseCpuWrite(at, length, backed: random.Next(2) == 0); + } + else + { + cache.OnCpuWrite(at, length); + } + + break; + } + case 4: + { + // A burst: enough pages to pass the limit now and then. + var pages = random.Next(1, 48); + var first = (ulong)random.Next(0, 800); + for (var page = 0; page < pages; page++) + { + var at = Range + (first + (ulong)page) * Page + 17; + if (at - address < size) + { + Store(at, (byte)random.Next(256)); + cache.OnCpuWrite(at, 4); + } + } + + break; + } + case 5: + { + var at = address + (ulong)random.NextInt64((long)size - 64); + Store(at, (byte)random.Next(256)); + if (random.Next(2) == 0) + { + cache.OnHostWrite(at, 64); + } + else + { + cache.OnGpuWrite(at, 64, MetalGpuWriteOrigin.Fill); + } + + break; + } + default: + { + written = random.Next(4) == 0; + var completed = random.Next(3) == 0 ? serial - 2 : serial - 1; + Bind(cache, address, size, written, serial, completed, out var entry); + Assert.True(Staging(entry, size).SequenceEqual(GuestBytes(address, size)), $"step {step}: staging differs"); + if (entry.Pristine.IsValid) + { + Assert.True(new ReadOnlySpan((void*)entry.PristineBuffer!.Contents, (int)size) + .SequenceEqual(GuestBytes(address, size)), $"step {step}: pristine differs"); + } + + serial++; + break; + } + } + + foreach (var count in _backend.WatchesByPage.Values) + { + Assert.Equal(1, count); + } + } + + if (partial) + { + Assert.True(cache.PartialCounters.Refills > 0); + } + } +} From 6564ccd030714c292b9a9477d2d26a032d9ed003 Mon Sep 17 00:00:00 2001 From: kuba Date: Mon, 26 Oct 2026 05:39:34 +0100 Subject: [PATCH 11/82] [GPU] Translated shaders compile in relaxed math, test depth early when the guest does, and resolve global records once GPU time of translated shaders was ~200 ms/frame in Demon's Souls (audit finding A2). Three changes, each A/B-timed on the device with MTLCommandBuffer GPUStartTime/GPUEndTime (MslGpuShaderTimeBenchTests, opt-in SHARPEMU_GPU_BENCH=1, same process, best of runs): * Math mode: libraries compile with MTLMathModeRelaxed + precise library functions instead of safe. ALU loop (fmac, mul+add, sqrt, rcp, sin): 9.3 -> 8.3 ms (-10%, quiet run); 10.1-11.2 -> 8.9-9.2 ms under load. Relaxed treats signed zeros as equal, which broke the omod "flush zero to +0" test (x == 0.0f); it now tests the bits. The RDNA2 ALU conformance suite and the numeric tests pass. SHARPEMU_MSL_MATHMODE=safe goes back. The mode is part of the binary-archive library key and the prewarm stamp, so safe-mode binaries are never reused. * Early Z: a pixel function gets [[early_fragment_tests]] when DB_SHADER_CONTROL.Z_ORDER is not LATE_Z and the shader has no kill, depth, stencil or coverage export. The existing PixelInputInfo.EarlyDepth (already in the stage key) now also covers RE_Z and EARLY_Z_THEN_RE_Z and reaches the translator as ShaderCompileRequest.PixelEarlyDepth. 8-layer overdraw of a buffer-storing fragment function: ~34 ms -> ~4.0 ms (synthetic upper bound). SHARPEMU_MSL_EARLYZ=0 leaves it to Metal. * Device spans: GLOBAL_LOAD/STORE_DWORDX2-X4 resolve the record once (page-crossing or unmapped records fall back per dword). Compute loop of global_load/store_dwordx4: 6.8 -> 5.2 ms (-24%). SHARPEMU_MSL_DEVICESPAN=0 resolves per dword. The s_load span and the one-entry range cache were already on main; descriptor-addressed buffer accesses place each dword through the runtime swizzle, so they stay per dword. Edits outside the owned files: ShaderCompileRequest.PixelEarlyDepth, ShaderProgramCache (one line), PixelStageInputResolver (Z_ORDER != 0), MetalBinaryArchive.LibraryKey, MetalShaderPrewarm stamp, test harness MetalNative compile options. --- .../Gpu/Pipelines/PixelStageInputResolver.cs | 5 +- .../Gpu/Pipelines/ShaderProgramCache.cs | 1 + .../Gen5MslCodegenOptions.cs | 18 +- .../Gen5MslTranslator.Alu.cs | 2 +- .../Gen5MslTranslator.Resources.cs | 63 ++++++ .../Gen5MslTranslator.cs | 9 +- .../MslMathMode.cs | 40 ++++ .../ShaderCompileRequest.cs | 7 + .../SharpEmu.Gpu.Metal/MetalNative.cs | 29 ++- .../V2/MetalBinaryArchive.cs | 13 +- .../V2/MetalShaderPrewarm.cs | 1 + .../MetalBinaryArchiveTests.cs | 11 + .../Gen5ComputeFixtures.cs | 13 +- .../GlobalMemoryRegisterTests.cs | 9 +- .../MetalNative.Benchmark.cs | 39 +++- .../MetalNative.EarlyZBench.cs | 134 +++++++++++ .../MetalNative.Render.cs | 1 + .../MetalNative.cs | 27 ++- .../MslGpuShaderTimeBenchTests.cs | 194 ++++++++++++++++ .../MslShaderTimeChangesTests.cs | 214 ++++++++++++++++++ 20 files changed, 806 insertions(+), 24 deletions(-) create mode 100644 src/SharpEmu.ShaderCompiler.Metal/MslMathMode.cs create mode 100644 tests/SharpEmu.ShaderCompiler.Metal.Tests/MetalNative.EarlyZBench.cs create mode 100644 tests/SharpEmu.ShaderCompiler.Metal.Tests/MslGpuShaderTimeBenchTests.cs create mode 100644 tests/SharpEmu.ShaderCompiler.Metal.Tests/MslShaderTimeChangesTests.cs diff --git a/src/SharpEmu.Libs/Gpu/Pipelines/PixelStageInputResolver.cs b/src/SharpEmu.Libs/Gpu/Pipelines/PixelStageInputResolver.cs index 3166491f9..0c74269a7 100644 --- a/src/SharpEmu.Libs/Gpu/Pipelines/PixelStageInputResolver.cs +++ b/src/SharpEmu.Libs/Gpu/Pipelines/PixelStageInputResolver.cs @@ -136,7 +136,10 @@ public static PixelInputInfo Resolve( DepthExportFormat = shaderInterface.DepthExportFormat is var zFormat and > 0 and <= 9 ? (SharpEmu.ShaderCompiler.Gen5ColorExportFormat)zFormat : null, - EarlyDepth = control.DepthExportOrder == 1 && !control.KillEnable && !control.DepthExportEnable && + // Z_ORDER 0 is LATE_Z; EARLY_Z_THEN_LATE_Z, RE_Z and EARLY_Z_THEN_RE_Z (1..3) all test + // before the shader. A shader that kills or exports depth, stencil or coverage is + // decided after it ran, whatever the order says. + EarlyDepth = control.DepthExportOrder != 0 && !control.KillEnable && !control.DepthExportEnable && !control.StencilExportEnable && !control.MaskExportEnable, ExecuteOnNoop = control.ExecuteOnNoop, }; diff --git a/src/SharpEmu.Libs/Gpu/Pipelines/ShaderProgramCache.cs b/src/SharpEmu.Libs/Gpu/Pipelines/ShaderProgramCache.cs index b50b182b5..f89a5f351 100644 --- a/src/SharpEmu.Libs/Gpu/Pipelines/ShaderProgramCache.cs +++ b/src/SharpEmu.Libs/Gpu/Pipelines/ShaderProgramCache.cs @@ -1683,6 +1683,7 @@ private ShaderCompileRequest BuildRequest( PixelDepthExportFormat = info.DepthExportFormat, PixelStencilExportEnable = info.StencilExportEnable, PixelSampleMaskExportEnable = info.SampleMaskExportEnable, + PixelEarlyDepth = info.EarlyDepth, PixelInputEnable = options.PixelInputEnable, PixelFrontFaceAllBits = info.FrontFaceAllBits, PixelCustomInterpolationMask = info.CustomInterpolationMask, diff --git a/src/SharpEmu.ShaderCompiler.Metal/Gen5MslCodegenOptions.cs b/src/SharpEmu.ShaderCompiler.Metal/Gen5MslCodegenOptions.cs index 2d8d91fa3..29b0e15f1 100644 --- a/src/SharpEmu.ShaderCompiler.Metal/Gen5MslCodegenOptions.cs +++ b/src/SharpEmu.ShaderCompiler.Metal/Gen5MslCodegenOptions.cs @@ -77,6 +77,16 @@ namespace SharpEmu.ShaderCompiler.Metal; /// guest writes and reads such a record with one instruction, which other CUs see whole; here /// each dword is its own access, and UE5's Nanite culling read queue records half written. /// SHARPEMU_MSL_PERSISTENT_RECORDS=0 accesses them dword by dword. +/// Resolve a multi-dword global or device-buffer access (GLOBAL_LOAD/STORE +/// DWORDX2-X4, BUFFER_LOAD/STORE through a device descriptor) once for the whole span instead of +/// walking the page table per dword: a span inside one 16 KB page indexes off the one pointer and +/// one that crosses a page falls back to the per-dword path. SHARPEMU_MSL_DEVICESPAN=0 +/// resolves every dword alone. +/// Declare [[early_fragment_tests]] on a pixel function whose +/// guest DB_SHADER_CONTROL asks for early Z (): Metal +/// only tests depth before the shader on its own for a function that writes no memory, so a shader +/// with buffer or image stores ran for fragments the guest hardware would have rejected first. +/// SHARPEMU_MSL_EARLYZ=0 leaves the choice to Metal. public sealed record Gen5MslCodegenOptions( bool StructuredControlFlow = true, bool RegisterAllocation = true, @@ -92,7 +102,9 @@ public sealed record Gen5MslCodegenOptions( bool GraphicsLoopStepCap = false, bool LaneSpillCopies = true, bool PersistentWaitFences = true, - bool PersistentRecordLocks = true) + bool PersistentRecordLocks = true, + bool DeviceAddressSpans = true, + bool EarlyFragmentTests = true) { public static Gen5MslCodegenOptions FromEnvironment { get; } = new( StructuredControlFlow: Environment.GetEnvironmentVariable("SHARPEMU_MSL_STRUCTURED") != "0", @@ -111,5 +123,7 @@ public sealed record Gen5MslCodegenOptions( GraphicsLoopStepCap: Environment.GetEnvironmentVariable("SHARPEMU_SHADER_MAX_STEPS_GRAPHICS") == "1", LaneSpillCopies: Environment.GetEnvironmentVariable("SHARPEMU_MSL_SPILLCOPY") != "0", PersistentWaitFences: Environment.GetEnvironmentVariable("SHARPEMU_MSL_PERSISTENT_FENCES") != "0", - PersistentRecordLocks: Environment.GetEnvironmentVariable("SHARPEMU_MSL_PERSISTENT_RECORDS") != "0"); + PersistentRecordLocks: Environment.GetEnvironmentVariable("SHARPEMU_MSL_PERSISTENT_RECORDS") != "0", + DeviceAddressSpans: Environment.GetEnvironmentVariable("SHARPEMU_MSL_DEVICESPAN") != "0", + EarlyFragmentTests: Environment.GetEnvironmentVariable("SHARPEMU_MSL_EARLYZ") != "0"); } diff --git a/src/SharpEmu.ShaderCompiler.Metal/Gen5MslTranslator.Alu.cs b/src/SharpEmu.ShaderCompiler.Metal/Gen5MslTranslator.Alu.cs index 0293e9399..e2aa86f3a 100644 --- a/src/SharpEmu.ShaderCompiler.Metal/Gen5MslTranslator.Alu.cs +++ b/src/SharpEmu.ShaderCompiler.Metal/Gen5MslTranslator.Alu.cs @@ -2965,7 +2965,7 @@ private string ApplyFloatOutputModifiers(Gen5ShaderInstruction instruction, stri // to an f16 denormal is flushed as well. var zero = half ? $"(as_type(half({scaled})) & 0x7C00u) == 0u" - : $"{scaled} == 0.0f"; + : $"(as_type({scaled}) & 0x7FFFFFFFu) == 0u"; // a bit test: the relaxed math mode ignores the sign of a zero expression = Temp("float", $"({zero}) ? 0.0f : {scaled}"); } diff --git a/src/SharpEmu.ShaderCompiler.Metal/Gen5MslTranslator.Resources.cs b/src/SharpEmu.ShaderCompiler.Metal/Gen5MslTranslator.Resources.cs index b5746f053..9c05bc5b2 100644 --- a/src/SharpEmu.ShaderCompiler.Metal/Gen5MslTranslator.Resources.cs +++ b/src/SharpEmu.ShaderCompiler.Metal/Gen5MslTranslator.Resources.cs @@ -1524,6 +1524,34 @@ private bool TryEmitLayoutGlobalMemory(Gen5ShaderInstruction instruction, Gen5Gl : $"(v[{control.SourceVectorRegister}] >> {sourceShift})"; Line($"sharpemu_store_device_bytes({DeviceArguments}, {address}, {source}, {byteCount}u);"); } + else if (_options.DeviceAddressSpans && control.DwordCount > 1) + { + // One page-table lookup for the whole record. A record inside one page is + // contiguous host memory, so the dwords index off the one pointer; one that + // crosses a page or is unmapped takes the per-dword stores. + var spanValid = Temp("bool", "false"); + var span = Temp("device uint*", $"sharpemu_resolve_device_span({DeviceArguments}, {address}, {control.DwordCount * sizeof(uint)}u, {spanValid})"); + Line($"if ({spanValid})"); + Line("{"); + _indent++; + for (uint index = 0; index < control.DwordCount; index++) + { + Line($"{span}[{index}] = v[{control.SourceVectorRegister + index}];"); + } + + _indent--; + Line("}"); + Line("else"); + Line("{"); + _indent++; + for (uint index = 0; index < control.DwordCount; index++) + { + Line($"sharpemu_store_device_dword({DeviceArguments}, {address} + {index * sizeof(uint)}ul, v[{control.SourceVectorRegister + index}]);"); + } + + _indent--; + Line("}"); + } else { for (uint index = 0; index < control.DwordCount; index++) @@ -1568,6 +1596,41 @@ private bool TryEmitLayoutGlobalMemory(Gen5ShaderInstruction instruction, Gen5Gl return true; } + if (_options.DeviceAddressSpans && control.DwordCount > 1 && control.DestinationVectorRegister + control.DwordCount <= VectorRegisterFileCount) + { + // One page-table lookup for the whole record (GLOBAL_LOAD_DWORDX2-X4); see the + // store above. Every destination is written after the lookup, under EXEC. + var guarded = !_execKnownFull; + if (guarded) + { + Line("if (exec)"); + Line("{"); + _indent++; + } + + var spanValid = Temp("bool", "false"); + var span = Temp("device uint*", $"sharpemu_resolve_device_span({DeviceArguments}, {aligned}, {control.DwordCount * sizeof(uint)}u, {spanValid})"); + for (uint index = 0; index < control.DwordCount; index++) + { + var componentAddress = index == 0 ? aligned : $"{aligned} + {index * sizeof(uint)}ul"; + var spanWord = control.Glc + ? $"atomic_load_explicit((device atomic_uint*)({span} + {index}), memory_order_relaxed)" + : $"{span}[{index}]"; + StoreVector( + control.DestinationVectorRegister + index, + $"{spanValid} ? {spanWord} : sharpemu_load_device_dword{coherent}({DeviceArguments}, {componentAddress})", + guardWithExec: false); + } + + if (guarded) + { + _indent--; + Line("}"); + } + + return true; + } + for (uint index = 0; index < control.DwordCount; index++) { var componentAddress = index == 0 ? aligned : $"{aligned} + {index * sizeof(uint)}ul"; diff --git a/src/SharpEmu.ShaderCompiler.Metal/Gen5MslTranslator.cs b/src/SharpEmu.ShaderCompiler.Metal/Gen5MslTranslator.cs index f69160c79..9848cf80f 100644 --- a/src/SharpEmu.ShaderCompiler.Metal/Gen5MslTranslator.cs +++ b/src/SharpEmu.ShaderCompiler.Metal/Gen5MslTranslator.cs @@ -883,7 +883,14 @@ private void EmitModule(StringBuilder source, IReadOnlyList blocks) source.AppendLine("};"); source.AppendLine(); - source.AppendLine($"fragment Gen5PsOut {EntryPointName}("); + // The guest asked for the depth test before the shader and nothing the shader does + // can change its outcome (the request clears kill and every export), so the test + // runs first here too: Metal does that on its own only for a function that + // writes no memory, and a shader with buffer or image stores would run, and + // store, for fragments the guest hardware rejects before it starts. + var earlyFragmentTests = _options.EarlyFragmentTests && _request.PixelEarlyDepth && + !_pixelDepthExport && !_pixelStencilExport && !_pixelSampleMaskExport; + source.AppendLine($"{(earlyFragmentTests ? "[[early_fragment_tests]] " : string.Empty)}fragment Gen5PsOut {EntryPointName}("); source.AppendLine(" Gen5PsIn sharpemu_in [[stage_in]],"); } else if (_stage != Gen5MslStage.Mesh) diff --git a/src/SharpEmu.ShaderCompiler.Metal/MslMathMode.cs b/src/SharpEmu.ShaderCompiler.Metal/MslMathMode.cs new file mode 100644 index 000000000..d5418bd2b --- /dev/null +++ b/src/SharpEmu.ShaderCompiler.Metal/MslMathMode.cs @@ -0,0 +1,40 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +namespace SharpEmu.ShaderCompiler.Metal; + +/// +/// The floating-point mode every translated-guest MSL library is compiled with, shared by +/// the emulator's compile sites and the codegen test harness so both build the same code. +/// +/// +/// is what the backend shipped with: no reassociation, no contraction. +/// (MTLMathModeRelaxed, MSL 3.2) lets the compiler contract a multiply +/// and an add into one fused operation and reassociate, but keeps NaN and infinity +/// semantics, which the translated GCN float code relies on. The transcendental library +/// functions stay precise (): the fast variants return wrong +/// results for infinite and huge inputs. SHARPEMU_MSL_MATHMODE=safe goes back to +/// . +/// +public static class MslMathMode +{ + /// MTLMathModeSafe. + public const nuint Safe = 0; + + /// MTLMathModeRelaxed. + public const nuint Relaxed = 1; + + /// MTLMathFloatingPointFunctionsPrecise. + public const nuint FunctionsPrecise = 1; + + /// The mode the emulator compiles with: unless + /// SHARPEMU_MSL_MATHMODE=safe. + public static nuint Default { get; } = + string.Equals(Environment.GetEnvironmentVariable("SHARPEMU_MSL_MATHMODE"), "safe", StringComparison.OrdinalIgnoreCase) + ? Safe + : Relaxed; + + /// The mode as a short token for cache keys and stamps: a binary built in one + /// mode is not reusable in another. + public static string Identity(nuint mode) => mode == Safe ? "math-safe" : "math-relaxed"; +} diff --git a/src/SharpEmu.ShaderCompiler/ShaderCompileRequest.cs b/src/SharpEmu.ShaderCompiler/ShaderCompileRequest.cs index f223dd434..854780265 100644 --- a/src/SharpEmu.ShaderCompiler/ShaderCompileRequest.cs +++ b/src/SharpEmu.ShaderCompiler/ShaderCompileRequest.cs @@ -381,6 +381,13 @@ [.. uses.Where(use => use.Callee == callee).Select(use => use.Field).Distinct() // DB_SHADER_CONTROL.MASK_EXPORT_ENABLE: the MRTZ export's sample mask is ANDed with the // fragment's coverage. public bool PixelSampleMaskExportEnable { get; init; } + + // DB_SHADER_CONTROL.Z_ORDER asks for the depth test before the shader (EARLY_Z_THEN_LATE_Z, + // RE_Z, EARLY_Z_THEN_RE_Z) and the program neither kills pixels (KILL_ENABLE) nor exports + // depth, stencil or a sample mask, so the early test cannot disagree with the late one. The + // Metal backend declares [[early_fragment_tests]] for it: a fragment function that writes + // memory is otherwise tested after the shader. + public bool PixelEarlyDepth { get; init; } public uint PixelInputEnable { get; init; } // SPI_BARYC_CNTL.FRONT_FACE_ALL_BITS: FRONT_FACE reads 0xFFFFFFFF/0 instead of +1.0/-1.0. diff --git a/src/graphics/SharpEmu.Gpu.Metal/MetalNative.cs b/src/graphics/SharpEmu.Gpu.Metal/MetalNative.cs index a4f7efd0b..eddb229dc 100644 --- a/src/graphics/SharpEmu.Gpu.Metal/MetalNative.cs +++ b/src/graphics/SharpEmu.Gpu.Metal/MetalNative.cs @@ -4,6 +4,7 @@ using System.Collections.Concurrent; using System.Runtime.InteropServices; using SharpEmu.Libs.Gpu; +using SharpEmu.ShaderCompiler.Metal; namespace SharpEmu.Gpu.Metal; @@ -278,18 +279,38 @@ public static void SetMslLanguageVersion(nint options, nuint version) => /// /// The compile options every translated-guest MSL library is built with: fast - /// math off, because the translated GCN float semantics do not survive it, an - /// explicit language version so MSL 3.1+ features are available, and invariance - /// preserved, so a [[position, invariant]] output is computed the same way in every - /// program (a depth-EQUAL pass against a prepass depends on it). + /// math off, because the translated GCN float semantics do not survive it, with the + /// relaxed math mode on top (: contraction and reassociation + /// allowed, NaN and infinity kept, library functions precise), an explicit language + /// version so MSL 3.1+ features are available, and invariance preserved, so a + /// [[position, invariant]] output is computed the same way in every program (a + /// depth-EQUAL pass against a prepass depends on it). /// public static void ApplyMslCompileOptions(nint options) { SendVoidBool(options, Selector("setFastMathEnabled:"), false); + ApplyMslMathMode(options, MslMathMode.Default); SendVoidBool(options, Selector("setPreserveInvariance:"), true); SetMslLanguageVersion(options); } + /// Sets the math mode on a compile options object; a driver without the + /// MSL 3.2 math-mode selectors keeps the safe mode setFastMathEnabled: false asked for. + public static void ApplyMslMathMode(nint options, nuint mode) + { + if (mode == MslMathMode.Safe || + !SendBool(options, Selector("respondsToSelector:"), Selector("setMathMode:"))) + { + return; + } + + SendVoidNUInt(options, Selector("setMathMode:"), mode); + if (SendBool(options, Selector("respondsToSelector:"), Selector("setMathFloatingPointFunctions:"))) + { + SendVoidNUInt(options, Selector("setMathFloatingPointFunctions:"), MslMathMode.FunctionsPrecise); + } + } + [LibraryImport(ObjCLibrary, EntryPoint = "objc_msgSend")] public static partial void SendVoidDouble(nint receiver, nint selector, double argument); diff --git a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalBinaryArchive.cs b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalBinaryArchive.cs index 5d08ae5a0..7f09c2f63 100644 --- a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalBinaryArchive.cs +++ b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalBinaryArchive.cs @@ -321,13 +321,18 @@ private static string DeviceStamp(nint device) => RuntimeInformation.OSArchitecture); /// - /// The library key of a source: its text and the options it compiles with (fast math - /// off, the MSL version the per-vertex-inputs flag picks). + /// The library key of a source: its text and the options it compiles with (the math mode, + /// the MSL version the per-vertex-inputs flag picks). A binary built in the safe math + /// mode is another binary than the relaxed one, so the mode is part of the seed; the + /// safe seed is the one every library was keyed with before the mode existed. /// - public static UInt128 LibraryKey(string source, bool usesPerVertexInputs) + public static UInt128 LibraryKey(string source, bool usesPerVertexInputs) => + LibraryKey(source, usesPerVertexInputs, MslMathMode.Default); + + internal static UInt128 LibraryKey(string source, bool usesPerVertexInputs, nuint mathMode) { var version = usesPerVertexInputs ? Gen5MslShader.PerVertexInputsLanguageVersion : MetalNative.MslLanguageVersion; - var seed = (long)(((ulong)FormatVersion << 48) ^ version); + var seed = (long)(((ulong)FormatVersion << 48) ^ version ^ ((ulong)mathMode << 40)); return XxHash128.HashToUInt128(MemoryMarshal.AsBytes(source.AsSpan()), seed); } diff --git a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalShaderPrewarm.cs b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalShaderPrewarm.cs index 814270e07..ee3c59806 100644 --- a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalShaderPrewarm.cs +++ b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalShaderPrewarm.cs @@ -706,6 +706,7 @@ private static string Stamp(IGuestGpuBackend compiler, nint device) => StampPart(typeof(Gen5ShaderTranslator)), StampPart(typeof(Gen5MslTranslator)), compiler.GetType().FullName, + MslMathMode.Identity(MslMathMode.Default), MetalNative.ReadNsString(MetalNative.Send(device, SelName)), RuntimeInformation.OSDescription); diff --git a/tests/SharpEmu.Gpu.Metal.Tests/MetalBinaryArchiveTests.cs b/tests/SharpEmu.Gpu.Metal.Tests/MetalBinaryArchiveTests.cs index f2de313aa..f8ca69709 100644 --- a/tests/SharpEmu.Gpu.Metal.Tests/MetalBinaryArchiveTests.cs +++ b/tests/SharpEmu.Gpu.Metal.Tests/MetalBinaryArchiveTests.cs @@ -72,6 +72,17 @@ public void Dispose() } } + // A binary compiled in the safe math mode must not be served for the relaxed one. + [Fact] + public void TheMathModeIsPartOfTheLibraryKey() + { + var safe = MetalBinaryArchive.LibraryKey(ComputeSource, false, MslMathMode.Safe); + var relaxed = MetalBinaryArchive.LibraryKey(ComputeSource, false, MslMathMode.Relaxed); + Assert.NotEqual(safe, relaxed); + Assert.Equal(relaxed, MetalBinaryArchive.LibraryKey(ComputeSource, false, MslMathMode.Relaxed)); + Assert.Equal(MetalBinaryArchive.LibraryKey(ComputeSource, false), MetalBinaryArchive.LibraryKey(ComputeSource, false, MslMathMode.Default)); + } + [Fact] public void ALibraryAndComputePipelineFromTheArchiveComputeWhatTheCompiledOnesDo() { diff --git a/tests/SharpEmu.ShaderCompiler.Metal.Tests/Gen5ComputeFixtures.cs b/tests/SharpEmu.ShaderCompiler.Metal.Tests/Gen5ComputeFixtures.cs index fd86c557a..549900e42 100644 --- a/tests/SharpEmu.ShaderCompiler.Metal.Tests/Gen5ComputeFixtures.cs +++ b/tests/SharpEmu.ShaderCompiler.Metal.Tests/Gen5ComputeFixtures.cs @@ -336,12 +336,16 @@ public static Gen5MslShader CompileVertexOrThrow(int requiredVertexOutputCount = public static Gen5MslShader CompilePixelOrThrow( Gen5PixelOutputKind outputKind = Gen5PixelOutputKind.Float, - Gen5ColorComponentMapping componentMapping = default) + Gen5ColorComponentMapping componentMapping = default, + bool pixelEarlyDepth = false, + bool pixelDepthExportEnable = false, + Gen5MslCodegenOptions? options = null) { var program = DecodeOrThrow(new Gen5ComputeFixture("pixel", PixelWords, 0, 0)); var request = RequestOrThrow(program, ShaderStage.Pixel, - pixelOutputs: [new Gen5PixelOutputBinding(0, 0, outputKind, componentMapping)]); - if (!Gen5MslTranslator.TryCompileProgram(request, out var shader, out var error)) + pixelOutputs: [new Gen5PixelOutputBinding(0, 0, outputKind, componentMapping)], + pixelDepthExportEnable: pixelDepthExportEnable, pixelEarlyDepth: pixelEarlyDepth); + if (!Gen5MslTranslator.TryCompileProgram(request, out var shader, out var error, options)) { throw new InvalidOperationException($"[pixel] MSL request emit failed: {error}"); } @@ -782,7 +786,7 @@ public static ShaderCompileRequest RequestOrThrow(Gen5ShaderProgram program, Sha bool pixelStencilExportEnable = false, bool pixelSampleMaskExportEnable = false, uint floatMode = ShaderCompileRequest.DefaultFloatMode, bool dx10Clamp = true, bool pixelGuestWave32 = false, uint meshPerVertexParameterMask = 0, IReadOnlyList? vertexInterpolants = null, - Gen5TessellationLaunch? tessellation = null) + Gen5TessellationLaunch? tessellation = null, bool pixelEarlyDepth = false) { var plan = ShaderResourcePlan.Extract(program, stage, RequestHash, userDataBase, UserDataCount, excludedUserDataRegisters: excludedUserDataRegisters); @@ -824,6 +828,7 @@ public static ShaderCompileRequest RequestOrThrow(Gen5ShaderProgram program, Sha PixelDepthExportFormat = pixelDepthExportFormat, PixelStencilExportEnable = pixelStencilExportEnable, PixelSampleMaskExportEnable = pixelSampleMaskExportEnable, + PixelEarlyDepth = pixelEarlyDepth, Tessellation = tessellation, }; } diff --git a/tests/SharpEmu.ShaderCompiler.Metal.Tests/GlobalMemoryRegisterTests.cs b/tests/SharpEmu.ShaderCompiler.Metal.Tests/GlobalMemoryRegisterTests.cs index 283a3dcd2..2145df49a 100644 --- a/tests/SharpEmu.ShaderCompiler.Metal.Tests/GlobalMemoryRegisterTests.cs +++ b/tests/SharpEmu.ShaderCompiler.Metal.Tests/GlobalMemoryRegisterTests.cs @@ -17,7 +17,14 @@ public void Loads_WriteTheDestinationRegisters(uint opcode, int componentCount) { var source = Compile(opcode); for (var component = 0; component < componentCount; component++) - Assert.Contains($"v{8 + component} = sharpemu_load_device_dword", source); + { + // A multi-dword load resolves its record once and falls back to the per-dword load + // for a record that crosses a page or is unmapped. + var load = componentCount > 1 + ? $@"v{8 + component} = t\d+ \? t\d+\[{component}\] : sharpemu_load_device_dword" + : $@"v{8 + component} = sharpemu_load_device_dword"; + Assert.Matches(load, source); + } Assert.DoesNotMatch(@"(?m)^\s*(if \(exec\) \{ )?v3 = ", source); } diff --git a/tests/SharpEmu.ShaderCompiler.Metal.Tests/MetalNative.Benchmark.cs b/tests/SharpEmu.ShaderCompiler.Metal.Tests/MetalNative.Benchmark.cs index 8c6282beb..bda5383e5 100644 --- a/tests/SharpEmu.ShaderCompiler.Metal.Tests/MetalNative.Benchmark.cs +++ b/tests/SharpEmu.ShaderCompiler.Metal.Tests/MetalNative.Benchmark.cs @@ -38,17 +38,20 @@ public static bool TryBenchmark( uint threadsPerThreadgroup, uint iterations, out BenchmarkResult result, - out string error) + out string error, + IReadOnlyList? tables = null, + nuint? mathMode = null) { result = default; - if (!TryCompileLibrary(shader.Source, out var library, out error)) + if (!TryCompileLibrary(shader.Source, out var library, out error, mathMode)) { return false; } var buffer = SharedBuffer(bufferContents); var stepReport = SharedBuffer(new byte[sizeof(uint)]); - if (buffer == 0 || stepReport == 0) + var pairLocks = SharedBuffer(new byte[Gen5MslArgumentLayout.PairLockTableWords * sizeof(uint)]); + if (buffer == 0 || stepReport == 0 || pairLocks == 0) { error = "failed to create the data buffers"; return false; @@ -58,7 +61,9 @@ public static bool TryBenchmark( shader.ArgumentLayout!, SendULong(buffer, Selector("gpuAddress")), (uint)bufferContents.Length, - SendULong(stepReport, Selector("gpuAddress"))); + SendULong(stepReport, Selector("gpuAddress")), + tables, + SendULong(pairLocks, Selector("gpuAddress"))); var argumentBuffer = SharedBuffer(arguments); var pushBuffer = SharedBuffer(pushData.Length == 0 ? new byte[4] : pushData); if (argumentBuffer == 0 || pushBuffer == 0) @@ -121,6 +126,15 @@ public static bool TryBenchmark( (nuint)Gen5MslArgumentLayout.PushDataBufferIndex); SendUseResource(encoder, Selector("useResource:usage:"), buffer, ReadWriteUsage); SendUseResource(encoder, Selector("useResource:usage:"), stepReport, ReadWriteUsage); + SendUseResource(encoder, Selector("useResource:usage:"), pairLocks, ReadWriteUsage); + SendUseResource(encoder, Selector("useResource:usage:"), SharedTable.Value, ReadWriteUsage); + foreach (var table in tables ?? []) + { + foreach (var resident in (nint[])[table.Buffer, .. table.Residents]) + { + SendUseResource(encoder, Selector("useResource:usage:"), resident, ReadWriteUsage); + } + } for (var iteration = 0u; iteration < iterations; iteration++) { SendDispatch(encoder, Selector("dispatchThreadgroups:threadsPerThreadgroup:"), groups, threads); @@ -156,7 +170,9 @@ private static byte[] BuildArgumentBuffer( Gen5MslArgumentLayout layout, ulong bufferAddress, uint bufferBytes, - ulong stepReportAddress) + ulong stepReportAddress, + IReadOnlyList? tables = null, + ulong pairLocksAddress = 0) { var arguments = new byte[Math.Max(layout.ByteSize, 8)]; foreach (var field in layout.Fields) @@ -176,6 +192,19 @@ private static byte[] BuildArgumentBuffer( case MslArgumentFieldKind.StepReport: BitConverter.TryWriteBytes(arguments.AsSpan((int)field.ByteOffset), stepReportAddress); break; + case MslArgumentFieldKind.PairLocks: + BitConverter.TryWriteBytes(arguments.AsSpan((int)field.ByteOffset), pairLocksAddress); + break; + case MslArgumentFieldKind.Pointer: + BitConverter.TryWriteBytes( + arguments.AsSpan((int)field.ByteOffset), + FindTable(tables, field.Kind) is { } table ? SendULong(table.Buffer, Selector("gpuAddress")) : SharedTableAddress()); + break; + case MslArgumentFieldKind.Word: + BitConverter.TryWriteBytes( + arguments.AsSpan((int)field.ByteOffset), + FindTable(tables, field.Kind) is { } counted ? counted.Count : SharedTableDwords); + break; } } } diff --git a/tests/SharpEmu.ShaderCompiler.Metal.Tests/MetalNative.EarlyZBench.cs b/tests/SharpEmu.ShaderCompiler.Metal.Tests/MetalNative.EarlyZBench.cs new file mode 100644 index 000000000..910631f69 --- /dev/null +++ b/tests/SharpEmu.ShaderCompiler.Metal.Tests/MetalNative.EarlyZBench.cs @@ -0,0 +1,134 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +namespace SharpEmu.ShaderCompiler.Metal.Tests; + +/// +/// A timed overdraw pass for the early-Z measurement: full-screen triangles drawn nearest +/// first against a Depth32Float target with a LESS test, so every layer after the first fails +/// the depth test. The fragment function stores to a device buffer and may discard, the shape +/// of a translated pixel shader with buffer stores, which Metal does not test early on its own. +/// +internal static partial class MetalNative +{ + /// GPU milliseconds (command buffer GPUStartTime to GPUEndTime) of the best of + /// runs of overdraw layers. + public static bool TryBenchmarkOverdraw( + string source, + uint width, + uint height, + uint layers, + uint passes, + out double gpuMilliseconds, + out string error) + { + gpuMilliseconds = 0; + if (!TryCompileLibrary(source, out var library, out error)) + { + return false; + } + + var vertexFunction = Send(library, Selector("newFunctionWithName:"), NsString("bench_vs")); + var fragmentFunction = Send(library, Selector("newFunctionWithName:"), NsString("bench_fs")); + if (vertexFunction == 0 || fragmentFunction == 0) + { + error = "bench entry points not found"; + return false; + } + + const nuint rgba8 = 70; + var descriptor = Send(Send(objc_getClass("MTLRenderPipelineDescriptor"), Selector("alloc")), Selector("init")); + SendVoid(descriptor, Selector("setVertexFunction:"), vertexFunction); + SendVoid(descriptor, Selector("setFragmentFunction:"), fragmentFunction); + var attachment = SendIndexed( + Send(descriptor, Selector("colorAttachments")), Selector("objectAtIndexedSubscript:"), 0); + SendInteger(attachment, Selector("setPixelFormat:"), rgba8); + SendInteger(descriptor, Selector("setDepthAttachmentPixelFormat:"), Depth32FloatPixelFormat); + nint nsError = 0; + var pipeline = SendRenderPipeline( + Device.Value, Selector("newRenderPipelineStateWithDescriptor:error:"), descriptor, ref nsError); + if (pipeline == 0) + { + error = $"render pipeline creation failed: {DescribeError(nsError)}"; + return false; + } + + var depthDescriptor = Send(Send(objc_getClass("MTLDepthStencilDescriptor"), Selector("alloc")), Selector("init")); + SendInteger(depthDescriptor, Selector("setDepthCompareFunction:"), 1); // MTLCompareFunctionLess + SendVoidBool(depthDescriptor, Selector("setDepthWriteEnabled:"), true); + var depthState = Send(Device.Value, Selector("newDepthStencilStateWithDescriptor:"), depthDescriptor); + + var color = CreateRenderTarget(width, height, pixelFormat: rgba8); + var textureDescriptor = Send(Send(objc_getClass("MTLTextureDescriptor"), Selector("alloc")), Selector("init")); + SendInteger(textureDescriptor, Selector("setPixelFormat:"), Depth32FloatPixelFormat); + SendInteger(textureDescriptor, Selector("setTextureType:"), 2); + SendInteger(textureDescriptor, Selector("setWidth:"), width); + SendInteger(textureDescriptor, Selector("setHeight:"), height); + SendInteger(textureDescriptor, Selector("setStorageMode:"), 2); // Private + SendInteger(textureDescriptor, Selector("setUsage:"), 4); // RenderTarget + var depth = Send(Device.Value, Selector("newTextureWithDescriptor:"), textureDescriptor); + var output = SharedBuffer(new byte[width * height * sizeof(uint)]); + if (color == 0 || depth == 0 || depthState == 0 || output == 0) + { + error = "failed to create the render targets"; + return false; + } + + var zBuffers = new nint[layers]; + for (var layer = 0; layer < layers; layer++) + { + zBuffers[layer] = SharedBuffer(BitConverter.GetBytes(0.05f + (0.08f * layer))); + } + + var queue = Send(Device.Value, Selector("newCommandQueue")); + var best = double.MaxValue; + for (var pass = 0u; pass <= passes; pass++) + { + var passDescriptor = Send(objc_getClass("MTLRenderPassDescriptor"), Selector("renderPassDescriptor")); + var passAttachment = SendIndexed( + Send(passDescriptor, Selector("colorAttachments")), Selector("objectAtIndexedSubscript:"), 0); + SendVoid(passAttachment, Selector("setTexture:"), color); + SendInteger(passAttachment, Selector("setLoadAction:"), 2); + SendInteger(passAttachment, Selector("setStoreAction:"), 1); + SendClearColor(passAttachment, Selector("setClearColor:"), default); + var depthAttachment = Send(passDescriptor, Selector("depthAttachment")); + SendVoid(depthAttachment, Selector("setTexture:"), depth); + SendInteger(depthAttachment, Selector("setLoadAction:"), 2); + SendInteger(depthAttachment, Selector("setStoreAction:"), 0); // DontCare + SendVoidDouble(depthAttachment, Selector("setClearDepth:"), 1.0); + + var commandBuffer = Send(queue, Selector("commandBuffer")); + var encoder = Send(commandBuffer, Selector("renderCommandEncoderWithDescriptor:"), passDescriptor); + SendVoid(encoder, Selector("setRenderPipelineState:"), pipeline); + SendVoid(encoder, Selector("setDepthStencilState:"), depthState); + SendSetBuffer(encoder, Selector("setFragmentBuffer:offset:atIndex:"), output, 0, 0); + for (var layer = 0; layer < layers; layer++) + { + SendSetBuffer(encoder, Selector("setVertexBuffer:offset:atIndex:"), zBuffers[layer], 0, 0); + SendDrawPrimitives(encoder, Selector("drawPrimitives:vertexStart:vertexCount:"), 3, 0, 3); + } + + SendVoid(encoder, Selector("endEncoding")); + SendVoid(commandBuffer, Selector("commit")); + SendVoid(commandBuffer, Selector("waitUntilCompleted")); + if (Send(commandBuffer, Selector("error")) != 0) + { + error = "GPU execution failed: " + DescribeError(Send(commandBuffer, Selector("error"))); + return false; + } + + if (pass == 0) + { + continue; // warm-up + } + + var elapsed = SendDouble(commandBuffer, Selector("GPUEndTime")) - + SendDouble(commandBuffer, Selector("GPUStartTime")); + best = Math.Min(best, elapsed * 1000.0); + } + + gpuMilliseconds = best; + error = string.Empty; + return true; + } +} diff --git a/tests/SharpEmu.ShaderCompiler.Metal.Tests/MetalNative.Render.cs b/tests/SharpEmu.ShaderCompiler.Metal.Tests/MetalNative.Render.cs index 3ed7fc01b..735b3f405 100644 --- a/tests/SharpEmu.ShaderCompiler.Metal.Tests/MetalNative.Render.cs +++ b/tests/SharpEmu.ShaderCompiler.Metal.Tests/MetalNative.Render.cs @@ -43,6 +43,7 @@ public static bool TryCompileLibrary(string source, nuint languageVersion, out n error = string.Empty; var options = Send(Send(objc_getClass("MTLCompileOptions"), Selector("alloc")), Selector("init")); SendVoidBool(options, Selector("setFastMathEnabled:"), false); + ApplyMathMode(options, MslMathMode.Default); SendVoidUInt(options, Selector("setLanguageVersion:"), languageVersion); nint nsError = 0; library = Send(Device.Value, Selector("newLibraryWithSource:options:error:"), NsString(source), options, ref nsError); diff --git a/tests/SharpEmu.ShaderCompiler.Metal.Tests/MetalNative.cs b/tests/SharpEmu.ShaderCompiler.Metal.Tests/MetalNative.cs index 551d98cb9..664d6259d 100644 --- a/tests/SharpEmu.ShaderCompiler.Metal.Tests/MetalNative.cs +++ b/tests/SharpEmu.ShaderCompiler.Metal.Tests/MetalNative.cs @@ -141,8 +141,32 @@ private static string DescribeError(nint error) return Marshal.PtrToStringUTF8(utf8) ?? "unknown error"; } + [LibraryImport(ObjCLibrary, EntryPoint = "objc_msgSend")] + [return: MarshalAs(UnmanagedType.I1)] + private static partial bool RespondsTo(nint receiver, nint selector, nint argument); + + private static bool RespondsTo(nint receiver, nint selector) => + RespondsTo(receiver, Selector("respondsToSelector:"), selector); + + /// The emulator's math mode on a compile options object (MetalNative.ApplyMslMathMode + /// in the Metal backend), so the harness compiles the way the backend does. + public static void ApplyMathMode(nint options, nuint mode) + { + if (mode == MslMathMode.Safe || + !RespondsTo(options, Selector("setMathMode:"))) + { + return; + } + + SendVoidUInt(options, Selector("setMathMode:"), mode); + if (RespondsTo(options, Selector("setMathFloatingPointFunctions:"))) + { + SendVoidUInt(options, Selector("setMathFloatingPointFunctions:"), MslMathMode.FunctionsPrecise); + } + } + /// Compiles MSL source with the OS runtime compiler. - public static bool TryCompileLibrary(string source, out nint library, out string error) + public static bool TryCompileLibrary(string source, out nint library, out string error, nuint? mathMode = null) { library = 0; error = string.Empty; @@ -153,6 +177,7 @@ public static bool TryCompileLibrary(string source, out nint library, out string // MSL 3.1 or later (texture atomics) compiles here too. var options = Send(Send(objc_getClass("MTLCompileOptions"), Selector("alloc")), Selector("init")); SendVoidBool(options, Selector("setFastMathEnabled:"), false); + ApplyMathMode(options, mathMode ?? MslMathMode.Default); SendVoidUInt(options, Selector("setLanguageVersion:"), MslLanguageVersion); nint nsError = 0; diff --git a/tests/SharpEmu.ShaderCompiler.Metal.Tests/MslGpuShaderTimeBenchTests.cs b/tests/SharpEmu.ShaderCompiler.Metal.Tests/MslGpuShaderTimeBenchTests.cs new file mode 100644 index 000000000..8c8d5054a --- /dev/null +++ b/tests/SharpEmu.ShaderCompiler.Metal.Tests/MslGpuShaderTimeBenchTests.cs @@ -0,0 +1,194 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using SharpEmu.ShaderCompiler.Resources; +using Xunit; +using Xunit.Abstractions; + +namespace SharpEmu.ShaderCompiler.Metal.Tests; + +/// +/// GPU time of translated shaders, before and after the three shader-time changes, on the +/// device: each test compiles the same guest program both ways in one process and times both +/// with the command buffer's own clock (best of several runs). Opt-in, because a timing is not +/// a pass/fail property: SHARPEMU_GPU_BENCH=1 with SHARPEMU_TEST_REQUIRE_DEVICE=1, and read the +/// output (dotnet test --logger "console;verbosity=detailed"). +/// +/// * math mode: an ALU-heavy loop (fmac, mul+add, sqrt, rcp, sin) in the safe and the relaxed +/// mode (); +/// * device-address spans: a compute loop of GLOBAL_LOAD_DWORDX4 / GLOBAL_STORE_DWORDX4 +/// through the guest page table, resolved per dword and per record; +/// * early Z: an eight-layer overdraw of a fragment function that stores to a device buffer, +/// with and without [[early_fragment_tests]]. +/// +public sealed class MslGpuShaderTimeBenchTests(ITestOutputHelper output) +{ + private const uint Groups = 2048; + private const uint Threads = 64; + private const uint Iterations = 4; + private const ulong GuestBase = 0x4000_0000; + private const int GuestBytes = 1 << 20; + + private static bool Enabled => + Environment.GetEnvironmentVariable("SHARPEMU_GPU_BENCH") == "1" && + Environment.GetEnvironmentVariable("SHARPEMU_TEST_REQUIRE_DEVICE") == "1" && + MetalNative.IsAvailable; + + // v1 = float(lane); a loop of 512 trips: v3 += v1 * v2 (fmac), v4 = v3 * v3 + v1 (a multiply + // and an add the relaxed mode can fuse), v5 = sqrt(v4), v6 = 1 / v5, v7 = sin(v6), + // v1 += v7 * v6; then the result is stored. + private static readonly Gen5ComputeFixture AluLoop = new( + "bench-alu-loop", + [ + 0x7E020D00, // v_cvt_f32_u32 v1, v0 + 0x7E0402F2, // v_mov_b32 v2, 1.0 + 0x7E060280, // v_mov_b32 v3, 0 + 0xBE8003FF, 0x00000200, // s_mov_b32 s0, 512 + // loop: + 0x56060501, // v_fmac_f32 v3, v1, v2 + 0x10080703, // v_mul_f32 v4, v3, v3 + 0x06080204, // v_add_f32 v4, v4, v1 + 0x7E0A6704, // v_sqrt_f32 v5, v4 + 0x7E0C5505, // v_rcp_f32 v6, v5 + 0x7E0E6B06, // v_sin_f32 v7, v6 + 0x10100D07, // v_mul_f32 v8, v7, v6 + 0x06020308, // v_add_f32 v1, v8, v1 + 0x81808100, // s_sub_i32 s0, s0, 1 + 0xBF078000, // s_cmp_lg_u32 s0, 0 + 0xBF85FFF5, // s_cbranch_scc1 loop + 0xE0700000, 0x80020100, // buffer_store_dword v1, off, s[8:11], 0 + 0xBF810000, // s_endpgm + ], + StoreScalarResourceBase: 8, + StoreBackingBytes: 16); + + // 256 trips: global_load_dwordx4 v[4:7], v[9:10], off; the four dwords summed into v1; the + // same record stored back with global_store_dwordx4; the address advances 1 KB. + private static readonly Gen5ComputeFixture GlobalRecords = new( + "bench-global-records", + [ + 0x34120084, // v_lshlrev_b32 v9, 4, v0 + 0x381212FF, 0x40000000, // v_or_b32 v9, 0x40000000, v9 + 0x7E140280, // v_mov_b32 v10, 0 + 0x7E020280, // v_mov_b32 v1, 0 + 0xBE8003FF, 0x00000100, // s_mov_b32 s0, 256 + // loop: + 0xDC388000, 0x047D0009, // global_load_dwordx4 v[4:7], v[9:10], off + 0xBF8C3F70, // s_waitcnt vmcnt(0) + 0x4A020304, // v_add_nc_u32 v1, v4, v1 + 0x4A020305, // v_add_nc_u32 v1, v5, v1 + 0x4A020306, // v_add_nc_u32 v1, v6, v1 + 0x4A020307, // v_add_nc_u32 v1, v7, v1 + 0xDC788000, 0x007D0409, // global_store_dwordx4 v[9:10], v[4:7], off + 0x4A1212FF, 0x00000400, // v_add_nc_u32 v9, 0x400, v9 + 0x81808100, // s_sub_i32 s0, s0, 1 + 0xBF078000, // s_cmp_lg_u32 s0, 0 + 0xBF85FFF2, // s_cbranch_scc1 loop + 0xE0700000, 0x80020100, // buffer_store_dword v1, off, s[8:11], 0 + 0xBF810000, // s_endpgm + ], + StoreScalarResourceBase: 8, + StoreBackingBytes: 16); + + private static Gen5MslShader Compile(Gen5ComputeFixture fixture, Gen5MslCodegenOptions? options = null) + { + var request = Gen5ComputeFixtures.CreateComputeRequest(fixture, localSizeX: Threads); + Assert.True(Gen5MslTranslator.TryCompileProgram(request, out var shader, out var error, options), error); + return shader; + } + + private MetalNative.BenchmarkResult Time( + Gen5MslShader shader, string label, IReadOnlyList? tables = null, nuint? mathMode = null) + { + Assert.True( + MetalNative.TryBenchmark( + shader, new byte[1024], new byte[PushData.ByteSize], Groups, Threads, Iterations, + out var result, out var error, tables, mathMode), + error); + output.WriteLine($"{label,-34} {result.GpuMilliseconds,9:F3} ms GPU maxThreads {result.MaxThreadsPerThreadgroup}"); + return result; + } + + [Fact] + public void AluLoopSafeVersusRelaxedMath() + { + if (!Enabled) + { + return; + } + + var shader = Compile(AluLoop); + for (var round = 0; round < 3; round++) + { + Time(shader, "alu loop, safe math", mathMode: MslMathMode.Safe); + Time(shader, "alu loop, relaxed math", mathMode: MslMathMode.Relaxed); + } + } + + [Fact] + public void GlobalRecordLoopPerDwordVersusPerRecord() + { + if (!Enabled) + { + return; + } + + var perDword = Compile(GlobalRecords, new Gen5MslCodegenOptions(DeviceAddressSpans: false)); + var perRecord = Compile(GlobalRecords, new Gen5MslCodegenOptions(DeviceAddressSpans: true)); + Assert.DoesNotContain("sharpemu_resolve_device_span(sharpemu_ranges", perDword.Source, StringComparison.Ordinal); + Assert.Contains("sharpemu_resolve_device_span(sharpemu_ranges", perRecord.Source, StringComparison.Ordinal); + var (tables, _) = MetalNative.CreateGuestMapping(GuestBase, new byte[GuestBytes], 0x1000); + for (var round = 0; round < 3; round++) + { + Time(perDword, "global records, per dword", tables); + Time(perRecord, "global records, per record", tables); + } + } + + // The fragment function stands in for a translated pixel shader that stores to a buffer + // and ends with the exit discard every translated pixel shader carries; only the attribute + // differs between the two. + private static string OverdrawSource(bool earlyFragmentTests) => $$""" + #include + using namespace metal; + struct VOut { float4 position [[position]]; }; + vertex VOut bench_vs(uint vid [[vertex_id]], constant float& z [[buffer(0)]]) + { + float2 p = float2(float((vid << 1) & 2u), float(vid & 2u)); + VOut o; + o.position = float4(p * 2.0f - 1.0f, z, 1.0f); + return o; + } + {{(earlyFragmentTests ? "[[early_fragment_tests]] " : string.Empty)}}fragment float4 bench_fs(VOut in [[stage_in]], device uint* out [[buffer(0)]]) + { + float v = in.position.x * 0.001f; + for (int i = 0; i < 96; i++) + { + v = sin(v) * 1.0001f + sqrt(abs(v) + 1.0f); + } + out[uint(in.position.y) * 1920u + uint(in.position.x)] = as_type(v); + if (v < -1.0e30f) + { + discard_fragment(); + } + return float4(v); + } + """; + + [Fact] + public void BufferStoringFragmentOverdrawWithAndWithoutEarlyFragmentTests() + { + if (!Enabled) + { + return; + } + + for (var round = 0; round < 3; round++) + { + Assert.True(MetalNative.TryBenchmarkOverdraw(OverdrawSource(false), 1920, 1080, 8, 4, out var late, out var lateError), lateError); + Assert.True(MetalNative.TryBenchmarkOverdraw(OverdrawSource(true), 1920, 1080, 8, 4, out var early, out var earlyError), earlyError); + output.WriteLine($"overdraw x8, tested after shader {late,9:F3} ms GPU"); + output.WriteLine($"overdraw x8, [[early_fragment_tests]] {early,6:F3} ms GPU"); + } + } +} diff --git a/tests/SharpEmu.ShaderCompiler.Metal.Tests/MslShaderTimeChangesTests.cs b/tests/SharpEmu.ShaderCompiler.Metal.Tests/MslShaderTimeChangesTests.cs new file mode 100644 index 000000000..8faef36f0 --- /dev/null +++ b/tests/SharpEmu.ShaderCompiler.Metal.Tests/MslShaderTimeChangesTests.cs @@ -0,0 +1,214 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using SharpEmu.ShaderCompiler.Resources; +using Xunit; + +namespace SharpEmu.ShaderCompiler.Metal.Tests; + +// The three changes that cut the GPU time of translated shaders: the relaxed math mode, the +// early-fragment-tests attribute, and one page-table lookup per global record. +public sealed class MslShaderTimeChangesTests +{ + private const ulong GuestBase = 0x4000_0000; + private const int GuestBytes = 4 << 14; + private const uint Lanes = 32; + + private static bool DeviceRequired => Environment.GetEnvironmentVariable("SHARPEMU_TEST_REQUIRE_DEVICE") == "1"; + + // ---- early fragment tests ---- + + [Fact] + public void APixelShaderWhoseGuestStateTestsDepthEarlyDeclaresEarlyFragmentTests() + { + var shader = Gen5ComputeFixtures.CompilePixelOrThrow(pixelEarlyDepth: true); + Assert.Contains("[[early_fragment_tests]] fragment Gen5PsOut gen5_ps(", shader.Source, StringComparison.Ordinal); + if (DeviceRequired) + { + Assert.True(MetalNative.TryCompileLibrary(shader.Source, out var library, out var error), error); + MetalNative.ReleaseCollisionLibrary(library); + } + } + + [Fact] + public void APixelShaderWithLateDepthTestKeepsMetalsChoice() + { + var shader = Gen5ComputeFixtures.CompilePixelOrThrow(pixelEarlyDepth: false); + Assert.DoesNotContain("early_fragment_tests", shader.Source, StringComparison.Ordinal); + } + + [Fact] + public void TheEarlyFragmentTestsSwitchTurnsTheAttributeOff() + { + var shader = Gen5ComputeFixtures.CompilePixelOrThrow( + pixelEarlyDepth: true, options: new Gen5MslCodegenOptions(EarlyFragmentTests: false)); + Assert.DoesNotContain("early_fragment_tests", shader.Source, StringComparison.Ordinal); + } + + // ---- math mode ---- + + [Fact] + public void TheRelaxedMathModeIsTheDefaultAndSafeIsTheSwitch() + { + // SHARPEMU_MSL_MATHMODE=safe is read once at startup, so the default is what a run + // without it gets. + var requested = Environment.GetEnvironmentVariable("SHARPEMU_MSL_MATHMODE"); + Assert.Equal( + string.Equals(requested, "safe", StringComparison.OrdinalIgnoreCase) ? MslMathMode.Safe : MslMathMode.Relaxed, + MslMathMode.Default); + Assert.NotEqual(MslMathMode.Identity(MslMathMode.Safe), MslMathMode.Identity(MslMathMode.Relaxed)); + } + + // omod flushes a zero result to +0. A test of the result against 0.0f cannot say which zero + // it is under the relaxed mode (signed zeros are not distinguished), so the flush tests the + // bits. + [Fact] + public void AnOutputModifierFlushesZeroByItsBits() + { + var program = new Gen5ComputeFixture( + "omod-negative-zero", + [ + 0xD5030001, 0x08020300, // v_add_f32_e64 v1, v0, v1 mul:2 + 0xBF810000, + ], + StoreScalarResourceBase: 0, + StoreBackingBytes: 0); + var shader = Gen5ComputeFixtures.CompileRequestOrThrow(program); + Assert.Contains("& 0x7FFFFFFFu) == 0u", shader.Source, StringComparison.Ordinal); + Assert.DoesNotContain("== 0.0f ? 0.0f", shader.Source, StringComparison.Ordinal); + } + + // ---- one lookup per global record ---- + + // global_load_dwordx4 v[4:7], v[9:10], off; global_store_dwordx4 v[11:12], v[4:7], off. + private static Gen5ComputeFixture Copy(uint[] addressSetup) => new( + "global-record-copy", + [ + .. addressSetup, + 0xDC388000, 0x047D0009, // global_load_dwordx4 v[4:7], v[9:10], off + 0xBF8C3F70, // s_waitcnt vmcnt(0) + 0xDC788000, 0x007D040B, // global_store_dwordx4 v[11:12], v[4:7], off + 0xBF810000, // s_endpgm + ], + StoreScalarResourceBase: 0, + StoreBackingBytes: 0); + + [Fact] + public void AGlobalRecordIsResolvedOnceAndFallsBackPerDword() + { + var setup = new uint[] { 0x7E1202FF, 0x40000000, 0x7E140280, 0x7E1602FF, 0x40001000, 0x7E180280 }; + var shader = Gen5ComputeFixtures.CompileRequestOrThrow(Copy(setup)); + + // One lookup for the record's load and one for its store, the per-dword forms as fallback. + Assert.Equal(2, CountOccurrences(shader.Source, "sharpemu_resolve_device_span(sharpemu_ranges")); + Assert.Equal(4, CountOccurrences(shader.Source, "sharpemu_load_device_dword(sharpemu_ranges")); + Assert.Equal(4, CountOccurrences(shader.Source, "sharpemu_store_device_dword(sharpemu_ranges")); + + var request = Gen5ComputeFixtures.CreateComputeRequest(Copy(setup)); + Assert.True(Gen5MslTranslator.TryCompileProgram(request, out var off, out var error, new Gen5MslCodegenOptions(DeviceAddressSpans: false)), error); + Assert.DoesNotContain("sharpemu_resolve_device_span(sharpemu_ranges", off.Source, StringComparison.Ordinal); + } + + private static byte[] Pattern(int length) + { + var bytes = new byte[length]; + for (var index = 0; index < length; index++) + { + bytes[index] = (byte)((index * 7) + ((index >> 8) * 13) + (index >> 14) + 3); + } + + return bytes; + } + + private static byte[] RunCopy(Gen5ComputeFixture fixture, byte[] guest, uint guestBytesMapped) + { + var shader = Gen5ComputeFixtures.CompileRequestOrThrow(fixture, localSizeX: Lanes); + Assert.True(MetalNative.TryCompileLibrary(shader.Source, out var library, out var compileError), compileError); + try + { + var (mapping, _) = MetalNative.CreateGuestMapping(GuestBase, guest[..(int)guestBytesMapped], 0x1000); + // A global store is performed inside the program's written range, slot 0 of the + // flattened table: base low, base high, size. + var written = MetalNative.CreateWordTable( + DescriptorBindingKind.FlattenedResourceTable, [(uint)GuestBase, (uint)(GuestBase >> 32), guestBytesMapped]); + IReadOnlyList tables = [.. mapping, written]; + Assert.True( + MetalNative.TryExecuteWithArgumentBuffer( + library, shader.EntryPoint, new byte[1024], new byte[PushData.ByteSize], shader.ArgumentLayout!, + Lanes, out _, out _, out var runError, tables: tables), + runError); + var words = MetalNative.ReadWords(mapping[0].Residents[0], (int)guestBytesMapped / 4); + var bytes = new byte[guestBytesMapped]; + Buffer.BlockCopy(words, 0, bytes, 0, bytes.Length); + return bytes; + } + finally + { + MetalNative.ReleaseCollisionLibrary(library); + } + } + + [Fact] + public void AGlobalRecordCopyMovesEveryByteIncludingRecordsAcrossAPageBoundary() + { + if (!DeviceRequired) + { + return; + } + + // Lane n copies the 16 bytes at base + 0x3FF8 + 16n to the same offset + 0x8000. Lane 0's + // record and its copy straddle a 16 KB page, the rest sit inside one. + var guest = Pattern(GuestBytes); + var fixture = Copy( + [ + 0x34120084, // v_lshlrev_b32 v9, 4, v0 + 0x4A1212FF, 0x40003FF8, // v_add_nc_u32 v9, 0x40003FF8, v9 + 0x7E140280, // v_mov_b32 v10, 0 + 0x4A1612FF, 0x00008000, // v_add_nc_u32 v11, 0x8000, v9 + 0x7E180280, // v_mov_b32 v12, 0 + ]); + var result = RunCopy(fixture, guest, GuestBytes); + for (var lane = 0; lane < Lanes; lane++) + { + var source = 0x3FF8 + (lane * 16); + Assert.True( + guest.AsSpan(source, 16).SequenceEqual(result.AsSpan(source + 0x8000, 16)), + $"lane {lane} did not copy its record"); + } + } + + [Fact] + public void AGlobalRecordRunningOffTheMappedRangeLoadsZeroAndStoresOnlyWhatIsMapped() + { + if (!DeviceRequired) + { + return; + } + + // One mapped page. Every lane reads 16 bytes at page offset 0x3FF8: two dwords are + // mapped and two are not, and the unmapped ones read as zero. The record is written to + // page offset 0x1000, inside the mapping. + var guest = Pattern(1 << 14); + var fixture = Copy( + [ + 0x7E1202FF, 0x40003FF8, // v_mov_b32 v9, 0x40003FF8 + 0x7E140280, // v_mov_b32 v10, 0 + 0x7E1602FF, 0x40001000, // v_mov_b32 v11, 0x40001000 + 0x7E180280, // v_mov_b32 v12, 0 + ]); + var result = RunCopy(fixture, guest, 1 << 14); + Assert.Equal(Convert.ToHexString(guest.AsSpan(0x3FF8, 8)), Convert.ToHexString(result.AsSpan(0x1000, 8))); + Assert.Equal("0000000000000000", Convert.ToHexString(result.AsSpan(0x1008, 8))); + } + + private static int CountOccurrences(string text, string value) + { + var count = 0; + for (var at = text.IndexOf(value, StringComparison.Ordinal); at >= 0; at = text.IndexOf(value, at + value.Length, StringComparison.Ordinal)) + { + count++; + } + + return count; + } +} From 63ad96d5fbc71a6722e0d4797cb68e3b33f81c1a Mon Sep 17 00:00:00 2001 From: kuba Date: Mon, 26 Oct 2026 05:47:24 +0100 Subject: [PATCH 12/82] [GPU] Draw lookups read each guest table once, resolve each color target once and take no lock on a cache hit Per-draw shader and pipeline resolution on the GPU worker made one guest read per pixel input (up to 32), two per vertex attribute, and a hash of the whole code for a shader without a declared hash, and resolved each bound color target (a several-hundred-byte request) up to four times. Each guest read takes the memory read lock. - PixelStageInputResolver reads the semantic words in one read. - VertexInputResolver reads the attribute words and the buffer descriptors in two reads and keeps the decoded layout with the bytes it came from; equal bytes at the same tables share the previous draw's attribute and buffer arrays, any other byte decodes afresh. - ShaderIdentity remembers the hash of a shader without a declared hash with a copy of its code, validated by an in-place compare (ICpuMemory.TryCompare) instead of read plus XxHash3. - ShaderPipelineCache resolves each bound target once per draw and passes the slots and extents to the dual-source, render-area and output steps. - The pipeline maps and the header registry's record map are concurrent: a hit takes no lock. The resolve gate and the program gate stay, they guard the program cache's mutation. Direct measurement (ShaderPipelineCache.GetGraphicsPrograms, vertex program with 3 attributes, pixel program with 4 custom inputs, 4 color targets, no declared hash, arm64, thread CPU time, host load ~180 so absolute numbers are inflated, A/B interleaved): 3.97 us -> 2.40 us per draw, 1368 -> 744 B per draw, 20 -> 7 guest reads (+2 in-place compares) per draw. The audit's 2a-2d and 8a-8e rows do not move: their rig has no attribute tables, no pixel semantics and declared hashes. No behaviour switch: the caches validate against guest bytes. --- .../Gpu/Pipelines/PixelStageInputResolver.cs | 52 ++- .../Gpu/Pipelines/ShaderHeaderRegistry.cs | 62 ++- .../Gpu/Pipelines/ShaderIdentity.cs | 127 ++++++ .../Gpu/Pipelines/ShaderPipelineCache.cs | 113 ++--- .../Gpu/Pipelines/VertexInputResolver.cs | 183 ++++++-- tests/SharpEmu.Libs.Tests/FakeCpuMemory.cs | 12 + .../ShaderPipelineLookupCacheTests.cs | 401 ++++++++++++++++++ 7 files changed, 805 insertions(+), 145 deletions(-) create mode 100644 tests/SharpEmu.Libs.Tests/Gpu/Pipelines/ShaderPipelineLookupCacheTests.cs diff --git a/src/SharpEmu.Libs/Gpu/Pipelines/PixelStageInputResolver.cs b/src/SharpEmu.Libs/Gpu/Pipelines/PixelStageInputResolver.cs index 3166491f9..1ebe07296 100644 --- a/src/SharpEmu.Libs/Gpu/Pipelines/PixelStageInputResolver.cs +++ b/src/SharpEmu.Libs/Gpu/Pipelines/PixelStageInputResolver.cs @@ -57,6 +57,43 @@ public static uint CountSystemInputRegisters(ShaderInterfaceRegisters shaderInte return registers; } + // The semantics a pixel program's header declares are static, but the guest owns the table, so every + // draw still reads it. One read for all of them: this was one guest read per input (up to 32), each + // taking the memory read lock, to decode a few words. + private static uint CustomInterpolationMask(CpuContext context, RegisteredShader shader, uint semanticCount) + { + if (semanticCount == 0) + { + return 0; + } + + Span words = stackalloc uint[PixelInputInfo.InterpolatorCount]; + words = words[..(int)semanticCount]; + if (!context.Memory.TryRead(shader.InputSemanticsAddress, System.Runtime.InteropServices.MemoryMarshal.AsBytes(words))) + { + // A word is unmapped: read them one at a time to name it. + for (var index = 0u; index < semanticCount; index++) + { + if (!context.TryReadUInt32(shader.InputSemanticsAddress + index * sizeof(uint), out words[(int)index])) + { + throw GpuFatal.Fatal($"The pixel input semantics are unreadable: shader=0x{shader.CodeAddress:X16} index={index}."); + } + } + } + + var customMask = 0u; + for (var index = 0; index < words.Length; index++) + { + var semantic = new ShaderInputSemantic(words[index]); + if (semantic.IsCustom && semantic.IsHalfFloat == 0) + { + customMask |= 1u << index; + } + } + + return customMask; + } + public static PixelInputInfo Resolve( CpuContext context, RegisteredShader shader, @@ -69,21 +106,8 @@ public static PixelInputInfo Resolve( PixelInputArrays? reuse = null) { var activeInputs = shaderInterface.PixelInputEnable & shaderInterface.PixelInputAddress; - var customMask = 0u; var semanticCount = Math.Min(Math.Min(shader.InputSemanticsCount, inputCount), (uint)PixelInputInfo.InterpolatorCount); - for (var index = 0u; index < semanticCount; index++) - { - if (!context.TryReadUInt32(shader.InputSemanticsAddress + index * sizeof(uint), out var word)) - { - throw GpuFatal.Fatal($"The pixel input semantics are unreadable: shader=0x{shader.CodeAddress:X16} index={index}."); - } - - var semantic = new ShaderInputSemantic(word); - if (semantic.IsCustom && semantic.IsHalfFloat == 0) - { - customMask |= 1u << (int)index; - } - } + var customMask = CustomInterpolationMask(context, shader, semanticCount); Span interpolators = stackalloc uint[PixelInputInfo.InterpolatorCount]; for (var index = 0u; index < inputCount; index++) diff --git a/src/SharpEmu.Libs/Gpu/Pipelines/ShaderHeaderRegistry.cs b/src/SharpEmu.Libs/Gpu/Pipelines/ShaderHeaderRegistry.cs index 40d446cd8..ad766d3a7 100644 --- a/src/SharpEmu.Libs/Gpu/Pipelines/ShaderHeaderRegistry.cs +++ b/src/SharpEmu.Libs/Gpu/Pipelines/ShaderHeaderRegistry.cs @@ -150,43 +150,39 @@ public RegisteredShader Require(ulong codeAddress, string label) // Every draw resolves its shaders again, so an unchanged header hands back the record // it made last time instead of a new one. The header is still read and compared above // and here, so a guest that rewrites it gets a fresh record as before. - RegisteredShader created; - lock (_recent) + // The map is concurrent, so the hit, which is every draw but the first of a shader, takes no lock. + if (_recent.TryGetValue(codeAddress, out var recent) && + recent.HeaderAddress == headerAddress && recent.CodeSizeBytes == codeSize && + recent.ScratchDwords == scratchDwords && recent.UserDataAddress == userDataAddress && + recent.InputSemanticsAddress == inputSemanticsAddress && recent.InputSemanticsCount == inputSemanticsCount && + recent.ContinuationAddress == continuationAddress && recent.ContinuationSizeBytes == continuationSize && + recent.ContinuationScratchDwords == continuationScratchDwords) { - if (_recent.TryGetValue(codeAddress, out var recent) && - recent.HeaderAddress == headerAddress && recent.CodeSizeBytes == codeSize && - recent.ScratchDwords == scratchDwords && recent.UserDataAddress == userDataAddress && - recent.InputSemanticsAddress == inputSemanticsAddress && recent.InputSemanticsCount == inputSemanticsCount && - recent.ContinuationAddress == continuationAddress && recent.ContinuationSizeBytes == continuationSize && - recent.ContinuationScratchDwords == continuationScratchDwords) - { - return recent; - } - - var registered = new RegisteredShader( - codeAddress, - headerAddress, - codeSize, - scratchDwords, - userDataAddress, - inputSemanticsAddress, - inputSemanticsCount, - continuationAddress, - continuationSize) - { - ContinuationScratchDwords = continuationScratchDwords, - }; + return recent; + } - // Bounded: dropping the records only costs rebuilding them. - if (_recent.Count >= MaxRecent && !_recent.ContainsKey(codeAddress)) - { - _recent.Clear(); - } + var created = new RegisteredShader( + codeAddress, + headerAddress, + codeSize, + scratchDwords, + userDataAddress, + inputSemanticsAddress, + inputSemanticsCount, + continuationAddress, + continuationSize) + { + ContinuationScratchDwords = continuationScratchDwords, + }; - _recent[codeAddress] = registered; - created = registered; + // Bounded: dropping the records only costs rebuilding them. + if (_recent.Count >= MaxRecent && !_recent.ContainsKey(codeAddress)) + { + _recent.Clear(); } + _recent[codeAddress] = created; + CheckSplitVisibility(created); return created; } @@ -208,7 +204,7 @@ private void CheckSplitVisibility(RegisteredShader shader) } private const int MaxRecent = 4096; - private readonly Dictionary _recent = new(); + private readonly System.Collections.Concurrent.ConcurrentDictionary _recent = new(); // A bounded, nonfatal description for diagnostics of draws that remain rejected. internal bool TryGetDiagnosticCodeRanges(ulong address, out (ulong Address, uint SizeBytes)[] ranges) diff --git a/src/SharpEmu.Libs/Gpu/Pipelines/ShaderIdentity.cs b/src/SharpEmu.Libs/Gpu/Pipelines/ShaderIdentity.cs index 1ccfb0b12..96af51e90 100644 --- a/src/SharpEmu.Libs/Gpu/Pipelines/ShaderIdentity.cs +++ b/src/SharpEmu.Libs/Gpu/Pipelines/ShaderIdentity.cs @@ -84,9 +84,136 @@ public static ulong Compute(ICpuMemory memory, ulong codeAddress, ReadOnlySpan<( return hash.GetCurrentHashAsUInt64(); } + // Compute with the content hashes of the shaders already hashed remembered, so a shader without a + // declared hash is not read out and hashed again on every draw. The remembered hash stands only while + // the guest still holds the bytes it was made from, which one in-place compare per code range settles; + // memory that cannot compare in place, and any difference, take the full path as before. + public static ulong Compute(ICpuMemory memory, ulong codeAddress, ReadOnlySpan<(ulong Address, uint SizeBytes)> ranges, string label, + ShaderIdentityMemo memo) + { + if (!TryReadDeclaredHash(memory, codeAddress, out var declaredHash)) + { + throw Scheduling.GpuFatal.Fatal($"The shader code is unreadable: label={label} shader=0x{codeAddress:X16}."); + } + + if (declaredHash != 0) + { + return declaredHash; + } + + if (memo.TryGet(memory, codeAddress, ranges, out var remembered)) + { + return remembered; + } + + if (!memo.CanKeep(ranges)) + { + return Compute(memory, codeAddress, ranges, label); + } + + // The hash is made from the very bytes kept, so the two cannot disagree if the guest writes meanwhile. + var code = new byte[ranges.Length][]; + var hash = new XxHash3(); + for (var index = 0; index < ranges.Length; index++) + { + var (address, sizeBytes) = ranges[index]; + code[index] = new byte[sizeBytes]; + if (!memory.TryRead(address, code[index])) + { + throw Scheduling.GpuFatal.Fatal($"The shader code is unreadable: label={label} shader=0x{address:X16} size=0x{sizeBytes:X8}."); + } + + hash.Append(code[index]); + } + + var contentHash = hash.GetCurrentHashAsUInt64(); + memo.Add(codeAddress, ranges, code, contentHash); + return contentHash; + } + [ThreadStatic] private static XxHash3? _hasher; [ThreadStatic] private static byte[]? _codeBuffer; } + +// The content hashes of shaders without a declared hash, with the code bytes each was made from +// (ShaderIdentity.Compute). Entries are immutable and the map is concurrent: the render thread and a +// thread resolving draws ahead of it share one. +public sealed class ShaderIdentityMemo +{ + private const int MaxEntries = 4096; + private const long MaxEntryBytes = 1 << 20; + private const long MaxTotalBytes = 64 << 20; + + private sealed class Entry(ulong codeAddress, (ulong Address, uint SizeBytes)[] ranges, byte[][] code, ulong hash) + { + public readonly ulong CodeAddress = codeAddress; + public readonly (ulong Address, uint SizeBytes)[] Ranges = ranges; + public readonly byte[][] Code = code; + public readonly ulong Hash = hash; + } + + private readonly System.Collections.Concurrent.ConcurrentDictionary _entries = new(); + private long _bytes; + + public int Count => _entries.Count; + + internal bool CanKeep(ReadOnlySpan<(ulong Address, uint SizeBytes)> ranges) + { + var total = 0L; + foreach (var (_, sizeBytes) in ranges) + { + total += sizeBytes; + } + + return total <= MaxEntryBytes; + } + + internal bool TryGet(ICpuMemory memory, ulong codeAddress, ReadOnlySpan<(ulong Address, uint SizeBytes)> ranges, out ulong hash) + { + hash = 0; + if (!_entries.TryGetValue(codeAddress, out var entry) || entry.Ranges.Length != ranges.Length) + { + return false; + } + + for (var index = 0; index < ranges.Length; index++) + { + if (entry.Ranges[index] != ranges[index]) + { + return false; + } + } + + for (var index = 0; index < ranges.Length; index++) + { + if (!memory.TryCompare(ranges[index].Address, entry.Code[index], out var equal) || !equal) + { + return false; + } + } + + hash = entry.Hash; + return true; + } + + internal void Add(ulong codeAddress, ReadOnlySpan<(ulong Address, uint SizeBytes)> ranges, byte[][] code, ulong hash) + { + var size = 0L; + foreach (var bytes in code) + { + size += bytes.Length; + } + + // Bounded: dropping the entries only costs hashing the code again. + if (_entries.Count >= MaxEntries || Interlocked.Add(ref _bytes, size) > MaxTotalBytes) + { + _entries.Clear(); + Interlocked.Exchange(ref _bytes, size); + } + + _entries[codeAddress] = new Entry(codeAddress, ranges.ToArray(), code, hash); + } +} diff --git a/src/SharpEmu.Libs/Gpu/Pipelines/ShaderPipelineCache.cs b/src/SharpEmu.Libs/Gpu/Pipelines/ShaderPipelineCache.cs index 852f42a57..1f8a70a63 100644 --- a/src/SharpEmu.Libs/Gpu/Pipelines/ShaderPipelineCache.cs +++ b/src/SharpEmu.Libs/Gpu/Pipelines/ShaderPipelineCache.cs @@ -28,6 +28,8 @@ internal sealed partial class ShaderPipelineCache : IShaderPipelineProvider private readonly ShaderHeaderRegistry _registry; private readonly ShaderProgramCache _programs; private readonly VertexTableMetadataCache _vertexTables = new(); + // The content hashes of the shaders that declare none, so a draw does not hash their code again. + private readonly ShaderIdentityMemo _identities = new(); // The last user data array of each graphics stage, kept under _resolveGate. private uint[]? _vertexUserData; private uint[]? _pixelUserData; @@ -38,8 +40,8 @@ internal sealed partial class ShaderPipelineCache : IShaderPipelineProvider // Nothing writes a resolved draw's arrays. private Gen5PixelOutputBinding[] _lastBoundOutputs = []; private readonly PixelInputArrays _pixelInputArrays = new(); - private readonly Dictionary _graphicsPipelines = new(); - private readonly Dictionary _computePipelines = new(); + private readonly System.Collections.Concurrent.ConcurrentDictionary _graphicsPipelines = new(); + private readonly System.Collections.Concurrent.ConcurrentDictionary _computePipelines = new(); private readonly Dictionary _rayTracingComputePrograms = new(); private readonly object _gate = new(); // The options of each stage's lookup, refilled under _gate (StageCompileOptions.Detached). @@ -47,9 +49,8 @@ internal sealed partial class ShaderPipelineCache : IShaderPipelineProvider private readonly StageCompileOptions _pixelOptions = StageCompileOptions.CreateReusable(); private readonly StageCompileOptions _computeOptions = StageCompileOptions.CreateReusable(); private readonly StageCompileOptions _hullOptions = StageCompileOptions.CreateReusable(); - // The pipeline maps: a lookup takes this lock or _gate, an addition both; a cached draw so does - // not wait for the program preparation a resolving thread holds _gate for. - private readonly object _pipelineGate = new(); + // The pipeline maps are concurrent: a lookup takes no lock, so a cached draw does not wait for the + // program preparation a resolving thread holds _gate for, and an addition holds _gate. private readonly bool _strictShaders = Environment.GetEnvironmentVariable("SHARPEMU_STRICT_COMPUTE") != "0"; private readonly bool _skipRayTracing = Environment.GetEnvironmentVariable("SHARPEMU_SKIP_RT") == "1"; private readonly HashSet<(ShaderStage Stage, ulong Hash, uint CodeSize)> _reportedShaderSkips = []; @@ -187,7 +188,7 @@ private ShaderSource PrepareSource(ulong codeAddress, ShaderStage stage, string _host.SynchronizeShaderCode(address, sizeBytes); } - var hash = ShaderIdentity.Compute(_context.Memory, codeAddress, ranges, label); + var hash = ShaderIdentity.Compute(_context.Memory, codeAddress, ranges, label, _identities); var userData = UserData(registers, declaredCount, probeWrittenRegisters, codeAddress, label, ref userDataMemo); if (stage == ShaderStage.Mesh && registered.IsFused) { @@ -219,7 +220,7 @@ private ShaderSource PrepareSource(ulong codeAddress, ShaderStage stage, string // Program resolution runs on the render thread and on a thread resolving draws ahead of it; // the program, decode and materialization caches it updates are used under this lock only. - // A pipeline lookup takes _pipelineGate instead, so it does not wait for a resolution. + // A pipeline lookup takes no lock, so it does not wait for a resolution. private readonly object _resolveGate = new(); public GraphicsPrograms GetGraphicsPrograms( @@ -467,13 +468,20 @@ private GraphicsPrograms ResolveGraphicsProgramsCore( throw GpuFatal.Fatal($"The pixel program declares too many interpolators: shader=0x{pixel.Address:X16} count={inputCount}."); } + // Each bound target is resolved once for the draw: the three questions below ask about the same + // targets, and every resolution hands back a request of several hundred bytes. + Span boundSlots = stackalloc uint[(int)ContextRegisters.ColorTargetCount]; + Span boundExtents = stackalloc GpuExtent2D[(int)ContextRegisters.ColorTargetCount]; + var boundCount = ResolveBoundSlots(context, boundSlots, boundExtents); + boundSlots = boundSlots[..boundCount]; + boundExtents = boundExtents[..boundCount]; dropUnwritten = depthBound || - LeavesSingleDualSourceTarget(context, targetExportMapping, pixelProgram.PixelColorExportMasks) || - UnwrittenTargetBoundsRenderArea(context, targetExportMapping, pixelProgram.PixelColorExportMasks); + LeavesSingleDualSourceTarget(context, targetExportMapping, pixelProgram.PixelColorExportMasks, boundSlots) || + UnwrittenTargetBoundsRenderArea(context, targetExportMapping, pixelProgram.PixelColorExportMasks, boundSlots, boundExtents); Span outputModes = stackalloc byte[PixelInputInfo.TargetCount]; Span outputMappings = stackalloc ColorComponentMap[PixelInputInfo.TargetCount]; pixelOutputs = ResolveBoundTargets(context, targetExportMapping, dropUnwritten ? pixelProgram.PixelColorExportMasks : null, - outputModes, outputMappings); + boundSlots, outputModes, outputMappings); pixelInfo = PixelStageInputResolver.Resolve(_context, pixelSource.Registered, shaderInterface, outputModes, outputMappings, inputCount, depthAspectBound, stencilAspectBound, _pixelInputArrays); // SPI_PS_INPUT_CNTL can map an input to any parameter export, beyond the input count; @@ -685,23 +693,13 @@ internal static uint InterpolatedAttributeCount(Gen5ShaderProgram program) // attachments alike (RenderExecutor.DropUnwrittenColorTargets): unwrittenExportMasks is the // pixel program's export mask then, null otherwise. private Gen5PixelOutputBinding[] ResolveBoundTargets(ContextRegisters context, ReadOnlySpan targetExportMapping, - uint? unwrittenExportMasks, Span outputModes, Span outputMappings) + uint? unwrittenExportMasks, ReadOnlySpan boundSlots, Span outputModes, Span outputMappings) { var outputs = _boundOutputs; outputs.Clear(); var location = 0u; - for (var slot = 0u; slot < ContextRegisters.ColorTargetCount; slot++) + foreach (var slot in boundSlots) { - if (slot != 0 && (context.RenderTargetMaskForSlot(slot) == 0 || context.ColorTargets[slot].BaseAddress == 0)) - { - continue; - } - - if (ColorTargetResolver.Resolve(context, slot, 0, ignoreTargetMask: false, out _) is null) - { - continue; - } - if (unwrittenExportMasks is { } exportMasks && RenderExecutor.IsUnwrittenColorTarget(context, slot, targetExportMapping[(int)slot], exportMasks)) { @@ -863,16 +861,15 @@ private static bool BlendsWithSourceAlpha(ContextRegisters context, uint slot) private bool LeavesSingleDualSourceTarget( ContextRegisters context, ReadOnlySpan mappingBySlot, - uint pixelColorExportMasks) + uint pixelColorExportMasks, + ReadOnlySpan boundSlots) { - // Most draws blend no target with source alpha; skip resolving their targets. + // Most draws blend no target with source alpha. if (!_host.SupportsDualSourceBlend || !AnyTargetBlendsWithSourceAlpha(context)) { return false; } - Span boundSlots = stackalloc uint[(int)ContextRegisters.ColorTargetCount]; - boundSlots = boundSlots[..BoundColorSlots(context, boundSlots)]; if (boundSlots.Length < 2) { return false; @@ -908,10 +905,10 @@ private static bool AnyTargetBlendsWithSourceAlpha(ContextRegisters context) private static bool UnwrittenTargetBoundsRenderArea( ContextRegisters context, ReadOnlySpan mappingBySlot, - uint pixelColorExportMasks) + uint pixelColorExportMasks, + ReadOnlySpan boundSlots, + ReadOnlySpan boundExtents) { - Span boundSlots = stackalloc uint[(int)ContextRegisters.ColorTargetCount]; - boundSlots = boundSlots[..BoundColorSlots(context, boundSlots)]; if (boundSlots.Length < 2) { return false; @@ -919,14 +916,10 @@ private static bool UnwrittenTargetBoundsRenderArea( uint writtenWidth = 0, writtenHeight = 0; uint unwrittenWidth = uint.MaxValue, unwrittenHeight = uint.MaxValue; - foreach (var slot in boundSlots) + for (var index = 0; index < boundSlots.Length; index++) { - if (ColorTargetResolver.Resolve(context, slot, 0, ignoreTargetMask: false, out _) is not { } resolution) - { - continue; - } - - var extent = resolution.Extent; + var slot = boundSlots[index]; + var extent = boundExtents[index]; if (RenderExecutor.IsUnwrittenColorTarget(context, slot, mappingBySlot[(int)slot], pixelColorExportMasks)) { unwrittenWidth = Math.Min(unwrittenWidth, extent.Width); @@ -942,8 +935,9 @@ private static bool UnwrittenTargetBoundsRenderArea( return writtenWidth > unwrittenWidth || writtenHeight > unwrittenHeight; } - // The slots ResolveBoundTargets considers bound, before any are dropped as unwritten. - private static int BoundColorSlots(ContextRegisters context, Span slots) + // The slots with a color target, and their extents, before any are dropped as unwritten. + // Each target is resolved here and nowhere else in the lookup. + private static int ResolveBoundSlots(ContextRegisters context, Span slots, Span extents) { var count = 0; for (var slot = 0u; slot < ContextRegisters.ColorTargetCount; slot++) @@ -953,12 +947,13 @@ private static int BoundColorSlots(ContextRegisters context, Span slots) continue; } - if (ColorTargetResolver.Resolve(context, slot, 0, ignoreTargetMask: false, out _) is null) + if (ColorTargetResolver.Resolve(context, slot, 0, ignoreTargetMask: false, out _) is not { } resolution) { continue; } - slots[count++] = slot; + slots[count] = slot; + extents[count++] = resolution.Extent; } return count; @@ -1255,12 +1250,9 @@ private bool CreateGraphicsPipeline( lookup.PixelProgramId = pixelInput is not null ? pixelProgram.Id : 0; var lodStatistics = pixelInput is not null && _host.PixelPipelinesFollowLodStatistics && MipStatistics.TitleUsesCounters; lookup.LodStatistics = lodStatistics; - lock (_pipelineGate) + if (_graphicsPipelines.TryGetValue(lookup, out handle)) { - if (_graphicsPipelines.TryGetValue(lookup, out handle)) - { - return true; - } + return true; } var description = BuildGraphicsDescription( @@ -1294,10 +1286,7 @@ private bool CreateGraphicsPipeline( return false; } - lock (_pipelineGate) - { - _graphicsPipelines.Add(key, created); - } + _graphicsPipelines[key] = created; ShaderCacheCounters.CountGraphicsPipeline(); handle = created; @@ -1593,12 +1582,9 @@ public PipelineHandle CreateComputePipeline(ComputeInputInfo input, ShaderProgra var stage = input.Stage.Program ?? throw GpuFatal.Fatal("The compute stage has no program."); var key = new ComputePipelineKey(program.Id); - lock (_pipelineGate) + if (_computePipelines.TryGetValue(key, out var existing)) { - if (_computePipelines.TryGetValue(key, out var cached)) - { - return cached; - } + return existing; } profileScope.SwitchPhase(RenderPhaseProfile.Phase.PipelineSharedGateWait); @@ -1616,10 +1602,7 @@ public PipelineHandle CreateComputePipeline(ComputeInputInfo input, ShaderProgra } var created = _host.CreateComputePipeline(new ComputePipelineDescription { Input = input, Program = program, Stage = stage }); - lock (_pipelineGate) - { - _computePipelines.Add(key, created); - } + _computePipelines[key] = created; ShaderCacheCounters.CountComputePipeline(); return created; @@ -1637,13 +1620,10 @@ public bool TryCreateComputePipeline(ComputeInputInfo input, ShaderProgram progr var stage = input.Stage.Program ?? throw GpuFatal.Fatal("The compute stage has no program."); var key = new ComputePipelineKey(program.Id); - lock (_pipelineGate) + if (_computePipelines.TryGetValue(key, out var existing)) { - if (_computePipelines.TryGetValue(key, out var cached)) - { - handle = cached; - return true; - } + handle = existing; + return true; } profileScope.SwitchPhase(RenderPhaseProfile.Phase.PipelineSharedGateWait); @@ -1663,10 +1643,7 @@ public bool TryCreateComputePipeline(ComputeInputInfo input, ShaderProgram progr return false; } - lock (_pipelineGate) - { - _computePipelines.Add(key, created); - } + _computePipelines[key] = created; ShaderCacheCounters.CountComputePipeline(); handle = created; diff --git a/src/SharpEmu.Libs/Gpu/Pipelines/VertexInputResolver.cs b/src/SharpEmu.Libs/Gpu/Pipelines/VertexInputResolver.cs index 26f1ef97d..4a1b8e795 100644 --- a/src/SharpEmu.Libs/Gpu/Pipelines/VertexInputResolver.cs +++ b/src/SharpEmu.Libs/Gpu/Pipelines/VertexInputResolver.cs @@ -15,6 +15,24 @@ public sealed class VertexTableMetadata public int VertexBufferRegister { get; init; } = NoRegister; public int VertexAttributeRegister { get; init; } = NoRegister; public uint[] InputSemantics { get; init; } = []; + + // The attributes and buffers decoded from the tables the guest held at the last draw, with the + // bytes they were decoded from. Replaced as a whole, never mutated. + internal VertexLayoutMemo? Layout; +} + +// A draw's decoded vertex layout and the guest bytes it came from: the attribute table words from the lowest +// to the highest semantic the program reads, and the buffer descriptors from index zero to the highest index +// an attribute names. A draw that reads the same bytes at the same tables has the same layout. +internal sealed class VertexLayoutMemo(ulong attributeTable, ulong bufferTable, byte[] attributeBytes, byte[] bufferBytes, + VertexAttributeResource[] attributes, VertexInputBuffer[] buffers) +{ + public readonly ulong AttributeTable = attributeTable; + public readonly ulong BufferTable = bufferTable; + public readonly byte[] AttributeBytes = attributeBytes; + public readonly byte[] BufferBytes = bufferBytes; + public readonly VertexAttributeResource[] Attributes = attributes; + public readonly VertexInputBuffer[] Buffers = buffers; } // The header-derived tables of vertex programs already read, keyed by the registered header @@ -84,13 +102,17 @@ public static VertexInputInfo ResolveVertexInputs(CpuContext context, Registered $"user_data=[{string.Join(",", userData.ToArray().Select(word => $"{word:X8}"))}]."); } - // Per-thread scratch: the list never outlives the call; the draw keeps the array. - var resources = _attributeScratch ??= new List(VertexInputInfo.MaxBuffers); - resources.Clear(); - ApplySemantics(context, metadata.InputSemantics, attributeTable, bufferTable, shader.CodeAddress, resources); - buffers = MergeBuffers(resources, shader.CodeAddress); - attributes = resources.ToArray(); - resources.Clear(); + if (metadataCache is null || + !TryResolveLayout(context, metadata, attributeTable, bufferTable, shader.CodeAddress, out attributes, out buffers)) + { + // Per-thread scratch: the list never outlives the call; the draw keeps the array. + var resources = _attributeScratch ??= new List(VertexInputInfo.MaxBuffers); + resources.Clear(); + ApplySemantics(context, metadata.InputSemantics, attributeTable, bufferTable, shader.CodeAddress, resources); + buffers = MergeBuffers(resources, shader.CodeAddress); + attributes = resources.ToArray(); + resources.Clear(); + } } return new VertexInputInfo @@ -298,9 +320,6 @@ private static void ApplySemantics( } var index = attribute & 0x1Fu; - var format = (attribute >> 5) & 0x1FFu; - var offset = (attribute >> 14) & 0xFFFu; - var fetchIndex = (attribute >> 26) & 0x1u; if (index >= VertexInputInfo.MaxBuffers) { throw GpuFatal.Fatal($"The vertex attribute names a buffer outside the table: shader=0x{shaderAddress:X16} index={index}."); @@ -319,35 +338,139 @@ private static void ApplySemantics( throw GpuFatal.Fatal($"The vertex buffer table is unreadable: shader=0x{shaderAddress:X16} address=0x{descriptorAddress:X16}."); } - var descriptor = BufferDescriptorWords.From(descriptorWords); - if (format != 0) + AddResource(semantic, attribute, descriptorWords, resources); + } + } + + // The attribute one semantic names: its attribute word's format and offset applied to the buffer descriptor it indexes. + private static void AddResource(ShaderInputSemantic semantic, uint attribute, ReadOnlySpan descriptorWords, + List resources) + { + var format = (attribute >> 5) & 0x1FFu; + var offset = (attribute >> 14) & 0xFFFu; + var fetchIndex = (attribute >> 26) & 0x1u; + var descriptor = BufferDescriptorWords.From(descriptorWords); + if (format != 0) + { + var bufferFormat = format >> 2; + var channels = (format & 0x3u) + 1u; + descriptor = descriptor with { - var bufferFormat = format >> 2; - var channels = (format & 0x3u) + 1u; - descriptor = descriptor with - { - Word3 = (descriptor.Word3 & ~((0x7Fu << 12) | 0xFFFu)) | - ((bufferFormat & 0x7Fu) << 12) | - DestinationSelectForChannels(channels), - }; + Word3 = (descriptor.Word3 & ~((0x7Fu << 12) | 0xFFFu)) | + ((bufferFormat & 0x7Fu) << 12) | + DestinationSelectForChannels(channels), + }; + } + + if (offset != 0) + { + descriptor = descriptor.WithAddress(descriptor.Address + offset); + } + + resources.Add(new VertexAttributeResource( + descriptor, + (int)semantic.HardwareMapping, + (int)semantic.SizeInElements, + (int)semantic.Semantic, + fetchIndex, + BufferIndex: -1, + OffsetBytes: 0)); + } + + // The tables of a draw in two guest reads, one over the attribute words the program names and one over the buffer + // descriptors they index, instead of two reads per attribute (each taking the memory read lock). A draw whose + // bytes equal the last draw's at the same tables gets that draw's attributes and buffers back without decoding or + // allocating them; any other byte decodes afresh, so a guest that rewrites a table is seen on the next draw. + // False leaves the word-by-word path to decode the draw, and to report whatever stopped this one. + private static bool TryResolveLayout(CpuContext context, VertexTableMetadata metadata, ulong attributeTable, ulong bufferTable, + ulong shaderAddress, out VertexAttributeResource[] attributes, out VertexInputBuffer[] buffers) + { + attributes = []; + buffers = []; + var semantics = metadata.InputSemantics; + if (semantics.Length > VertexInputInfo.MaxBuffers) + { + return false; + } + + var lowest = uint.MaxValue; + var highest = 0u; + foreach (var word in semantics) + { + var semantic = new ShaderInputSemantic(word); + if (semantic.StaticBufferIndex || semantic.StaticAttribute) + { + return false; } - if (offset != 0) + lowest = Math.Min(lowest, semantic.Semantic); + highest = Math.Max(highest, semantic.Semantic); + } + + // A semantic is eight bits, so the words it can name span at most 256. + Span attributeBytes = stackalloc byte[256 * sizeof(uint)]; + attributeBytes = attributeBytes[..(int)((highest - lowest + 1) * sizeof(uint))]; + if (!context.Memory.TryRead(attributeTable + lowest * sizeof(uint), attributeBytes)) + { + return false; + } + + var highestBuffer = 0u; + foreach (var word in semantics) + { + var attribute = System.Buffers.Binary.BinaryPrimitives.ReadUInt32LittleEndian( + attributeBytes[(int)((new ShaderInputSemantic(word).Semantic - lowest) * sizeof(uint))..]); + var index = attribute & 0x1Fu; + if (index >= VertexInputInfo.MaxBuffers) { - descriptor = descriptor.WithAddress(descriptor.Address + offset); + return false; } - resources.Add(new VertexAttributeResource( - descriptor, - (int)semantic.HardwareMapping, - (int)semantic.SizeInElements, - (int)semantic.Semantic, - fetchIndex, - BufferIndex: -1, - OffsetBytes: 0)); + highestBuffer = Math.Max(highestBuffer, index); + } + + Span bufferBytes = stackalloc byte[VertexInputInfo.MaxBuffers * BufferDescriptorBytes]; + bufferBytes = bufferBytes[..(int)((highestBuffer + 1) * BufferDescriptorBytes)]; + if (!context.Memory.TryRead(bufferTable, bufferBytes)) + { + return false; } + + var memo = metadata.Layout; + if (memo is not null && memo.AttributeTable == attributeTable && memo.BufferTable == bufferTable && + attributeBytes.SequenceEqual(memo.AttributeBytes) && bufferBytes.SequenceEqual(memo.BufferBytes)) + { + attributes = memo.Attributes; + buffers = memo.Buffers; + return true; + } + + var resources = _attributeScratch ??= new List(VertexInputInfo.MaxBuffers); + resources.Clear(); + Span descriptorWords = stackalloc uint[BufferDescriptorBytes / sizeof(uint)]; + foreach (var word in semantics) + { + var semantic = new ShaderInputSemantic(word); + var attribute = System.Buffers.Binary.BinaryPrimitives.ReadUInt32LittleEndian( + attributeBytes[(int)((semantic.Semantic - lowest) * sizeof(uint))..]); + var descriptorBytes = bufferBytes.Slice((int)((attribute & 0x1Fu) * BufferDescriptorBytes), BufferDescriptorBytes); + for (var index = 0; index < descriptorWords.Length; index++) + { + descriptorWords[index] = System.Buffers.Binary.BinaryPrimitives.ReadUInt32LittleEndian(descriptorBytes[(index * sizeof(uint))..]); + } + + AddResource(semantic, attribute, descriptorWords, resources); + } + + buffers = MergeBuffers(resources, shaderAddress); + attributes = resources.ToArray(); + resources.Clear(); + metadata.Layout = new VertexLayoutMemo(attributeTable, bufferTable, attributeBytes.ToArray(), bufferBytes.ToArray(), attributes, buffers); + return true; } + private const int BufferDescriptorBytes = 16; + // Attributes with one stride and fetch rate whose bases lie within one record share a buffer. public static VertexInputBuffer[] MergeBuffers(List resources, ulong shaderAddress) { diff --git a/tests/SharpEmu.Libs.Tests/FakeCpuMemory.cs b/tests/SharpEmu.Libs.Tests/FakeCpuMemory.cs index 720c1b120..462e0ddf2 100644 --- a/tests/SharpEmu.Libs.Tests/FakeCpuMemory.cs +++ b/tests/SharpEmu.Libs.Tests/FakeCpuMemory.cs @@ -69,6 +69,18 @@ public bool TryWrite(ulong virtualAddress, ReadOnlySpan source) return true; } + public bool TryCompare(ulong virtualAddress, ReadOnlySpan expected, out bool equal) + { + equal = false; + if (!TryResolve(virtualAddress, expected.Length, out var offset)) + { + return false; + } + + equal = _storage.AsSpan(offset, expected.Length).SequenceEqual(expected); + return true; + } + public bool CanRead(ulong address, ulong size) => size != 0 && size <= int.MaxValue && TryResolve(address, (int)size, out _); diff --git a/tests/SharpEmu.Libs.Tests/Gpu/Pipelines/ShaderPipelineLookupCacheTests.cs b/tests/SharpEmu.Libs.Tests/Gpu/Pipelines/ShaderPipelineLookupCacheTests.cs new file mode 100644 index 000000000..48b80658d --- /dev/null +++ b/tests/SharpEmu.Libs.Tests/Gpu/Pipelines/ShaderPipelineLookupCacheTests.cs @@ -0,0 +1,401 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using System.Buffers.Binary; +using SharpEmu.HLE; +using SharpEmu.Libs.Gpu.Abstractions; +using SharpEmu.Libs.Gpu.GpuCommands.Registers; +using SharpEmu.Libs.Gpu.Images; +using SharpEmu.Libs.Gpu.Pipelines; +using SharpEmu.Libs.Gpu.Rendering; +using SharpEmu.Libs.Tests.Gpu.Images; +using SharpEmu.Libs.Tests.Gpu.Scheduling; +using Xunit; +using static SharpEmu.Libs.Tests.Gpu.Rendering.RenderExecutorFixtures; + +namespace SharpEmu.Libs.Tests.Gpu.Pipelines; + +// Counts the reads a lookup makes, so a test can tell a cached lookup from a decoded one. +internal sealed class ReadCountingMemory(ICpuMemory inner) : ICpuMemory +{ + public long Reads { get; private set; } + + public bool TryRead(ulong virtualAddress, Span destination) + { + Reads++; + return inner.TryRead(virtualAddress, destination); + } + + public bool TryWrite(ulong virtualAddress, ReadOnlySpan source) => inner.TryWrite(virtualAddress, source); + + public bool TryCompare(ulong virtualAddress, ReadOnlySpan expected, out bool equal) => + inner.TryCompare(virtualAddress, expected, out equal); + + public bool CanRead(ulong address, ulong size) => inner.CanRead(address, size); +} + +// What the per-draw lookups of a graphics draw remember between draws (the pixel semantics, the vertex layout, the +// shader identity, the header records and the pipeline maps), and that a guest that changes the bytes one of them +// came from is seen on the next draw. +[Collection(SchedulingStateCollection.Name)] +public sealed class ShaderPipelineLookupCacheTests : IDisposable +{ + private const ulong B = PipelineTestGuest.MemoryBase; + private const ulong VertexCode = B + 0x1000; + private const ulong PixelCode = B + 0x4000; + private const ulong VertexSemantics = B + 0x9200; + private const ulong AttributeTable = B + 0x9300; + private const ulong BufferTable = B + 0x9400; + private const ulong PixelSemantics = B + 0x9500; + private const uint CustomSemantic = 1u << 24; + private const uint AttributeFormat32x2Float = 257; + + private readonly FatalScope _fatal = new(); + private readonly FakeCpuMemory _memory = new(B, PipelineTestGuest.MemorySize); + private readonly ReadCountingMemory _counting; + private readonly Dictionary _headers = new(); + private readonly FakePipelineHost _host; + private readonly ShaderPipelineCache _cache; + private readonly RegisterBanks _banks = Banks(); + private readonly ColorComponentMap[] _mappings = Enumerable.Repeat(ColorComponentMap.Identity, 8).ToArray(); + + public ShaderPipelineLookupCacheTests() + { + _counting = new ReadCountingMemory(_memory); + var context = new CpuContext(_counting, Generation.Gen5); + var registry = new ShaderHeaderRegistry(context, code => _headers.TryGetValue(code, out var header) ? header : 0, _ => null); + _host = new FakePipelineHost(_counting); + _cache = new ShaderPipelineCache(context, _host, new FakeShaderCompiler(), registry); + RegisterProgram(VertexCode, B + 0x8000, 64, B + 0x9000, VertexSemantics, 3); + RegisterProgram(PixelCode, B + 0x8100, 128, 0, PixelSemantics, 4); + // The user data block of the vertex program: buffer table at s[10:11], attribute table at s[8:9]. + WriteQword(B + 0x9000, B + 0x9100); + WriteHalf(B + 0x9000 + 0x2C, 11); + for (uint type = 0; type < 11; type++) + { + WriteHalf(B + 0x9100 + type * 2, 0xFFFF); + } + + WriteHalf(B + 0x9100 + 8 * 2, 10); + WriteHalf(B + 0x9100 + 10 * 2, 8); + for (uint index = 0; index < 3; index++) + { + WriteWord(VertexSemantics + index * 4, index | (4u << 8) | (4u << 16)); + WriteAttribute(AttributeTable, index, index, offset: 0); + WriteBuffer(index, B + 0x20000 + index * 0x1000); + } + + for (uint index = 0; index < 4; index++) + { + WriteWord(PixelSemantics + index * 4, index | CustomSemantic); + } + + for (uint slot = 0; slot < 4; slot++) + { + _banks.Context.ColorTargets[slot] = RegisterWords.Color(B + 0x40000 + slot * 0x10000, 64, 64); + } + + _banks.Context.RenderTargetMask = 0xFFFF; + _banks.Context.ShaderInterface.ColorShaderMask = 0xFFFF; + _banks.Context.ShaderInterface.PixelInputControl = 4; + _banks.Shader.Vertex.ExportAddress = VertexCode; + _banks.Shader.Pixel.Address = PixelCode; + SetAttributeTable(AttributeTable); + } + + public void Dispose() => _fatal.Dispose(); + + private void SetAttributeTable(ulong address) + { + var user = _banks.Shader.Vertex.GeometryUserScalars; + user.Set(8, unchecked((uint)address), UserScalarKind.Unknown); + user.Set(9, (uint)(address >> 32), UserScalarKind.Unknown); + user.Set(10, unchecked((uint)BufferTable), UserScalarKind.Unknown); + user.Set(11, (uint)(BufferTable >> 32), UserScalarKind.Unknown); + } + + private void WriteWord(ulong address, uint value) + { + Span bytes = stackalloc byte[sizeof(uint)]; + BinaryPrimitives.WriteUInt32LittleEndian(bytes, value); + Assert.True(_memory.TryWrite(address, bytes)); + } + + private void WriteQword(ulong address, ulong value) + { + Span bytes = stackalloc byte[sizeof(ulong)]; + BinaryPrimitives.WriteUInt64LittleEndian(bytes, value); + Assert.True(_memory.TryWrite(address, bytes)); + } + + private void WriteHalf(ulong address, ushort value) + { + Span bytes = stackalloc byte[sizeof(ushort)]; + BinaryPrimitives.WriteUInt16LittleEndian(bytes, value); + Assert.True(_memory.TryWrite(address, bytes)); + } + + private void WriteAttribute(ulong table, uint semantic, uint bufferIndex, uint offset) => + WriteWord(table + semantic * 4, bufferIndex | (AttributeFormat32x2Float << 5) | (offset << 14)); + + private void WriteBuffer(uint index, ulong address, uint records = 100) + { + var entry = BufferTable + index * 16; + WriteWord(entry, unchecked((uint)address)); + WriteWord(entry + 4, (uint)((address >> 32) & 0xFFFF) | (16u << 16)); + WriteWord(entry + 8, records); + WriteWord(entry + 12, 0xFAC | (64u << 12)); + } + + // Code of the given size that starts with a no-op and ends the program. + private static byte[] Code(uint words) + { + var code = new byte[words * 4]; + BinaryPrimitives.WriteUInt32LittleEndian(code, 0xBF800000); + BinaryPrimitives.WriteUInt32LittleEndian(code.AsSpan((int)(words - 1) * 4), 0xBF810000); + return code; + } + + private void RegisterProgram(ulong code, ulong header, uint words, ulong userData, ulong semantics, uint semanticCount) + { + Assert.True(_memory.TryWrite(code, Code(words))); + var bytes = new byte[PipelineTestGuest.HeaderBytes]; + BinaryPrimitives.WriteUInt32LittleEndian(bytes.AsSpan(0x44), words * 4); + BinaryPrimitives.WriteUInt64LittleEndian(bytes.AsSpan(0x08), userData); + BinaryPrimitives.WriteUInt64LittleEndian(bytes.AsSpan(0x30), semantics); + BinaryPrimitives.WriteUInt32LittleEndian(bytes.AsSpan(0x50), semanticCount); + Assert.True(_memory.TryWrite(header, bytes)); + _headers[code] = header; + } + + private GraphicsPrograms Draw(bool pixelActive = true) + { + var programs = _cache.GetGraphicsPrograms(_banks.Shader.Vertex, _banks.Shader.Pixel, _banks.Context.ShaderInterface, + _banks.Context, _mappings, pixelActive, depthBound: false, depthAspectBound: false); + Assert.True(programs.Available); + return programs; + } + + private static ulong AddressOf(GraphicsPrograms programs, int attributeId) => + programs.VertexInput.Attributes.Single(attribute => attribute.AttributeId == attributeId).Descriptor.Address; + + [Fact] + public void ARepeatedDrawSharesTheVertexLayoutAndReadsTheTablesOnce() + { + var first = Draw(); + Draw(); + var before = _counting.Reads; + var second = Draw(); + var reads = _counting.Reads - before; + + Assert.Same(first.VertexInput.Attributes, second.VertexInput.Attributes); + Assert.Same(first.VertexInput.Buffers, second.VertexInput.Buffers); + Assert.Equal(3, second.VertexInput.Attributes.Length); + // Two header reads, two declared-hash probes, the pixel semantics and the two vertex tables: not one + // read per attribute or input. + Assert.True(reads <= 7, $"reads={reads}"); + } + + [Fact] + public void ARewrittenAttributeWordIsSeenOnTheNextDraw() + { + var first = Draw(); + Assert.Equal(B + 0x21000, AddressOf(first, 1)); + + WriteAttribute(AttributeTable, 1, 1, offset: 32); + var second = Draw(); + + Assert.NotSame(first.VertexInput.Attributes, second.VertexInput.Attributes); + Assert.Equal(B + 0x21020, AddressOf(second, 1)); + Assert.Equal(B + 0x21000, AddressOf(first, 1)); + } + + [Fact] + public void ARewrittenAttributeTablePointerIsSeenOnTheNextDraw() + { + var first = Draw(); + var moved = B + 0x9600; + for (uint index = 0; index < 3; index++) + { + WriteAttribute(moved, index, index, offset: (index + 1) * 16); + } + + SetAttributeTable(moved); + var second = Draw(); + + Assert.Equal(B + 0x22000, AddressOf(first, 2)); + Assert.Equal(B + 0x22030, AddressOf(second, 2)); + } + + [Fact] + public void ARewrittenBufferDescriptorIsSeenOnTheNextDraw() + { + var first = Draw(); + WriteBuffer(2, B + 0x30000); + var second = Draw(); + + Assert.NotSame(first.VertexInput.Buffers, second.VertexInput.Buffers); + Assert.Contains(second.VertexInput.Buffers, buffer => buffer.Address == B + 0x30000); + Assert.DoesNotContain(first.VertexInput.Buffers, buffer => buffer.Address == B + 0x30000); + } + + [Fact] + public void ARewrittenPixelSemanticIsSeenOnTheNextDraw() + { + var reads = _counting.Reads; + Assert.Equal(0xFu, Draw().PixelInput.CustomInterpolationMask); + + WriteWord(PixelSemantics + 2 * 4, 2); + Assert.Equal(0xBu, Draw().PixelInput.CustomInterpolationMask); + + // A half-float input is not a custom interpolant. + WriteWord(PixelSemantics + 2 * 4, 2u | CustomSemantic | (1u << 20)); + Assert.Equal(0xBu, Draw().PixelInput.CustomInterpolationMask); + + WriteWord(PixelSemantics + 2 * 4, 2u | CustomSemantic); + Assert.Equal(0xFu, Draw().PixelInput.CustomInterpolationMask); + Assert.True(_counting.Reads > reads); + } + + [Fact] + public void AnUnreadablePixelSemanticTableStopsTheDraw() + { + // The header names a table that is not mapped. + RegisterProgram(PixelCode, B + 0x8100, 128, 0, B + 0xFFFFF0, 8); + var failure = Assert.Throws(() => Draw()); + Assert.Contains("pixel input semantics are unreadable", failure.Message); + } + + [Fact] + public void ShaderCodeWithoutADeclaredHashIsHashedOnceAndRehashedWhenTheGuestChangesIt() + { + var memo = new ShaderIdentityMemo(); + var ranges = new[] { (VertexCode, 64u * 4) }; + var first = ShaderIdentity.Compute(_memory, VertexCode, ranges, "vertex", memo); + Assert.Equal(ShaderIdentity.Compute(_memory, VertexCode, ranges, "vertex"), first); + Assert.Equal(1, memo.Count); + + // Unchanged: remembered. The code is compared in place, not read out and hashed again. + var before = _counting.Reads; + Assert.Equal(first, ShaderIdentity.Compute(_counting, VertexCode, ranges, "vertex", memo)); + Assert.Equal(1, _counting.Reads - before); + + // One word of the code changes in place. + WriteWord(VertexCode + 8, 0xBF800001); + var second = ShaderIdentity.Compute(_counting, VertexCode, ranges, "vertex", memo); + Assert.NotEqual(first, second); + Assert.Equal(ShaderIdentity.Compute(_memory, VertexCode, ranges, "vertex"), second); + + // And back: the content decides. + WriteWord(VertexCode + 8, 0); + Assert.Equal(first, ShaderIdentity.Compute(_counting, VertexCode, ranges, "vertex", memo)); + } + + [Fact] + public void ACodeRangeThatMovesOrGrowsIsHashedAgain() + { + var memo = new ShaderIdentityMemo(); + var small = ShaderIdentity.Compute(_memory, VertexCode, [(VertexCode, 32u * 4)], "vertex", memo); + var large = ShaderIdentity.Compute(_memory, VertexCode, [(VertexCode, 64u * 4)], "vertex", memo); + + Assert.NotEqual(small, large); + Assert.Equal(ShaderIdentity.Compute(_memory, VertexCode, [(VertexCode, 64u * 4)], "vertex"), large); + } + + [Fact] + public void ADeclaredHashIsNotRemembered() + { + var memo = new ShaderIdentityMemo(); + var code = Code(64); + // The marker's info block follows (word 1 + 1) pairs of dwords from the start; its hash is at +16 and +20. + BinaryPrimitives.WriteUInt32LittleEndian(code, ShaderBinaryInfo.MarkerWord); + BinaryPrimitives.WriteUInt32LittleEndian(code.AsSpan(4), 7); + BinaryPrimitives.WriteUInt32LittleEndian(code.AsSpan((7 + 1) * 8 + 16), 0x1234); + BinaryPrimitives.WriteUInt32LittleEndian(code.AsSpan((7 + 1) * 8 + 20), 0x5678); + Assert.True(_memory.TryWrite(VertexCode, code)); + + Assert.Equal(0x5678_0000_1234UL, ShaderIdentity.Compute(_memory, VertexCode, [(VertexCode, 64u * 4)], "vertex", memo)); + Assert.Equal(0, memo.Count); + } + + [Fact] + public void ARewrittenShaderCodeIsSeenByTheNextDraw() + { + var first = Draw(); + WriteWord(PixelCode + 8, 0xBF800002); + var second = Draw(); + + Assert.NotEqual(first.Pixel.Id, second.Pixel.Id); + Assert.Equal(first.Vertex.Id, second.Vertex.Id); + } + + [Fact] + public void ARepeatedHeaderLookupSharesItsRecordAndAChangedHeaderReplacesIt() + { + var registry = new ShaderHeaderRegistry(new CpuContext(_memory, Generation.Gen5), + code => _headers.TryGetValue(code, out var header) ? header : 0, _ => null); + var first = registry.Require(PixelCode, "pixel"); + Assert.Same(first, registry.Require(PixelCode, "pixel")); + + RegisterProgram(PixelCode, B + 0x8100, 96, 0, PixelSemantics, 4); + var changed = registry.Require(PixelCode, "pixel"); + Assert.NotSame(first, changed); + Assert.Equal(96u * 4, changed.CodeSizeBytes); + Assert.Same(changed, registry.Require(PixelCode, "pixel")); + } + + [Fact] + public void ConcurrentHeaderLookupsAllSeeTheCurrentRecord() + { + var registry = new ShaderHeaderRegistry(new CpuContext(_memory, Generation.Gen5), + code => _headers.TryGetValue(code, out var header) ? header : 0, _ => null); + Parallel.For(0, 8, _ => + { + for (var round = 0; round < 2000; round++) + { + Assert.Equal(128u * 4, registry.Require(PixelCode, "pixel").CodeSizeBytes); + } + }); + + // Racing first lookups may each make a record; afterwards every lookup gets the one that stuck. + var settled = registry.Require(PixelCode, "pixel"); + Assert.Same(settled, registry.Require(PixelCode, "pixel")); + } + + [Fact] + public void ADroppedColorTargetLeavesTheNextDrawsPixelOutputs() + { + Assert.Equal(4, Draw().PixelInput.TargetOutputModes.Count(mode => mode != 0)); + + _banks.Context.ColorTargets[2] = default; + Assert.Equal(3, Draw().PixelInput.TargetOutputModes.Count(mode => mode != 0)); + + _banks.Context.ColorTargets[2] = RegisterWords.Color(B + 0x60000, 64, 64); + Assert.Equal(4, Draw().PixelInput.TargetOutputModes.Count(mode => mode != 0)); + } + + [Fact] + public void APipelineIsLookedUpWithoutBeingCreatedAgain() + { + var programs = Draw(); + var depth = default(DepthAttachmentState); + var rendering = default(RenderingState); + Assert.True(_cache.TryCreateGraphicsPipeline([], in depth, programs.VertexInput, programs.PixelInput, _banks.Context, in rendering, + GpuPrimitiveTopology.TriangleList, false, false, programs.Vertex, programs.Pixel, out var first)); + var created = _host.GraphicsPipelines.Count; + + var again = Draw(); + Assert.True(_cache.TryCreateGraphicsPipeline([], in depth, again.VertexInput, again.PixelInput, _banks.Context, in rendering, + GpuPrimitiveTopology.TriangleList, false, false, again.Vertex, again.Pixel, out var second)); + + Assert.Equal(first, second); + Assert.Equal(created, _host.GraphicsPipelines.Count); + Assert.Equal(1, _cache.GraphicsPipelineCount); + + // A different pipeline state is a different pipeline. + Assert.True(_cache.TryCreateGraphicsPipeline([], in depth, again.VertexInput, again.PixelInput, _banks.Context, in rendering, + GpuPrimitiveTopology.LineList, false, false, again.Vertex, again.Pixel, out var other)); + Assert.NotEqual(first, other); + Assert.Equal(2, _cache.GraphicsPipelineCount); + } +} From 15ae93f38485c146907c7d454df85b35bf0c0c48 Mon Sep 17 00:00:00 2001 From: kuba Date: Mon, 26 Oct 2026 05:53:16 +0100 Subject: [PATCH 13/82] [Kernel] Short sleeps and yields nap on an escalating schedule; native QoS threads; exit ends the session; thread-time and memset usleep/nanosleep/sched_yield/scePthreadYield: once a thread had polled for 500 us every sub-ms sleep slept max(requested, 250 us), so back-to-back usleep(100) cost ~290 us (the game's GPU-label poll). Requests at or above SHARPEMU_USLEEP_NAP_FLOOR_US (50) are now honoured; tiny requests and yields nap from the floor, rising linearly to SHARPEMU_USLEEP_NAP_US (250) over SHARPEMU_USLEEP_NAP_RAMP_US (4000). Existing env names unchanged; SHARPEMU_USLEEP_NAP_LEGACY=1 restores the old fixed policy. Measured (test eboot, usleep(100) back-to-back p50): 288-298 us -> 130 us; threadsim d streak 320 -> 128 us; threadsim b usleep(100) poll start p50 553 -> 388 us, p99 1820 -> 1096 us, idle worker CPU 0.19 -> 0.51 cores (13 workers, the cost of really sleeping 100 us). QoS: CommandStreamWorker and the CLI emulation thread now start as native HostQosThread (user-interactive) threads; pthread_set_qos_class_self_np returns EPERM on CoreCLR threads. CommandStreamWorker.WorkerQosClass reads the class back (0x21, tested). Managed children of a native thread get no class (measured), so guest workers are not promoted. exit()/_exit() end the host session (SHARPEMU_EXIT_TERMINATES=0 restores the old behaviour); the emulator now exits by itself after the guest's exit(). Adds sceKernelGetThreadTime and pthread_getcpuclockid (calling thread). Large memset fills the mapping in place (managed export 18-20 -> 24-30 GB/s). --- src/SharpEmu.CLI/Program.cs | 51 ++++--- .../Gpu/GpuCommands/CommandStreamWorker.cs | 51 +++++-- src/SharpEmu.Libs/Kernel/KernelExports.cs | 39 ++++- .../Kernel/KernelMemoryCompatExports.cs | 39 ++++- .../Kernel/KernelPthreadCompatExports.cs | 16 +- .../Kernel/KernelRuntimeCompatExports.cs | 135 +++++++++++++++-- .../Threading/HostThreadCpuTime.cs | 61 ++++++++ .../Kernel/KernelExitAndThreadTimeTests.cs | 138 ++++++++++++++++++ .../Kernel/SleepAndPriorityQosTests.cs | 57 +++++++- 9 files changed, 537 insertions(+), 50 deletions(-) create mode 100644 src/SharpEmu.Libs/Threading/HostThreadCpuTime.cs create mode 100644 tests/SharpEmu.Libs.Tests/Kernel/KernelExitAndThreadTimeTests.cs diff --git a/src/SharpEmu.CLI/Program.cs b/src/SharpEmu.CLI/Program.cs index f6df90faf..f127cdcbb 100644 --- a/src/SharpEmu.CLI/Program.cs +++ b/src/SharpEmu.CLI/Program.cs @@ -139,17 +139,8 @@ private static int Run(string[] args) // event ownership. Emulation remains on a worker thread. var exitCode = 0; HostMainThread.Enable(); - var emulation = new Thread(() => - { - // This thread runs the guest's main thread: it drives the ELF - // entry point and everything the title does before it spawns a - // pthread of its own. Frame deadlines live here, so it asks for - // the same QoS class as the presenter. Guest pthreads inherit - // their class from whoever creates them and are deliberately - // left alone — a title with thirty worker threads must not have - // all of them outranking the presenter. - SharpEmu.HLE.Host.HostThreadQualityOfService.TryApplyToCurrentThread( - SharpEmu.HLE.Host.HostThreadQos.UserInteractive); + void RunEmulation() + { try { exitCode = RunEmulator(args, isMitigatedChild); @@ -158,13 +149,39 @@ private static int Run(string[] args) { HostMainThread.Shutdown(); } - }, 32 * 1024 * 1024) - { - Name = "SharpEmu Emulation", - }; - emulation.Start(); + } + + // This thread runs the guest's main thread: it drives the ELF entry point and + // everything the title does before it spawns a pthread of its own. Frame deadlines + // live here, so it takes the same QoS class as the presenter. Darwin refuses + // pthread_set_qos_class_self_np on a thread CoreCLR created (EPERM, measured), so + // the class goes into the attributes of a natively created thread instead. + // Guest pthreads are managed threads and do not inherit it (measured: a managed + // child of a user-interactive thread reports no class), which is what keeps a + // title with thirty worker threads from outranking the presenter. + var emulationQos = SharpEmu.HLE.Host.HostQosThread.TryStart( + "SharpEmu Emulation", SharpEmu.HLE.Host.HostThreadQos.UserInteractive, 32 * 1024 * 1024, + RunEmulation, out _); + Thread? emulation = null; + if (emulationQos is null) + { + emulation = new Thread(RunEmulation, 32 * 1024 * 1024) + { + Name = "SharpEmu Emulation", + }; + emulation.Start(); + } + HostMainThread.Pump(); - emulation.Join(); + if (emulationQos is not null) + { + emulationQos.Join(Timeout.Infinite); + } + else + { + emulation!.Join(); + } + return exitCode; } diff --git a/src/SharpEmu.Libs/Gpu/GpuCommands/CommandStreamWorker.cs b/src/SharpEmu.Libs/Gpu/GpuCommands/CommandStreamWorker.cs index 5a4deb944..94034a986 100644 --- a/src/SharpEmu.Libs/Gpu/GpuCommands/CommandStreamWorker.cs +++ b/src/SharpEmu.Libs/Gpu/GpuCommands/CommandStreamWorker.cs @@ -10,7 +10,9 @@ public sealed class CommandStreamWorker private readonly ICommandStreamHost _host; private readonly Func _hasPendingCommands; private readonly bool _cancelBlockedAtStop; - private readonly Thread _thread; + private readonly string _threadName; + private Thread? _thread; + private SharpEmu.HLE.Host.HostQosThread? _qosThread; private readonly Action? _onTerminalFailure; private Exception? _failure; @@ -21,21 +23,51 @@ public CommandStreamWorker(CommandStreamQueue queue, ICommandStreamHost host, Fu _hasPendingCommands = hasPendingCommands; _cancelBlockedAtStop = cancelBlockedAtStop; _onTerminalFailure = onTerminalFailure; - _thread = new Thread(Run) { IsBackground = true, Name = threadName }; + _threadName = threadName; } - public bool IsWorkerThread => Thread.CurrentThread == _thread; + /// The host QoS class the worker thread runs at (Darwin qos_class_t; 0x21 is user-interactive), read back from the thread itself; 0 until it starts or off macOS. + public uint WorkerQosClass => Volatile.Read(ref _workerQosClass); - public void Start() => _thread.Start(); + private uint _workerQosClass; + + // Recorded by Run itself so the answer is right from the worker's first instruction, + // before Start has stored the native thread handle. + private volatile Thread? _runThread; + + public bool IsWorkerThread => ReferenceEquals(Thread.CurrentThread, _runThread); + + public void Start() + { + // The GPU submission thread: every guest draw and every flip passes through here on its way + // to the backend, so it sits on the frame deadline just as much as the presenter does. + // pthread_set_qos_class_self_np fails with EPERM on a CoreCLR-created thread, so the class + // goes into the pthread attributes of a thread created natively; the managed thread below + // is the fallback for hosts without that (every non-macOS host). + _qosThread = SharpEmu.HLE.Host.HostQosThread.TryStart( + _threadName, SharpEmu.HLE.Host.HostThreadQos.UserInteractive, 16 * 1024 * 1024, Run, out _); + if (_qosThread is null) + { + _thread = new Thread(Run) { IsBackground = true, Name = _threadName }; + _thread.Start(); + } + } // Stops accepting work, drains the queue and joins; a worker failure is rethrown here. public IdleOutcome Stop() { _queue.StopAccepting(); _queue.Wake(); - if (_thread.IsAlive && !IsWorkerThread) + if (!IsWorkerThread) { - _thread.Join(); + if (_qosThread is { } qosThread) + { + qosThread.Join(Timeout.Infinite); + } + else if (_thread is { IsAlive: true } thread) + { + thread.Join(); + } } if (_failure is not null) @@ -48,11 +80,8 @@ public IdleOutcome Stop() private void Run() { - // The GPU submission thread: every guest draw and every flip passes - // through here on its way to the backend, so it sits on the frame - // deadline just as much as the presenter does. - SharpEmu.HLE.Host.HostThreadQualityOfService.TryApplyToCurrentThread( - SharpEmu.HLE.Host.HostThreadQos.UserInteractive); + _runThread = Thread.CurrentThread; + Volatile.Write(ref _workerQosClass, SharpEmu.HLE.Host.HostThreadQualityOfService.GetCurrentThreadClassForDiagnostics()); try { for (;;) diff --git a/src/SharpEmu.Libs/Kernel/KernelExports.cs b/src/SharpEmu.Libs/Kernel/KernelExports.cs index 61eb66c97..3b835cc9e 100644 --- a/src/SharpEmu.Libs/Kernel/KernelExports.cs +++ b/src/SharpEmu.Libs/Kernel/KernelExports.cs @@ -100,10 +100,43 @@ public static int Exit(CpuContext ctx) var status = unchecked((int)ctx[CpuRegister.Rdi]); Console.Error.WriteLine($"[LOADER][INFO] exit(status={status})"); GuestThreadExecution.RequestCurrentEntryExit("exit", status); + TerminateProcessForGuestExit(status); ctx[CpuRegister.Rax] = unchecked((ulong)status); return (int)OrbisGen2Result.ORBIS_GEN2_OK; } + // exit()/_exit() end the whole process on the console. Unwinding only the calling + // thread left every other guest thread running and the emulator alive until the user + // stopped it. SHARPEMU_EXIT_TERMINATES=0 restores the old behaviour. + private static readonly bool _exitTerminatesProcess = + !string.Equals(Environment.GetEnvironmentVariable("SHARPEMU_EXIT_TERMINATES"), "0", StringComparison.Ordinal); + + /// What a guest exit does to the host session; tests replace it. + internal static Action GuestExitTermination = RequestHostSessionEnd; + + private static void TerminateProcessForGuestExit(int status) + { + if (_exitTerminatesProcess) + { + GuestExitTermination(status); + } + } + + private static void RequestHostSessionEnd(int status) + { + Console.Error.WriteLine($"[LOADER][INFO] guest exit(status={status}): ending the host session"); + if (HostMainThread.IsAvailable) + { + // The command-line host: the same cooperative stop as SIGTERM (guest threads + // unwind, audio and the GPU worker drain, the window closes, the process exits). + VideoOut.VideoOutExports.NotifyHostInterrupt(); + return; + } + + // An embedding host (the GUI) owns the process and ends the session itself. + HostSessionControl.RequestShutdown("guest-exit"); + } + [SysAbiExport( Nid = "XKRegsFpEpk", ExportName = "catchReturnFromMain", @@ -217,7 +250,11 @@ public static int PthreadCxaFinalize(CpuContext ctx) LibraryName = "libKernel")] public static int UnderscoreExit(CpuContext ctx) { - _ = ctx; + var status = unchecked((int)ctx[CpuRegister.Rdi]); + Console.Error.WriteLine($"[LOADER][INFO] _exit(status={status})"); + GuestThreadExecution.RequestCurrentEntryExit("_exit", status); + TerminateProcessForGuestExit(status); + ctx[CpuRegister.Rax] = unchecked((ulong)status); return (int)OrbisGen2Result.ORBIS_GEN2_OK; } diff --git a/src/SharpEmu.Libs/Kernel/KernelMemoryCompatExports.cs b/src/SharpEmu.Libs/Kernel/KernelMemoryCompatExports.cs index 004ae2d54..7ef30fb37 100644 --- a/src/SharpEmu.Libs/Kernel/KernelMemoryCompatExports.cs +++ b/src/SharpEmu.Libs/Kernel/KernelMemoryCompatExports.cs @@ -28,6 +28,9 @@ public static partial class KernelMemoryCompatExports internal const FileShare GuestFileShare = FileShare.ReadWrite | FileShare.Delete; private const int WideCharSize = sizeof(ushort); private const int MemsetChunkSize = 16 * 1024; + // Clears at least this large fill the guest mapping in place; smaller ones stay on the staged path. + private const ulong MemsetInPlaceThreshold = 4096; + private const int MemsetInPlaceChunkSize = 256 * 1024 * 1024; private static readonly byte[] _zeroChunk = new byte[MemsetChunkSize]; private const int O_WRONLY = 0x1; private const int O_RDWR = 0x2; @@ -453,9 +456,39 @@ public static int Memset(CpuContext ctx) return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; } + // A large clear fills the guest's host mapping in place (the same canonical alias the + // memcpy import copies through): a native vector fill with the same write tracking as + // TryWrite, instead of re-copying a staging chunk across it, which ran at a quarter of + // the speed of a guest-side fill. A range this memory cannot hand out in place, or a + // watched one, declines and the staged loop below writes the rest. + var filledInPlace = 0UL; + if (length >= MemsetInPlaceThreshold) + { + while (filledInPlace < length) + { + var take = (int)Math.Min(length - filledInPlace, MemsetInPlaceChunkSize); + if (!ctx.Memory.TryFillInPlace( + destination + filledInPlace, take, value, + static (fillValue, span) => { span.Fill(fillValue); return span.Length; }, + out var filled) || + filled <= 0) + { + break; + } + + filledInPlace += (ulong)filled; + } + + if (filledInPlace == length) + { + ctx[CpuRegister.Rax] = destination; + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + } + } + // Rent may hand back a larger array than requested; only the first chunkLength // bytes are filled, so the loop must cap at chunkLength rather than chunk.Length. - var chunkLength = (int)Math.Min(length, (ulong)MemsetChunkSize); + var chunkLength = (int)Math.Min(length - filledInPlace, (ulong)MemsetChunkSize); var chunk = value == 0 ? _zeroChunk : ArrayPool.Shared.Rent(chunkLength); if (value != 0) { @@ -464,8 +497,8 @@ public static int Memset(CpuContext ctx) try { - var remaining = length; - var cursor = destination; + var remaining = length - filledInPlace; + var cursor = destination + filledInPlace; while (remaining > 0) { var take = (int)Math.Min((ulong)chunkLength, remaining); diff --git a/src/SharpEmu.Libs/Kernel/KernelPthreadCompatExports.cs b/src/SharpEmu.Libs/Kernel/KernelPthreadCompatExports.cs index 3fa54ae17..ef850dd90 100644 --- a/src/SharpEmu.Libs/Kernel/KernelPthreadCompatExports.cs +++ b/src/SharpEmu.Libs/Kernel/KernelPthreadCompatExports.cs @@ -333,7 +333,9 @@ public static int PthreadEqual(CpuContext ctx) // A guest that polls with sched_yield / scePthreadYield burns a core exactly like a short-usleep // poller. Same streak policy as KernelRuntimeCompatExports.ShortUsleep: yields less than 2 ms // apart form a streak; once a thread has yielded continuously for SHARPEMU_USLEEP_NAP_AFTER_US - // (default 500 us) each further yield sleeps SHARPEMU_USLEEP_NAP_US (default 250 us) instead. + // (default 500 us) each further yield naps instead, escalating from SHARPEMU_USLEEP_NAP_FLOOR_US + // (50 us) to SHARPEMU_USLEEP_NAP_US (250 us) over SHARPEMU_USLEEP_NAP_RAMP_US (4 ms); with + // SHARPEMU_USLEEP_NAP_LEGACY=1 every yield naps the full SHARPEMU_USLEEP_NAP_US. // SHARPEMU_YIELD_NAP=0 restores the bare Thread.Yield(). private static readonly bool _yieldNapEnabled = !string.Equals(Environment.GetEnvironmentVariable("SHARPEMU_YIELD_NAP"), "0", StringComparison.Ordinal) && @@ -369,9 +371,15 @@ internal static bool YieldNapPolicy(ref long streakStart, ref long lastYield, lo public static int PthreadYield(CpuContext ctx) { _ = ctx; - if (_yieldNapEnabled && YieldNapStreak(Stopwatch.GetTimestamp())) - { - HostTiming.SleepMicroseconds(KernelRuntimeCompatExports.UsleepNapMicroseconds); + var now = Stopwatch.GetTimestamp(); + if (_yieldNapEnabled && YieldNapStreak(now)) + { + // A yield carries no duration, so it naps for the escalating floor-to-ceiling + // short-sleep schedule (KernelRuntimeCompatExports.NapPolicyMicroseconds). + HostTiming.SleepMicroseconds(KernelRuntimeCompatExports.NapPolicyMicroseconds( + now - _yieldStreakStart, 0, KernelRuntimeCompatExports.UsleepNapMicroseconds, + KernelRuntimeCompatExports.UsleepNapFloorMicroseconds, KernelRuntimeCompatExports.UsleepNapRampMicroseconds, + KernelRuntimeCompatExports._usleepNapAfterTicks, KernelRuntimeCompatExports.UsleepNapLegacy, out _)); _lastYield = Stopwatch.GetTimestamp(); return (int)OrbisGen2Result.ORBIS_GEN2_OK; } diff --git a/src/SharpEmu.Libs/Kernel/KernelRuntimeCompatExports.cs b/src/SharpEmu.Libs/Kernel/KernelRuntimeCompatExports.cs index 6c46db6bf..9e2a5dbf1 100644 --- a/src/SharpEmu.Libs/Kernel/KernelRuntimeCompatExports.cs +++ b/src/SharpEmu.Libs/Kernel/KernelRuntimeCompatExports.cs @@ -94,12 +94,25 @@ public static class KernelRuntimeCompatExports // Adaptive nap for usleep(<1 ms) polling loops (a worker pool waiting for jobs). The // first short sleeps of a streak sleep exactly what they ask for, so a wait that ends - // quickly stays fast; once one thread has been polling for SHARPEMU_USLEEP_NAP_AFTER_US (default 500 us) - // its short sleeps really sleep, for max(requested, SHARPEMU_USLEEP_NAP_US) (default - // 250 us; 0 turns naps off). A streak ends when the thread spends more than 2 ms - // between short sleeps or sleeps for 1 ms or more. Per thread, not per title. + // quickly stays fast. Once one thread has been polling for SHARPEMU_USLEEP_NAP_AFTER_US + // (default 500 us) its short sleeps start to nap, but the nap escalates instead of + // jumping to its ceiling: it starts at SHARPEMU_USLEEP_NAP_FLOOR_US (default 50 us) and + // rises linearly to SHARPEMU_USLEEP_NAP_US (default 250 us; 0 turns naps off) over the + // next SHARPEMU_USLEEP_NAP_RAMP_US (default 4000 us), so a short wait stays responsive + // and only a long idle spin settles at the cheap ceiling. A request at or above the + // floor is never stretched: usleep(100) sleeps ~100 us (a guest's GPU-label poll used to + // pay 285 us per iteration). A streak ends when the thread spends more than 2 ms between + // short sleeps or sleeps for 1 ms or more. Per thread, not per title. + // SHARPEMU_USLEEP_NAP_LEGACY=1 restores the old fixed policy: after the streak window + // every short sleep sleeps max(requested, SHARPEMU_USLEEP_NAP_US). internal static readonly long UsleepNapMicroseconds = ReadNonNegativeEnvironmentInteger("SHARPEMU_USLEEP_NAP_US", 250); + internal static readonly long UsleepNapFloorMicroseconds = + Math.Min(ReadNonNegativeEnvironmentInteger("SHARPEMU_USLEEP_NAP_FLOOR_US", 50), UsleepNapMicroseconds); + internal static readonly long UsleepNapRampMicroseconds = + ReadNonNegativeEnvironmentInteger("SHARPEMU_USLEEP_NAP_RAMP_US", 4000); + internal static readonly bool UsleepNapLegacy = + string.Equals(Environment.GetEnvironmentVariable("SHARPEMU_USLEEP_NAP_LEGACY"), "1", StringComparison.Ordinal); internal static readonly long _usleepNapAfterTicks = ReadNonNegativeEnvironmentInteger("SHARPEMU_USLEEP_NAP_AFTER_US", 500) * Stopwatch.Frequency / 1_000_000; internal static readonly long _usleepStreakGapTicks = Stopwatch.Frequency * 2 / 1000; @@ -115,6 +128,54 @@ private static long ReadNonNegativeEnvironmentInteger(string name, long fallback ? value : fallback; + /// + /// How long a short sleep (or a yield, 0) that + /// belongs to a streak of should really sleep. Returns + /// and sets false + /// while no nap applies; the arguments are the policy's tunables so tests can pin them. + /// + internal static long NapPolicyMicroseconds( + long streakAgeTicks, + long requestedMicroseconds, + long napMicroseconds, + long floorMicroseconds, + long rampMicroseconds, + long afterTicks, + bool legacy, + out bool napped) + { + napped = false; + if (napMicroseconds <= 0 || streakAgeTicks < afterTicks) + { + return requestedMicroseconds; + } + + if (legacy) + { + napped = true; + return Math.Max(requestedMicroseconds, napMicroseconds); + } + + // A request the guest made on purpose is honoured; only tiny requests and yields, + // which carry no duration, are the busy-poll backoff the nap exists for. + if (requestedMicroseconds >= floorMicroseconds && requestedMicroseconds > 0) + { + return requestedMicroseconds; + } + + var floor = Math.Min(floorMicroseconds, napMicroseconds); + var nap = napMicroseconds; + if (rampMicroseconds > 0 && napMicroseconds > floor) + { + var rampTicks = rampMicroseconds * Stopwatch.Frequency / 1_000_000; + var progress = Math.Min(1.0, (double)(streakAgeTicks - afterTicks) / rampTicks); + nap = floor + (long)((napMicroseconds - floor) * progress); + } + + napped = true; + return Math.Max(requestedMicroseconds, nap); + } + /// /// The sub-millisecond usleep policy (see ): returns /// true when this call napped, false when it slept just the requested time. @@ -128,13 +189,6 @@ internal static bool ShortUsleep(ulong micros) } _lastShortUsleep = now; - if (UsleepNapMicroseconds > 0 && now - _shortUsleepStreakStart >= _usleepNapAfterTicks) - { - HostTiming.SleepMicroseconds(Math.Max((long)micros, UsleepNapMicroseconds)); - _lastShortUsleep = Stopwatch.GetTimestamp(); - return true; - } - // Like the FreeBSD kernel (kern_clock_nanosleep always blocks for a nonzero // interval), a short sleep never returns before its requested time. A bare yield // returned in well under a microsecond, so a guest that counts usleep(1) calls as @@ -142,8 +196,16 @@ internal static bool ShortUsleep(ulong micros) // and before the streak above was old enough to nap. The wait is an // absolute-deadline host wait (mach_wait_until on macOS), never a millisecond // round-up. - HostTiming.SleepMicroseconds((long)micros); - return false; + var sleepMicros = NapPolicyMicroseconds( + now - _shortUsleepStreakStart, (long)micros, UsleepNapMicroseconds, UsleepNapFloorMicroseconds, + UsleepNapRampMicroseconds, _usleepNapAfterTicks, UsleepNapLegacy, out var napped); + HostTiming.SleepMicroseconds(sleepMicros); + if (napped) + { + _lastShortUsleep = Stopwatch.GetTimestamp(); + } + + return napped; } private static readonly bool _stopwatchTicksAreNanoseconds = @@ -464,6 +526,53 @@ public static int KernelGetProcessTime(CpuContext ctx) return (int)OrbisGen2Result.ORBIS_GEN2_OK; } + /// + /// The CPU time the calling thread has consumed, in microseconds (the unit of + /// sceKernelGetProcessTime). The SDK headers name no such function; this is the + /// thread-scoped sibling of the process-time exports, kept for titles that profile their + /// own worker threads with it. + /// + [SysAbiExport( + Nid = "afZu6Sswlr4", + ExportName = "sceKernelGetThreadTime", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libKernel")] + public static int KernelGetThreadTime(CpuContext ctx) + { + ctx[CpuRegister.Rax] = unchecked((ulong)Math.Max(0, Threading.HostThreadCpuTime.ReadCurrentMicroseconds())); + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + } + + /// + /// int pthread_getcpuclockid(pthread_t, clockid_t *), the one thread-CPU-time API the + /// SDK headers (pthread.h) declare. The clock id of the calling thread is + /// CLOCK_THREAD_CPUTIME_ID (14), which clock_gettime reads; another thread's CPU clock has + /// no id the emulator's clock_gettime could serve, so that reports ENOTSUP. + /// + [SysAbiExport( + Nid = "nnoxZUHK+SA", + ExportName = "pthread_getcpuclockid", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libKernel")] + public static int PthreadGetCpuClockId(CpuContext ctx) + { + var thread = ctx[CpuRegister.Rdi]; + var clockIdAddress = ctx[CpuRegister.Rsi]; + if (clockIdAddress == 0) + { + return Threading.GuestErrno.EINVAL; + } + + if (thread != KernelPthreadState.GetCurrentThreadHandle()) + { + return Threading.GuestErrno.ENOTSUP; + } + + return TryWriteInt32(ctx, clockIdAddress, KernelClocks.ThreadCputime) + ? 0 + : Threading.GuestErrno.EFAULT; + } + internal static ulong ReadProcessTimeMicroseconds() { var elapsedTicks = Stopwatch.GetTimestamp() - _processStartCounter; diff --git a/src/SharpEmu.Libs/Threading/HostThreadCpuTime.cs b/src/SharpEmu.Libs/Threading/HostThreadCpuTime.cs new file mode 100644 index 000000000..df4406540 --- /dev/null +++ b/src/SharpEmu.Libs/Threading/HostThreadCpuTime.cs @@ -0,0 +1,61 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using System.Runtime.InteropServices; + +namespace SharpEmu.Libs.Threading; + +/// +/// CPU time (user + system) consumed by the calling host thread. On macOS this is +/// thread_info(THREAD_BASIC_INFO), not CLOCK_THREAD_CPUTIME_ID: under Rosetta +/// the clock charges a translated thread only a few percent of the CPU it really used. +/// Elsewhere it is the host's per-thread CPU clock, so a guest that profiles its own +/// threads sees monotonic time that tracks the work it did. +/// +internal static unsafe partial class HostThreadCpuTime +{ + private const int ThreadBasicInfoFlavor = 3; // THREAD_BASIC_INFO + private const int ThreadBasicInfoCount = 10; // sizeof(thread_basic_info) / sizeof(natural_t) + + [ThreadStatic] + private static uint _machThread; + + /// Microseconds of CPU time the calling thread has used, or -1 when the host cannot say. + internal static long ReadCurrentMicroseconds() + { + if (OperatingSystem.IsMacOS()) + { + try + { + if (_machThread == 0) + { + _machThread = mach_thread_self(); + } + + var info = stackalloc int[ThreadBasicInfoCount]; + var count = (uint)ThreadBasicInfoCount; + if (thread_info(_machThread, ThreadBasicInfoFlavor, info, &count) == 0) + { + // user_time {seconds, microseconds}, system_time {seconds, microseconds} + return ((long)info[0] + info[2]) * 1_000_000L + info[1] + info[3]; + } + } + catch (Exception) + { + } + } + + return KernelClocksThreadMicroseconds(); + } + + private static long KernelClocksThreadMicroseconds() => + Kernel.KernelClocks.TryGetTime(Kernel.KernelClocks.ThreadCputime, out var seconds, out var nanoseconds) == 0 + ? seconds * 1_000_000L + nanoseconds / 1_000L + : -1; + + [LibraryImport("libSystem.B.dylib")] + private static partial uint mach_thread_self(); + + [LibraryImport("libSystem.B.dylib")] + private static partial int thread_info(uint thread, int flavor, int* info, uint* count); +} diff --git a/tests/SharpEmu.Libs.Tests/Kernel/KernelExitAndThreadTimeTests.cs b/tests/SharpEmu.Libs.Tests/Kernel/KernelExitAndThreadTimeTests.cs new file mode 100644 index 000000000..b29b7e3af --- /dev/null +++ b/tests/SharpEmu.Libs.Tests/Kernel/KernelExitAndThreadTimeTests.cs @@ -0,0 +1,138 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using SharpEmu.HLE; +using SharpEmu.HLE.Host; +using SharpEmu.Libs.Gpu.GpuCommands; +using SharpEmu.Libs.Kernel; +using SharpEmu.Libs.Tests.Gpu.GpuCommands; +using Xunit; + +namespace SharpEmu.Libs.Tests.Kernel; + +public sealed class KernelExitAndThreadTimeTests +{ + private const ulong MemoryBase = 0x1_0000_0000; + + private static CpuContext NewContext() => new(new FakeCpuMemory(MemoryBase, 0x20000), Generation.Gen5); + + [Fact] + public void ExitAndUnderscoreExitEndTheHostSessionWithTheGuestStatus() + { + if (string.Equals(Environment.GetEnvironmentVariable("SHARPEMU_EXIT_TERMINATES"), "0", StringComparison.Ordinal)) + { + return; // disabled by the environment + } + + var previous = KernelExports.GuestExitTermination; + var seen = new List(); + KernelExports.GuestExitTermination = seen.Add; + try + { + var ctx = NewContext(); + ctx[CpuRegister.Rdi] = 3; + Assert.Equal((int)OrbisGen2Result.ORBIS_GEN2_OK, KernelExports.Exit(ctx)); + Assert.True(GuestThreadExecution.TryConsumeCurrentEntryExit(out _, out var reason)); + Assert.Equal("exit", reason); + + ctx[CpuRegister.Rdi] = 5; + Assert.Equal((int)OrbisGen2Result.ORBIS_GEN2_OK, KernelExports.UnderscoreExit(ctx)); + Assert.True(GuestThreadExecution.TryConsumeCurrentEntryExit(out _, out reason)); + Assert.Equal("_exit", reason); + Assert.Equal(new[] { 3, 5 }, seen); + } + finally + { + KernelExports.GuestExitTermination = previous; + } + } + + [Fact] + public void GetThreadTimeReportsCpuTimeTheThreadSpends() + { + var ctx = NewContext(); + KernelRuntimeCompatExports.KernelGetThreadTime(ctx); + var before = ctx[CpuRegister.Rax]; + + // Burn CPU on this thread (bounded by wall time so a loaded runner still ends). + var wall = System.Diagnostics.Stopwatch.StartNew(); + var sink = 0L; + ulong after = before; + while (after - before < 5_000 && wall.ElapsedMilliseconds < 5_000) + { + for (var i = 0; i < 100_000; i++) + { + sink += i; + } + + KernelRuntimeCompatExports.KernelGetThreadTime(ctx); + after = ctx[CpuRegister.Rax]; + } + + GC.KeepAlive(sink); + Assert.True(after >= before + 1_000, $"thread time moved {after - before} us after burning CPU"); + } + + [Fact] + public void PthreadGetCpuClockIdAnswersForTheCallingThreadOnly() + { + var ctx = NewContext(); + const ulong ClockIdAddress = MemoryBase + 0x100; + ctx[CpuRegister.Rdi] = KernelPthreadState.GetCurrentThreadHandle(); + ctx[CpuRegister.Rsi] = ClockIdAddress; + Assert.Equal(0, KernelRuntimeCompatExports.PthreadGetCpuClockId(ctx)); + Span value = stackalloc byte[4]; + Assert.True(ctx.Memory.TryRead(ClockIdAddress, value)); + Assert.Equal(14, BitConverter.ToInt32(value)); // CLOCK_THREAD_CPUTIME_ID + + ctx[CpuRegister.Rdi] = KernelPthreadState.GetCurrentThreadHandle() + 0x1000; + Assert.NotEqual(0, KernelRuntimeCompatExports.PthreadGetCpuClockId(ctx)); + } + + [Fact] + public void CommandStreamWorkerRunsAtUserInteractiveQos() + { + if (!OperatingSystem.IsMacOS()) + { + return; // QoS classes are a Darwin concept + } + + var host = new RecordingCommandStreamHost(); + var queue = new CommandStreamQueue(host); + var worker = new CommandStreamWorker(queue, host, () => false, true, "qos check"); + worker.Start(); + try + { + var wait = System.Diagnostics.Stopwatch.StartNew(); + while (worker.WorkerQosClass == 0 && wait.Elapsed < TimeSpan.FromSeconds(30)) + { + Thread.Sleep(1); + } + + // 0x21 is QOS_CLASS_USER_INTERACTIVE, read back from the worker thread itself. + Assert.Equal(0x21u, worker.WorkerQosClass); + } + finally + { + worker.Stop(); + } + } + + [Fact] + public void LargeMemsetFillsTheWholeRangeWithTheByte() + { + var ctx = NewContext(); + const ulong Destination = MemoryBase + 0x1000; + ctx[CpuRegister.Rdi] = Destination; + ctx[CpuRegister.Rsi] = 0xAB; + ctx[CpuRegister.Rdx] = 0x10000; + Assert.Equal((int)OrbisGen2Result.ORBIS_GEN2_OK, KernelMemoryCompatExports.Memset(ctx)); + Assert.Equal(Destination, ctx[CpuRegister.Rax]); + var bytes = new byte[0x10000]; + Assert.True(ctx.Memory.TryRead(Destination, bytes)); + Assert.All(bytes, b => Assert.Equal(0xAB, b)); + var after = new byte[1]; + Assert.True(ctx.Memory.TryRead(Destination + 0x10000, after)); + Assert.Equal(0, after[0]); + } +} diff --git a/tests/SharpEmu.Libs.Tests/Kernel/SleepAndPriorityQosTests.cs b/tests/SharpEmu.Libs.Tests/Kernel/SleepAndPriorityQosTests.cs index 8866eccfb..4f137a56e 100644 --- a/tests/SharpEmu.Libs.Tests/Kernel/SleepAndPriorityQosTests.cs +++ b/tests/SharpEmu.Libs.Tests/Kernel/SleepAndPriorityQosTests.cs @@ -45,7 +45,11 @@ public void ShortUsleepStreakStartsNappingAfterTheSpinWindow() { napped++; var tookUs = (Stopwatch.GetTimestamp() - before) * 1_000_000 / Stopwatch.Frequency; - Assert.True(tookUs >= KernelRuntimeCompatExports.UsleepNapMicroseconds / 2, $"nap took {tookUs} us"); + // The first nap of a streak is the floor (the ceiling under the legacy switch). + var shortestNap = KernelRuntimeCompatExports.UsleepNapLegacy + ? KernelRuntimeCompatExports.UsleepNapMicroseconds + : KernelRuntimeCompatExports.UsleepNapFloorMicroseconds; + Assert.True(tookUs >= shortestNap / 2, $"nap took {tookUs} us"); } } @@ -81,6 +85,57 @@ public void YieldStreakNapsOnlyAfterSustainedPollingAndRestartsOnAGap() private static bool KernelRuntimeCompatExports_YieldNap(ref long start, ref long last, long now, long gap, long after) => KernelPthreadCompatExports.YieldNapPolicy(ref start, ref last, now, gap, after); + // Policy tunables pinned to the defaults: 250 us ceiling, 50 us floor, 4 ms ramp, nap after 500 us. + private static long Policy(long ageUs, long requestedUs, bool legacy, out bool napped) => + KernelRuntimeCompatExports.NapPolicyMicroseconds( + ageUs * System.Diagnostics.Stopwatch.Frequency / 1_000_000, requestedUs, + napMicroseconds: 250, floorMicroseconds: 50, rampMicroseconds: 4000, + afterTicks: 500 * System.Diagnostics.Stopwatch.Frequency / 1_000_000, legacy, out napped); + + [Fact] + public void NapPolicyHonoursRequestsAtOrAboveTheFloorEvenInALongStreak() + { + Assert.Equal(100, Policy(ageUs: 60_000, requestedUs: 100, legacy: false, out var napped)); + Assert.False(napped); + Assert.Equal(50, Policy(ageUs: 60_000, requestedUs: 50, legacy: false, out napped)); + Assert.False(napped); + // The legacy policy stretches the same request to the ceiling (the measured 285 us). + Assert.Equal(250, Policy(ageUs: 60_000, requestedUs: 100, legacy: true, out napped)); + Assert.True(napped); + } + + [Fact] + public void NapPolicyEscalatesTinyRequestsAndYieldsFromFloorToCeiling() + { + // Before the streak window nothing naps. + Assert.Equal(1, Policy(ageUs: 400, requestedUs: 1, legacy: false, out var napped)); + Assert.False(napped); + + // At the window the nap starts at the floor, not the ceiling. + Assert.Equal(50, Policy(ageUs: 500, requestedUs: 1, legacy: false, out napped)); + Assert.True(napped); + Assert.Equal(50, Policy(ageUs: 500, requestedUs: 0, legacy: false, out _)); + + // It rises monotonically over the ramp and then stays at the ceiling. + var halfway = Policy(ageUs: 500 + 2000, requestedUs: 1, legacy: false, out _); + Assert.InRange(halfway, 140, 160); + Assert.Equal(250, Policy(ageUs: 500 + 4000, requestedUs: 1, legacy: false, out _)); + Assert.Equal(250, Policy(ageUs: 1_000_000, requestedUs: 0, legacy: false, out _)); + + // Legacy: straight to the ceiling. + Assert.Equal(250, Policy(ageUs: 500, requestedUs: 1, legacy: true, out _)); + } + + [Fact] + public void NapPolicyIsOffWhenTheCeilingIsZero() + { + var micros = KernelRuntimeCompatExports.NapPolicyMicroseconds( + streakAgeTicks: 1_000_000_000, requestedMicroseconds: 7, napMicroseconds: 0, floorMicroseconds: 0, rampMicroseconds: 4000, + afterTicks: 1, legacy: false, out var napped); + Assert.Equal(7, micros); + Assert.False(napped); + } + [Fact] public void ShortUsleepNeverReturnsBeforeTheRequestedTime() { From 6ba3553d43aea59dd28b77699480122bd3a0d6bd Mon Sep 17 00:00:00 2001 From: kuba Date: Mon, 26 Oct 2026 05:54:47 +0100 Subject: [PATCH 14/82] [Kernel] Drop the sceKernelGetThreadTime export: neither the SDK headers nor any title's imports name it The export was added with a NID derived from a guessed name. Demon's Souls and Grand Theft Auto V import neither it nor pthread_getcpuclockid, and only the latter is declared by the SDK (pthread.h), so only the latter stays. The host thread-CPU-time helper it used has no other caller and goes with it. --- .../Kernel/KernelRuntimeCompatExports.cs | 17 ------ .../Threading/HostThreadCpuTime.cs | 61 ------------------- .../Kernel/KernelExitAndThreadTimeTests.cs | 26 -------- 3 files changed, 104 deletions(-) delete mode 100644 src/SharpEmu.Libs/Threading/HostThreadCpuTime.cs diff --git a/src/SharpEmu.Libs/Kernel/KernelRuntimeCompatExports.cs b/src/SharpEmu.Libs/Kernel/KernelRuntimeCompatExports.cs index 9e2a5dbf1..a4467ad9c 100644 --- a/src/SharpEmu.Libs/Kernel/KernelRuntimeCompatExports.cs +++ b/src/SharpEmu.Libs/Kernel/KernelRuntimeCompatExports.cs @@ -526,23 +526,6 @@ public static int KernelGetProcessTime(CpuContext ctx) return (int)OrbisGen2Result.ORBIS_GEN2_OK; } - /// - /// The CPU time the calling thread has consumed, in microseconds (the unit of - /// sceKernelGetProcessTime). The SDK headers name no such function; this is the - /// thread-scoped sibling of the process-time exports, kept for titles that profile their - /// own worker threads with it. - /// - [SysAbiExport( - Nid = "afZu6Sswlr4", - ExportName = "sceKernelGetThreadTime", - Target = Generation.Gen4 | Generation.Gen5, - LibraryName = "libKernel")] - public static int KernelGetThreadTime(CpuContext ctx) - { - ctx[CpuRegister.Rax] = unchecked((ulong)Math.Max(0, Threading.HostThreadCpuTime.ReadCurrentMicroseconds())); - return (int)OrbisGen2Result.ORBIS_GEN2_OK; - } - /// /// int pthread_getcpuclockid(pthread_t, clockid_t *), the one thread-CPU-time API the /// SDK headers (pthread.h) declare. The clock id of the calling thread is diff --git a/src/SharpEmu.Libs/Threading/HostThreadCpuTime.cs b/src/SharpEmu.Libs/Threading/HostThreadCpuTime.cs deleted file mode 100644 index df4406540..000000000 --- a/src/SharpEmu.Libs/Threading/HostThreadCpuTime.cs +++ /dev/null @@ -1,61 +0,0 @@ -// Copyright (C) 2026 SharpEmu Emulator Project -// SPDX-License-Identifier: GPL-2.0-or-later - -using System.Runtime.InteropServices; - -namespace SharpEmu.Libs.Threading; - -/// -/// CPU time (user + system) consumed by the calling host thread. On macOS this is -/// thread_info(THREAD_BASIC_INFO), not CLOCK_THREAD_CPUTIME_ID: under Rosetta -/// the clock charges a translated thread only a few percent of the CPU it really used. -/// Elsewhere it is the host's per-thread CPU clock, so a guest that profiles its own -/// threads sees monotonic time that tracks the work it did. -/// -internal static unsafe partial class HostThreadCpuTime -{ - private const int ThreadBasicInfoFlavor = 3; // THREAD_BASIC_INFO - private const int ThreadBasicInfoCount = 10; // sizeof(thread_basic_info) / sizeof(natural_t) - - [ThreadStatic] - private static uint _machThread; - - /// Microseconds of CPU time the calling thread has used, or -1 when the host cannot say. - internal static long ReadCurrentMicroseconds() - { - if (OperatingSystem.IsMacOS()) - { - try - { - if (_machThread == 0) - { - _machThread = mach_thread_self(); - } - - var info = stackalloc int[ThreadBasicInfoCount]; - var count = (uint)ThreadBasicInfoCount; - if (thread_info(_machThread, ThreadBasicInfoFlavor, info, &count) == 0) - { - // user_time {seconds, microseconds}, system_time {seconds, microseconds} - return ((long)info[0] + info[2]) * 1_000_000L + info[1] + info[3]; - } - } - catch (Exception) - { - } - } - - return KernelClocksThreadMicroseconds(); - } - - private static long KernelClocksThreadMicroseconds() => - Kernel.KernelClocks.TryGetTime(Kernel.KernelClocks.ThreadCputime, out var seconds, out var nanoseconds) == 0 - ? seconds * 1_000_000L + nanoseconds / 1_000L - : -1; - - [LibraryImport("libSystem.B.dylib")] - private static partial uint mach_thread_self(); - - [LibraryImport("libSystem.B.dylib")] - private static partial int thread_info(uint thread, int flavor, int* info, uint* count); -} diff --git a/tests/SharpEmu.Libs.Tests/Kernel/KernelExitAndThreadTimeTests.cs b/tests/SharpEmu.Libs.Tests/Kernel/KernelExitAndThreadTimeTests.cs index b29b7e3af..009ee9f78 100644 --- a/tests/SharpEmu.Libs.Tests/Kernel/KernelExitAndThreadTimeTests.cs +++ b/tests/SharpEmu.Libs.Tests/Kernel/KernelExitAndThreadTimeTests.cs @@ -47,32 +47,6 @@ public void ExitAndUnderscoreExitEndTheHostSessionWithTheGuestStatus() } } - [Fact] - public void GetThreadTimeReportsCpuTimeTheThreadSpends() - { - var ctx = NewContext(); - KernelRuntimeCompatExports.KernelGetThreadTime(ctx); - var before = ctx[CpuRegister.Rax]; - - // Burn CPU on this thread (bounded by wall time so a loaded runner still ends). - var wall = System.Diagnostics.Stopwatch.StartNew(); - var sink = 0L; - ulong after = before; - while (after - before < 5_000 && wall.ElapsedMilliseconds < 5_000) - { - for (var i = 0; i < 100_000; i++) - { - sink += i; - } - - KernelRuntimeCompatExports.KernelGetThreadTime(ctx); - after = ctx[CpuRegister.Rax]; - } - - GC.KeepAlive(sink); - Assert.True(after >= before + 1_000, $"thread time moved {after - before} us after burning CPU"); - } - [Fact] public void PthreadGetCpuClockIdAnswersForTheCallingThreadOnly() { From eac89b59183b6c4c62dcbc87a0d7e2834c2c1fcd Mon Sep 17 00:00:00 2001 From: kuba Date: Mon, 26 Oct 2026 06:06:34 +0100 Subject: [PATCH 15/82] [Perf] Guest-to-HLE crossings: AGC builder leaf, native clock stubs, host-thread pthread_self, one thread-state object, in-place large memset Test eboot (osx-x64 under Rosetta, warm pass, host load ~120 so compare same-session pairs; logs w2-before2/3 vs w2-after1/2, w2-agc-*): - clock_gettime(MONOTONIC) 300/231 -> 16 ns; sceKernelClockGettime 500 -> 18; clock_gettime(REALTIME) 298/239 -> 22-30; gettimeofday 305/237 -> 22; sceKernelGetProcessTime 157/108 -> 15-18 (stubs, SHARPEMU_HLE_FAST_CLOCK=0 disables; same epoch/units as the managed exports) - scePthreadSelf on the main thread 284/192 -> 3.2-3.6 ns (SHARPEMU_HLE_FAST_SELF_HOST=0 disables) - generic crossing (getpid, GetCurrentCpu) 257/244 -> 167-220 ns: the ~25 thread-statics per crossing are one per-thread object read once, ImportCount no longer takes a locked add - AGC builders through the real dispatcher (w2-eboot, SetShRegisterDirect/SetCxRegisterDirect/DrawIndexAuto/ DrawIndex/CbDispatch/EventWrite): 320-360 ns full path -> 220-235 ns leaf, DcbWriteData 367-440 -> 313-326 (SHARPEMU_AGC_BUILDER_LEAF=0 disables). A leaf never calls guest code: a full buffer declines before anything is written and the full path repeats the call; tests compare leaf and full bytes per builder. - memset >= 64 KiB fills the guest range in place (ICpuMemory.TryFillInPlace) instead of staging 16 KiB chunks: hle_memset_32MB 6-12 GB/s -> 12-21 GB/s warm (SHARPEMU_MEMSET_INPLACE=0 disables) - lazy trace strings in the AGC builders, single guest write for register packets, AcbWaitRegMem and AcbEventWrite as one packet write --- .../Native/DirectExecutionBackend.FastPath.cs | 214 +++++++- .../Native/DirectExecutionBackend.Imports.cs | 359 +++++++++---- .../Cpu/Native/DirectExecutionBackend.cs | 15 +- .../Platform/IDirectExecutionPlatform.cs | 15 + src/SharpEmu.HLE/CpuContext.cs | 14 + src/SharpEmu.HLE/GuestFastPath.cs | 28 + src/SharpEmu.HLE/GuestThreadExecution.cs | 477 ++++++++++-------- .../Agc/AgcExports.CommandPackets.cs | 66 +-- .../Agc/AgcExports.ControlFlow.cs | 3 +- .../Agc/AgcExports.DmaPackets.cs | 8 +- .../Agc/AgcExports.DrawPackets.cs | 24 +- .../Agc/AgcExports.DriverResources.cs | 18 +- src/SharpEmu.Libs/Agc/AgcExports.Events.cs | 14 +- .../Agc/AgcExports.RegisterPackets.cs | 59 ++- .../Agc/AgcExports.ShaderState.cs | 25 +- src/SharpEmu.Libs/Agc/AgcExports.cs | 30 +- .../Kernel/KernelMemoryCompatExports.cs | 22 + .../Kernel/KernelPthreadCompatExports.cs | 2 + .../Kernel/KernelRuntimeCompatExports.cs | 3 + .../MacOSExecutionPlatform.cs | 31 ++ .../Agc/AgcBuilderLeafTests.cs | 236 +++++++++ .../Cpu/FaultThreadImportTraceTests.cs | 14 +- .../Cpu/GuestClockStubTests.cs | 220 ++++++++ .../Cpu/GuestFastPathStubTests.cs | 31 ++ .../Cpu/GuestThreadStateTests.cs | 62 +++ .../Kernel/MemsetInPlaceTests.cs | 89 ++++ 26 files changed, 1619 insertions(+), 460 deletions(-) create mode 100644 tests/SharpEmu.Libs.Tests/Agc/AgcBuilderLeafTests.cs create mode 100644 tests/SharpEmu.Libs.Tests/Cpu/GuestClockStubTests.cs create mode 100644 tests/SharpEmu.Libs.Tests/Cpu/GuestThreadStateTests.cs create mode 100644 tests/SharpEmu.Libs.Tests/Kernel/MemsetInPlaceTests.cs diff --git a/src/SharpEmu.Core/Cpu/Native/DirectExecutionBackend.FastPath.cs b/src/SharpEmu.Core/Cpu/Native/DirectExecutionBackend.FastPath.cs index fe42fd06b..aeaf989e3 100644 --- a/src/SharpEmu.Core/Cpu/Native/DirectExecutionBackend.FastPath.cs +++ b/src/SharpEmu.Core/Cpu/Native/DirectExecutionBackend.FastPath.cs @@ -32,8 +32,36 @@ internal enum GuestFastPathStub /// xor eax,eax; ret, so the guest never crosses into managed code. /// ConstantZero, + + /// clock_gettime / sceKernelClockGettime for the realtime and monotonic clock ids. + ClockGettime, + + /// gettimeofday(tv, NULL). + GetTimeOfDay, + + /// sceKernelGetProcessTime: microseconds since process start. + ProcessTimeMicroseconds, } + // clock_gettime / sceKernelClockGettime / gettimeofday / sceKernelGetProcessTime. + private const string ClockGettimeNid = "lLMT9vJAck0"; + private const string KernelClockGettimeNid = "QBi7HCK03hw"; + private const string GettimeofdayNid = "n88vx3C5nW8"; + private const string KernelGetProcessTimeNid = "4J2sUJmuHZQ"; + + /// + /// What a clock stub calls and subtracts: the host's monotonic counter (the same + /// mach_absolute_time the managed Stopwatch reads, which makes the stub's answer + /// exactly the managed export's), the process start in that counter, and the host realtime clock. + /// + internal readonly record struct ClockStubTargets(nint MachAbsoluteTime, long ProcessStartCounter, nint ClockGettime); + + // SHARPEMU_HLE_FAST_CLOCK=0 keeps the clock reads on the managed exports. + private static readonly bool ClockStubsEnabled = + !string.Equals(Environment.GetEnvironmentVariable("SHARPEMU_HLE_FAST_CLOCK"), "0", StringComparison.Ordinal); + + private ClockStubTargets? _clockStubTargets; + // scePthreadSelf / pthread_self. private const string PthreadSelfNid = "aI+OeCz8xrQ"; private const string PosixPthreadSelfNid = "EotR8a3ASf4"; @@ -122,6 +150,8 @@ private void ConfigureGuestFastPath() !_logAllImports && string.IsNullOrEmpty(_importFilter); + _clockStubTargets = ResolveClockStubTargets(); + // Emission needs a TLS read the platform can express as inline code // (the Darwin pthread gs-indexed key array). Platforms whose TLS read is // a real call keep the managed gateway. @@ -160,12 +190,42 @@ private void ConfigureGuestFastPath() _guestFastPathEnabled = true; } + private ClockStubTargets? ResolveClockStubTargets() + { + // The stubs read the host clocks without entering managed code, so anything that must see + // every import call - and the import-loop guard, which time reads reset - rules them out. + if (!ClockStubsEnabled || + RuntimeInformation.ProcessArchitecture != Architecture.X64 || + !_nativeConstantThunksAllowed || + !NativeImportThunksEnabled || + GuestWriteWatch.Armed || + string.Equals(Environment.GetEnvironmentVariable("SHARPEMU_DISABLE_IMPORT_LOOP_GUARD"), "0", StringComparison.Ordinal) || + !SharpEmu.Libs.Kernel.KernelRuntimeCompatExports.HostCounterMatchesProcessTimeFrequency) + { + return null; + } + + // The platform names the routines (and vouches that its counter ticks in nanoseconds, as + // Stopwatch does), so no OS-specific code lives here. + if (!ExecutionPlatform.TryGetHostClockEntryPoints(out var counter, out var clockGettime) || + counter == 0 || + clockGettime == 0) + { + return null; + } + + return new ClockStubTargets(counter, SharpEmu.Libs.Kernel.KernelRuntimeCompatExports.ProcessStartCounter, clockGettime); + } + internal static GuestFastPathStub ClassifyGuestFastPathStub(string nid) => nid switch { PthreadSelfNid or PosixPthreadSelfNid => GuestFastPathStub.PthreadSelf, PthreadGetspecificNid or PosixPthreadGetspecificNid => GuestFastPathStub.PthreadGetspecific, MemcpyNid or MemmoveNid => GuestFastPathStub.MemoryCopy, MemsetNid => GuestFastPathStub.MemorySet, + ClockGettimeNid or KernelClockGettimeNid => GuestFastPathStub.ClockGettime, + GettimeofdayNid => GuestFastPathStub.GetTimeOfDay, + KernelGetProcessTimeNid => GuestFastPathStub.ProcessTimeMicroseconds, _ => IsConstantZeroImport(nid) ? GuestFastPathStub.ConstantZero : GuestFastPathStub.None, }; @@ -201,13 +261,23 @@ private unsafe bool TryCreateGuestFastPathStub(string nid, nint fallbackTrampoli return false; } } + else if (kind is GuestFastPathStub.ClockGettime or GuestFastPathStub.GetTimeOfDay or GuestFastPathStub.ProcessTimeMicroseconds) + { + // Needs neither the per-thread block nor the emitted TLS read. + if (_clockStubTargets is null) + { + return false; + } + } else if (!_guestFastPathEnabled) { return false; } - var code = EmitGuestFastPathStub(kind, _fastPathBlockTlsIndex, fallbackTrampoline); - const uint stubAllocationSize = 128u; + var code = kind is GuestFastPathStub.ClockGettime or GuestFastPathStub.GetTimeOfDay or GuestFastPathStub.ProcessTimeMicroseconds + ? EmitGuestClockStub(kind, _clockStubTargets!.Value, fallbackTrampoline) + : EmitGuestFastPathStub(kind, _fastPathBlockTlsIndex, fallbackTrampoline); + const uint stubAllocationSize = 192u; if (code.Count > stubAllocationSize) { return false; @@ -362,6 +432,146 @@ void EmitBranchToSlowPath(byte opcode) return code; } + /// + /// Emits a clock stub. Each one checks, before touching anything, that the call is one it can + /// answer exactly as the managed export does, and jumps to the import's trampoline otherwise. + /// The answer is computed from the host clocks and written straight to the guest's timespec or + /// timeval; RAX is 0 (success) and only caller-saved registers and the flags change. + /// + internal static List EmitGuestClockStub(GuestFastPathStub kind, ClockStubTargets targets, nint fallbackTrampoline) + { + var code = new List(160); + var slowPathFixups = new List(); + void Emit(params byte[] bytes) => code.AddRange(bytes); + void EmitImm64(long value) => code.AddRange(BitConverter.GetBytes(value)); + // Conditional near branch (0F 8x rel32) to a label patched once its offset is known. + void EmitBranch(byte shortOpcode, List fixups) + { + Emit(0x0F, (byte)(shortOpcode + 0x10), 0, 0, 0, 0); + fixups.Add(code.Count - 4); + } + + void Patch(List fixups) + { + foreach (var fixup in fixups) + { + var delta = BitConverter.GetBytes(code.Count - (fixup + 4)); + for (var i = 0; i < 4; i++) + { + code[fixup + i] = delta[i]; + } + } + } + + // call mach_absolute_time, leaving the nanoseconds since process start in RAX. + void EmitElapsedNanoseconds() + { + Emit(0x48, 0xB8); EmitImm64(targets.MachAbsoluteTime); // mov rax, imm64 + Emit(0xFF, 0xD0); // call rax + Emit(0x48, 0xB9); EmitImm64(targets.ProcessStartCounter); // mov rcx, imm64 + Emit(0x48, 0x29, 0xC8); // sub rax, rcx + } + + // call clock_gettime(CLOCK_REALTIME, rsp) into a 16-byte scratch the caller already reserved. + void EmitRealtimeIntoStack() + { + Emit(0x31, 0xFF); // xor edi, edi + Emit(0x48, 0x89, 0xE6); // mov rsi, rsp + Emit(0x48, 0xB8); EmitImm64(targets.ClockGettime); // mov rax, imm64 + Emit(0xFF, 0xD0); // call rax + } + + var realtimeFixups = new List(); + switch (kind) + { + case GuestFastPathStub.ClockGettime: + { + // rdi = clock id, rsi = timespec*. A null pointer, an id outside the realtime and + // monotonic families, or one with high bits set is the managed export's business. + Emit(0x48, 0x85, 0xF6); // test rsi, rsi + EmitBranch(0x74, slowPathFixups); // jz slow + Emit(0x48, 0x83, 0xFF, 0x0C); // cmp rdi, 12 + EmitBranch(0x77, slowPathFixups); // ja slow + Emit(0xB8, 0xB0, 0x19, 0x00, 0x00); // mov eax, 0x19B0 (monotonic ids 4 5 7 8 11 12) + Emit(0x0F, 0xA3, 0xF8); // bt eax, edi + EmitBranch(0x73, realtimeFixups); // jnc not-monotonic + Emit(0x56); // push rsi (keeps the stack 16-aligned for the call) + EmitElapsedNanoseconds(); + Emit(0x31, 0xD2); // xor edx, edx + Emit(0xB9, 0x00, 0xCA, 0x9A, 0x3B); // mov ecx, 1000000000 + Emit(0x48, 0xF7, 0xF1); // div rcx -> rax = seconds, rdx = nanoseconds + Emit(0x5E); // pop rsi + Emit(0x48, 0x89, 0x06); // mov [rsi], rax + Emit(0x48, 0x89, 0x56, 0x08); // mov [rsi+8], rdx + Emit(0x31, 0xC0); // xor eax, eax + Emit(0xC3); // ret + Patch(realtimeFixups); + Emit(0xB8, 0x01, 0x06, 0x00, 0x00); // mov eax, 0x601 (realtime ids 0 9 10) + Emit(0x0F, 0xA3, 0xF8); // bt eax, edi + EmitBranch(0x73, slowPathFixups); // jnc slow + Emit(0x56); // push rsi + Emit(0x48, 0x83, 0xEC, 0x10); // sub rsp, 16 + EmitRealtimeIntoStack(); + Emit(0x48, 0x8B, 0x04, 0x24); // mov rax, [rsp] + Emit(0x48, 0x8B, 0x54, 0x24, 0x08); // mov rdx, [rsp+8] + Emit(0x48, 0x83, 0xC4, 0x10); // add rsp, 16 + Emit(0x5E); // pop rsi + Emit(0x48, 0x89, 0x06); // mov [rsi], rax + Emit(0x48, 0x89, 0x56, 0x08); // mov [rsi+8], rdx + Emit(0x31, 0xC0); // xor eax, eax + Emit(0xC3); // ret + break; + } + + case GuestFastPathStub.GetTimeOfDay: + { + // rdi = timeval*, rsi = timezone*. Only the common (tv, NULL) call is answered here. + Emit(0x48, 0x85, 0xFF); // test rdi, rdi + EmitBranch(0x74, slowPathFixups); // jz slow + Emit(0x48, 0x85, 0xF6); // test rsi, rsi + EmitBranch(0x75, slowPathFixups); // jnz slow + Emit(0x57); // push rdi + Emit(0x48, 0x83, 0xEC, 0x10); // sub rsp, 16 + EmitRealtimeIntoStack(); + Emit(0x48, 0x8B, 0x44, 0x24, 0x08); // mov rax, [rsp+8] (nanoseconds) + Emit(0x31, 0xD2); // xor edx, edx + Emit(0xB9, 0xE8, 0x03, 0x00, 0x00); // mov ecx, 1000 + Emit(0x48, 0xF7, 0xF1); // div rcx -> rax = microseconds + Emit(0x48, 0x89, 0xC2); // mov rdx, rax + Emit(0x48, 0x8B, 0x04, 0x24); // mov rax, [rsp] (seconds) + Emit(0x48, 0x83, 0xC4, 0x10); // add rsp, 16 + Emit(0x5F); // pop rdi + Emit(0x48, 0x89, 0x07); // mov [rdi], rax + Emit(0x48, 0x89, 0x57, 0x08); // mov [rdi+8], rdx + Emit(0x31, 0xC0); // xor eax, eax + Emit(0xC3); // ret + break; + } + + case GuestFastPathStub.ProcessTimeMicroseconds: + { + Emit(0x48, 0x83, 0xEC, 0x08); // sub rsp, 8 (16-align for the call) + EmitElapsedNanoseconds(); + Emit(0x31, 0xD2); // xor edx, edx + Emit(0xB9, 0xE8, 0x03, 0x00, 0x00); // mov ecx, 1000 + Emit(0x48, 0xF7, 0xF1); // div rcx + Emit(0x48, 0x83, 0xC4, 0x08); // add rsp, 8 + Emit(0xC3); // ret + // Nothing to decline: the stub answers every call, but the fallback keeps the layout uniform. + break; + } + + default: + throw new ArgumentOutOfRangeException(nameof(kind), kind, "Not a clock stub kind."); + } + + Patch(slowPathFixups); + // jmp qword [rip+0] with the absolute trampoline address after it. + Emit(0xFF, 0x25, 0x00, 0x00, 0x00, 0x00); + code.AddRange(BitConverter.GetBytes((long)fallbackTrampoline)); + return code; + } + /// /// memcpy/memmove(dst, src, n) as a forward 32-byte SSE loop (16-byte step, then /// rep movsb for the last 0-15 bytes), returning dst. Rosetta translates diff --git a/src/SharpEmu.Core/Cpu/Native/DirectExecutionBackend.Imports.cs b/src/SharpEmu.Core/Cpu/Native/DirectExecutionBackend.Imports.cs index 979808f25..64e502604 100644 --- a/src/SharpEmu.Core/Cpu/Native/DirectExecutionBackend.Imports.cs +++ b/src/SharpEmu.Core/Cpu/Native/DirectExecutionBackend.Imports.cs @@ -51,13 +51,13 @@ internal static string DescribeUnresolvedImport(string nid) => private const int UnresolvedImportSampleKey = 0; private int _il2CppExceptionDiagnosticCount; - [ThreadStatic] - private static nint _preparedContextTransferStub; - private static unsafe ulong ImportDispatchGatewayManaged(nint backendHandle, int importIndex, nint argPackPtr, nint transferStubAddress) { *(nint*)transferStubAddress = 0; - _preparedContextTransferStub = 0; + // One thread-static read for the whole crossing: everything the gateway keeps per host thread + // hangs off this object. A stub left by an abandoned crossing is cleared with it. + var threadState = GuestThreadExecution.State; + threadState.PreparedContextTransferStub = 0; try { if (!(GCHandle.FromIntPtr(backendHandle).Target is DirectExecutionBackend directExecutionBackend)) @@ -70,14 +70,14 @@ private static unsafe ulong ImportDispatchGatewayManaged(nint backendHandle, int if (DsTrace.Hle && DsTrace.Armed) { var traced = DispatchImportWithDsTrace(directExecutionBackend, importIndex, argPackPtr); - CompleteImportContextTransfer(transferStubAddress); + CompleteImportContextTransfer(threadState, transferStubAddress); return traced; } if (_perfHleHistogram) { var profiledHle = DispatchImportWithProfile(directExecutionBackend, importIndex, argPackPtr); - CompleteImportContextTransfer(transferStubAddress); + CompleteImportContextTransfer(threadState, transferStubAddress); return profiledHle; } @@ -88,7 +88,7 @@ private static unsafe ulong ImportDispatchGatewayManaged(nint backendHandle, int { var profiledResult = directExecutionBackend.DispatchImport(importIndex, argPackPtr); directExecutionBackend.ClearActiveImportIndex(); - CompleteImportContextTransfer(transferStubAddress); + CompleteImportContextTransfer(threadState, transferStubAddress); return profiledResult; } finally @@ -99,7 +99,7 @@ private static unsafe ulong ImportDispatchGatewayManaged(nint backendHandle, int var result = directExecutionBackend.DispatchImport(importIndex, argPackPtr); directExecutionBackend.ClearActiveImportIndex(); - CompleteImportContextTransfer(transferStubAddress); + CompleteImportContextTransfer(threadState, transferStubAddress); return result; } catch (Exception ex) @@ -111,13 +111,13 @@ private static unsafe ulong ImportDispatchGatewayManaged(nint backendHandle, int } } - private static unsafe void CompleteImportContextTransfer(nint transferStubAddress) + private static unsafe void CompleteImportContextTransfer(GuestThreadExecution.ThreadState threadState, nint transferStubAddress) { - var stub = _preparedContextTransferStub; - _preparedContextTransferStub = 0; + var stub = threadState.PreparedContextTransferStub; + threadState.PreparedContextTransferStub = 0; // The transfer exit uses only the host stack after this publication. // Keep every read of the outgoing guest pack before this point. - GuestThreadExecution.CompleteCurrentContextTransfer(stub != 0); + threadState.CompleteContextTransfer(stub != 0); *(nint*)transferStubAddress = stub; } @@ -491,7 +491,8 @@ private unsafe ulong DispatchImportCore(int importIndex, nint argPackPtr) { // Any HLE entry ends this thread's polling streak in the spin-nap locked-op probe. SharpEmu.Core.Loader.SpinNapRuntime.NoteHleEntry(argPackPtr); - long num = NextImportDispatchIndex(); + var guestState = GuestThreadExecution.State; + long num = NextImportDispatchIndex(guestState); if ((num & 0x3F) == 0) { MarkExecutionProgress(); @@ -566,6 +567,18 @@ private unsafe ulong DispatchImportCore(int importIndex, nint argPackPtr) leafReturnRip = *(ulong*)(argPackPtr + 96); } + if (importStubEntry.IsBuilderLeaf && + TryDispatchBuilderLeaf(cpuContext, in importStubEntry, argPackPtr, num, out var builderResult)) + { + if (traceLeaf) + { + TraceLeafImport( + in importStubEntry, num, + leafArg0, leafArg1, leafArg2, leafArg3, leafReturnRip, builderResult); + } + return builderResult; + } + // The leaf paths skip the rest of the gateway entirely, so they mark // their own body: without this the profile charges a memcpy's whole cost // to "transition" and hides the work it actually did. @@ -681,7 +694,7 @@ private unsafe ulong DispatchImportCore(int importIndex, nint argPackPtr) stackArg5); ulong num7 = *(ulong*)(argPackPtr + 96); var probeTarget = _probeImportReturnAddress != 0 && num7 == _probeImportReturnAddress; - var isGuestWorker = GuestThreadExecution.IsGuestThread; + var isGuestWorker = guestState.GuestThreadHandle != 0; if (!IsLikelyReturnAddress(num7)) { for (int i = 1; i <= 4; i++) @@ -723,7 +736,9 @@ private unsafe ulong DispatchImportCore(int importIndex, nint argPackPtr) } if (_activeGuestThreadState is { } activeGuestThreadState) { - Interlocked.Increment(ref activeGuestThreadState.ImportCount); + // Only the host thread running this guest thread writes its count, so a plain increment + // is exact; readers use Interlocked.Read. The locked add was ~20 cycles per crossing. + activeGuestThreadState.ImportCount++; activeGuestThreadState.LastImportRdi = value; activeGuestThreadState.LastImportRsi = value2; activeGuestThreadState.LastImportRdx = num3; @@ -957,7 +972,7 @@ private unsafe ulong DispatchImportCore(int importIndex, nint argPackPtr) { OrbisGen2Result orbisGen2Result; bool dispatchResolved = true; - var previousImportCallFrame = GuestThreadExecution.EnterImportCallFrame( + var previousImportCallFrame = guestState.EnterImportCallFrame( num7, (ulong)argPackPtr + 104uL, ActiveGuestReturnSlotAddress); @@ -995,7 +1010,7 @@ private unsafe ulong DispatchImportCore(int importIndex, nint argPackPtr) } finally { - GuestThreadExecution.RestoreImportCallFrame(previousImportCallFrame); + guestState.RestoreImportCallFrame(previousImportCallFrame); } if (Volatile.Read(ref _pendingGuestExceptionCount) != 0) { @@ -1061,86 +1076,91 @@ private unsafe ulong DispatchImportCore(int importIndex, nint argPackPtr) value8, value, value2); - if (GuestThreadExecution.TryConsumeCurrentContextTransfer(out var transferTarget)) + // Almost no import leaves a context transfer, an entry exit or a block pending; one flag read + // replaces the three consume calls. + if (guestState.HasPendingDispatchOutcome) { - if (DsTrace.Hle) - { - DsTrace.LastOutcome = 2; - } - var dsPrepStarted = DsTrace.Fiber ? Stopwatch.GetTimestamp() : 0; - if (!TryPrepareGuestContextTransfer( - transferTarget, - out var transferFrame, - out var transferStub, - out var transferError)) + if (guestState.TryConsumeContextTransfer(out var transferTarget)) { - LastError = transferError ?? "failed to prepare guest context transfer"; - ActiveForcedGuestExit = true; - cpuContext[CpuRegister.Rax] = 18446744071562199298uL; - return cpuContext[CpuRegister.Rax]; - } - if (DsTrace.Fiber) - { - DsFiber.OnTransferPrepared(cpuContext, dsPrepStarted); - } + if (DsTrace.Hle) + { + DsTrace.LastOutcome = 2; + } + var dsPrepStarted = DsTrace.Fiber ? Stopwatch.GetTimestamp() : 0; + if (!TryPrepareGuestContextTransfer( + transferTarget, + out var transferFrame, + out var transferStub, + out var transferError)) + { + LastError = transferError ?? "failed to prepare guest context transfer"; + ActiveForcedGuestExit = true; + cpuContext[CpuRegister.Rax] = 18446744071562199298uL; + return cpuContext[CpuRegister.Rax]; + } + if (DsTrace.Fiber) + { + DsFiber.OnTransferPrepared(cpuContext, dsPrepStarted); + } - if (_logFiber) - { - Console.Error.WriteLine( - $"[LOADER][TRACE] fiber.context-transfer rip=0x{transferTarget.Rip:X16} " + - $"rsp=0x{transferTarget.Rsp:X16} guest=0x{GuestThreadExecution.CurrentGuestThreadHandle:X16} " + - $"fiber=0x{GuestThreadExecution.CurrentFiberAddress:X16}"); - } + if (_logFiber) + { + Console.Error.WriteLine( + $"[LOADER][TRACE] fiber.context-transfer rip=0x{transferTarget.Rip:X16} " + + $"rsp=0x{transferTarget.Rsp:X16} guest=0x{GuestThreadExecution.CurrentGuestThreadHandle:X16} " + + $"fiber=0x{GuestThreadExecution.CurrentFiberAddress:X16}"); + } - if (_activeGuestThreadState is { } transferGuestThreadState) - { - Volatile.Write(ref transferGuestThreadState.LastImportRax, transferTarget.Rax); - Volatile.Write(ref transferGuestThreadState.LastImportResultValid, 1); - } - _preparedContextTransferStub = transferStub; - return unchecked((ulong)transferFrame); - } - if (GuestThreadExecution.TryConsumeCurrentEntryExit(out var exitValue, out var exitReason)) - { - if (DsTrace.Hle) - { - DsTrace.LastOutcome = 3; + if (_activeGuestThreadState is { } transferGuestThreadState) + { + Volatile.Write(ref transferGuestThreadState.LastImportRax, transferTarget.Rax); + Volatile.Write(ref transferGuestThreadState.LastImportResultValid, 1); + } + guestState.PreparedContextTransferStub = transferStub; + return unchecked((ulong)transferFrame); } - if (TryCompleteGuestEntryToHostStub(argPackPtr, num, num7, importStubEntry.Nid, exitReason, exitValue)) + if (guestState.TryConsumeEntryExit(out var exitValue, out var exitReason)) { - cpuContext[CpuRegister.Rax] = exitValue; + if (DsTrace.Hle) + { + DsTrace.LastOutcome = 3; + } + if (TryCompleteGuestEntryToHostStub(argPackPtr, num, num7, importStubEntry.Nid, exitReason, exitValue)) + { + cpuContext[CpuRegister.Rax] = exitValue; + } + else + { + LastError = $"Failed to complete guest entry after {importStubEntry.Nid}: missing host return sentinel"; + cpuContext[CpuRegister.Rax] = 18446744071562199298uL; + } } - else + var consumedThreadBlock = guestState.TryConsumeThreadBlock( + out var blockReason, + out var blockContinuation, + out var hasBlockContinuation, + out var blockWakeKey, + out var blockWaiter, + out var blockDeadlineTimestamp); + if (consumedThreadBlock && (DsTrace.Hle || DsTrace.Sched)) { - LastError = $"Failed to complete guest entry after {importStubEntry.Nid}: missing host return sentinel"; - cpuContext[CpuRegister.Rax] = 18446744071562199298uL; + NoteDsTraceBlockConsumed(in importStubEntry, num7, cpuContext, argPackPtr); } - } - var consumedThreadBlock = GuestThreadExecution.TryConsumeCurrentThreadBlock( - out var blockReason, - out var blockContinuation, - out var hasBlockContinuation, - out var blockWakeKey, - out var blockWaiter, - out var blockDeadlineTimestamp); - if (consumedThreadBlock && (DsTrace.Hle || DsTrace.Sched)) - { - NoteDsTraceBlockConsumed(in importStubEntry, num7, cpuContext, argPackPtr); - } - if (consumedThreadBlock && - TryYieldGuestThreadToHostStub(argPackPtr, num, num7, importStubEntry.Nid, blockReason)) - { - if (hasBlockContinuation) + if (consumedThreadBlock && + TryYieldGuestThreadToHostStub(argPackPtr, num, num7, importStubEntry.Nid, blockReason)) { - RegisterBlockedGuestThreadContinuation( - GuestThreadExecution.CurrentGuestThreadHandle, - blockContinuation, - blockWakeKey, - blockWaiter, - blockDeadlineTimestamp); - } + if (hasBlockContinuation) + { + RegisterBlockedGuestThreadContinuation( + GuestThreadExecution.CurrentGuestThreadHandle, + blockContinuation, + blockWakeKey, + blockWaiter, + blockDeadlineTimestamp); + } - cpuContext[CpuRegister.Rax] = 0uL; + cpuContext[CpuRegister.Rax] = 0uL; + } } if (flag || flag2 || flag3) { @@ -1821,6 +1841,7 @@ private unsafe bool TryDispatchLeafImport( return false; } + var leafGuestState = GuestThreadExecution.State; var arg0 = *(ulong*)argPackPtr; var returnRip = *(ulong*)(argPackPtr + 96); var leafStackPointer = (ulong)argPackPtr + 96UL; @@ -1860,7 +1881,9 @@ private unsafe bool TryDispatchLeafImport( if (_activeGuestThreadState is { } activeGuestThreadState) { - Interlocked.Increment(ref activeGuestThreadState.ImportCount); + // Only the host thread running this guest thread writes its count, so a plain increment + // is exact; readers use Interlocked.Read. The locked add was ~20 cycles per crossing. + activeGuestThreadState.ImportCount++; activeGuestThreadState.LastImportRdi = arg0; activeGuestThreadState.LastImportRsi = *(ulong*)(argPackPtr + 8); activeGuestThreadState.LastImportRdx = *(ulong*)(argPackPtr + 16); @@ -1898,7 +1921,7 @@ private unsafe bool TryDispatchLeafImport( } else { - var previousImportCallFrame = GuestThreadExecution.EnterImportCallFrame( + var previousImportCallFrame = leafGuestState.EnterImportCallFrame( returnRip, (ulong)argPackPtr + 104uL, ActiveGuestReturnSlotAddress); @@ -1913,7 +1936,7 @@ private unsafe bool TryDispatchLeafImport( } finally { - GuestThreadExecution.RestoreImportCallFrame(previousImportCallFrame); + leafGuestState.RestoreImportCallFrame(previousImportCallFrame); } } if (Volatile.Read(ref _pendingGuestExceptionCount) != 0) @@ -1939,13 +1962,23 @@ private unsafe bool TryDispatchLeafImport( } } - var consumedThreadBlock = GuestThreadExecution.TryConsumeCurrentThreadBlock( - out var blockReason, - out var blockContinuation, - out var hasBlockContinuation, - out var blockWakeKey, - out var blockWaiter, - out var blockDeadlineTimestamp); + var consumedThreadBlock = false; + var blockReason = string.Empty; + var blockContinuation = default(GuestCpuContinuation); + var hasBlockContinuation = false; + var blockWakeKey = string.Empty; + IGuestThreadBlockWaiter? blockWaiter = null; + var blockDeadlineTimestamp = 0L; + if (leafGuestState.HasPendingDispatchOutcome) + { + consumedThreadBlock = leafGuestState.TryConsumeThreadBlock( + out blockReason, + out blockContinuation, + out hasBlockContinuation, + out blockWakeKey, + out blockWaiter, + out blockDeadlineTimestamp); + } if (consumedThreadBlock && (DsTrace.Hle || DsTrace.Sched)) { NoteDsTraceBlockConsumed(in importStubEntry, returnRip, cpuContext, argPackPtr); @@ -1983,6 +2016,126 @@ private unsafe bool TryDispatchLeafImport( return true; } + // SHARPEMU_AGC_BUILDER_LEAF=0 sends every sceAgc builder through the full import path. + private static readonly bool _builderLeafEnabled = !string.Equals( + Environment.GetEnvironmentVariable("SHARPEMU_AGC_BUILDER_LEAF"), "0", StringComparison.Ordinal); + + // The command-buffer builders a title calls thousands of times a frame. Each reads its + // arguments from registers (and, for a few, the first stack slots, which the leaf publishes + // through RSP), writes its packet into the guest command buffer and returns the packet's + // address. None blocks or yields. The one thing they can do that a leaf may not is call the + // buffer's "full" callback, which is guest code: a leaf runs them with + // CpuContext.DeclineGuestCallbacks set, so they back out before writing anything and the full + // path repeats the call. A builder that allocates in more than one step has to decline before + // its first step (see TryDeclineLeaf's callers); the ones listed here were checked for that. + internal static bool IsAgcBuilderLeafImport(string nid) => + nid is + "q88lQ+GP5Yk" or // sceAgcDcbDrawIndex + "Yw0jKSqop+E" or // sceAgcDcbDrawIndexAuto + "B+aG9DUnTKA" or // sceAgcDcbDrawIndexOffset + "t1vNu082-jM" or // sceAgcDcbDrawIndexIndirect + "l4fM9K-Lyks" or // sceAgcDcbSetIndexBuffer + "8N2tmT3jmC8" or // sceAgcDcbSetIndexCount + "GIIW2J37e70" or // sceAgcDcbSetIndexSize + "tSBxhAPyytQ" or // sceAgcDcbSetNumInstances + "pFLArOT53+w" or // sceAgcDcbSetShRegisterDirect + "LHFXRrlTPD8" or // sceAgcDcbSetCxRegisterDirect + "w4-d0n60hdo" or // sceAgcDcbSetUcRegisterDirect + "BVFg3CWU6Eo" or // sceAgcDcbSetCfRegisterRangeDirect + "n2fD4A+pb+g" or // sceAgcCbSetShRegisterRangeDirect + "-HOOCn0JY48" or // sceAgcDcbSetShRegistersIndirect + "ZvwO9euwYzc" or // sceAgcDcbSetCxRegistersIndirect + "hvUfkUIQcOE" or // sceAgcDcbSetUcRegistersIndirect + "k3GhuSNmBLU" or // sceAgcCbDispatch + "CtB+A9-VxO0" or // sceAgcDcbDispatchIndirect + "j3EtxFkSIhQ" or // sceAgcAcbDispatchIndirect + "RmaJwLtc8rY" or // sceAgcDcbSetBaseIndirectArgs + "VmW0Tdpy420" or // sceAgcDcbWaitRegMem + "htn36gPnBk4" or // sceAgcAcbWaitRegMem + "i1jyy49AjXU" or // sceAgcDcbWriteData + "eZ4+17OQz4Q" or // sceAgcAcbWriteData + "aJf+j5yntiU" or // sceAgcDcbEventWrite + "cFazmnXpJOE" or // sceAgcAcbEventWrite + "57labkp+rSQ" or // sceAgcDcbAcquireMem + "KT-hTp-Ch14" or // sceAgcAcbAcquireMem + "wr23dPKyWc0"; // sceAgcCbReleaseMem + + /// + /// Runs a command-buffer builder with the bookkeeping of a trivial leaf plus the stack pointer + /// (some builders read their seventh and later arguments from the guest stack), declining - + /// returning false with nothing written - when the builder would have to call the buffer's + /// guest callback or when anything that needs the full path is pending. + /// + private unsafe bool TryDispatchBuilderLeaf( + CpuContext cpuContext, + in ImportStubEntry importStubEntry, + nint argPackPtr, + long dispatchIndex, + out ulong result) + { + result = 0; + // The full path delivers pending guest exceptions at its safe point and ends a guest that + // is shutting down; the all-imports and filtered traces see only calls that take it. + if (!_builderLeafEnabled || + _logAllImports || + _profileGuestRip || + !string.IsNullOrWhiteSpace(_importFilter) || + Volatile.Read(ref _pendingGuestExceptionCount) != 0 || + Volatile.Read(ref _forcedGuestExit) || + importStubEntry.Export is not { } export || + (export.Target & cpuContext.TargetGeneration) == 0) + { + return false; + } + + cpuContext.Rip = importStubEntry.Address; + cpuContext[CpuRegister.Rdi] = *(ulong*)argPackPtr; + cpuContext[CpuRegister.Rsi] = *(ulong*)(argPackPtr + 8); + cpuContext[CpuRegister.Rdx] = *(ulong*)(argPackPtr + 16); + cpuContext[CpuRegister.Rcx] = *(ulong*)(argPackPtr + 24); + cpuContext[CpuRegister.R8] = *(ulong*)(argPackPtr + 32); + cpuContext[CpuRegister.R9] = *(ulong*)(argPackPtr + 40); + cpuContext[CpuRegister.Rsp] = (ulong)argPackPtr + 96uL; + cpuContext.ClearRaxWriteFlag(); + cpuContext.GuestCallbackDeclined = false; + cpuContext.DeclineGuestCallbacks = true; + int returnValue; + ImportProfile.MarkBodyStart(); + try + { + returnValue = export.Function(cpuContext); + } + finally + { + cpuContext.DeclineGuestCallbacks = false; + ImportProfile.MarkBodyEnd(); + } + + if (cpuContext.GuestCallbackDeclined) + { + cpuContext.GuestCallbackDeclined = false; + return false; + } + + if (!cpuContext.WasRaxWritten) + { + cpuContext[CpuRegister.Rax] = unchecked((ulong)returnValue); + } + + if (returnValue != (int)OrbisGen2Result.ORBIS_GEN2_OK && + ShouldLogImportResult(in importStubEntry, (OrbisGen2Result)returnValue, out var mutexOccurrence)) + { + Console.Error.WriteLine( + $"[LOADER][WARN] Import#{dispatchIndex} result: {(OrbisGen2Result)returnValue} ({importStubEntry.Nid}) " + + $"rdi=0x{cpuContext[CpuRegister.Rdi]:X16} rsi=0x{cpuContext[CpuRegister.Rsi]:X16} " + + $"ret=0x{*(ulong*)(argPackPtr + 96):X16}" + + (mutexOccurrence != 0 ? $" occurrence={mutexOccurrence}" : string.Empty)); + } + + result = cpuContext[CpuRegister.Rax]; + return true; + } + private static readonly bool _disableTrivialLeafDispatch = string.Equals( Environment.GetEnvironmentVariable("SHARPEMU_DISABLE_TRIVIAL_LEAF"), "1", StringComparison.Ordinal); @@ -2326,18 +2479,18 @@ private bool IsLeafImport(string nid) "kwGyyjohI50"; // sceKernelGetEventData } - private long NextImportDispatchIndex() + private long NextImportDispatchIndex(GuestThreadExecution.ThreadState state) { - if (!ReferenceEquals(_importCounterOwner, this) || - _nextImportDispatchIndex >= _importDispatchBlockEnd) + if (!ReferenceEquals(state.ImportCounterOwner, this) || + state.NextImportDispatchIndex >= state.ImportDispatchBlockEnd) { var blockEnd = Interlocked.Add(ref _importDispatchCount, ImportDispatchBlockSize); - _importCounterOwner = this; - _nextImportDispatchIndex = blockEnd - ImportDispatchBlockSize + 1; - _importDispatchBlockEnd = blockEnd + 1; + state.ImportCounterOwner = this; + state.NextImportDispatchIndex = blockEnd - ImportDispatchBlockSize + 1; + state.ImportDispatchBlockEnd = blockEnd + 1; } - return _nextImportDispatchIndex++; + return state.NextImportDispatchIndex++; } private void TraceImportFrameChain(CpuContext context, long dispatchIndex) diff --git a/src/SharpEmu.Core/Cpu/Native/DirectExecutionBackend.cs b/src/SharpEmu.Core/Cpu/Native/DirectExecutionBackend.cs index b1e7e59f6..677b68043 100644 --- a/src/SharpEmu.Core/Cpu/Native/DirectExecutionBackend.cs +++ b/src/SharpEmu.Core/Cpu/Native/DirectExecutionBackend.cs @@ -118,6 +118,11 @@ private readonly struct ImportStubEntry // stack; DispatchImport runs them without the full import bookkeeping. public bool IsTrivialLeaf { get; } + // A sceAgc command-buffer builder that DispatchBuilderLeaf may run without the + // import-boundary bookkeeping; it declines (and the full path runs) when it would + // have to call back into the guest. + public bool IsBuilderLeaf { get; } + public bool SuppressStrlenTrace { get; } public bool IsLoopGuardBoundary { get; } @@ -152,6 +157,7 @@ public ImportStubEntry( IsLeaf = isLeaf; IsNoBlockLeaf = isNoBlockLeaf; IsTrivialLeaf = export is not null && IsTrivialLeafImport(nid); + IsBuilderLeaf = export is not null && IsAgcBuilderLeafImport(nid); SuppressStrlenTrace = suppressStrlenTrace; IsLoopGuardBoundary = isLoopGuardBoundary; NidHash = nidHash; @@ -445,15 +451,6 @@ private static bool TryTakeReusableNestedCallbackStack(IVirtualMemory memory, ou [ThreadStatic] private static GuestThreadState? _activeGuestThreadState; - [ThreadStatic] - private static DirectExecutionBackend? _importCounterOwner; - - [ThreadStatic] - private static long _nextImportDispatchIndex; - - [ThreadStatic] - private static long _importDispatchBlockEnd; - private ImportStubEntry[] _importEntries = Array.Empty(); private readonly object _runtimeModuleGate = new object(); diff --git a/src/SharpEmu.Core/Cpu/Native/Platform/IDirectExecutionPlatform.cs b/src/SharpEmu.Core/Cpu/Native/Platform/IDirectExecutionPlatform.cs index 522a4cec9..6712f9c1a 100644 --- a/src/SharpEmu.Core/Cpu/Native/Platform/IDirectExecutionPlatform.cs +++ b/src/SharpEmu.Core/Cpu/Native/Platform/IDirectExecutionPlatform.cs @@ -73,6 +73,21 @@ public unsafe interface IDirectExecutionPlatform /// nint GetRuntimeRoutine(HostRuntimeFunction routine); + /// + /// The host's clock entry points the emitted guest clock stubs call directly, so a clock read + /// does not cross into managed code. is a no-argument SysV routine + /// returning a monotonic count in nanoseconds in RAX, on the same timebase as + /// ; is the C + /// clock_gettime(clockid, timespec*). Returns false when the platform has no such + /// routines, and the clock imports then keep going through the managed exports. + /// + bool TryGetHostClockEntryPoints(out nint counter, out nint clockGettime) + { + counter = 0; + clockGettime = 0; + return false; + } + /// /// Makes a managed callback callable with the Win64 ABI the emitted call /// sites use. Identity where managed code already uses that ABI; a diff --git a/src/SharpEmu.HLE/CpuContext.cs b/src/SharpEmu.HLE/CpuContext.cs index 9aff39230..952ea8748 100644 --- a/src/SharpEmu.HLE/CpuContext.cs +++ b/src/SharpEmu.HLE/CpuContext.cs @@ -31,6 +31,20 @@ public sealed class CpuContext(ICpuMemory memory, Generation generation) /// public int ActiveImportIndex { get; set; } = -1; + /// + /// Set by a leaf dispatcher for the duration of one export call: the export must not run guest + /// code (a command buffer's "full" callback, say) and instead sets + /// and returns before it has changed any state, so the dispatcher can run the call again through + /// the full import path, which can. + /// + public bool DeclineGuestCallbacks { get; set; } + + /// + /// Set by an export that declined to call into the guest under . + /// The dispatcher clears it and discards the call's result. + /// + public bool GuestCallbackDeclined { get; set; } + public ulong Rflags { get; set; } public ulong FsBase { get; set; } diff --git a/src/SharpEmu.HLE/GuestFastPath.cs b/src/SharpEmu.HLE/GuestFastPath.cs index 18912b3af..5551edc68 100644 --- a/src/SharpEmu.HLE/GuestFastPath.cs +++ b/src/SharpEmu.HLE/GuestFastPath.cs @@ -146,6 +146,34 @@ public static void BindGuestThread(ulong guestThreadHandle) *(ulong*)(block + BlockSelfHandleOffset) = guestThreadHandle; } + // SHARPEMU_HLE_FAST_SELF_HOST=0 keeps scePthreadSelf on the managed export for host threads + // (the entry thread among them) the guest scheduler does not own. + private static readonly bool HostSelfEnabled = + !string.Equals(Environment.GetEnvironmentVariable("SHARPEMU_HLE_FAST_SELF_HOST"), "0", StringComparison.Ordinal); + + /// + /// Lets the native pthread_self stub answer on a host thread with no guest thread bound - the + /// entry (main) thread - once the managed export has answered there and registered the thread's + /// CPU context with the scheduler. Until then the slot stays zero, so the first call on such a + /// thread still reaches the managed export; a guest thread binding over the thread replaces the + /// handle and unbinding clears it, which sends the next call back through the export. + /// + public static void PublishHostSelf(ulong hostThreadHandle) + { + if (!_enabled || !HostSelfEnabled || hostThreadHandle == 0) + { + return; + } + + var block = _block; + if (block == 0 || *(ulong*)(block + BlockSelfHandleOffset) != 0) + { + return; + } + + *(ulong*)(block + BlockSelfHandleOffset) = hostThreadHandle; + } + /// /// Records the calling host thread's own pthread handle - the one /// KernelPthreadState reports when no guest thread is bound - so diff --git a/src/SharpEmu.HLE/GuestThreadExecution.cs b/src/SharpEmu.HLE/GuestThreadExecution.cs index 9b99d7373..813a688c1 100644 --- a/src/SharpEmu.HLE/GuestThreadExecution.cs +++ b/src/SharpEmu.HLE/GuestThreadExecution.cs @@ -235,59 +235,158 @@ public DelegateGuestThreadBlockWaiter(Func resume, Func tryWake) public bool TryWake() => _tryWake(); } - [ThreadStatic] - private static ulong _currentGuestThreadHandle; - - [ThreadStatic] - private static ulong _currentFiberAddress; - - [ThreadStatic] - private static string? _pendingBlockReason; - - [ThreadStatic] - private static bool _pendingBlockContinuationValid; - - [ThreadStatic] - private static GuestCpuContinuation _pendingBlockContinuation; - - [ThreadStatic] - private static string? _pendingBlockWakeKey; - - [ThreadStatic] - private static IGuestThreadBlockWaiter? _pendingBlockWaiter; - - [ThreadStatic] - private static long _pendingBlockDeadlineTimestamp; - - [ThreadStatic] - private static bool _pendingEntryExit; - - [ThreadStatic] - private static ulong _pendingEntryExitValue; + /// + /// Every per-host-thread field of the guest execution state in one object. + /// + /// + /// These used to be seventeen separate [ThreadStatic] fields, and the import gateway + /// touched a dozen of them on every guest call: a thread-static read is a TLS lookup, which + /// is not free under Rosetta, and the gateway pays it again for each field. One thread-static + /// holds the object, so a crossing resolves the thread's state once and reads plain fields. + /// + internal sealed class ThreadState + { + public ulong GuestThreadHandle; + public ulong FiberAddress; + public string? PendingBlockReason; + public bool PendingBlockContinuationValid; + public GuestCpuContinuation PendingBlockContinuation; + public string? PendingBlockWakeKey; + public IGuestThreadBlockWaiter? PendingBlockWaiter; + public long PendingBlockDeadlineTimestamp; + public bool PendingEntryExit; + public ulong PendingEntryExitValue; + public string? PendingEntryExitReason; + public bool PendingContextTransfer; + public GuestCpuContinuation PendingContextTransferTarget; + public Action? ContextTransferCompletion; + public bool HasImportCallFrame; + public ulong ImportReturnRip; + public ulong ImportResumeRsp; + public ulong ImportReturnSlotAddress; + + // Slots the native import gateway keeps per host thread. They live here so that a crossing + // resolves this object once instead of reading a thread-static for each of them. + public object? ImportCounterOwner; + public long NextImportDispatchIndex; + public long ImportDispatchBlockEnd; + public nint PreparedContextTransferStub; + + /// + /// True when an import left a block, an entry exit or a context transfer pending. False for + /// almost every crossing, which then skips the three consume calls the gateway otherwise makes. + /// + public bool HasPendingDispatchOutcome => + PendingBlockReason is not null || PendingEntryExit || PendingContextTransfer; + + public GuestImportCallFrame EnterImportCallFrame(ulong returnRip, ulong resumeRsp, ulong returnSlotAddress) + { + var previous = new GuestImportCallFrame( + HasImportCallFrame, + ImportReturnRip, + ImportResumeRsp, + ImportReturnSlotAddress); + HasImportCallFrame = true; + ImportReturnRip = returnRip; + ImportResumeRsp = resumeRsp; + ImportReturnSlotAddress = returnSlotAddress; + return previous; + } - [ThreadStatic] - private static string? _pendingEntryExitReason; + public void RestoreImportCallFrame(GuestImportCallFrame previous) + { + HasImportCallFrame = previous.IsValid; + ImportReturnRip = previous.ReturnRip; + ImportResumeRsp = previous.ResumeRsp; + ImportReturnSlotAddress = previous.ReturnSlotAddress; + } - [ThreadStatic] - private static bool _pendingContextTransfer; + public bool TryConsumeContextTransfer(out GuestCpuContinuation target) + { + target = PendingContextTransferTarget; + if (!PendingContextTransfer) + { + return false; + } + + PendingContextTransfer = false; + PendingContextTransferTarget = default; + return true; + } - [ThreadStatic] - private static GuestCpuContinuation _pendingContextTransferTarget; + public bool TryConsumeEntryExit(out ulong value, out string reason) + { + value = PendingEntryExitValue; + reason = PendingEntryExitReason ?? string.Empty; + if (!PendingEntryExit) + { + return false; + } + + PendingEntryExit = false; + PendingEntryExitValue = 0; + PendingEntryExitReason = null; + return true; + } - [ThreadStatic] - private static Action? _contextTransferCompletion; + public bool TryConsumeThreadBlock( + out string reason, + out GuestCpuContinuation continuation, + out bool hasContinuation, + out string wakeKey, + out IGuestThreadBlockWaiter? waiter, + out long blockDeadlineTimestamp) + { + reason = PendingBlockReason ?? string.Empty; + if (string.IsNullOrEmpty(reason)) + { + continuation = default; + hasContinuation = false; + wakeKey = string.Empty; + waiter = null; + blockDeadlineTimestamp = 0; + return false; + } + + continuation = PendingBlockContinuation; + hasContinuation = PendingBlockContinuationValid; + wakeKey = PendingBlockWakeKey ?? reason; + waiter = PendingBlockWaiter; + blockDeadlineTimestamp = PendingBlockDeadlineTimestamp; + PendingBlockReason = null; + PendingBlockContinuation = default; + PendingBlockContinuationValid = false; + PendingBlockWakeKey = null; + PendingBlockWaiter = null; + PendingBlockDeadlineTimestamp = 0; + return true; + } - [ThreadStatic] - private static bool _hasCurrentImportCallFrame; + public void CompleteContextTransfer(bool succeeded) + { + // The target is only ever non-default while PendingContextTransfer is set, so with + // neither a pending transfer nor a completion armed there is nothing to clear - the + // common case, which used to zero the continuation on every crossing. + var completion = ContextTransferCompletion; + if (completion is null && !PendingContextTransfer) + { + return; + } + + ContextTransferCompletion = null; + PendingContextTransfer = false; + PendingContextTransferTarget = default; + completion?.Invoke(succeeded); + } + } [ThreadStatic] - private static ulong _currentImportReturnRip; + private static ThreadState? t_state; - [ThreadStatic] - private static ulong _currentImportResumeRsp; + /// The calling host thread's guest execution state; created on first use. + internal static ThreadState State => t_state ?? CreateState(); - [ThreadStatic] - private static ulong _currentImportReturnSlotAddress; + private static ThreadState CreateState() => t_state = new ThreadState(); public static IGuestThreadScheduler? Scheduler { get; set; } @@ -356,71 +455,75 @@ public static void NotifyHostWorkProgress() } } - public static bool IsGuestThread => _currentGuestThreadHandle != 0; + public static bool IsGuestThread => State.GuestThreadHandle != 0; - public static ulong CurrentGuestThreadHandle => _currentGuestThreadHandle; + public static ulong CurrentGuestThreadHandle => State.GuestThreadHandle; - public static bool HasPendingCurrentThreadBlock => _pendingBlockReason is not null; + public static bool HasPendingCurrentThreadBlock => State.PendingBlockReason is not null; - public static ulong CurrentFiberAddress => _currentFiberAddress; + public static ulong CurrentFiberAddress => State.FiberAddress; public static ulong EnterGuestThread(ulong threadHandle) { + var s = State; CompleteCurrentContextTransfer(false); GpuMemory.GpuMemoryAccessProfile.InitializeCurrentThread(); - var previous = _currentGuestThreadHandle; - _currentGuestThreadHandle = threadHandle; + var previous = s.GuestThreadHandle; + s.GuestThreadHandle = threadHandle; GuestFastPath.BindGuestThread(threadHandle); - _pendingBlockReason = null; - _pendingBlockContinuationValid = false; - _pendingBlockContinuation = default; - _pendingBlockWakeKey = null; - _pendingBlockWaiter = null; - _pendingBlockDeadlineTimestamp = 0; - _pendingEntryExit = false; - _pendingEntryExitValue = 0; - _pendingEntryExitReason = null; - _pendingContextTransfer = false; - _pendingContextTransferTarget = default; - _hasCurrentImportCallFrame = false; - _currentImportReturnRip = 0; - _currentImportResumeRsp = 0; - _currentImportReturnSlotAddress = 0; + s.PendingBlockReason = null; + s.PendingBlockContinuationValid = false; + s.PendingBlockContinuation = default; + s.PendingBlockWakeKey = null; + s.PendingBlockWaiter = null; + s.PendingBlockDeadlineTimestamp = 0; + s.PendingEntryExit = false; + s.PendingEntryExitValue = 0; + s.PendingEntryExitReason = null; + s.PendingContextTransfer = false; + s.PendingContextTransferTarget = default; + s.HasImportCallFrame = false; + s.ImportReturnRip = 0; + s.ImportResumeRsp = 0; + s.ImportReturnSlotAddress = 0; return previous; } public static void RestoreGuestThread(ulong previousThreadHandle) { + var s = State; CompleteCurrentContextTransfer(false); - _currentGuestThreadHandle = previousThreadHandle; + s.GuestThreadHandle = previousThreadHandle; GuestFastPath.BindGuestThread(previousThreadHandle); - _pendingBlockReason = null; - _pendingBlockContinuationValid = false; - _pendingBlockContinuation = default; - _pendingBlockWakeKey = null; - _pendingBlockWaiter = null; - _pendingBlockDeadlineTimestamp = 0; - _pendingEntryExit = false; - _pendingEntryExitValue = 0; - _pendingEntryExitReason = null; - _pendingContextTransfer = false; - _pendingContextTransferTarget = default; - _hasCurrentImportCallFrame = false; - _currentImportReturnRip = 0; - _currentImportResumeRsp = 0; - _currentImportReturnSlotAddress = 0; + s.PendingBlockReason = null; + s.PendingBlockContinuationValid = false; + s.PendingBlockContinuation = default; + s.PendingBlockWakeKey = null; + s.PendingBlockWaiter = null; + s.PendingBlockDeadlineTimestamp = 0; + s.PendingEntryExit = false; + s.PendingEntryExitValue = 0; + s.PendingEntryExitReason = null; + s.PendingContextTransfer = false; + s.PendingContextTransferTarget = default; + s.HasImportCallFrame = false; + s.ImportReturnRip = 0; + s.ImportResumeRsp = 0; + s.ImportReturnSlotAddress = 0; } public static ulong EnterFiber(ulong fiberAddress) { - var previous = _currentFiberAddress; - _currentFiberAddress = fiberAddress; + var s = State; + var previous = s.FiberAddress; + s.FiberAddress = fiberAddress; return previous; } public static void RestoreFiber(ulong previousFiberAddress) { - _currentFiberAddress = previousFiberAddress; + var s = State; + s.FiberAddress = previousFiberAddress; } // A nested guest exception handler (delivered synchronously from @@ -476,45 +579,47 @@ internal PendingBlockSnapshot( public static PendingBlockSnapshot SuspendPendingBlock() { + var s = State; var snapshot = new PendingBlockSnapshot( - _pendingBlockReason, - _pendingBlockContinuationValid, - _pendingBlockContinuation, - _pendingBlockWakeKey, - _pendingBlockWaiter, - _pendingBlockDeadlineTimestamp, - _pendingEntryExit, - _pendingEntryExitValue, - _pendingEntryExitReason, - _pendingContextTransfer, - _pendingContextTransferTarget); - _pendingBlockReason = null; - _pendingBlockContinuation = default; - _pendingBlockContinuationValid = false; - _pendingBlockWakeKey = null; - _pendingBlockWaiter = null; - _pendingBlockDeadlineTimestamp = 0; - _pendingEntryExit = false; - _pendingEntryExitValue = 0; - _pendingEntryExitReason = null; - _pendingContextTransfer = false; - _pendingContextTransferTarget = default; + s.PendingBlockReason, + s.PendingBlockContinuationValid, + s.PendingBlockContinuation, + s.PendingBlockWakeKey, + s.PendingBlockWaiter, + s.PendingBlockDeadlineTimestamp, + s.PendingEntryExit, + s.PendingEntryExitValue, + s.PendingEntryExitReason, + s.PendingContextTransfer, + s.PendingContextTransferTarget); + s.PendingBlockReason = null; + s.PendingBlockContinuation = default; + s.PendingBlockContinuationValid = false; + s.PendingBlockWakeKey = null; + s.PendingBlockWaiter = null; + s.PendingBlockDeadlineTimestamp = 0; + s.PendingEntryExit = false; + s.PendingEntryExitValue = 0; + s.PendingEntryExitReason = null; + s.PendingContextTransfer = false; + s.PendingContextTransferTarget = default; return snapshot; } public static void RestorePendingBlock(PendingBlockSnapshot snapshot) { - _pendingEntryExit = snapshot.EntryExitPending; - _pendingEntryExitValue = snapshot.EntryExitValue; - _pendingEntryExitReason = snapshot.EntryExitReason; - _pendingContextTransfer = snapshot.ContextTransferPending; - _pendingContextTransferTarget = snapshot.ContextTransferTarget; - _pendingBlockReason = snapshot.Reason; - _pendingBlockContinuationValid = snapshot.ContinuationValid; - _pendingBlockContinuation = snapshot.Continuation; - _pendingBlockWakeKey = snapshot.WakeKey; - _pendingBlockWaiter = snapshot.Waiter; - _pendingBlockDeadlineTimestamp = snapshot.DeadlineTimestamp; + var s = State; + s.PendingEntryExit = snapshot.EntryExitPending; + s.PendingEntryExitValue = snapshot.EntryExitValue; + s.PendingEntryExitReason = snapshot.EntryExitReason; + s.PendingContextTransfer = snapshot.ContextTransferPending; + s.PendingContextTransferTarget = snapshot.ContextTransferTarget; + s.PendingBlockReason = snapshot.Reason; + s.PendingBlockContinuationValid = snapshot.ContinuationValid; + s.PendingBlockContinuation = snapshot.Continuation; + s.PendingBlockWakeKey = snapshot.WakeKey; + s.PendingBlockWaiter = snapshot.Waiter; + s.PendingBlockDeadlineTimestamp = snapshot.DeadlineTimestamp; } public static bool RequestCurrentThreadBlock(string reason) => RequestCurrentThreadBlock(null, reason); @@ -526,24 +631,25 @@ public static bool RequestCurrentThreadBlock( IGuestThreadBlockWaiter? waiter = null, long blockDeadlineTimestamp = 0) { + var s = State; if (!IsGuestThread) { return false; } - _pendingBlockReason = string.IsNullOrWhiteSpace(reason) ? "guest_thread_blocked" : reason; - _pendingBlockWakeKey = string.IsNullOrWhiteSpace(wakeKey) ? _pendingBlockReason : wakeKey; - _pendingBlockWaiter = waiter; - _pendingBlockDeadlineTimestamp = blockDeadlineTimestamp; + s.PendingBlockReason = string.IsNullOrWhiteSpace(reason) ? "guest_thread_blocked" : reason; + s.PendingBlockWakeKey = string.IsNullOrWhiteSpace(wakeKey) ? s.PendingBlockReason : wakeKey; + s.PendingBlockWaiter = waiter; + s.PendingBlockDeadlineTimestamp = blockDeadlineTimestamp; if (context is not null && TryCaptureCurrentBlockContinuation(context, out var continuation)) { - _pendingBlockContinuation = continuation; - _pendingBlockContinuationValid = true; + s.PendingBlockContinuation = continuation; + s.PendingBlockContinuationValid = true; } else { - _pendingBlockContinuation = default; - _pendingBlockContinuationValid = false; + s.PendingBlockContinuation = default; + s.PendingBlockContinuationValid = false; } return true; @@ -607,32 +713,14 @@ public static bool TryConsumeCurrentThreadBlock( out bool hasContinuation, out string wakeKey, out IGuestThreadBlockWaiter? waiter, - out long blockDeadlineTimestamp) - { - reason = _pendingBlockReason ?? string.Empty; - if (string.IsNullOrEmpty(reason)) - { - continuation = default; - hasContinuation = false; - wakeKey = string.Empty; - waiter = null; - blockDeadlineTimestamp = 0; - return false; - } - - continuation = _pendingBlockContinuation; - hasContinuation = _pendingBlockContinuationValid; - wakeKey = _pendingBlockWakeKey ?? reason; - waiter = _pendingBlockWaiter; - blockDeadlineTimestamp = _pendingBlockDeadlineTimestamp; - _pendingBlockReason = null; - _pendingBlockContinuation = default; - _pendingBlockContinuationValid = false; - _pendingBlockWakeKey = null; - _pendingBlockWaiter = null; - _pendingBlockDeadlineTimestamp = 0; - return true; - } + out long blockDeadlineTimestamp) => + State.TryConsumeThreadBlock( + out reason, + out continuation, + out hasContinuation, + out wakeKey, + out waiter, + out blockDeadlineTimestamp); public static long ComputeDeadlineTimestamp(TimeSpan timeout) { @@ -699,83 +787,42 @@ public static void RequestCurrentEntryExit(string reason, int status) public static void RequestCurrentEntryExit(string reason, ulong value) { - _pendingEntryExit = true; - _pendingEntryExitValue = value; - _pendingEntryExitReason = string.IsNullOrWhiteSpace(reason) ? "guest_entry_exit" : reason; + var s = State; + s.PendingEntryExit = true; + s.PendingEntryExitValue = value; + s.PendingEntryExitReason = string.IsNullOrWhiteSpace(reason) ? "guest_entry_exit" : reason; } - public static bool TryConsumeCurrentEntryExit(out ulong value, out string reason) - { - value = _pendingEntryExitValue; - reason = _pendingEntryExitReason ?? string.Empty; - if (!_pendingEntryExit) - { - return false; - } - - _pendingEntryExit = false; - _pendingEntryExitValue = 0; - _pendingEntryExitReason = null; - return true; - } + public static bool TryConsumeCurrentEntryExit(out ulong value, out string reason) => + State.TryConsumeEntryExit(out value, out reason); public static void RequestCurrentContextTransfer(GuestCpuContinuation target, Action? completion = null) { - _pendingContextTransferTarget = target; - _pendingContextTransfer = true; - _contextTransferCompletion = completion; + var s = State; + s.PendingContextTransferTarget = target; + s.PendingContextTransfer = true; + s.ContextTransferCompletion = completion; } - public static void CompleteCurrentContextTransfer(bool succeeded) - { - var completion = _contextTransferCompletion; - _contextTransferCompletion = null; - _pendingContextTransfer = false; - _pendingContextTransferTarget = default; - completion?.Invoke(succeeded); - } + public static void CompleteCurrentContextTransfer(bool succeeded) => + State.CompleteContextTransfer(succeeded); - public static bool TryConsumeCurrentContextTransfer(out GuestCpuContinuation target) - { - target = _pendingContextTransferTarget; - if (!_pendingContextTransfer) - { - return false; - } - - _pendingContextTransfer = false; - _pendingContextTransferTarget = default; - return true; - } + public static bool TryConsumeCurrentContextTransfer(out GuestCpuContinuation target) => + State.TryConsumeContextTransfer(out target); public static GuestImportCallFrame EnterImportCallFrame( ulong returnRip, ulong resumeRsp, - ulong returnSlotAddress) - { - var previous = new GuestImportCallFrame( - _hasCurrentImportCallFrame, - _currentImportReturnRip, - _currentImportResumeRsp, - _currentImportReturnSlotAddress); - _hasCurrentImportCallFrame = true; - _currentImportReturnRip = returnRip; - _currentImportResumeRsp = resumeRsp; - _currentImportReturnSlotAddress = returnSlotAddress; - return previous; - } + ulong returnSlotAddress) => + State.EnterImportCallFrame(returnRip, resumeRsp, returnSlotAddress); - public static void RestoreImportCallFrame(GuestImportCallFrame previous) - { - _hasCurrentImportCallFrame = previous.IsValid; - _currentImportReturnRip = previous.ReturnRip; - _currentImportResumeRsp = previous.ResumeRsp; - _currentImportReturnSlotAddress = previous.ReturnSlotAddress; - } + public static void RestoreImportCallFrame(GuestImportCallFrame previous) => + State.RestoreImportCallFrame(previous); public static bool TryGetCurrentImportCallFrame(out GuestImportCallFrame frame) { - if (!_hasCurrentImportCallFrame) + var s = State; + if (!s.HasImportCallFrame) { frame = default; return false; @@ -783,9 +830,9 @@ public static bool TryGetCurrentImportCallFrame(out GuestImportCallFrame frame) frame = new GuestImportCallFrame( true, - _currentImportReturnRip, - _currentImportResumeRsp, - _currentImportReturnSlotAddress); + s.ImportReturnRip, + s.ImportResumeRsp, + s.ImportReturnSlotAddress); return true; } } diff --git a/src/SharpEmu.Libs/Agc/AgcExports.CommandPackets.cs b/src/SharpEmu.Libs/Agc/AgcExports.CommandPackets.cs index e841356d9..95dfed3ff 100644 --- a/src/SharpEmu.Libs/Agc/AgcExports.CommandPackets.cs +++ b/src/SharpEmu.Libs/Agc/AgcExports.CommandPackets.cs @@ -70,8 +70,7 @@ public static int GetDataPacketPayloadAddress(CpuContext ctx) if (ShouldTraceHotPath(ref _packetPayloadTraceCount)) { TraceAgc( - $"agc.get_packet_payload out=0x{outputAddress:X16} cmd=0x{commandAddress:X16} " + - $"type={type} payload=0x{payloadAddress:X16}"); + $"agc.get_packet_payload out=0x{outputAddress:X16} cmd=0x{commandAddress:X16} type={type} payload=0x{payloadAddress:X16}"); } ctx[CpuRegister.Rax] = 0; @@ -271,29 +270,20 @@ public static int AcbWaitRegMem(CpuContext ctx) var packetDwords = size == 0 ? 7u : 9u; var packetRegister = size == 0 ? RWaitMem32 : RWaitMem64; - if (!TryAllocateCommandDwords(ctx, commandBufferAddress, packetDwords, out var commandAddress) || - !TryWriteUInt32(ctx, commandAddress, Pm4(packetDwords, ItNop, packetRegister)) || - !TryWriteUInt32(ctx, commandAddress + 4, (uint)address & (size == 0 ? ~0x3u : ~0x7u)) || - !TryWriteUInt32(ctx, commandAddress + 8, (uint)(address >> 32) & 0x3FFFFu) || - !TryWriteUInt32(ctx, commandAddress + 12, (uint)mask)) - { - return ReturnPointer(ctx, 0); - } - - if (size == 0) - { - if (!TryWriteUInt32(ctx, commandAddress + 16, (uint)reference) || - !TryWriteUInt32(ctx, commandAddress + 20, EncodeWaitRegMem32Control(compareFunction, 0, cachePolicy)) || - !TryWriteUInt32(ctx, commandAddress + 24, EncodeWaitRegMemPoll(pollCycles))) - { - return ReturnPointer(ctx, 0); - } - } - else if (!TryWriteUInt32(ctx, commandAddress + 16, (uint)(mask >> 32)) || - !TryWriteUInt32(ctx, commandAddress + 20, (uint)reference) || - !TryWriteUInt32(ctx, commandAddress + 24, (uint)(reference >> 32)) || - !TryWriteUInt32(ctx, commandAddress + 28, EncodeWaitRegMem64Control(compareFunction, 0, cachePolicy)) || - !TryWriteUInt32(ctx, commandAddress + 32, EncodeWaitRegMemPoll(pollCycles))) + var header = Pm4(packetDwords, ItNop, packetRegister); + var addressLow = (uint)address & (size == 0 ? ~0x3u : ~0x7u); + var addressHigh = (uint)(address >> 32) & 0x3FFFFu; + // One write for the whole packet, as DcbWaitRegMem does. + var emitted = size == 0 + ? TryEmitCommandDwords( + ctx, commandBufferAddress, out var commandAddress, + header, addressLow, addressHigh, (uint)mask, (uint)reference, + EncodeWaitRegMem32Control(compareFunction, 0, cachePolicy), EncodeWaitRegMemPoll(pollCycles)) + : TryEmitCommandDwords( + ctx, commandBufferAddress, out commandAddress, + header, addressLow, addressHigh, (uint)mask, (uint)(mask >> 32), (uint)reference, (uint)(reference >> 32), + EncodeWaitRegMem64Control(compareFunction, 0, cachePolicy), EncodeWaitRegMemPoll(pollCycles)); + if (!emitted) { return ReturnPointer(ctx, 0); } @@ -381,8 +371,7 @@ public static int CbReleaseMem(CpuContext ctx) } TraceAgc( - $"agc.cb_release_mem buf=0x{commandBufferAddress:X16} cmd=0x{commandAddress:X16} " + - $"action=0x{action:X2} gcr=0x{gcrControl:X4} dst=0x{destinationAddress:X16} data_sel={dataSelection} data=0x{data:X16}"); + $"agc.cb_release_mem buf=0x{commandBufferAddress:X16} cmd=0x{commandAddress:X16} action=0x{action:X2} gcr=0x{gcrControl:X4} dst=0x{destinationAddress:X16} data_sel={dataSelection} data=0x{data:X16}"); return ReturnPointer(ctx, commandAddress); } @@ -458,8 +447,7 @@ public static int DcbGetLodStats(CpuContext ctx) } TraceAgc( - $"agc.dcb_get_lod_stats buf=0x{commandBufferAddress:X16} cmd=0x{commandAddress:X16} " + - $"dst=0x{destinationAddress:X16} control=0x{control:X8} counters=0x{counterMask:X2}"); + $"agc.dcb_get_lod_stats buf=0x{commandBufferAddress:X16} cmd=0x{commandAddress:X16} dst=0x{destinationAddress:X16} control=0x{control:X8} counters=0x{counterMask:X2}"); return ReturnPointer(ctx, commandAddress); } @@ -494,8 +482,7 @@ public static int DcbAcquireMem(CpuContext ctx) } TraceAgc( - $"agc.dcb_acquire_mem buf=0x{commandBufferAddress:X16} cmd=0x{commandAddress:X16} " + - $"engine={engine} cbdb=0x{cbDbOp:X8} gcr=0x{gcrControl:X8} base=0x{baseAddress:X16} size=0x{sizeBytes:X16}"); + $"agc.dcb_acquire_mem buf=0x{commandBufferAddress:X16} cmd=0x{commandAddress:X16} engine={engine} cbdb=0x{cbDbOp:X8} gcr=0x{gcrControl:X8} base=0x{baseAddress:X16} size=0x{sizeBytes:X16}"); return ReturnPointer(ctx, commandAddress); } @@ -580,9 +567,7 @@ public static int DcbWriteData(CpuContext ctx) if (ShouldTraceHotPath(ref _dcbWriteDataTraceCount)) { TraceAgc( - $"agc.dcb_write_data buf=0x{commandBufferAddress:X16} cmd=0x{commandAddress:X16} " + - $"dst={destination} cache={cachePolicy} addr=0x{destinationAddress:X16} count={dwordCount} " + - $"increment={increment} confirm={writeConfirm}"); + $"agc.dcb_write_data buf=0x{commandBufferAddress:X16} cmd=0x{commandAddress:X16} dst={destination} cache={cachePolicy} addr=0x{destinationAddress:X16} count={dwordCount} increment={increment} confirm={writeConfirm}"); } return ReturnPointer(ctx, commandAddress); @@ -635,11 +620,7 @@ writeSpace is not 1 and not 2 || } TraceAgc( - $"agc.cb_cond_write buf=0x{commandBufferAddress:X16} " + - $"cmd=0x{commandAddress:X16} compare={compareFunction} " + - $"space={writeSpace} read=0x{readAddress:X16} " + - $"ref=0x{reference:X8} mask=0x{mask:X8} " + - $"write=0x{writeAddress:X16} data=0x{writeValue:X8}"); + $"agc.cb_cond_write buf=0x{commandBufferAddress:X16} cmd=0x{commandAddress:X16} compare={compareFunction} space={writeSpace} read=0x{readAddress:X16} ref=0x{reference:X8} mask=0x{mask:X8} write=0x{writeAddress:X16} data=0x{writeValue:X8}"); return ReturnPointer(ctx, commandAddress); } @@ -696,9 +677,7 @@ public static int DcbWaitRegMem(CpuContext ctx) if (ShouldTraceHotPath(ref _dcbWaitRegMemTraceCount)) { TraceAgc( - $"agc.dcb_wait_reg_mem buf=0x{commandBufferAddress:X16} cmd=0x{commandAddress:X16} " + - $"size={size} compare={compareFunction} op={operation} cache={cachePolicy} " + - $"addr=0x{address:X16} ref=0x{reference:X16} mask=0x{mask:X16} poll={pollCycles}"); + $"agc.dcb_wait_reg_mem buf=0x{commandBufferAddress:X16} cmd=0x{commandAddress:X16} size={size} compare={compareFunction} op={operation} cache={cachePolicy} addr=0x{address:X16} ref=0x{reference:X16} mask=0x{mask:X16} poll={pollCycles}"); } return ReturnPointer(ctx, commandAddress); @@ -742,8 +721,7 @@ public static int DcbStallCommandBufferParser(CpuContext ctx) // hardware command processor to stall. Keep a well-formed no-op in the DCB // so packet addresses and the command-buffer cursor remain coherent. TraceAgc( - $"agc.dcb_stall_parser buf=0x{commandBufferAddress:X16} cmd=0x{commandAddress:X16} " + - $"size={size} addr=0x{address:X16} reference=0x{reference:X16}"); + $"agc.dcb_stall_parser buf=0x{commandBufferAddress:X16} cmd=0x{commandAddress:X16} size={size} addr=0x{address:X16} reference=0x{reference:X16}"); return ReturnPointer(ctx, commandAddress); } diff --git a/src/SharpEmu.Libs/Agc/AgcExports.ControlFlow.cs b/src/SharpEmu.Libs/Agc/AgcExports.ControlFlow.cs index c14ae7709..3c89fabe2 100644 --- a/src/SharpEmu.Libs/Agc/AgcExports.ControlFlow.cs +++ b/src/SharpEmu.Libs/Agc/AgcExports.ControlFlow.cs @@ -255,8 +255,7 @@ public static int CbBranch(CpuContext ctx) } TraceAgc( - $"agc.cb_branch buf=0x{commandBufferAddress:X16} cmd=0x{commandAddress:X16} " + - $"target=0x{target:X16} dwords={targetDwords}"); + $"agc.cb_branch buf=0x{commandBufferAddress:X16} cmd=0x{commandAddress:X16} target=0x{target:X16} dwords={targetDwords}"); return ReturnPointer(ctx, commandAddress); } diff --git a/src/SharpEmu.Libs/Agc/AgcExports.DmaPackets.cs b/src/SharpEmu.Libs/Agc/AgcExports.DmaPackets.cs index 19fb15349..90327ec55 100644 --- a/src/SharpEmu.Libs/Agc/AgcExports.DmaPackets.cs +++ b/src/SharpEmu.Libs/Agc/AgcExports.DmaPackets.cs @@ -56,9 +56,7 @@ public static int DcbCopyData(CpuContext ctx) } TraceAgc( - $"agc.dcb_copy_data buf=0x{commandBufferAddress:X16} cmd=0x{packetAddress:X16} " + - $"dstSel=0x{destinationSelector:X2} srcSel=0x{sourceSelector:X2} " + - $"dst=0x{destinationAddress:X16} src=0x{sourceValue:X16} control=0x{control:X8}"); + $"agc.dcb_copy_data buf=0x{commandBufferAddress:X16} cmd=0x{packetAddress:X16} dstSel=0x{destinationSelector:X2} srcSel=0x{sourceSelector:X2} dst=0x{destinationAddress:X16} src=0x{sourceValue:X16} control=0x{control:X8}"); return ReturnPointer(ctx, packetAddress); } @@ -113,9 +111,7 @@ public static int DcbDmaData(CpuContext ctx) } TraceAgc( - $"agc.dcb_dma_data buf=0x{commandBufferAddress:X16} cmd=0x{commandAddress:X16} " + - $"dst=0x{destinationAddress:X16} src=0x{sourceAddress:X16} bytes={byteCount} " + - $"control0=0x{destination | (destinationCachePolicy << 8) | (source << 16) | (sourceCachePolicy << 24):X8}"); + $"agc.dcb_dma_data buf=0x{commandBufferAddress:X16} cmd=0x{commandAddress:X16} dst=0x{destinationAddress:X16} src=0x{sourceAddress:X16} bytes={byteCount} control0=0x{destination | (destinationCachePolicy << 8) | (source << 16) | (sourceCachePolicy << 24):X8}"); return ReturnPointer(ctx, commandAddress); } diff --git a/src/SharpEmu.Libs/Agc/AgcExports.DrawPackets.cs b/src/SharpEmu.Libs/Agc/AgcExports.DrawPackets.cs index 02801e8c7..9871ef295 100644 --- a/src/SharpEmu.Libs/Agc/AgcExports.DrawPackets.cs +++ b/src/SharpEmu.Libs/Agc/AgcExports.DrawPackets.cs @@ -52,9 +52,7 @@ private static int DcbSetIndexSizePacket( } TraceAgc( - $"agc.dcb_set_index_size buf=0x{commandBufferAddress:X16} " + - $"cmd=0x{commandAddress:X16} size={indexSize & 0x3u} " + - $"cache={cachePolicy & 0x3u} instance_id={perInstanceObjectIdSupport & 0x1u}"); + $"agc.dcb_set_index_size buf=0x{commandBufferAddress:X16} cmd=0x{commandAddress:X16} size={indexSize & 0x3u} cache={cachePolicy & 0x3u} instance_id={perInstanceObjectIdSupport & 0x1u}"); return ReturnPointer(ctx, commandAddress); } @@ -149,6 +147,12 @@ public static int DcbDrawIndex(CpuContext ctx) return ReturnPointer(ctx, 0); } } + else if (TryDeclineLeaf(ctx)) + { + // The two packets would be allocated one at a time, the second after the first was + // written, so a leaf backs out before either. + return ReturnPointer(ctx, 0); + } else if (!TryEmitCommandDwords(ctx, commandBufferAddress, out baseCommand, Pm4(3, ItIndexBase, 0), (uint)indexAddress, (uint)(indexAddress >> 32), Pm4(2, ItIndexBufferSize, 0), indexCount) || !TryEmitCommandDwords(ctx, commandBufferAddress, out drawCommand, Pm4(6, ItDrawIndex2, 0), indexCount, (uint)indexAddress, (uint)(indexAddress >> 32), indexCount, drawInitiator)) { @@ -156,9 +160,7 @@ public static int DcbDrawIndex(CpuContext ctx) } TraceAgc( - $"agc.dcb_draw_index buf=0x{commandBufferAddress:X16} " + - $"base=0x{baseCommand:X16} draw=0x{drawCommand:X16} " + - $"count={indexCount} index=0x{indexAddress:X16} modifier=0x{modifier:X}"); + $"agc.dcb_draw_index buf=0x{commandBufferAddress:X16} base=0x{baseCommand:X16} draw=0x{drawCommand:X16} count={indexCount} index=0x{indexAddress:X16} modifier=0x{modifier:X}"); return ReturnPointer(ctx, drawCommand); } @@ -207,8 +209,7 @@ public static int DcbDrawIndirect(CpuContext ctx) } TraceAgc( - $"agc.dcb_draw_indirect buf=0x{commandBufferAddress:X16} " + - $"draw=0x{drawCommand:X16} offset=0x{dataOffset:X} modifier=0x{drawModifier:X}"); + $"agc.dcb_draw_indirect buf=0x{commandBufferAddress:X16} draw=0x{drawCommand:X16} offset=0x{dataOffset:X} modifier=0x{drawModifier:X}"); return ReturnPointer(ctx, drawCommand); } @@ -282,8 +283,7 @@ public static int DcbDrawIndexIndirect(CpuContext ctx) } TraceAgc( - $"agc.dcb_draw_index_indirect buf=0x{commandBufferAddress:X16} " + - $"cmd=0x{commandAddress:X16} offset=0x{dataOffset:X8} modifier=0x{modifier:X8}"); + $"agc.dcb_draw_index_indirect buf=0x{commandBufferAddress:X16} cmd=0x{commandAddress:X16} offset=0x{dataOffset:X8} modifier=0x{modifier:X8}"); return ReturnPointer(ctx, commandAddress); } @@ -332,9 +332,7 @@ public static int DcbDrawIndexIndirectMulti(CpuContext ctx) } TraceAgc( - $"agc.dcb_draw_index_indirect_multi buf=0x{commandBufferAddress:X16} " + - $"cmd=0x{commandAddress:X16} offset=0x{dataOffset:X8} draws={drawCount} " + - $"stride={stride} modifier=0x{modifier:X8}"); + $"agc.dcb_draw_index_indirect_multi buf=0x{commandBufferAddress:X16} cmd=0x{commandAddress:X16} offset=0x{dataOffset:X8} draws={drawCount} stride={stride} modifier=0x{modifier:X8}"); return ReturnPointer(ctx, commandAddress); } diff --git a/src/SharpEmu.Libs/Agc/AgcExports.DriverResources.cs b/src/SharpEmu.Libs/Agc/AgcExports.DriverResources.cs index c6c2ddddc..97c509126 100644 --- a/src/SharpEmu.Libs/Agc/AgcExports.DriverResources.cs +++ b/src/SharpEmu.Libs/Agc/AgcExports.DriverResources.cs @@ -83,8 +83,7 @@ public static int DriverRegisterResource(CpuContext ctx) var flags = (uint)ctx[CpuRegister.R9]; TraceAgc( - $"agc.driver_register_resource resource=0x{resourceAddress:X16} owner={owner} " + - $"name=0x{nameAddress:X16} type={type} flags={flags}"); + $"agc.driver_register_resource resource=0x{resourceAddress:X16} owner={owner} name=0x{nameAddress:X16} type={type} flags={flags}"); return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_OK); } @@ -122,8 +121,7 @@ public static int DriverQueryResourceRegistrationUserMemoryRequirements(CpuConte } TraceAgc( - $"agc.driver_query_resource_registration_memory resources={resourceCount} " + - $"owners={ownerCount} bytes=0x{requiredSize:X}"); + $"agc.driver_query_resource_registration_memory resources={resourceCount} owners={ownerCount} bytes=0x{requiredSize:X}"); return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_OK); } @@ -157,8 +155,7 @@ public static int DriverInitResourceRegistration(CpuContext ctx) } TraceAgc( - $"agc.driver_init_resource_registration memory=0x{memoryAddress:X16} " + - $"bytes=0x{memorySize:X} owners={ownerCount}"); + $"agc.driver_init_resource_registration memory=0x{memoryAddress:X16} bytes=0x{memorySize:X} owners={ownerCount}"); return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_OK); } @@ -235,8 +232,7 @@ public static int DriverRegisterOwner(CpuContext ctx) } TraceAgc( - $"agc.driver_register_owner out=0x{ownerAddress:X16} owner={owner} " + - $"name={System.Text.Encoding.UTF8.GetString(nameBytes)}"); + $"agc.driver_register_owner out=0x{ownerAddress:X16} owner={owner} name={System.Text.Encoding.UTF8.GetString(nameBytes)}"); return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_OK); } @@ -342,8 +338,7 @@ public static int DriverSetTFRing(CpuContext ctx) Console.Error.WriteLine( $"[AGC][INFO] Tessellation-factor ring: address=0x{ctx[CpuRegister.Rdi]:X16} size=0x{(uint)ctx[CpuRegister.Rsi]:X8}."); TraceAgc( - $"agc.driver_set_tf_ring ring=0x{ctx[CpuRegister.Rdi]:X16} " + - $"size=0x{(uint)ctx[CpuRegister.Rsi]:X8}"); + $"agc.driver_set_tf_ring ring=0x{ctx[CpuRegister.Rdi]:X16} size=0x{(uint)ctx[CpuRegister.Rsi]:X8}"); return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_OK); } @@ -360,8 +355,7 @@ public static int DriverSetHsOffchipParam(CpuContext ctx) Console.Error.WriteLine( $"[AGC][INFO] Hull-shader off-chip buffer: address=0x{ctx[CpuRegister.Rdi]:X16} param=0x{(uint)ctx[CpuRegister.Rsi]:X8}."); TraceAgc( - $"agc.driver_set_hs_offchip_param buffer=0x{ctx[CpuRegister.Rdi]:X16} " + - $"param=0x{(uint)ctx[CpuRegister.Rsi]:X8}"); + $"agc.driver_set_hs_offchip_param buffer=0x{ctx[CpuRegister.Rdi]:X16} param=0x{(uint)ctx[CpuRegister.Rsi]:X8}"); return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_OK); } } diff --git a/src/SharpEmu.Libs/Agc/AgcExports.Events.cs b/src/SharpEmu.Libs/Agc/AgcExports.Events.cs index 806361df3..95892d46b 100644 --- a/src/SharpEmu.Libs/Agc/AgcExports.Events.cs +++ b/src/SharpEmu.Libs/Agc/AgcExports.Events.cs @@ -38,16 +38,12 @@ public static int AcbEventWrite(CpuContext ctx) var hasAddress = (eventType & ~1u) == 0x38; var packetDwords = hasAddress ? 4u : 2u; - if (!TryAllocateCommandDwords(ctx, commandBufferAddress, packetDwords, out var commandAddress) || - !TryWriteUInt32(ctx, commandAddress, Pm4(packetDwords, ItEventWrite, 0)) || - !TryWriteUInt32(ctx, commandAddress + 4, hasAddress ? eventType | 0x100u : eventType & 0x3Fu)) + Span packet = stackalloc uint[4] { - return ReturnPointer(ctx, 0); - } - - if (hasAddress && - (!TryWriteUInt32(ctx, commandAddress + 8, (uint)eventAddress & ~7u) || - !TryWriteUInt32(ctx, commandAddress + 12, (uint)(eventAddress >> 32)))) + Pm4(packetDwords, ItEventWrite, 0), hasAddress ? eventType | 0x100u : eventType & 0x3Fu, + (uint)eventAddress & ~7u, (uint)(eventAddress >> 32), + }; + if (!TryEmitCommandDwords(ctx, commandBufferAddress, out var commandAddress, packet[..(int)packetDwords])) { return ReturnPointer(ctx, 0); } diff --git a/src/SharpEmu.Libs/Agc/AgcExports.RegisterPackets.cs b/src/SharpEmu.Libs/Agc/AgcExports.RegisterPackets.cs index 8b22040ab..118e3b0fb 100644 --- a/src/SharpEmu.Libs/Agc/AgcExports.RegisterPackets.cs +++ b/src/SharpEmu.Libs/Agc/AgcExports.RegisterPackets.cs @@ -148,6 +148,12 @@ public static int CbSetShRegisterRangeDirect(CpuContext ctx) return ReturnPointer(ctx, 0); } } + else if (TryDeclineLeaf(ctx)) + { + // The marker and the packet would be allocated one after the other: a leaf backs out + // before either is written. + return ReturnPointer(ctx, 0); + } else if (!TryEmitCommandDwords(ctx, commandBufferAddress, out _, Pm4(2, ItNop, RZero), CbSetShRegisterRangeMarker) || !TryAllocateCommandDwords(ctx, commandBufferAddress, packetDwords, out commandAddress) || !TryWriteCommandDwords(ctx, commandAddress, Pm4(packetDwords, ItSetShReg, 0), offset)) @@ -362,9 +368,7 @@ private static int DcbSetRegistersIndirect(CpuContext ctx, uint packetRegister, Span tableSnapshot = stackalloc byte[(int)Math.Min(registerCount & 0x3FFFu, 16u) * 8]; var tableAddress = registersAddress & ~3UL; var captured = ctx.Memory.TryRead(tableAddress, tableSnapshot); - TraceAgc($"agc.register_table_created cmd=0x{commandAddress:X16} bank={registerSpace} " + - $"table=0x{tableAddress:X16} count={registerCount & 0x3FFFu} captured={captured} " + - $"bytes={(captured ? Convert.ToHexString(tableSnapshot) : string.Empty)}"); + TraceAgc($"agc.register_table_created cmd=0x{commandAddress:X16} bank={registerSpace} table=0x{tableAddress:X16} count={registerCount & 0x3FFFu} captured={captured} bytes={(captured ? Convert.ToHexString(tableSnapshot) : string.Empty)}"); } return ReturnPointer(ctx, commandAddress); } @@ -381,18 +385,15 @@ private static int DcbSetRegisterDirect(CpuContext ctx, uint op, string register return ReturnPointer(ctx, 0); } + // The three dwords are one allocation and one guest write, like the other fixed-size builders. const uint packetDwords = 3; - if (!TryAllocateCommandDwords(ctx, commandBufferAddress, packetDwords, out var commandAddress) || - !TryWriteUInt32(ctx, commandAddress, Pm4(packetDwords, op, 0)) || - !TryWriteUInt32(ctx, commandAddress + 4, registerOffset & 0xFFFFu) || - !TryWriteUInt32(ctx, commandAddress + 8, registerValue)) + if (!TryEmitCommandDwords(ctx, commandBufferAddress, out var commandAddress, Pm4(packetDwords, op, 0), registerOffset & 0xFFFFu, registerValue)) { return ReturnPointer(ctx, 0); } TraceAgc( - $"agc.dcb_set_{registerSpace}_direct buf=0x{commandBufferAddress:X16} " + - $"cmd=0x{commandAddress:X16} offset=0x{registerOffset:X4} value=0x{registerValue:X8}"); + $"agc.dcb_set_{registerSpace}_direct buf=0x{commandBufferAddress:X16} cmd=0x{commandAddress:X16} offset=0x{registerOffset:X4} value=0x{registerValue:X8}"); return ReturnPointer(ctx, commandAddress); } @@ -426,31 +427,49 @@ private static int DcbSetRegisterRangeDirect(CpuContext ctx, uint op, string reg } var packetDwords = valueCount + 2; - if (!TryAllocateCommandDwords(ctx, commandBufferAddress, packetDwords, out var commandAddress) || - !TryWriteUInt32(ctx, commandAddress, Pm4(packetDwords, op, 0)) || - !TryWriteUInt32(ctx, commandAddress + 4, offset & 0xFFFFu)) + if (!TryAllocateCommandDwords(ctx, commandBufferAddress, packetDwords, out var commandAddress)) { return ReturnPointer(ctx, 0); } - for (uint i = 0; i < valueCount; i++) + // The header, the offset and the payload are one guest write: read the values first (a null + // source reserves the payload as zeros), then write the whole packet. + uint[]? rented = null; + try { - var value = 0u; - if (valuesAddress != 0 && - !TryReadUInt32(ctx, valuesAddress + (i * sizeof(uint)), out value)) + Span packet = packetDwords <= 2 + InlinePayloadDwords + ? stackalloc uint[(int)packetDwords] + : (rented = System.Buffers.ArrayPool.Shared.Rent((int)packetDwords)).AsSpan(0, (int)packetDwords); + packet[0] = Pm4(packetDwords, op, 0); + packet[1] = offset & 0xFFFFu; + var payload = packet[2..]; + if (valuesAddress != 0) { - return ReturnPointer(ctx, 0); + if (!ctx.Memory.TryRead(valuesAddress, System.Runtime.InteropServices.MemoryMarshal.AsBytes(payload))) + { + return ReturnPointer(ctx, 0); + } + } + else + { + payload.Clear(); } - if (!TryWriteUInt32(ctx, commandAddress + 8 + (i * sizeof(uint)), value)) + if (!TryWriteCommandDwords(ctx, commandAddress, packet)) { return ReturnPointer(ctx, 0); } } + finally + { + if (rented is not null) + { + System.Buffers.ArrayPool.Shared.Return(rented); + } + } TraceAgc( - $"agc.dcb_set_{registerSpace}_range buf=0x{commandBufferAddress:X16} " + - $"cmd=0x{commandAddress:X16} offset=0x{offset:X4} count={valueCount}"); + $"agc.dcb_set_{registerSpace}_range buf=0x{commandBufferAddress:X16} cmd=0x{commandAddress:X16} offset=0x{offset:X4} count={valueCount}"); return ReturnPointer(ctx, commandAddress); } } diff --git a/src/SharpEmu.Libs/Agc/AgcExports.ShaderState.cs b/src/SharpEmu.Libs/Agc/AgcExports.ShaderState.cs index ce0dbcc60..3e5e41587 100644 --- a/src/SharpEmu.Libs/Agc/AgcExports.ShaderState.cs +++ b/src/SharpEmu.Libs/Agc/AgcExports.ShaderState.cs @@ -332,8 +332,7 @@ public static int GetFusedShaderSize(CpuContext ctx) } TraceAgc( - $"agc.get_fused_shader_size front=0x{frontAddress:X16} back=0x{backAddress:X16} " + - $"types={frontType}/{backType} registers={registerCount}"); + $"agc.get_fused_shader_size front=0x{frontAddress:X16} back=0x{backAddress:X16} types={frontType}/{backType} registers={registerCount}"); ctx[CpuRegister.Rax] = 0; return (int)OrbisGen2Result.ORBIS_GEN2_OK; } @@ -494,10 +493,7 @@ private static int FuseShaderHalvesCore(CpuContext ctx, bool legacy) } TraceAgc( - $"agc.fuse_shader_halves fused=0x{fusedAddress:X16} front=0x{frontAddress:X16} " + - $"back=0x{backAddress:X16} scratch=0x{scratchAddress:X16} types={frontType}/{backType} " + - $"registers={registerCount} entry=0x{frontCodeAddress:X16} " + - $"continuation=0x{backCodeAddress:X16}"); + $"agc.fuse_shader_halves fused=0x{fusedAddress:X16} front=0x{frontAddress:X16} back=0x{backAddress:X16} scratch=0x{scratchAddress:X16} types={frontType}/{backType} registers={registerCount} entry=0x{frontCodeAddress:X16} continuation=0x{backCodeAddress:X16}"); ctx[CpuRegister.Rax] = 0; return (int)OrbisGen2Result.ORBIS_GEN2_OK; } @@ -667,8 +663,7 @@ public static int CreatePrimState(CpuContext ctx) } TraceAgc( - $"agc.create_prim_state cx=0x{cxRegistersAddress:X16} uc=0x{ucRegistersAddress:X16} " + - $"hull=0x{hullShaderAddress:X16} gs=0x{geometryShaderAddress:X16} type={shaderType} prim=0x{primitiveType:X8}"); + $"agc.create_prim_state cx=0x{cxRegistersAddress:X16} uc=0x{ucRegistersAddress:X16} hull=0x{hullShaderAddress:X16} gs=0x{geometryShaderAddress:X16} type={shaderType} prim=0x{primitiveType:X8}"); ctx[CpuRegister.Rax] = 0; return (int)OrbisGen2Result.ORBIS_GEN2_OK; } @@ -751,8 +746,7 @@ public static int CreateInterpolantMapping2(CpuContext ctx) } TraceAgc( - $"agc.create_interpolant_mapping2 regs=0x{registersAddress:X16} " + - $"gs=0x{geometryShaderAddress:X16} ps=0x{pixelShaderAddress:X16} inputs=0"); + $"agc.create_interpolant_mapping2 regs=0x{registersAddress:X16} gs=0x{geometryShaderAddress:X16} ps=0x{pixelShaderAddress:X16} inputs=0"); ctx[CpuRegister.Rax] = 0; return (int)OrbisGen2Result.ORBIS_GEN2_OK; } @@ -806,9 +800,7 @@ public static int CreateInterpolantMapping2(CpuContext ctx) } TraceAgc( - $"agc.create_interpolant_mapping2 regs=0x{registersAddress:X16} " + - $"gs=0x{geometryShaderAddress:X16} ps=0x{pixelShaderAddress:X16} " + - $"inputs={inputSemanticsCount} outputs={outputSemanticsCount}"); + $"agc.create_interpolant_mapping2 regs=0x{registersAddress:X16} gs=0x{geometryShaderAddress:X16} ps=0x{pixelShaderAddress:X16} inputs={inputSemanticsCount} outputs={outputSemanticsCount}"); ctx[CpuRegister.Rax] = 0; return (int)OrbisGen2Result.ORBIS_GEN2_OK; } @@ -1007,8 +999,7 @@ public static int CreateInterpolantMapping(CpuContext ctx) } TraceAgc( - $"agc.create_interpolant_mapping regs=0x{registersAddress:X16} " + - $"gs=0x{geometryShaderAddress:X16} ps=0x{pixelShaderAddress:X16} inputs=0"); + $"agc.create_interpolant_mapping regs=0x{registersAddress:X16} gs=0x{geometryShaderAddress:X16} ps=0x{pixelShaderAddress:X16} inputs=0"); ctx[CpuRegister.Rax] = 0; return (int)OrbisGen2Result.ORBIS_GEN2_OK; } @@ -1063,9 +1054,7 @@ public static int CreateInterpolantMapping(CpuContext ctx) } TraceAgc( - $"agc.create_interpolant_mapping regs=0x{registersAddress:X16} " + - $"gs=0x{geometryShaderAddress:X16} ps=0x{pixelShaderAddress:X16} " + - $"inputs={inputSemanticsCount} outputs={outputSemanticsCount}"); + $"agc.create_interpolant_mapping regs=0x{registersAddress:X16} gs=0x{geometryShaderAddress:X16} ps=0x{pixelShaderAddress:X16} inputs={inputSemanticsCount} outputs={outputSemanticsCount}"); ctx[CpuRegister.Rax] = 0; return (int)OrbisGen2Result.ORBIS_GEN2_OK; } diff --git a/src/SharpEmu.Libs/Agc/AgcExports.cs b/src/SharpEmu.Libs/Agc/AgcExports.cs index d475d9c2e..aaf079375 100644 --- a/src/SharpEmu.Libs/Agc/AgcExports.cs +++ b/src/SharpEmu.Libs/Agc/AgcExports.cs @@ -552,6 +552,13 @@ private static bool TryPrepareCommandDwords(CpuContext ctx, ulong commandBufferA var remainingDwords = GetRemainingCommandDwords(cursorUp, cursorDown, reservedDwords); if (sizeDwords > remainingDwords) { + // A leaf dispatch cannot run the buffer's callback, which is guest code: it backs out + // before anything is written and the call is made again through the full import path. + if (TryDeclineLeaf(ctx)) + { + return false; + } + TraceAgc($"agc.cmd_alloc_full buf=0x{commandBufferAddress:X16} need={sizeDwords} remaining={remainingDwords} callback=0x{callback:X16}"); var scheduler = GuestThreadExecution.Scheduler; ulong callbackResult = 0; @@ -571,15 +578,12 @@ scheduler is null || out callbackError)) { TraceAgc( - $"agc.cmd_alloc_callback_failed buf=0x{commandBufferAddress:X16} " + - $"callback=0x{callback:X16} result=0x{callbackResult:X16} " + - $"error={callbackError ?? "none"}"); + $"agc.cmd_alloc_callback_failed buf=0x{commandBufferAddress:X16} callback=0x{callback:X16} result=0x{callbackResult:X16} error={callbackError ?? "none"}"); return false; } TraceAgc( - $"agc.cmd_alloc_callback_complete buf=0x{commandBufferAddress:X16} " + - $"callback=0x{callback:X16} result=0x{callbackResult:X16}"); + $"agc.cmd_alloc_callback_complete buf=0x{commandBufferAddress:X16} callback=0x{callback:X16} result=0x{callbackResult:X16}"); if (!TryReadUInt64(ctx, commandBufferAddress + CommandBufferCursorUpOffset, out cursorUp) || !TryReadUInt64(ctx, commandBufferAddress + CommandBufferCursorDownOffset, out cursorDown) || @@ -601,6 +605,22 @@ scheduler is null || return true; } + /// + /// True when the call is running as a leaf, which cannot call back into the guest. The caller + /// must return at once, with no command-buffer state changed; the dispatcher then repeats the + /// call on the full import path. + /// + private static bool TryDeclineLeaf(CpuContext ctx) + { + if (!ctx.DeclineGuestCallbacks) + { + return false; + } + + ctx.GuestCallbackDeclined = true; + return true; + } + private static uint GetRemainingCommandDwords( ulong cursorUp, ulong cursorDown, diff --git a/src/SharpEmu.Libs/Kernel/KernelMemoryCompatExports.cs b/src/SharpEmu.Libs/Kernel/KernelMemoryCompatExports.cs index 004ae2d54..d8df4c183 100644 --- a/src/SharpEmu.Libs/Kernel/KernelMemoryCompatExports.cs +++ b/src/SharpEmu.Libs/Kernel/KernelMemoryCompatExports.cs @@ -29,6 +29,15 @@ public static partial class KernelMemoryCompatExports private const int WideCharSize = sizeof(ushort); private const int MemsetChunkSize = 16 * 1024; private static readonly byte[] _zeroChunk = new byte[MemsetChunkSize]; + private const ulong InPlaceMemsetThreshold = 64 * 1024; + // SHARPEMU_MEMSET_INPLACE=0 sends every memset through the staged chunk loop again. + private static readonly bool InPlaceMemsetEnabled = + !string.Equals(Environment.GetEnvironmentVariable("SHARPEMU_MEMSET_INPLACE"), "0", StringComparison.Ordinal); + private static readonly GuestSpanFill FillWithByte = static (value, destination) => + { + destination.Fill(value); + return destination.Length; + }; private const int O_WRONLY = 0x1; private const int O_RDWR = 0x2; private const int O_APPEND = 0x8; @@ -453,6 +462,19 @@ public static int Memset(CpuContext ctx) return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; } + // A large fill into a single backed view is written in place with a vectorised span fill: + // no staging chunk to read back, no per-16-KiB write transaction (10 GB/s staged, memory + // bandwidth in place). Anything the memory cannot hand out in place, a range that crosses + // views, or an armed write watch declines and takes the staged loop below, which also + // keeps the clamp-to-mapped-range behaviour. + if (length >= InPlaceMemsetThreshold && length <= int.MaxValue && InPlaceMemsetEnabled && + ctx.Memory.TryFillInPlace(destination, (int)length, value, FillWithByte, out var filledInPlace) && + (ulong)filledInPlace == length) + { + ctx[CpuRegister.Rax] = destination; + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + } + // Rent may hand back a larger array than requested; only the first chunkLength // bytes are filled, so the loop must cap at chunkLength rather than chunk.Length. var chunkLength = (int)Math.Min(length, (ulong)MemsetChunkSize); diff --git a/src/SharpEmu.Libs/Kernel/KernelPthreadCompatExports.cs b/src/SharpEmu.Libs/Kernel/KernelPthreadCompatExports.cs index 3fa54ae17..d8e863274 100644 --- a/src/SharpEmu.Libs/Kernel/KernelPthreadCompatExports.cs +++ b/src/SharpEmu.Libs/Kernel/KernelPthreadCompatExports.cs @@ -297,6 +297,8 @@ public static int PthreadSelf(CpuContext ctx) if (GuestThreadExecution.CurrentGuestThreadHandle != currentThreadHandle) { GuestThreadExecution.Scheduler?.RegisterGuestThreadContext(currentThreadHandle, ctx); + // Registered: later calls on this host thread can be answered by the native stub. + GuestFastPath.PublishHostSelf(currentThreadHandle); } ctx[CpuRegister.Rax] = currentThreadHandle; TracePthreadSelf(ctx, currentThreadHandle); diff --git a/src/SharpEmu.Libs/Kernel/KernelRuntimeCompatExports.cs b/src/SharpEmu.Libs/Kernel/KernelRuntimeCompatExports.cs index 6c46db6bf..3019f5162 100644 --- a/src/SharpEmu.Libs/Kernel/KernelRuntimeCompatExports.cs +++ b/src/SharpEmu.Libs/Kernel/KernelRuntimeCompatExports.cs @@ -61,6 +61,9 @@ public static class KernelRuntimeCompatExports private const uint PageExecuteReadWrite = 0x40; private static readonly object _stateGate = new(); private static readonly long _processStartCounter = Stopwatch.GetTimestamp(); + + /// The Stopwatch reading the process-relative clocks count from (native clock stubs subtract it). + public static long ProcessStartCounter => _processStartCounter; private static readonly RdtscDelegate? _rdtscReader = CreateRdtscReader(); private static readonly ulong _kernelTscFrequency = ResolveKernelTscFrequency(); private static readonly ulong _stackChkGuardValue = 0xC0DEC0DECAFEBA00UL; diff --git a/src/platforms/macos/SharpEmu.Cpu.DirectExecution.MacOS.X64/MacOSExecutionPlatform.cs b/src/platforms/macos/SharpEmu.Cpu.DirectExecution.MacOS.X64/MacOSExecutionPlatform.cs index 0d7db3f15..249c86d33 100644 --- a/src/platforms/macos/SharpEmu.Cpu.DirectExecution.MacOS.X64/MacOSExecutionPlatform.cs +++ b/src/platforms/macos/SharpEmu.Cpu.DirectExecution.MacOS.X64/MacOSExecutionPlatform.cs @@ -64,6 +64,37 @@ public sealed unsafe class MacOSExecutionPlatform : IDirectExecutionPlatform _ => 0, }; + /// + /// mach_absolute_time counts nanoseconds only when the timebase is 1:1 (Intel, and Apple + /// silicon under Rosetta 2, which reports it that way); any other timebase reports no entry + /// points so the managed clocks keep their own conversion. + /// + public bool TryGetHostClockEntryPoints(out nint counter, out nint clockGettime) + { + counter = 0; + clockGettime = 0; + try + { + var libSystem = System.Runtime.InteropServices.NativeLibrary.Load("/usr/lib/libSystem.B.dylib"); + var timebaseInfo = System.Runtime.InteropServices.NativeLibrary.GetExport(libSystem, "mach_timebase_info"); + var timebase = stackalloc uint[2]; + if (((delegate* unmanaged)timebaseInfo)(timebase) != 0 || timebase[0] != timebase[1] || timebase[0] == 0) + { + return false; + } + + counter = System.Runtime.InteropServices.NativeLibrary.GetExport(libSystem, "mach_absolute_time"); + clockGettime = System.Runtime.InteropServices.NativeLibrary.GetExport(libSystem, "clock_gettime"); + return true; + } + catch (Exception ex) when (ex is DllNotFoundException or EntryPointNotFoundException) + { + counter = 0; + clockGettime = 0; + return false; + } + } + /// /// Managed code is compiled for the SysV ABI on POSIX .NET while the /// emitted call sites use Win64, so every managed callback the emitted code diff --git a/tests/SharpEmu.Libs.Tests/Agc/AgcBuilderLeafTests.cs b/tests/SharpEmu.Libs.Tests/Agc/AgcBuilderLeafTests.cs new file mode 100644 index 000000000..62b5ac182 --- /dev/null +++ b/tests/SharpEmu.Libs.Tests/Agc/AgcBuilderLeafTests.cs @@ -0,0 +1,236 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using System.Buffers.Binary; +using SharpEmu.Core.Cpu.Native; +using SharpEmu.HLE; +using Xunit; + +namespace SharpEmu.Libs.Tests.Agc; + +/// +/// The sceAgc builders the import dispatcher runs as leaves: with +/// set they must write exactly the bytes the full path +/// writes, and when the command buffer is full they must back out with nothing written so the +/// dispatcher can repeat the call through the path that may call the buffer's guest callback. +/// +[Collection(AgcCommandBufferChainCollection.Name)] +public sealed class AgcBuilderLeafTests +{ + private const ulong BaseAddress = 0x2_6000_0000; + private const ulong CommandBufferAddress = BaseAddress + 0x100; + private const ulong PacketAddress = BaseAddress + 0x400; + private const ulong ScratchAddress = BaseAddress + 0x900; + private const ulong StackAddress = BaseAddress + 0xC00; + private const int MemorySize = 0x1000; + + private delegate void Arguments(CpuContext context, FakeCpuMemory memory); + + // Every builder the leaf class lists, with arguments that make it emit a packet. Stack + // arguments sit at StackAddress + 8 onwards (StackAddress itself is the return slot). + private static readonly (string Nid, string Name, Arguments Setup)[] Builders = + [ + ("q88lQ+GP5Yk", "DrawIndex", (c, m) => { c[CpuRegister.Rsi] = 6; c[CpuRegister.Rdx] = 0x2_6000_0800; c[CpuRegister.Rcx] = 0x28; }), + ("Yw0jKSqop+E", "DrawIndexAuto", (c, m) => { c[CpuRegister.Rsi] = 3; c[CpuRegister.Rdx] = 0; }), + ("B+aG9DUnTKA", "DrawIndexOffset", (c, m) => { c[CpuRegister.Rsi] = 4; c[CpuRegister.Rdx] = 9; c[CpuRegister.Rcx] = 0x28; }), + ("t1vNu082-jM", "DrawIndexIndirect", (c, m) => { c[CpuRegister.Rsi] = 0x40; c[CpuRegister.Rdx] = 0x1_0000_0005; }), + ("l4fM9K-Lyks", "SetIndexBuffer", (c, m) => { c[CpuRegister.Rsi] = 0x2_6000_0800; c[CpuRegister.Rdx] = 12; }), + ("8N2tmT3jmC8", "SetIndexCount", (c, m) => { c[CpuRegister.Rsi] = 33; }), + ("GIIW2J37e70", "SetIndexSize", (c, m) => { c[CpuRegister.Rsi] = 1; c[CpuRegister.Rdx] = 2; }), + ("tSBxhAPyytQ", "SetNumInstances", (c, m) => { c[CpuRegister.Rsi] = 7; }), + ("pFLArOT53+w", "SetShRegisterDirect", (c, m) => c[CpuRegister.Rsi] = (0xFEDC_BA98UL << 32) | 0xABCD_00C8), + ("LHFXRrlTPD8", "SetCxRegisterDirect", (c, m) => c[CpuRegister.Rsi] = (0x1357_9BDFUL << 32) | 0x1_0028), + ("w4-d0n60hdo", "SetUcRegisterDirect", (c, m) => c[CpuRegister.Rsi] = (0x2468_ACE0UL << 32) | 0x0309), + ("BVFg3CWU6Eo", "SetCfRegisterRangeDirect", (c, m) => { Fill(m, ScratchAddress, 5); c[CpuRegister.Rsi] = 0x44; c[CpuRegister.Rdx] = ScratchAddress; c[CpuRegister.Rcx] = 5; }), + ("n2fD4A+pb+g", "CbSetShRegisterRangeDirect", (c, m) => { Fill(m, ScratchAddress, 3); c[CpuRegister.Rsi] = 0x40; c[CpuRegister.Rdx] = ScratchAddress; c[CpuRegister.Rcx] = 3; }), + ("-HOOCn0JY48", "SetShRegistersIndirect", (c, m) => { c[CpuRegister.Rsi] = 0x2_6000_0A00; c[CpuRegister.Rdx] = 4; }), + ("ZvwO9euwYzc", "SetCxRegistersIndirect", (c, m) => { c[CpuRegister.Rsi] = 0x2_6000_0A00; c[CpuRegister.Rdx] = 4; }), + ("hvUfkUIQcOE", "SetUcRegistersIndirect", (c, m) => { c[CpuRegister.Rsi] = 0x2_6000_0A00; c[CpuRegister.Rdx] = 4; }), + ("k3GhuSNmBLU", "CbDispatch", (c, m) => { c[CpuRegister.Rsi] = 8; c[CpuRegister.Rdx] = 4; c[CpuRegister.Rcx] = 2; c[CpuRegister.R8] = 0x8000; }), + ("CtB+A9-VxO0", "DcbDispatchIndirect", (c, m) => { c[CpuRegister.Rsi] = 0x20; c[CpuRegister.Rdx] = 0x8000; }), + ("j3EtxFkSIhQ", "AcbDispatchIndirect", (c, m) => { c[CpuRegister.Rsi] = 0x2_6000_0A00; c[CpuRegister.Rdx] = 0x20; }), + ("RmaJwLtc8rY", "SetBaseIndirectArgs", (c, m) => { c[CpuRegister.Rsi] = 1; c[CpuRegister.Rdx] = 0x2_6000_0A00; }), + ("VmW0Tdpy420", "DcbWaitRegMem32", (c, m) => DcbWaitRegMem(c, m, size: 0)), + ("VmW0Tdpy420", "DcbWaitRegMem64", (c, m) => DcbWaitRegMem(c, m, size: 1)), + ("htn36gPnBk4", "AcbWaitRegMem32", (c, m) => AcbWaitRegMem(c, m, size: 0)), + ("htn36gPnBk4", "AcbWaitRegMem64", (c, m) => AcbWaitRegMem(c, m, size: 1)), + ("i1jyy49AjXU", "DcbWriteData", WriteData), + ("eZ4+17OQz4Q", "AcbWriteData", WriteData), + ("aJf+j5yntiU", "DcbEventWrite", (c, m) => { c[CpuRegister.Rsi] = 0x07; c[CpuRegister.Rdx] = 0; }), + ("aJf+j5yntiU", "DcbEventWriteWithAddress", (c, m) => { c[CpuRegister.Rsi] = 0x38; c[CpuRegister.Rdx] = 0x2_6000_0A08; }), + ("cFazmnXpJOE", "AcbEventWrite", (c, m) => { c[CpuRegister.Rsi] = 0x07; c[CpuRegister.Rdx] = 0; }), + ("cFazmnXpJOE", "AcbEventWriteWithAddress", (c, m) => { c[CpuRegister.Rsi] = 0x38; c[CpuRegister.Rdx] = 0x2_6000_0A08; }), + ("57labkp+rSQ", "DcbAcquireMem", (c, m) => + { + Stack(m, 400); + c[CpuRegister.Rsi] = 1; c[CpuRegister.Rdx] = 0x2000; c[CpuRegister.Rcx] = 0x5555; c[CpuRegister.R8] = 0x2_6000_0000; c[CpuRegister.R9] = 0x1000; + }), + ("KT-hTp-Ch14", "AcbAcquireMem", (c, m) => { c[CpuRegister.Rsi] = 0x5555; c[CpuRegister.Rdx] = 0x2_6000_0000; c[CpuRegister.Rcx] = 0x1000; c[CpuRegister.R8] = 800; }), + ("wr23dPKyWc0", "CbReleaseMem", (c, m) => + { + Stack(m, 1, 0xAAAA_BBBB_CCCC_DDDD, 0, 0, 0, 0); + c[CpuRegister.Rsi] = 5; c[CpuRegister.Rdx] = 0x1234; c[CpuRegister.Rcx] = 1; c[CpuRegister.R8] = 2; c[CpuRegister.R9] = 0x2_6000_0A00; + }), + ]; + + public static IEnumerable BuilderNames() => Builders.Select(builder => new object[] { builder.Name }); + + [Theory] + [MemberData(nameof(BuilderNames))] + public void LeafModeWritesTheBytesTheFullPathWrites(string name) + { + var builder = Builders.Single(candidate => candidate.Name == name); + var full = Run(builder, leaf: false); + var leaf = Run(builder, leaf: true); + + Assert.False(leaf.Declined); + Assert.NotEqual(0UL, full.Rax); + Assert.Equal(full.Rax, leaf.Rax); + Assert.Equal(full.Memory, leaf.Memory); + // The builder really emitted: the cursor moved and the packet is not blank. + Assert.NotEqual(PacketAddress, BinaryPrimitives.ReadUInt64LittleEndian(leaf.Memory.AsSpan((int)(CommandBufferAddress - BaseAddress) + 0x10))); + } + + [Theory] + [MemberData(nameof(BuilderNames))] + public void LeafModeDeclinesWithNothingWrittenWhenTheBufferIsFull(string name) + { + var builder = Builders.Single(candidate => candidate.Name == name); + // Room for one dword: smaller than any packet, with a callback the full path would call. + var leaf = Run(builder, leaf: true, remainingBytes: 4); + var reference = Run(builder, leaf: false, remainingBytes: 4, runBuilder: false); + + Assert.True(leaf.Declined, name + " ran to completion with no room instead of declining"); + Assert.Equal(reference.Memory, leaf.Memory); + } + + [Theory] + [InlineData("DrawIndex", 40u)] + [InlineData("CbSetShRegisterRangeDirect", 16u)] + public void SplitAllocationBuildersDeclineBeforeTheirFirstPacket(string name, uint remainingBytes) + { + // Room for the first packet of a builder that allocates its packets one at a time but not + // for all of them: declining only at the second would leave the first written twice. + var builder = Builders.Single(candidate => candidate.Name == name); + var leaf = Run(builder, leaf: true, remainingBytes: remainingBytes); + var reference = Run(builder, leaf: false, remainingBytes: remainingBytes, runBuilder: false); + + Assert.True(leaf.Declined); + Assert.Equal(reference.Memory, leaf.Memory); + } + + [Fact] + public void EveryLeafBuilderIsListedAndEveryListedBuilderIsCovered() + { + var covered = Builders.Select(builder => builder.Nid).ToHashSet(StringComparer.Ordinal); + foreach (var nid in covered) + { + Assert.True(DirectExecutionBackend.IsAgcBuilderLeafImport(nid), nid + " is tested as a leaf builder but not classified as one"); + } + + var manager = CreateManager(); + foreach (var nid in covered) + { + Assert.True(manager.TryGetExport(nid, out _), nid + " exports nothing"); + } + } + + private readonly record struct Result(byte[] Memory, ulong Rax, bool Declined); + + private static Result Run( + (string Nid, string Name, Arguments Setup) builder, + bool leaf, + uint remainingBytes = 0x300, + bool runBuilder = true) + { + var memory = new FakeCpuMemory(BaseAddress, MemorySize); + WriteUInt64(memory, CommandBufferAddress + 0x10, PacketAddress); + WriteUInt64(memory, CommandBufferAddress + 0x18, PacketAddress + remainingBytes); + // A callback that is never called here (Scheduler is null): present so a full buffer would use it. + WriteUInt64(memory, CommandBufferAddress + 0x20, 0x1234); + WriteUInt64(memory, CommandBufferAddress + 0x28, 0x5678); + WriteUInt32(memory, ScratchAddress + 0x100, 0xDEADBEEF); + var context = new CpuContext(memory, Generation.Gen5); + context[CpuRegister.Rdi] = CommandBufferAddress; + context[CpuRegister.Rsp] = StackAddress; + builder.Setup(context, memory); + var previousScheduler = GuestThreadExecution.Scheduler; + GuestThreadExecution.Scheduler = null; + try + { + if (runBuilder) + { + context.DeclineGuestCallbacks = leaf; + CreateManager().Dispatch(builder.Nid, context); + context.DeclineGuestCallbacks = false; + } + } + finally + { + GuestThreadExecution.Scheduler = previousScheduler; + } + + var bytes = new byte[MemorySize]; + Assert.True(memory.TryRead(BaseAddress, bytes)); + return new Result(bytes, context[CpuRegister.Rax], context.GuestCallbackDeclined); + } + + private static void DcbWaitRegMem(CpuContext c, FakeCpuMemory m, ulong size) + { + // reference, mask, poll cycles + Stack(m, 0x1111_2222_3333_4444, 0x5555_6666_7777_8888, 320); + c[CpuRegister.Rsi] = size; c[CpuRegister.Rdx] = 3; c[CpuRegister.Rcx] = 0; c[CpuRegister.R8] = 1; c[CpuRegister.R9] = 0x2_6000_0A08; + } + + private static void AcbWaitRegMem(CpuContext c, FakeCpuMemory m, ulong size) + { + // mask, poll cycles + Stack(m, 0x5555_6666_7777_8888, 320); + c[CpuRegister.Rsi] = size; c[CpuRegister.Rdx] = 3; c[CpuRegister.Rcx] = 1; c[CpuRegister.R8] = 0x2_6000_0A08; c[CpuRegister.R9] = 0x1111_2222_3333_4444; + } + + private static void WriteData(CpuContext c, FakeCpuMemory m) + { + Fill(m, ScratchAddress, 4); + // increment, write confirm + Stack(m, 1, 1); + c[CpuRegister.Rsi] = 5; c[CpuRegister.Rdx] = 2; c[CpuRegister.Rcx] = 0x2_6000_0A08; c[CpuRegister.R8] = ScratchAddress; c[CpuRegister.R9] = 4; + } + + private static void Stack(FakeCpuMemory memory, params ulong[] arguments) + { + for (var index = 0; index < arguments.Length; index++) + { + WriteUInt64(memory, StackAddress + 8 + (ulong)index * 8, arguments[index]); + } + } + + private static void Fill(FakeCpuMemory memory, ulong address, int dwords) + { + for (var index = 0; index < dwords; index++) + { + WriteUInt32(memory, address + (ulong)index * 4, 0xC0DE0000u + (uint)index); + } + } + + private static ModuleManager CreateManager() + { + var manager = new ModuleManager(); + manager.RegisterExports(SharpEmu.Generated.SysAbiExportRegistry.CreateExports(Generation.Gen5)); + return manager; + } + + private static void WriteUInt32(FakeCpuMemory memory, ulong address, uint value) + { + Span bytes = stackalloc byte[sizeof(uint)]; + BinaryPrimitives.WriteUInt32LittleEndian(bytes, value); + Assert.True(memory.TryWrite(address, bytes)); + } + + private static void WriteUInt64(FakeCpuMemory memory, ulong address, ulong value) + { + Span bytes = stackalloc byte[sizeof(ulong)]; + BinaryPrimitives.WriteUInt64LittleEndian(bytes, value); + Assert.True(memory.TryWrite(address, bytes)); + } +} diff --git a/tests/SharpEmu.Libs.Tests/Cpu/FaultThreadImportTraceTests.cs b/tests/SharpEmu.Libs.Tests/Cpu/FaultThreadImportTraceTests.cs index dc83cb2b5..22121e526 100644 --- a/tests/SharpEmu.Libs.Tests/Cpu/FaultThreadImportTraceTests.cs +++ b/tests/SharpEmu.Libs.Tests/Cpu/FaultThreadImportTraceTests.cs @@ -212,15 +212,20 @@ private sealed unsafe class DispatchMetadataFixture : IDisposable private const BindingFlags Static = BindingFlags.Static | BindingFlags.NonPublic; private static readonly Type BackendType = typeof(DirectExecutionBackend); private readonly Dictionary saved = new(); + private readonly (object? Owner, long Next, long BlockEnd) savedCounters; internal readonly DirectExecutionBackend Backend = (DirectExecutionBackend)RuntimeHelpers.GetUninitializedObject(BackendType); internal readonly TrackedCpuMemory Memory = new(new RefusingMemory()); internal DispatchMetadataFixture() { - foreach (var name in new[] { "_activeExecutionBackend", "_activeCpuContext", "_activeGuestThreadState", "_importCounterOwner", "_nextImportDispatchIndex", "_importDispatchBlockEnd", "_faultThreadImports" }) + foreach (var name in new[] { "_activeExecutionBackend", "_activeCpuContext", "_activeGuestThreadState", "_faultThreadImports" }) { var field = BackendType.GetField(name, Static)!; saved.Add(field, field.GetValue(null)); field.SetValue(null, field.FieldType == typeof(long) ? (object)0L : null); } + // The import counters moved from DirectExecutionBackend statics to the per-thread state. + var state = GuestThreadExecution.State; + savedCounters = (state.ImportCounterOwner, state.NextImportDispatchIndex, state.ImportDispatchBlockEnd); + state.ImportCounterOwner = null; state.NextImportDispatchIndex = 0; state.ImportDispatchBlockEnd = 0; Context(new CpuContext(Memory, Generation.Gen5)); Export(_ => 0); } internal void Context(CpuContext? context) => BackendType.GetField("_cpuContext", Instance)!.SetValue(Backend, context); @@ -242,7 +247,12 @@ internal ulong Invoke() internal ulong InvokeCore(nint pointer, int importIndex = 0) => (ulong)BackendType.GetMethod("DispatchImportWithFaultThreadDiagnostics", Instance)!.Invoke(Backend, [importIndex, pointer])!; internal FaultThreadImportEntry[] Snapshot() => BackendType.GetField("_faultThreadImports", Static)!.GetValue(null) is FaultThreadImportTrace trace ? trace.Snapshot(Backend, Environment.CurrentManagedThreadId, GuestThreadExecution.CurrentGuestThreadHandle) : []; - public void Dispose() { foreach (var field in saved) field.Key.SetValue(null, field.Value); } + public void Dispose() + { + foreach (var field in saved) field.Key.SetValue(null, field.Value); + var state = GuestThreadExecution.State; + state.ImportCounterOwner = savedCounters.Owner; state.NextImportDispatchIndex = savedCounters.Next; state.ImportDispatchBlockEnd = savedCounters.BlockEnd; + } } private sealed class RefusingMemory : ICpuMemory { diff --git a/tests/SharpEmu.Libs.Tests/Cpu/GuestClockStubTests.cs b/tests/SharpEmu.Libs.Tests/Cpu/GuestClockStubTests.cs new file mode 100644 index 000000000..55ce4a984 --- /dev/null +++ b/tests/SharpEmu.Libs.Tests/Cpu/GuestClockStubTests.cs @@ -0,0 +1,220 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using SharpEmu.Core.Cpu.Native; +using SharpEmu.HLE; +using SharpEmu.Libs.Kernel; +using System.Runtime.InteropServices; +using Xunit; + +namespace SharpEmu.Libs.Tests.Cpu; + +/// +/// Runs the emitted clock stubs (clock_gettime, sceKernelClockGettime, gettimeofday, +/// sceKernelGetProcessTime) as machine code and holds their answers to the managed exports': +/// same epoch, same units, and a hand-off to the trampoline for every call the stub does not +/// answer. +/// +[Collection("GuestFastPath")] +public sealed unsafe class GuestClockStubTests : IDisposable +{ + private const ulong FallbackSentinel = 0x5A5A_1234_5678_9ABCUL; + private const long SlackNanoseconds = 50_000_000; + + private readonly byte* _page; + private readonly bool _canRun = + OperatingSystem.IsMacOS() && RuntimeInformation.ProcessArchitecture == Architecture.X64; + + public GuestClockStubTests() + { + if (_canRun) + { + _page = (byte*)HostMemory.Alloc( + null, + 4096, + HostMemory.MEM_COMMIT | HostMemory.MEM_RESERVE, + HostMemory.PAGE_EXECUTE_READWRITE); + } + } + + public void Dispose() + { + if (_page != null) + { + _ = HostMemory.Free(_page, 0, HostMemory.MEM_RELEASE); + } + } + + [Fact] + public void ClockGettime_MonotonicIdsMatchTheManagedClock() + { + if (!_canRun) + { + return; + } + + var stub = Build(DirectExecutionBackend.GuestFastPathStub.ClockGettime); + foreach (var clockId in new ulong[] { 4, 5, 7, 8, 11, 12 }) + { + var timespec = stackalloc long[2]; + KernelRuntimeCompatExports.ReadMonotonicSinceStart(out var beforeSeconds, out var beforeNanos); + var result = ((delegate* unmanaged)stub)(clockId, timespec); + KernelRuntimeCompatExports.ReadMonotonicSinceStart(out var afterSeconds, out var afterNanos); + + Assert.Equal(0UL, result); + var before = beforeSeconds * 1_000_000_000L + beforeNanos; + var after = afterSeconds * 1_000_000_000L + afterNanos; + var actual = timespec[0] * 1_000_000_000L + timespec[1]; + Assert.InRange(timespec[1], 0, 999_999_999); + Assert.InRange(actual, before, after); + } + } + + [Fact] + public void ClockGettime_RealtimeIdsReportUnixEpochNanoseconds() + { + if (!_canRun) + { + return; + } + + var stub = Build(DirectExecutionBackend.GuestFastPathStub.ClockGettime); + foreach (var clockId in new ulong[] { 0, 9, 10 }) + { + var timespec = stackalloc long[2]; + var result = ((delegate* unmanaged)stub)(clockId, timespec); + var now = DateTimeOffset.UtcNow; + + Assert.Equal(0UL, result); + Assert.InRange(timespec[1], 0, 999_999_999); + var actualTicks = timespec[0] * TimeSpan.TicksPerSecond + timespec[1] / 100; + var expectedTicks = now.ToUnixTimeSeconds() * TimeSpan.TicksPerSecond + now.Ticks % TimeSpan.TicksPerSecond; + Assert.InRange(actualTicks, expectedTicks - SlackNanoseconds / 100, expectedTicks + SlackNanoseconds / 100); + } + } + + [Theory] + [InlineData(1UL)] + [InlineData(2UL)] + [InlineData(3UL)] + [InlineData(13UL)] + [InlineData(14UL)] + [InlineData(15UL)] + [InlineData(99UL)] + [InlineData(0x1_0000_0004UL)] + public void ClockGettime_LeavesOtherClockIdsToTheManagedExport(ulong clockId) + { + if (!_canRun) + { + return; + } + + var stub = Build(DirectExecutionBackend.GuestFastPathStub.ClockGettime); + var timespec = stackalloc long[2]; + timespec[0] = 111; + timespec[1] = 222; + + Assert.Equal(FallbackSentinel, ((delegate* unmanaged)stub)(clockId, timespec)); + Assert.Equal(111, timespec[0]); + Assert.Equal(222, timespec[1]); + } + + [Fact] + public void ClockGettime_LeavesANullTimespecToTheManagedExport() + { + if (!_canRun) + { + return; + } + + var stub = Build(DirectExecutionBackend.GuestFastPathStub.ClockGettime); + Assert.Equal(FallbackSentinel, ((delegate* unmanaged)stub)(4, null)); + } + + [Fact] + public void GetTimeOfDay_WritesSecondsAndMicroseconds() + { + if (!_canRun) + { + return; + } + + var stub = Build(DirectExecutionBackend.GuestFastPathStub.GetTimeOfDay); + var timeval = stackalloc long[2]; + var result = ((delegate* unmanaged)stub)(timeval, null); + var now = DateTimeOffset.UtcNow; + + Assert.Equal(0UL, result); + Assert.InRange(timeval[1], 0, 999_999); + var actualMicros = timeval[0] * 1_000_000L + timeval[1]; + var expectedMicros = now.ToUnixTimeSeconds() * 1_000_000L + now.Ticks % TimeSpan.TicksPerSecond / 10; + Assert.InRange(actualMicros, expectedMicros - SlackNanoseconds / 1000, expectedMicros + SlackNanoseconds / 1000); + } + + [Fact] + public void GetTimeOfDay_LeavesATimezoneOrNullTimevalToTheManagedExport() + { + if (!_canRun) + { + return; + } + + var stub = Build(DirectExecutionBackend.GuestFastPathStub.GetTimeOfDay); + var timeval = stackalloc long[2]; + var timezone = stackalloc int[2]; + Assert.Equal(FallbackSentinel, ((delegate* unmanaged)stub)(timeval, timezone)); + Assert.Equal(FallbackSentinel, ((delegate* unmanaged)stub)(null, null)); + } + + [Fact] + public void ProcessTime_MatchesTheManagedMicrosecondCounter() + { + if (!_canRun) + { + return; + } + + var stub = Build(DirectExecutionBackend.GuestFastPathStub.ProcessTimeMicroseconds); + var before = KernelRuntimeCompatExports.ReadProcessTimeMicroseconds(); + var actual = ((delegate* unmanaged)stub)(); + var after = KernelRuntimeCompatExports.ReadProcessTimeMicroseconds(); + + Assert.InRange(actual, before, after); + } + + [Fact] + public void Classification_RoutesTheFourClockNids() + { + Assert.Equal(DirectExecutionBackend.GuestFastPathStub.ClockGettime, DirectExecutionBackend.ClassifyGuestFastPathStub("lLMT9vJAck0")); + Assert.Equal(DirectExecutionBackend.GuestFastPathStub.ClockGettime, DirectExecutionBackend.ClassifyGuestFastPathStub("QBi7HCK03hw")); + Assert.Equal(DirectExecutionBackend.GuestFastPathStub.GetTimeOfDay, DirectExecutionBackend.ClassifyGuestFastPathStub("n88vx3C5nW8")); + Assert.Equal(DirectExecutionBackend.GuestFastPathStub.ProcessTimeMicroseconds, DirectExecutionBackend.ClassifyGuestFastPathStub("4J2sUJmuHZQ")); + } + + // Emits the stub at +0x200 of an executable page and a fallback trampoline at +0x100 that + // returns a recognisable value. + private nint Build(DirectExecutionBackend.GuestFastPathStub kind) + { + var libSystem = NativeLibrary.Load("/usr/lib/libSystem.B.dylib"); + var targets = new DirectExecutionBackend.ClockStubTargets( + NativeLibrary.GetExport(libSystem, "mach_absolute_time"), + KernelRuntimeCompatExports.ProcessStartCounter, + NativeLibrary.GetExport(libSystem, "clock_gettime")); + + // mov rax, imm64 / ret + _page[0x100] = 0x48; + _page[0x101] = 0xB8; + *(ulong*)(_page + 0x102) = FallbackSentinel; + _page[0x10A] = 0xC3; + var code = DirectExecutionBackend.EmitGuestClockStub(kind, targets, (nint)(_page + 0x100)); + Assert.True(code.Count < 0x200); + for (var i = 0; i < code.Count; i++) + { + _page[0x200 + i] = code[i]; + } + + Assert.True(HostMemory.Protect(_page, 4096, HostMemory.PAGE_EXECUTE_READ, out _)); + HostMemory.FlushInstructionCache(_page, 4096); + return (nint)(_page + 0x200); + } +} diff --git a/tests/SharpEmu.Libs.Tests/Cpu/GuestFastPathStubTests.cs b/tests/SharpEmu.Libs.Tests/Cpu/GuestFastPathStubTests.cs index a7bb7ec75..457b4b6c9 100644 --- a/tests/SharpEmu.Libs.Tests/Cpu/GuestFastPathStubTests.cs +++ b/tests/SharpEmu.Libs.Tests/Cpu/GuestFastPathStubTests.cs @@ -159,6 +159,37 @@ public void SelfStub_StaysOnTheManagedPathForHostThreads() Assert.Equal(RaxSentinel, harness.Call(0)); } + [Fact] + public void SelfStub_AnswersAHostThreadOnceItsHandleIsPublished() + { + if (!CanRunStubs()) + { + return; + } + + // scePthreadSelf publishes the handle after registering the thread's context, so the + // primary thread's later calls are answered without the managed crossing. + using var harness = new StubHarness(DirectExecutionBackend.GuestFastPathStub.PthreadSelf); + var handle = NewHandle(); + harness.Bind(0); + Assert.Equal(RaxSentinel, harness.Call(0)); + + GuestFastPath.PublishHostSelf(handle); + Assert.Equal(handle, harness.Call(0)); + + // A guest thread binding over it replaces the handle; unbinding sends the call back. + var guest = NewHandle(); + harness.Bind(guest); + Assert.Equal(guest, harness.Call(0)); + harness.Bind(0); + Assert.Equal(RaxSentinel, harness.Call(0)); + + // An already-published handle is not overwritten by a second publish. + GuestFastPath.PublishHostSelf(handle); + GuestFastPath.PublishHostSelf(NewHandle()); + Assert.Equal(handle, harness.Call(0)); + } + [Fact] public void GetspecificStub_ReadsTheThreadCurrentlyBound() { diff --git a/tests/SharpEmu.Libs.Tests/Cpu/GuestThreadStateTests.cs b/tests/SharpEmu.Libs.Tests/Cpu/GuestThreadStateTests.cs new file mode 100644 index 000000000..cf5087dc6 --- /dev/null +++ b/tests/SharpEmu.Libs.Tests/Cpu/GuestThreadStateTests.cs @@ -0,0 +1,62 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using SharpEmu.HLE; +using Xunit; + +namespace SharpEmu.Libs.Tests.Cpu; + +/// +/// GuestThreadExecution keeps one per-thread state object (one thread-static read per crossing); +/// its public statics must still behave per thread. +/// +public sealed class GuestThreadStateTests +{ + [Fact] + public void GuestThreadHandleAndPendingOutcomeAreNotSharedBetweenThreads() + { + var previous = GuestThreadExecution.EnterGuestThread(0xABC0); + try + { + Assert.Equal(0xABC0UL, GuestThreadExecution.CurrentGuestThreadHandle); + Assert.True(GuestThreadExecution.RequestCurrentThreadBlock("state-test")); + Assert.True(GuestThreadExecution.HasPendingCurrentThreadBlock); + Assert.True(GuestThreadExecution.State.HasPendingDispatchOutcome); + + ulong otherHandle = 1; + var otherPending = true; + var thread = new Thread(() => + { + otherHandle = GuestThreadExecution.CurrentGuestThreadHandle; + otherPending = GuestThreadExecution.HasPendingCurrentThreadBlock || GuestThreadExecution.State.HasPendingDispatchOutcome; + }); + thread.Start(); + thread.Join(); + + Assert.Equal(0UL, otherHandle); + Assert.False(otherPending); + + Assert.True(GuestThreadExecution.TryConsumeCurrentThreadBlock(out var reason)); + Assert.Equal("state-test", reason); + Assert.False(GuestThreadExecution.State.HasPendingDispatchOutcome); + } + finally + { + GuestThreadExecution.RestoreGuestThread(previous); + } + } + + [Fact] + public void EntryExitAndContextTransferCountAsPendingOutcomes() + { + Assert.False(GuestThreadExecution.State.HasPendingDispatchOutcome); + + GuestThreadExecution.RequestCurrentEntryExit("state-test-exit", 7UL); + Assert.True(GuestThreadExecution.State.HasPendingDispatchOutcome); + Assert.True(GuestThreadExecution.TryConsumeCurrentEntryExit(out var value, out var reason)); + Assert.Equal(7UL, value); + Assert.Equal("state-test-exit", reason); + Assert.False(GuestThreadExecution.State.HasPendingDispatchOutcome); + Assert.False(GuestThreadExecution.TryConsumeCurrentEntryExit(out _, out _)); + } +} diff --git a/tests/SharpEmu.Libs.Tests/Kernel/MemsetInPlaceTests.cs b/tests/SharpEmu.Libs.Tests/Kernel/MemsetInPlaceTests.cs new file mode 100644 index 000000000..a08fc6205 --- /dev/null +++ b/tests/SharpEmu.Libs.Tests/Kernel/MemsetInPlaceTests.cs @@ -0,0 +1,89 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using SharpEmu.HLE; +using SharpEmu.Libs.Kernel; +using Xunit; + +namespace SharpEmu.Libs.Tests.Kernel; + +public sealed class MemsetInPlaceTests +{ + private const ulong Base = 0x2_7000_0000; + + [Theory] + [InlineData(0x10000UL, 0x7Fu)] + [InlineData(0x30001UL, 0x00u)] + [InlineData(0x1000UL, 0xA5u)] + public void LargeAndSmallFillsWriteTheSameBytesInPlaceOrStaged(ulong length, uint value) + { + var inPlace = Run(length, (byte)value, supportsInPlace: true, out var usedInPlace); + var staged = Run(length, (byte)value, supportsInPlace: false, out _); + + Assert.Equal(staged, inPlace); + Assert.Equal(length >= 0x10000, usedInPlace); + // The fill covers exactly [dst, dst + length) and leaves the neighbours alone. + Assert.Equal(0x11, inPlace[0x1FF]); + Assert.Equal((byte)value, inPlace[0x200]); + Assert.Equal((byte)value, inPlace[0x200 + (int)length - 1]); + Assert.Equal(0x11, inPlace[0x200 + (int)length]); + } + + private static byte[] Run(ulong length, byte value, bool supportsInPlace, out bool usedInPlace) + { + var memory = new InPlaceMemory(Base, 0x40000, supportsInPlace); + memory.Storage.AsSpan().Fill(0x11); + var context = new CpuContext(memory, Generation.Gen5) + { + [CpuRegister.Rdi] = Base + 0x200, + [CpuRegister.Rsi] = value, + [CpuRegister.Rdx] = length, + }; + Assert.Equal((int)OrbisGen2Result.ORBIS_GEN2_OK, KernelMemoryCompatExports.Memset(context)); + Assert.Equal(Base + 0x200, context[CpuRegister.Rax]); + usedInPlace = memory.InPlaceFills == 1; + return memory.Storage; + } + + private sealed class InPlaceMemory(ulong baseAddress, int size, bool supportsInPlace) : ICpuMemory + { + public byte[] Storage { get; } = new byte[size]; + + public int InPlaceFills { get; private set; } + + public bool TryRead(ulong address, Span destination) + { + if (address < baseAddress || address - baseAddress + (ulong)destination.Length > (ulong)Storage.Length) + { + return false; + } + + Storage.AsSpan((int)(address - baseAddress), destination.Length).CopyTo(destination); + return true; + } + + public bool TryWrite(ulong address, ReadOnlySpan source) + { + if (address < baseAddress || address - baseAddress + (ulong)source.Length > (ulong)Storage.Length) + { + return false; + } + + source.CopyTo(Storage.AsSpan((int)(address - baseAddress))); + return true; + } + + public bool TryFillInPlace(ulong address, int length, TState state, GuestSpanFill fill, out int filled) + { + filled = 0; + if (!supportsInPlace || address < baseAddress || address - baseAddress + (ulong)length > (ulong)Storage.Length) + { + return false; + } + + InPlaceFills++; + filled = fill(state, Storage.AsSpan((int)(address - baseAddress), length)); + return true; + } + } +} From 183f7ded5786d4345da2fe40044df0c25c78d0de Mon Sep 17 00:00:00 2001 From: kuba Date: Mon, 26 Oct 2026 06:17:50 +0100 Subject: [PATCH 16/82] [Fiber] Replace the global fiber gate with per-fiber lock stripes Every fiber switch in the process serialised on one gate, and the run path re-read all fiber fields under it. Fibers now hash to 64 lock stripes; the caller-continuation capture and previous-fiber checks run before the lock and only the target's state word is read under its stripe. A source fiber in Run is touched only by its own thread; its continuation is published before Complete writes Idle under the source stripe, and runners read the state under that stripe, so the next runner sees the continuation. Rollback takes both stripes in index order. New test: 8 workers x 1500 iterations over 6 fibers never run the same fiber twice at once and leave every fiber Idle. Switch: SHARPEMU_FIBER_GLOBAL_GATE=1 maps every fiber to one stripe. --- src/SharpEmu.Libs/Fiber/FiberExports.cs | 112 +++++++++++++----- .../Fiber/FiberSwitchLoopTests.cs | 99 ++++++++++++++++ 2 files changed, 184 insertions(+), 27 deletions(-) diff --git a/src/SharpEmu.Libs/Fiber/FiberExports.cs b/src/SharpEmu.Libs/Fiber/FiberExports.cs index 121ca472e..210f3fbd7 100644 --- a/src/SharpEmu.Libs/Fiber/FiberExports.cs +++ b/src/SharpEmu.Libs/Fiber/FiberExports.cs @@ -53,7 +53,16 @@ public static class FiberExports private static int _contextSizeCheck; private static readonly bool TraceEnabled = Environment.GetEnvironmentVariable("SHARPEMU_LOG_FIBER") == "1"; - private static readonly object _fiberGate = new(); + // A fiber's state word and saved continuation are guarded by the lock of that fiber's stripe, + // so switches on different workers between different fibers do not serialise on one gate. + // A switch locks only its target: the source is the calling thread's current fiber (state Run), + // which no other thread may run or change until the transfer commits it to Idle under the + // source's own stripe (PendingFiberTransfer.Complete). _threadStates entries are per thread. + // SHARPEMU_FIBER_GLOBAL_GATE=1 maps every fiber to one stripe (the old single gate). + private const int FiberLockStripes = 64; + private static readonly bool _fiberGlobalGate = + Environment.GetEnvironmentVariable("SHARPEMU_FIBER_GLOBAL_GATE") == "1"; + private static readonly object[] _fiberLocks = CreateFiberLocks(); private static readonly ConcurrentDictionary _continuations = new(); private static readonly ConcurrentDictionary _stackRanges = new(); private static readonly ConcurrentDictionary _threadStates = new(); @@ -79,15 +88,31 @@ private sealed class PendingFiberTransfer private void Complete(bool succeeded) { var commitFailed = false; - lock (_fiberGate) + if (succeeded && Source != 0) { - if (succeeded && Source != 0 && - !TryWriteUInt32(Context, Source + FiberStateOffset, FiberStateIdle)) + // Publishing Idle makes the source runnable by other threads; its continuation + // was stored before the transfer, so a runner that sees Idle under this stripe + // also sees the continuation. + lock (FiberLock(Source)) { - succeeded = false; - commitFailed = true; + if (!TryWriteUInt32(Context, Source + FiberStateOffset, FiberStateIdle)) + { + succeeded = false; + commitFailed = true; + } + } + } + if (!succeeded) + { + // Rollback (rare): hold both fibers' stripes, taken in stripe order. + var first = FiberLockIndex(Source); + var second = FiberLockIndex(Target); + if (second < first) + { + (first, second) = (second, first); } - if (!succeeded) + lock (_fiberLocks[first]) + lock (_fiberLocks[second]) { if (ResumeArgumentAddress != 0) _ = TryWriteUInt64(Context, ResumeArgumentAddress, PreviousResumeArgument); @@ -114,15 +139,43 @@ static FiberExports() RunFiberSelfChecks(); } + private static object[] CreateFiberLocks() + { + var locks = new object[FiberLockStripes]; + for (var i = 0; i < locks.Length; i++) + { + locks[i] = new object(); + } + + return locks; + } + + // Fiber objects are 8-aligned guest structures; mix the address so neighbours spread. + private static int FiberLockIndex(ulong fiber) => + _fiberGlobalGate ? 0 : (int)((fiber >> 3) * 0x9E3779B97F4A7C15UL >> 58) & (FiberLockStripes - 1); + + private static object FiberLock(ulong fiber) => _fiberLocks[FiberLockIndex(fiber)]; + public static void ResetRuntimeState() { - lock (_fiberGate) + foreach (var gate in _fiberLocks) + { + Monitor.Enter(gate); + } + try { _continuations.Clear(); _stackRanges.Clear(); _threadStates.Clear(); Volatile.Write(ref _contextSizeCheck, 0); } + finally + { + for (var i = _fiberLocks.Length - 1; i >= 0; i--) + { + Monitor.Exit(_fiberLocks[i]); + } + } } [SysAbiExport( @@ -331,7 +384,7 @@ public static int FiberReturnToThread(CpuContext ctx) var threadKey = GetThreadKey(ctx); FiberThreadState threadState; ulong previousResumeArgument = 0; - lock (_fiberGate) + lock (FiberLock(fiberAddress)) { _continuations[fiberAddress] = new FiberContinuation( CaptureContinuation(ctx, frame.ReturnRip, frame.ResumeRsp, frame.ReturnSlotAddress), @@ -652,23 +705,37 @@ private static int FiberRunCore( GuestCpuContinuation transferTarget; var resumed = false; FiberContinuation targetContinuation = default; - FiberThreadState? priorThreadState = null; var threadKey = GetThreadKey(ctx); ulong previousResumeArgument = 0; + // Everything that belongs to the calling thread is read before the target's lock: + // its _threadStates entry, its register state and its current (source) fiber, whose + // state only this thread changes while it runs. + _threadStates.TryGetValue(threadKey, out var priorThreadState); + var callerContinuation = new FiberContinuation( + CaptureContinuation(ctx, frame.ReturnRip, frame.ResumeRsp, frame.ReturnSlotAddress), + outArgumentAddress); + if (previousFiber != 0 && + (!TryReadUInt32(ctx, previousFiber + FiberStateOffset, out var previousState) || + previousState != FiberStateRun)) + { + return SetReturn(ctx, FiberErrorState); + } // Monitor enter/exit spelled out so DSTRACE can time the gate wait; // with DSTRACE off EnterGate is Monitor.Enter. - DsFiber.EnterGate(_fiberGate); + var fiberLock = FiberLock(fiber); + DsFiber.EnterGate(fiberLock); try { - _threadStates.TryGetValue(threadKey, out priorThreadState); - if (!TryReadFiberFields(ctx, fiber, out fields)) + // Only the state word can change under another thread (a concurrent run of the same + // fiber); the other fields were read above and are the guest's to keep stable. + if (!TryReadUInt32(ctx, fiber + FiberStateOffset, out var targetState)) { return SetReturn(ctx, FiberErrorInvalid); } - if (fields.State != FiberStateIdle) + if (targetState != FiberStateIdle) { if (TraceEnabled) - TraceFiber($"run-state-error reason={reason} fiber=0x{fiber:X16} state=0x{fields.State:X8}"); + TraceFiber($"run-state-error reason={reason} fiber=0x{fiber:X16} state=0x{targetState:X8}"); return SetReturn(ctx, FiberErrorState); } @@ -677,14 +744,11 @@ private static int FiberRunCore( targetContinuation = savedContinuation; resumed = true; } - var callerContinuation = new FiberContinuation( - CaptureContinuation(ctx, frame.ReturnRip, frame.ResumeRsp, frame.ReturnSlotAddress), - outArgumentAddress); if (!resumed) { - var rootStackTop = _threadStates.TryGetValue(threadKey, out var existingThreadState) - ? existingThreadState.RootContinuation.Context.Rsp + var rootStackTop = priorThreadState is not null + ? priorThreadState.RootContinuation.Context.Rsp : callerContinuation.Context.Rsp; if (!TryCreateInitialContinuation( ctx, @@ -705,12 +769,6 @@ private static int FiberRunCore( if (previousFiber != 0) { - if (!TryReadUInt32(ctx, previousFiber + FiberStateOffset, out var previousState) || - previousState != FiberStateRun) - { - return SetReturn(ctx, FiberErrorState); - } - _continuations[previousFiber] = callerContinuation; } else @@ -756,7 +814,7 @@ private static int FiberRunCore( } finally { - Monitor.Exit(_fiberGate); + Monitor.Exit(fiberLock); } _ = GuestThreadExecution.EnterFiber(fiber); diff --git a/tests/SharpEmu.Libs.Tests/Fiber/FiberSwitchLoopTests.cs b/tests/SharpEmu.Libs.Tests/Fiber/FiberSwitchLoopTests.cs index 54be8fde5..f05735a08 100644 --- a/tests/SharpEmu.Libs.Tests/Fiber/FiberSwitchLoopTests.cs +++ b/tests/SharpEmu.Libs.Tests/Fiber/FiberSwitchLoopTests.cs @@ -291,6 +291,105 @@ public void RepeatedSwitch_KeepsResumeStackInsideTheOwningFiberContext() Assert.NotEqual(toA.Rsp, toB.Rsp); } + // Fibers are locked per fiber: workers running different fibers do not serialise, and + // workers racing for the same idle fiber must still see exactly one winner until the + // winner's return commits the fiber back to Idle. + [Fact] + public void ConcurrentWorkersNeverRunTheSameFiberTwice() + { + const ulong poolBase = 0x3_1000_0000UL; + const int fibers = 6; + const int workers = 8; + const int iterations = 1500; + var memory = new FakeCpuMemory(poolBase, 0x20000); + var setup = new CpuContext(memory, Generation.Gen5); + ulong FiberAt(int index) => poolBase + (ulong)index * 0x100; + ulong ContextAt(int index) => poolBase + 0x8000 + (ulong)index * ContextSize; + for (var i = 0; i < fibers; i++) + { + InitializeFiber(memory, setup, FiberAt(i), poolBase + 0x4000 + (ulong)i * 0x40, + EntryA + (ulong)i * 0x100, (ulong)i, ContextAt(i)); + } + + var owners = new int[fibers]; + var runs = 0; + var errors = new System.Collections.Concurrent.ConcurrentQueue(); + void Worker(int worker) + { + var previousThread = GuestThreadExecution.EnterGuestThread(ThreadHandle + 0x100 + (ulong)worker); + var previousFiber = GuestThreadExecution.EnterFiber(0); + var rng = new System.Random(worker * 31 + 7); + try + { + var ctx = new CpuContext(memory, Generation.Gen5); + for (var n = 0; n < iterations; n++) + { + var index = rng.Next(fibers); + var rootStack = poolBase + 0x1F000 - (ulong)worker * 0x200; + GuestThreadExecution.EnterImportCallFrame(RootReturnRip, rootStack, rootStack - 8); + ctx[CpuRegister.Rdi] = FiberAt(index); + ctx[CpuRegister.Rsi] = 0; + ctx[CpuRegister.Rdx] = 0; + var result = FiberExports.FiberRun(ctx); + if (result != 0) + { + continue; // another worker runs it: the state check rejected this run + } + if (!ConsumeCompletedTransfer(out _)) + { + errors.Enqueue("run without transfer"); + return; + } + if (Interlocked.CompareExchange(ref owners[index], worker + 1, 0) != 0) + { + errors.Enqueue($"fiber {index} entered by two workers"); + } + Interlocked.Increment(ref runs); + Thread.SpinWait(rng.Next(1, 200)); + Interlocked.Exchange(ref owners[index], 0); + + var fiberStack = ContextAt(index) + ContextSize - 0x200; + GuestThreadExecution.EnterImportCallFrame(ResumeRipA, fiberStack, fiberStack - 8); + ctx[CpuRegister.Rdi] = 0; + ctx[CpuRegister.Rsi] = 0; + if (FiberExports.FiberReturnToThread(ctx) != 0 || !ConsumeCompletedTransfer(out _)) + { + errors.Enqueue($"return from fiber {index} failed"); + return; + } + } + } + catch (Exception error) + { + errors.Enqueue(error.ToString()); + } + finally + { + GuestThreadExecution.EnterFiber(previousFiber); + GuestThreadExecution.RestoreGuestThread(previousThread); + } + } + + var threads = new Thread[workers]; + for (var w = 0; w < workers; w++) + { + var worker = w; + threads[w] = new Thread(() => Worker(worker)) { IsBackground = true }; + threads[w].Start(); + } + foreach (var thread in threads) + { + Assert.True(thread.Join(TimeSpan.FromSeconds(60)), "A fiber worker did not finish."); + } + + Assert.Empty(errors); + Assert.True(runs > 0); + for (var i = 0; i < fibers; i++) + { + Assert.Equal(StateIdle, ReadUInt32(memory, FiberAt(i) + 4)); + } + } + private static bool ConsumeCompletedTransfer(out GuestCpuContinuation continuation) { var consumed = GuestThreadExecution.TryConsumeCurrentContextTransfer(out continuation); From c8b7a7c34b3e72404e521ba87cea84e25f69360f Mon Sep 17 00:00:00 2001 From: kuba Date: Mon, 26 Oct 2026 06:17:50 +0100 Subject: [PATCH 17/82] [HLE] Resolve __tls_get_addr from a per-thread cache without the template lock Every __tls_get_addr took the TLS template lock. Each host thread now caches the resolved block address and size per module for its current thread pointer, validated by an epoch that is bumped under the lock whenever a module is registered, the init image changes, a thread block is seeded or the template is reset; cache fills also happen under the lock, so a stale entry can never pass validation. Out-of-block offsets still take the slow path. New test covers cached resolves, past-block offsets, a lazily allocated block for a late module, re-seeding moving it into static TLS, and reset. No switch (pure cache). --- src/SharpEmu.HLE/GuestTlsTemplate.cs | 62 +++++++++++++++++++ .../Loader/TlsInitializationTests.cs | 33 ++++++++++ 2 files changed, 95 insertions(+) diff --git a/src/SharpEmu.HLE/GuestTlsTemplate.cs b/src/SharpEmu.HLE/GuestTlsTemplate.cs index 4d6180448..5286cc749 100644 --- a/src/SharpEmu.HLE/GuestTlsTemplate.cs +++ b/src/SharpEmu.HLE/GuestTlsTemplate.cs @@ -3,6 +3,7 @@ using System.Runtime.InteropServices; using System.Diagnostics; +using System.Numerics; namespace SharpEmu.HLE; @@ -25,6 +26,23 @@ public static class GuestTlsTemplate private static ulong _maximumAlignment = 1; private static ulong _generation; + // Bumped under _gate by every change that can move or drop a resolved TLS address (module + // registration, initialization image change, thread block seeding, reset). A per-host-thread + // cache of resolved module blocks is valid while its epoch matches, so __tls_get_addr does not + // take _gate once a thread has resolved a module. + private static long _resolveEpoch; + + [ThreadStatic] + private static ResolveCache? _resolveCache; + + private sealed class ResolveCache + { + public ulong ThreadPointer; + public long Epoch; + public ulong[] Addresses = new ulong[8]; + public ulong[] Sizes = new ulong[8]; + } + static GuestTlsTemplate() { RunTlsLayoutSelfChecks(); @@ -189,6 +207,7 @@ public static ulong RegisterModule( _staticTlsSize = staticOffset; _maximumAlignment = Math.Max(_maximumAlignment, normalizedAlignment); _generation++; + Interlocked.Increment(ref _resolveEpoch); return staticOffset; } } @@ -206,6 +225,7 @@ public static void UpdateInitializationImage(ulong moduleId, ReadOnlySpan throw new InvalidOperationException("Cannot change TLS initialization after a thread uses the module."); module.InitImage = initializationImage.ToArray(); + Interlocked.Increment(ref _resolveEpoch); } } @@ -240,6 +260,7 @@ public static void Reset() _staticTlsSize = 0; _maximumAlignment = 1; _generation++; + Interlocked.Increment(ref _resolveEpoch); } } @@ -257,6 +278,7 @@ public static void SeedThreadBlock(CpuContext context, ulong threadPointer) lock (_gate) { + Interlocked.Increment(ref _resolveEpoch); if (_threadDtvs.Remove(threadPointer, out var oldDtv)) { FreeDynamicEntries(oldDtv); @@ -284,6 +306,17 @@ public static ulong ResolveAddress(CpuContext context, ulong moduleId, ulong off return 0; } + var cache = _resolveCache; + if (cache is not null && + cache.ThreadPointer == context.FsBase && + cache.Epoch == Volatile.Read(ref _resolveEpoch) && + moduleId < (ulong)cache.Addresses.Length && + cache.Addresses[moduleId] != 0 && + offset < cache.Sizes[moduleId]) + { + return cache.Addresses[moduleId] + offset; + } + lock (_gate) { if (!_modules.TryGetValue(moduleId, out var module) || offset >= module.MemorySize) @@ -318,10 +351,39 @@ public static ulong ResolveAddress(CpuContext context, ulong moduleId, ulong off RebuildGuestDtv(context, context.FsBase, dtv); } + RememberResolvedBlock(context.FsBase, moduleId, entry.Address, module.MemorySize); return checked(entry.Address + offset); } } + // Caller holds _gate, so the epoch read here orders after every change it has seen. + private static void RememberResolvedBlock(ulong threadPointer, ulong moduleId, ulong address, ulong size) + { + if (moduleId > 4096) + { + return; + } + + var cache = _resolveCache ??= new ResolveCache(); + var epoch = Volatile.Read(ref _resolveEpoch); + if (cache.ThreadPointer != threadPointer || cache.Epoch != epoch) + { + Array.Clear(cache.Addresses); + Array.Clear(cache.Sizes); + cache.ThreadPointer = threadPointer; + cache.Epoch = epoch; + } + if (moduleId >= (ulong)cache.Addresses.Length) + { + var length = (int)BitOperations.RoundUpToPowerOf2((uint)moduleId + 1); + Array.Resize(ref cache.Addresses, length); + Array.Resize(ref cache.Sizes, length); + } + + cache.Addresses[moduleId] = address; + cache.Sizes[moduleId] = size; + } + private static DtvEntry CreateStaticOrDynamicEntry( CpuContext context, ulong threadPointer, diff --git a/tests/SharpEmu.Libs.Tests/Loader/TlsInitializationTests.cs b/tests/SharpEmu.Libs.Tests/Loader/TlsInitializationTests.cs index c8482ffc5..e48c7135c 100644 --- a/tests/SharpEmu.Libs.Tests/Loader/TlsInitializationTests.cs +++ b/tests/SharpEmu.Libs.Tests/Loader/TlsInitializationTests.cs @@ -74,6 +74,39 @@ public void InitializationUpdatePreservesLayoutAndRejectsLiveThreadChanges() } } + [Fact] + public void ResolvedAddressCacheFollowsSeedingRegistrationAndReset() + { + try + { + GuestTlsTemplate.Reset(); + var offset1 = GuestTlsTemplate.RegisterModule(1, new byte[8], 0x20, 0x10); + var context = new CpuContext(new FakeCpuMemory(0x10000, 0x20000), Generation.Gen5) { FsBase = 0x20000 }; + GuestTlsTemplate.SeedThreadBlock(context, 0x20000); + Assert.Equal(0x20000UL - offset1 + 4, GuestTlsTemplate.ResolveAddress(context, 1, 4)); + Assert.Equal(0x20000UL - offset1 + 4, GuestTlsTemplate.ResolveAddress(context, 1, 4)); // cached + Assert.Equal(0UL, GuestTlsTemplate.ResolveAddress(context, 1, 0x20)); // past the block + + // A module registered after the thread was seeded gets a lazily allocated block... + var offset2 = GuestTlsTemplate.RegisterModule(2, new byte[8], 0x20, 0x10); + var dynamic2 = GuestTlsTemplate.ResolveAddress(context, 2, 0); + Assert.NotEqual(0UL, dynamic2); + Assert.NotEqual(0x20000UL - offset2, dynamic2); + Assert.Equal(dynamic2 + 8, GuestTlsTemplate.ResolveAddress(context, 2, 8)); + + // ...and re-seeding the thread block moves it into static TLS: no stale cached block. + GuestTlsTemplate.SeedThreadBlock(context, 0x20000); + Assert.Equal(0x20000UL - offset2, GuestTlsTemplate.ResolveAddress(context, 2, 0)); + + GuestTlsTemplate.Reset(); + Assert.Equal(0UL, GuestTlsTemplate.ResolveAddress(context, 1, 4)); + } + finally + { + GuestTlsTemplate.Reset(); + } + } + [Theory] [InlineData(0x21UL, 0x20UL, 0x500UL)] [InlineData(8UL, 0x20UL, 0x2000UL)] From 9cb412fe8ba970c997ea0bcc173bb33875f07cbd Mon Sep 17 00:00:00 2001 From: kuba Date: Mon, 26 Oct 2026 06:17:50 +0100 Subject: [PATCH 18/82] [Ult] Wake only the waiters of the ULT mutex or queue that changed Every ULT mutex unlock, queue push/pop and condition signal woke every blocked guest thread parked on any ULT object, each re-ran its predicate and all but one went back to sleep. Mutexes and queues now park their waiters on a per-object wake key, condition waiters on their mutex's key, and a release wakes one waiter on that key: each release frees one unit and a woken waiter consumes it in its predicate, so waking one cannot lose a wakeup. Joins keep the shared key and wake-all semantics. Covered by the ULT case of the scheduler wake stress tests. No switch (wake targeting only). --- src/SharpEmu.Libs/Ult/UltExports.Sync.cs | 91 ++++++++++++++++++------ 1 file changed, 70 insertions(+), 21 deletions(-) diff --git a/src/SharpEmu.Libs/Ult/UltExports.Sync.cs b/src/SharpEmu.Libs/Ult/UltExports.Sync.cs index 01d93a45e..028916495 100644 --- a/src/SharpEmu.Libs/Ult/UltExports.Sync.cs +++ b/src/SharpEmu.Libs/Ult/UltExports.Sync.cs @@ -11,21 +11,28 @@ namespace SharpEmu.Libs.Ult; // and waits go through the scheduler's cooperative block. public static partial class UltExports { - private const string SyncWakeKey = "sceUlt:sync"; + // Each mutex and queue has its own wake key, so a release readies only the threads parked + // on that object (and at most the one the release can satisfy) instead of re-testing every + // ULT waiter in the process. Condition waiters park on their mutex's key: a signal only + // marks them, and they complete by re-acquiring that mutex. // The work areas only have to look plausible; see WorkAreaHeaderSize. private const ulong WaitingQueuePerThread = 0x40; private const ulong WaitingQueuePerSyncObject = 0x40; private const ulong QueueDataPerQueueObject = 0x80; - private sealed class MutexState + private sealed class MutexState(ulong address) { public ulong Owner; + + public string WakeKey { get; } = $"sceUlt:mutex:{address:X}"; } - private sealed class ConditionWaiter + private sealed class ConditionWaiter(MutexState mutex) { public bool Signaled; + + public MutexState Mutex { get; } = mutex; } private sealed class ConditionVariableState @@ -33,10 +40,12 @@ private sealed class ConditionVariableState public LinkedList Waiters { get; } = new(); } - private sealed class QueueState(ulong dataSize) + private sealed class QueueState(ulong address, ulong dataSize) { public ulong DataSize { get; } = dataSize; + public string WakeKey { get; } = $"sceUlt:queue:{address:X}"; + public Queue Items { get; } = new(); } @@ -204,7 +213,7 @@ public static int MutexCreate(CpuContext ctx) var name = ReadName(ctx, ctx[CpuRegister.Rsi]); lock (SyncGate) { - Mutexes[mutex] = new MutexState(); + Mutexes[mutex] = new MutexState(mutex); } TraceUlt($"mutex_create: mutex=0x{mutex:X16} name='{name}'"); @@ -266,7 +275,7 @@ public static int MutexLock(CpuContext ctx) } } - return BlockUntil(ctx, "sceUltMutexLock", () => TryAcquireMutex(state, self)); + return BlockUntil(ctx, "sceUltMutexLock", state.WakeKey, () => TryAcquireMutex(state, self)); } [SysAbiExport( @@ -309,9 +318,10 @@ public static int MutexUnlock(CpuContext ctx) } var self = CurrentSyncOwner(); + MutexState? state; lock (SyncGate) { - if (!Mutexes.TryGetValue(mutex, out var state) || state.Owner != self) + if (!Mutexes.TryGetValue(mutex, out state) || state.Owner != self) { return ctx.SetReturn(ErrorPermission, typeof(long)); } @@ -319,7 +329,7 @@ public static int MutexUnlock(CpuContext ctx) state.Owner = 0; } - WakeSyncWaiters(); + WakeSyncWaiters(state.WakeKey); return ctx.SetReturn(0, typeof(long)); } @@ -392,7 +402,7 @@ public static int ConditionVariableWait(CpuContext ctx) } var self = CurrentSyncOwner(); - var waiter = new ConditionWaiter(); + ConditionWaiter waiter; MutexState mutexState; lock (SyncGate) { @@ -401,17 +411,19 @@ public static int ConditionVariableWait(CpuContext ctx) return ctx.SetReturn(ErrorPermission, typeof(long)); } + waiter = new ConditionWaiter(mutexState); GetConditionVariable(conditionVariable).Waiters.AddLast(waiter); mutexState.Owner = 0; } - WakeSyncWaiters(); + WakeSyncWaiters(mutexState.WakeKey); // The wait completes once a signal has picked this waiter and the mutex could be re-acquired, // mirroring pthread_cond_wait. return BlockUntil( ctx, "sceUltConditionVariableWait", + mutexState.WakeKey, () => waiter.Signaled && TryAcquireMutex(mutexState, self)); } @@ -443,7 +455,8 @@ private static int SignalCore(CpuContext ctx, bool all) return ctx.SetReturn(ErrorNull, typeof(long)); } - var woke = false; + MutexState? signaledMutex = null; + List? moreMutexes = null; lock (SyncGate) { var waiters = GetConditionVariable(conditionVariable).Waiters; @@ -451,7 +464,16 @@ private static int SignalCore(CpuContext ctx, bool all) { first.Value.Signaled = true; waiters.RemoveFirst(); - woke = true; + var waiterMutex = first.Value.Mutex; + if (signaledMutex is null) + { + signaledMutex = waiterMutex; + } + else if (!ReferenceEquals(signaledMutex, waiterMutex) && + (moreMutexes is null || !moreMutexes.Contains(waiterMutex))) + { + (moreMutexes ??= new List()).Add(waiterMutex); + } if (!all) { break; @@ -459,9 +481,18 @@ private static int SignalCore(CpuContext ctx, bool all) } } - if (woke) + // Signaled waiters complete by taking their mutex: one per mutex can, the rest are + // readied by that mutex's unlocks. + if (signaledMutex is not null) { - WakeSyncWaiters(); + WakeSyncWaiters(signaledMutex.WakeKey); + if (moreMutexes is not null) + { + foreach (var waiterMutex in moreMutexes) + { + WakeSyncWaiters(waiterMutex.WakeKey); + } + } } return ctx.SetReturn(0, typeof(long)); @@ -489,7 +520,7 @@ public static int QueueCreate(CpuContext ctx) var name = ReadName(ctx, ctx[CpuRegister.Rsi]); lock (SyncGate) { - Queues[queue] = new QueueState(dataSize); + Queues[queue] = new QueueState(queue, dataSize); } TraceUlt($"queue_create: queue=0x{queue:X16} name='{name}' data_size=0x{dataSize:X}"); @@ -557,7 +588,7 @@ public static int QueuePush(CpuContext ctx) state.Items.Enqueue(item); } - WakeSyncWaiters(); + WakeSyncWaiters(state.WakeKey); return ctx.SetReturn(0, typeof(long)); } @@ -605,7 +636,7 @@ private static int PopCore(CpuContext ctx, bool tryOnly) } var memory = ctx.Memory; - return BlockUntil(ctx, "sceUltQueuePop", () => TryDequeueInto(memory, state, data)); + return BlockUntil(ctx, "sceUltQueuePop", state.WakeKey, () => TryDequeueInto(memory, state, data)); } private static bool TryDequeueInto(ICpuMemory memory, QueueState state, ulong destination) @@ -640,7 +671,7 @@ private static MutexState GetMutex(ulong mutex) { if (!Mutexes.TryGetValue(mutex, out var state)) { - state = new MutexState(); + state = new MutexState(mutex); Mutexes[mutex] = state; } @@ -664,6 +695,9 @@ private static ulong CurrentSyncOwner() return handle != 0 ? handle : (ulong)(uint)Environment.CurrentManagedThreadId | (1UL << 63); } + // Ulthread joins keep one shared key and wake every joiner (several may join one ulthread). + private const string JoinWakeKey = "sceUlt:join"; + private static void WakeSyncWaiters() { lock (SyncGate) @@ -671,10 +705,25 @@ private static void WakeSyncWaiters() Monitor.PulseAll(SyncGate); } - _ = GuestThreadExecution.Scheduler?.WakeBlockedThreads(SyncWakeKey); + _ = GuestThreadExecution.Scheduler?.WakeBlockedThreads(JoinWakeKey); + } + + private static int BlockUntil(CpuContext ctx, string reason, Func tryComplete) => + BlockUntil(ctx, reason, JoinWakeKey, tryComplete); + + // Every release frees exactly one unit (one owner slot, one queue item), and a parked + // waiter's TryWake takes that unit, so readying the first satisfiable waiter is enough. + private static void WakeSyncWaiters(string wakeKey) + { + lock (SyncGate) + { + Monitor.PulseAll(SyncGate); + } + + _ = GuestThreadExecution.Scheduler?.WakeBlockedThreads(wakeKey, 1); } - private static int BlockUntil(CpuContext ctx, string reason, Func tryComplete) + private static int BlockUntil(CpuContext ctx, string reason, string wakeKey, Func tryComplete) { if (tryComplete()) { @@ -684,7 +733,7 @@ private static int BlockUntil(CpuContext ctx, string reason, Func tryCompl if (GuestThreadExecution.RequestCurrentThreadBlock( ctx, reason, - SyncWakeKey, + wakeKey, static () => 0, tryComplete)) { From c1426a0f15d1a3bc1346f0d1d55f5d59614140af Mon Sep 17 00:00:00 2001 From: kuba Date: Mon, 26 Oct 2026 06:17:50 +0100 Subject: [PATCH 19/82] [Sched] Hand a woken guest thread straight to its runner and resume it through one shared stub A cross-thread wake used to queue the thread, pulse the ready dispatcher, let the dispatcher claim it and only then signal the thread's runner, and every resume emitted, protected, ran and freed a fresh continuation trampoline. Under Rosetta each fresh executable page is retranslated (~17 us p50 in a standalone mmap/mprotect/call/munmap loop), which was most of a wake. Direct handoff (F1): the waker claims the executor token under the scheduler gate when the thread's executor is idle and signals only that thread's runner; a releasing executor passes the token straight to its own next slice. The park-time re-check is unchanged: a parking thread becomes Blocked and then re-runs its wake predicate under the gate, and wakers only ready threads that are Blocked, so a wake can neither be lost nor double-scheduled. Otherwise the old queue + dispatcher path is used. Shared resume stub: a one-time entry stub saves the host frame and jumps into the existing context-transfer stub with a per-call data frame. Not used where emitted code must swap stack bounds or for the native-worker path. Also adds IGuestThreadScheduler.IsGuestThreadExecuting (lock-free) for the adaptive mutex spin, and a scheduler test seam with wake stress tests (semaphore producers/timed consumers, blocking ping-pong pairs, a contended mutex whose owners park while holding it, ULT per-object wakes). With the park-time re-check disabled all three scheduler stress tests fail. Test eboot pass 2, p50 us, 3 interleaved runs each (host load 10-100): sema.N4.wake 50.1/99.4/30.3 -> 7.4/71.2/7.0 sema.N4.roundtrip 77.6/49.5/32.0 -> 21.1/26.1/8.9 cond.N4.roundtrip 70.1/69.6/33.6 -> 8.6/11.6/13.9 evf.N4.wake 47.9/44.9/48.1 -> 7.3/8.0/8.0 sema.N13.wake 59.7/23.2/26.8 -> 7.1/9.0/8.1 sema.N13.fanout 892/329/327 -> 59/52/48 Stub off: N4 wakes ~21 us. Handoff off: N4 ~9-11 us, sema.N13.fanout 237 us. Switches: SHARPEMU_DIRECT_WAKE_HANDOFF=0, SHARPEMU_SHARED_RESUME_STUB=0. --- .../Cpu/Native/DirectExecutionBackend.cs | 422 ++++++++++++++- src/SharpEmu.HLE/GuestThreadExecution.cs | 7 + .../Cpu/GuestSchedulerWakeStressTests.cs | 480 ++++++++++++++++++ 3 files changed, 891 insertions(+), 18 deletions(-) create mode 100644 tests/SharpEmu.Libs.Tests/Cpu/GuestSchedulerWakeStressTests.cs diff --git a/src/SharpEmu.Core/Cpu/Native/DirectExecutionBackend.cs b/src/SharpEmu.Core/Cpu/Native/DirectExecutionBackend.cs index b1e7e59f6..947b0dc79 100644 --- a/src/SharpEmu.Core/Cpu/Native/DirectExecutionBackend.cs +++ b/src/SharpEmu.Core/Cpu/Native/DirectExecutionBackend.cs @@ -991,6 +991,18 @@ public void Dispose() private int _readyGuestThreadCount; private readonly Dictionary _guestThreads = new Dictionary(); + + // Lock-free mirror of _guestThreads (written under the gate next to it) for queries that + // must not take the gate, such as a contended mutex asking whether its owner is on a CPU. + private readonly System.Collections.Concurrent.ConcurrentDictionary _guestThreadsLockFree = new(); + + // A wake readies a thread whose executor is idle by claiming it under the gate and + // signalling only that thread's runner; a wake that lands while the executor is still + // unwinding is handed over by that executor when it releases the thread. Neither goes + // through the ready queue and the SharpEmu-ReadyDispatch thread, which cost a futile + // dispatcher wake per resume. SHARPEMU_DIRECT_WAKE_HANDOFF=0 restores queue + dispatcher. + private static readonly bool _directWakeHandoff = + !string.Equals(Environment.GetEnvironmentVariable("SHARPEMU_DIRECT_WAKE_HANDOFF"), "0", StringComparison.Ordinal); private readonly Queue<(IVirtualMemory Memory, ulong StackBase, ulong StackSize, ulong TlsBase)> _reusableGuestThreadRegions = new(); private readonly Dictionary _externalGuestThreads = new Dictionary(); @@ -4041,6 +4053,7 @@ public bool TryStartThread(CpuContext creatorContext, GuestThreadStartRequest re using (LockGate("TryStartThread")) { _guestThreads[request.ThreadHandle] = thread; + _guestThreadsLockFree[request.ThreadHandle] = thread; EnqueueReadyGuestThreadLocked(thread); } Console.Error.WriteLine( @@ -4291,6 +4304,9 @@ public int WakeBlockedThreads(string wakeKey, int maxCount = int.MaxValue) var dsWakeScanned = 0; var dsWakeFirst = 0UL; var dsWakePumped = false; + GuestThreadState? handoff = null; + List? moreHandoffs = null; + var queued = false; using (LockGate("WakeBlockedThreads")) { if (dsWake) @@ -4324,7 +4340,21 @@ public int WakeBlockedThreads(string wakeKey, int maxCount = int.MaxValue) thread.BlockReason = null; thread.BlockDeadlineTimestamp = 0; CancelBlockDeadlineTimerLocked(thread); - EnqueueReadyGuestThreadLocked(thread, GuestThreadExecution.IsGuestThread ? "signal" : "host_signal"); + if (TryReadyBlockedThreadLocked(thread, GuestThreadExecution.IsGuestThread ? "signal" : "host_signal")) + { + if (handoff is null) + { + handoff = thread; + } + else + { + (moreHandoffs ??= new List()).Add(thread); + } + } + else + { + queued = true; + } if (dsWake && wakeCount == 0) { dsWakeFirst = thread.ThreadHandle; @@ -4333,6 +4363,18 @@ public int WakeBlockedThreads(string wakeKey, int maxCount = int.MaxValue) } } + if (handoff is not null) + { + ScheduleGuestThreadExecution(handoff, "wake"); + if (moreHandoffs is not null) + { + foreach (var thread in moreHandoffs) + { + ScheduleGuestThreadExecution(thread, "wake"); + } + } + } + if (wakeCount != 0) { if (_logGuestThreads) @@ -4341,7 +4383,8 @@ public int WakeBlockedThreads(string wakeKey, int maxCount = int.MaxValue) } // Pump or the readied thread waits for an import dispatch that never comes. - if (_cpuContext is { } wakeContext) + // Under direct handoff a queued thread still owns its executor, which hands it over. + if (queued && !_directWakeHandoff && _cpuContext is { } wakeContext) { Pump(wakeContext, "wake"); if (dsWake) @@ -4439,6 +4482,7 @@ private void RegisterBlockedGuestThreadContinuation( public bool WakeBlockedWaiter(ulong threadHandle, IGuestThreadBlockWaiter waiter, string wakeKey) { + GuestThreadState? handoff; using (LockGate("WakeBlockedWaiter")) { if (!_guestThreads.TryGetValue(threadHandle, out var thread) || @@ -4455,7 +4499,9 @@ public bool WakeBlockedWaiter(ulong threadHandle, IGuestThreadBlockWaiter waiter thread.BlockReason = null; thread.BlockDeadlineTimestamp = 0; CancelBlockDeadlineTimerLocked(thread); - EnqueueReadyGuestThreadLocked(thread, GuestThreadExecution.IsGuestThread ? "signal" : "host_signal"); + handoff = TryReadyBlockedThreadLocked(thread, GuestThreadExecution.IsGuestThread ? "signal" : "host_signal") + ? thread + : null; } if (_logGuestThreads) @@ -4463,15 +4509,48 @@ public bool WakeBlockedWaiter(ulong threadHandle, IGuestThreadBlockWaiter waiter Console.Error.WriteLine($"[LOADER][INFO] guest_threads.wake_waiter handle=0x{threadHandle:X16} key={wakeKey}"); } - // Same hand-off as the key-based wake: dispatch from the waker right away. - if (_cpuContext is { } wakeContext) + if (handoff is not null) + { + // Claimed under the gate: signal only this thread's runner. + ScheduleGuestThreadExecution(handoff, "wake"); + } + else if (!_directWakeHandoff && _cpuContext is { } wakeContext) { + // Same hand-off as the key-based wake: dispatch from the waker right away. Pump(wakeContext, "wake"); } return true; } + public bool IsGuestThreadExecuting(ulong threadHandle) => + !_guestThreadsLockFree.TryGetValue(threadHandle, out var thread) || + Volatile.Read(ref thread.HostThreadId) != 0; + + // Caller holds the gate and has just moved a blocked thread to Ready. With direct handoff, + // a thread whose executor is idle is claimed here (the single-owner ExecutorActive token is + // taken under the gate, exactly as TryClaimReadyGuestThreadLocked does) and the caller + // schedules it on its runner after releasing the gate: returns true. Otherwise the thread + // is queued and the gate pulsed (dispatcher, or a nested-callback waiter that owns the + // executor); an executor that is still unwinding hands the thread over when it releases it + // (RunGuestThread). Returns false. + private bool TryReadyBlockedThreadLocked(GuestThreadState thread, string source) + { + // A thread without a runner yet is left to the dispatcher, so the runner (and the QoS + // class it inherits from its creator) is never created on a waker or timer thread. + if (_directWakeHandoff && !thread.ExecutorActive && thread.ExecutionRunner is not null) + { + ProfileGuestThreadReady(thread, source); + thread.ExecutorActive = true; + thread.State = GuestThreadRunState.Running; + ProfileGuestThreadClaimed(thread); + return true; + } + + EnqueueReadyGuestThreadLocked(thread, source); + return false; + } + // Deadlines given to RequestCurrentThreadBlock are served by the shared timer service // (one precise heap thread) instead of a polling scan. private void ArmBlockDeadlineTimerLocked(GuestThreadState thread) @@ -4503,6 +4582,7 @@ private static void CancelBlockDeadlineTimerLocked(GuestThreadState thread) private void WakeExpiredBlockedGuestThread(GuestThreadState thread) { + bool handoff; using (LockGate("WakeExpiredBlockedGuestThread")) { if (thread.State != GuestThreadRunState.Blocked || @@ -4517,7 +4597,12 @@ private void WakeExpiredBlockedGuestThread(GuestThreadState thread) thread.State = GuestThreadRunState.Ready; thread.BlockReason = null; thread.BlockDeadlineTimestamp = 0; - EnqueueReadyGuestThreadLocked(thread, "timeout"); + handoff = TryReadyBlockedThreadLocked(thread, "timeout"); + } + + if (handoff) + { + ScheduleGuestThreadExecution(thread, "timeout"); } if (_logGuestThreads) @@ -5702,6 +5787,7 @@ private void ClearGuestThreads() _readyGuestThreads.Clear(); Interlocked.Exchange(ref _readyGuestThreadCount, 0); _guestThreads.Clear(); + _guestThreadsLockFree.Clear(); _reusableGuestThreadRegions.Clear(); _externalGuestThreads.Clear(); _pendingGuestExceptions.Clear(); @@ -6261,9 +6347,12 @@ private void RunGuestThread(GuestThreadState thread, string reason) ? $"[LOADER][INFO] Pumping guest thread '{thread.Name}' reason={reason} resume=0x{continuation.Rip:X16}" : $"[LOADER][INFO] Pumping guest thread '{thread.Name}' reason={reason} entry=0x{thread.EntryPoint:X16}"); } - var exitReason = resumeContinuation - ? ExecuteBlockedGuestThreadContinuation(thread.Context, continuation, thread.Name, out var blockReason) - : ExecuteGuestThreadEntry(thread.Context, thread.EntryPoint, thread.Name, out blockReason); + string? blockReason; + var exitReason = TestGuestSlice is { } testSlice + ? RunTestGuestSlice(testSlice, thread, resumeContinuation, continuation.Rax, out blockReason) + : resumeContinuation + ? ExecuteBlockedGuestThreadContinuation(thread.Context, continuation, thread.Name, out blockReason) + : ExecuteGuestThreadEntry(thread.Context, thread.EntryPoint, thread.Name, out blockReason); if (exitReason == GuestNativeCallExitReason.Returned && thread.Recycler is null) { // Thread exit runs TLS-key and thread_atexit destructors (pthread_exit already @@ -6305,7 +6394,16 @@ thread.BlockWaiter is not null && thread.BlockReason = null; thread.BlockDeadlineTimestamp = 0; CancelBlockDeadlineTimerLocked(thread); - EnqueueReadyGuestThreadLocked(thread, "block_recheck"); + if (_directWakeHandoff && thread.ExecutionRunner is not null) + { + // This executor still owns the thread and hands it over to + // the next slice when it releases it below (same condition). + ProfileGuestThreadReady(thread, "block_recheck"); + } + else + { + EnqueueReadyGuestThreadLocked(thread, "block_recheck"); + } } break; default: @@ -6328,6 +6426,7 @@ thread.BlockWaiter is not null && finally { PendingGuestException? pendingAfterExecutorRelease = null; + var handOffToNextSlice = false; _activeGuestThreadState = previousGuestThreadState; Volatile.Write(ref thread.HostThreadId, 0); GuestThreadExecution.RestoreGuestThread(previousGuestThreadHandle); @@ -6340,10 +6439,22 @@ thread.BlockWaiter is not null && { thread.HostThread = null; } + if (_directWakeHandoff && thread.State == GuestThreadRunState.Ready && thread.ExecutionRunner is not null) + { + // The wait was satisfied while this executor unwound (a wake that found + // ExecutorActive, or the park-time re-check above). Keep the single-owner + // token and pass it straight to the next slice instead of releasing it and + // waking the dispatcher to claim it back. Any queue entry left by the wake + // is skipped by TryClaimReadyGuestThreadLocked because the state is Running. + RecordGuestThreadFlow(thread, GuestThreadFlowProfile.EventKind.ExecutorReleased); + thread.State = GuestThreadRunState.Running; + ProfileGuestThreadClaimed(thread); + handOffToNextSlice = true; + } // A signal can arrive after guest code blocks but before this // executor releases the thread. Deliver that signal from the saved // continuation after the thread becomes parked. - if (TryReleaseGuestThreadExecutorLocked(thread, out var pending)) + else if (TryReleaseGuestThreadExecutorLocked(thread, out var pending)) { pendingAfterExecutorRelease = pending; } @@ -6356,6 +6467,11 @@ thread.BlockWaiter is not null && thread.ExecutionRunner?.RequestStop(); } } + if (handOffToNextSlice) + { + // On the thread's own runner this queues the next slice behind the current one. + ScheduleGuestThreadExecution(thread, "handoff"); + } if (pendingAfterExecutorRelease is { } pendingException && !TryRaiseGuestException( thread.Context, @@ -6375,6 +6491,51 @@ thread.BlockWaiter is not null && parkedRecycler?.OnEntryReturned(thread.ThreadHandle, thread.ExitValue); } + /// + /// Test seam: when set, a guest thread slice runs this managed callback instead of native + /// guest code. Arguments: thread handle, its context, whether the slice resumes a blocked + /// continuation, and the resumed import's return value. The callback returns true when it + /// requested a block (GuestThreadExecution.RequestCurrentThreadBlock inside a simulated + /// import frame), which is registered exactly as the import dispatcher registers it. + /// + internal Func? TestGuestSlice; + + private GuestNativeCallExitReason RunTestGuestSlice( + Func slice, + GuestThreadState thread, + bool resumed, + ulong resumeValue, + out string? reason) + { + if (!slice(thread.ThreadHandle, thread.Context, resumed, resumeValue)) + { + reason = "returned"; + return GuestNativeCallExitReason.Returned; + } + + if (!GuestThreadExecution.TryConsumeCurrentThreadBlock( + out var blockReason, + out var blockContinuation, + out var hasBlockContinuation, + out var blockWakeKey, + out var blockWaiter, + out var blockDeadlineTimestamp) || + !hasBlockContinuation) + { + reason = "test slice blocked without a block request"; + return GuestNativeCallExitReason.Exception; + } + + RegisterBlockedGuestThreadContinuation( + thread.ThreadHandle, + blockContinuation, + blockWakeKey, + blockWaiter, + blockDeadlineTimestamp); + reason = blockReason; + return GuestNativeCallExitReason.Blocked; + } + private bool TryReleaseGuestThreadExecutorLocked( GuestThreadState thread, out PendingGuestException pending) @@ -6717,6 +6878,10 @@ private unsafe GuestNativeCallExitReason ExecuteGuestContinuationEntry( reason = "guest thread stack pointer is zero"; return GuestNativeCallExitReason.Exception; } + if (_sharedResumeStub && !SupportsSwappableStackBounds && name != "tbb_thead") + { + return ExecuteGuestContinuationEntryShared(context, entryPoint, returnSlotAddress, out reason); + } const uint stubSize = 512u; void* ptr = VirtualAlloc(null, stubSize, 12288u, 4u); if (ptr == null) @@ -6876,6 +7041,217 @@ private unsafe GuestNativeCallExitReason ExecuteGuestContinuationEntry( } } + // Resuming a blocked guest thread used to emit a fresh continuation trampoline per resume + // (VirtualAlloc, emit, VirtualProtect, run, VirtualFree). Under Rosetta every fresh executable + // page is translated again: ~17 us p50 per resume in a standalone mmap/mprotect/call/munmap + // loop, most of a cross-thread wake. This entry stub is emitted once; it saves the host frame + // exactly like the per-resume trampoline did and then jumps into the shared context-transfer + // stub, which loads the target registers from a per-call data frame. Not used where emitted + // code must swap thread stack bounds or for the native-worker path. The frame is passed as + // argument 1 and 4 so the stub can read it from rcx under either host calling convention. + // SHARPEMU_SHARED_RESUME_STUB=0 restores the per-resume trampoline. + private static readonly bool _sharedResumeStub = + !string.Equals(Environment.GetEnvironmentVariable("SHARPEMU_SHARED_RESUME_STUB"), "0", StringComparison.Ordinal); + + // Host-RSP state (HostStackStateBytes, padded to keep the frame 16-byte aligned), then the + // transfer frame, then the host-RSP slot address the entry stub stores the host stack into. + private const int ResumeBlockHostStateQwords = 4; + private const int ResumeFrameHostRspSlotIndex = GuestContextTransferFrameQwords; + + private nint _guestResumeEntryStub; + + private unsafe GuestNativeCallExitReason ExecuteGuestContinuationEntryShared( + CpuContext context, + ulong entryPoint, + ulong returnSlotAddress, + out string? reason) + { + reason = null; + var entryStub = GetOrCreateGuestResumeEntryStub(); + if (entryStub == 0) + { + reason = "failed to allocate guest resume entry stub"; + return GuestNativeCallExitReason.Exception; + } + var guestRsp = context[CpuRegister.Rsp]; + var block = (ulong*)NativeMemory.AllocZeroed( + (nuint)(ResumeBlockHostStateQwords + ResumeFrameHostRspSlotIndex + 1), + sizeof(ulong)); + if (block == null) + { + reason = "failed to allocate writable host-RSP storage for guest continuation stub"; + return GuestNativeCallExitReason.Exception; + } + var hostRspSlot = (ulong)block; + var frame = block + ResumeBlockHostStateQwords; + frame[0] = entryPoint; + frame[1] = guestRsp; + frame[2] = context[CpuRegister.Rax]; + frame[3] = context[CpuRegister.Rcx]; + frame[4] = context[CpuRegister.Rdx]; + frame[5] = context[CpuRegister.Rbx]; + frame[6] = context[CpuRegister.Rbp]; + frame[7] = context[CpuRegister.Rsi]; + frame[8] = context[CpuRegister.Rdi]; + frame[9] = context[CpuRegister.R8]; + frame[10] = context[CpuRegister.R9]; + frame[11] = context[CpuRegister.R10]; + frame[12] = context[CpuRegister.R11]; + frame[13] = context[CpuRegister.R12]; + frame[14] = context[CpuRegister.R13]; + frame[15] = context[CpuRegister.R14]; + frame[16] = context[CpuRegister.R15]; + frame[17] = context.Mxcsr; + frame[18] = context.FpuControlWord; + frame[19] = 1; // load MXCSR verbatim, as the per-resume trampoline did + frame[ResumeFrameHostRspSlotIndex] = hostRspSlot; + + var previousActiveBackend = _activeExecutionBackend; + var previousActiveContext = _activeCpuContext; + var previousSentinel = _activeEntryReturnSentinelRip; + var previousReturnSlotAddress = _activeGuestReturnSlotAddress; + var previousForcedExit = _activeForcedGuestExit; + var previousYieldRequested = _activeGuestThreadYieldRequested; + var previousYieldReason = _activeGuestThreadYieldReason; + nint previousHostRspSlotValue = TlsGetValue(_hostRspSlotTlsIndex); + var previousExecutingHostThreadId = Interlocked.Exchange( + ref _executingHostThreadId, unchecked((int)GetCurrentThreadId())); + try + { + _activeExecutionBackend = this; + _activeCpuContext = context; + _activeEntryReturnSentinelRip = 0; + _activeGuestReturnSlotAddress = returnSlotAddress; + _activeForcedGuestExit = false; + _activeGuestThreadYieldRequested = false; + _activeGuestThreadYieldReason = null; + BindTlsBase(context); + ExecutionPlatform.RouteGuestFaultsOnCurrentThread(); + // The transfer stub returns through [rsp-8], the slot the trampoline used to fill. + if (!context.TryWriteUInt64(guestRsp - sizeof(ulong), entryPoint)) + { + reason = $"guest continuation transfer slot is not writable at 0x{guestRsp - sizeof(ulong):X16}"; + return GuestNativeCallExitReason.Exception; + } + ActiveEntryReturnSentinelRip = (ulong)_guestReturnStub; + if (returnSlotAddress == 0 || !context.TryWriteUInt64(returnSlotAddress, (ulong)_guestReturnStub)) + { + reason = $"failed to patch guest continuation return slot at 0x{returnSlotAddress:X16}"; + return GuestNativeCallExitReason.Exception; + } + ActiveGuestThreadYieldRequested = false; + ActiveGuestThreadYieldReason = null; + try + { + if (!TlsSetValue(_hostRspSlotTlsIndex, (nint)hostRspSlot)) + { + reason = "failed to bind host-RSP storage for guest continuation stub"; + return GuestNativeCallExitReason.Exception; + } + var nativeReturn = ((delegate* unmanaged[Cdecl])entryStub)(frame, 0, 0, frame); + if (ActiveGuestThreadYieldRequested) + { + reason = ActiveGuestThreadYieldReason ?? "guest thread blocked"; + return GuestNativeCallExitReason.Blocked; + } + if (ActiveForcedGuestExit) + { + reason = LastError ?? "guest thread forced exit"; + return GuestNativeCallExitReason.ForcedExit; + } + context[CpuRegister.Rax] = nativeReturn; + reason = $"returned 0x{nativeReturn:X16}"; + return GuestNativeCallExitReason.Returned; + } + catch (AccessViolationException ex) + { + reason = "access violation: " + ex.Message; + return GuestNativeCallExitReason.Exception; + } + catch (Exception ex) + { + reason = ex.GetType().Name + ": " + ex.Message; + return GuestNativeCallExitReason.Exception; + } + } + finally + { + TlsSetValue(_hostRspSlotTlsIndex, previousHostRspSlotValue); + Volatile.Write(ref _executingHostThreadId, previousExecutingHostThreadId); + RestoreActiveExecutionThread( + previousActiveBackend, + previousActiveContext, + previousSentinel, + previousReturnSlotAddress, + previousForcedExit, + previousYieldRequested, + previousYieldReason); + NativeMemory.Free(block); + } + } + + // SysV entry: rdi = transfer frame. Saves the host non-volatile state in the same layout as + // the per-resume trampoline (the guest return stub restores it from the host-RSP slot), + // stores the host stack into frame[ResumeFrameHostRspSlotIndex] and continues in the shared + // context-transfer stub with rax = frame. + private unsafe nint GetOrCreateGuestResumeEntryStub() + { + if (Volatile.Read(ref _guestResumeEntryStub) != 0) + { + return _guestResumeEntryStub; + } + + var transferStub = GetOrCreateGuestContextTransferStub(); + if (transferStub == 0) + { + return 0; + } + + lock (_guestContextTransferStubGate) + { + if (_guestResumeEntryStub != 0) + { + return _guestResumeEntryStub; + } + + const uint stubSize = 256; + var code = (byte*)VirtualAlloc(null, stubSize, 12288u, 64u); + if (code == null) + { + return 0; + } + + var emitter = new NativeCodeEmitter(code); + emitter.Emit(0x53); // push rbx + emitter.Emit(0x55); // push rbp + emitter.Emit(0x57); // push rdi + emitter.Emit(0x56); // push rsi + emitter.Emit(0x41); emitter.Emit(0x54); // push r12 + emitter.Emit(0x41); emitter.Emit(0x55); // push r13 + emitter.Emit(0x41); emitter.Emit(0x56); // push r14 + emitter.Emit(0x41); emitter.Emit(0x57); // push r15 + EmitHostNonvolatileXmmSave(code, ref emitter.Offset); + emitter.Emit(0x48); emitter.Emit(0x89); emitter.Emit(0xC8); // mov rax, rcx (frame: SysV arg 4, Win64 arg 1) + emitter.Emit(0x4C); emitter.Emit(0x8B); emitter.Emit(0x90); // mov r10, [rax+disp32] + emitter.Emit((uint)(ResumeFrameHostRspSlotIndex * sizeof(ulong))); + emitter.Emit(0x49); emitter.Emit(0x89); emitter.Emit(0x22); // mov [r10], rsp + emitter.EmitMovR64Immediate(0x49, 0xBB, (ulong)transferStub); // mov r11, transfer stub + emitter.Emit(0x41); emitter.Emit(0xFF); emitter.Emit(0xE3); // jmp r11 + Debug.Assert(emitter.Offset <= stubSize, "Guest resume entry stub exceeded its allocation."); + + uint oldProtect = 0; + if (!VirtualProtect(code, stubSize, 32u, &oldProtect)) + { + VirtualFree(code, 0u, 32768u); + return 0; + } + + FlushInstructionCache(GetCurrentProcess(), code, stubSize); + Volatile.Write(ref _guestResumeEntryStub, (nint)code); + return (nint)code; + } + } + // Guest threads a stall snapshot lists; SHARPEMU_SNAPSHOT_MAX_THREADS= raises it. private static readonly int SnapshotMaxThreads = int.TryParse(Environment.GetEnvironmentVariable("SHARPEMU_SNAPSHOT_MAX_THREADS"), out var snapshotThreads) && snapshotThreads > 0 @@ -7600,14 +7976,19 @@ private void DispatchReadyGuestThreads() private void ScheduleGuestThreadExecution(GuestThreadState thread, string reason) { - GuestExecutionRunner runner; - lock (_guestThreadGate) + // The runner is created once under the gate and never replaced, so a set reference + // can be used without taking the gate again. + var runner = thread.ExecutionRunner; + if (runner is null) { - runner = thread.ExecutionRunner ??= new GuestExecutionRunner( - thread.ThreadHandle, - thread.Name, - MapGuestThreadPriority(thread.Priority), - thread.Priority); + lock (_guestThreadGate) + { + runner = thread.ExecutionRunner ??= new GuestExecutionRunner( + thread.ThreadHandle, + thread.Name, + MapGuestThreadPriority(thread.Priority), + thread.Priority); + } } ProfileGuestThreadScheduled(thread); runner.Schedule(() => @@ -7985,6 +8366,11 @@ public unsafe void Dispose() VirtualFree((void*)_guestContextTransferStub, 0u, 32768u); _guestContextTransferStub = 0; } + if (_guestResumeEntryStub != 0) + { + VirtualFree((void*)_guestResumeEntryStub, 0u, 32768u); + _guestResumeEntryStub = 0; + } foreach (var frame in _guestContextTransferFrames.Values) { if (frame != 0) diff --git a/src/SharpEmu.HLE/GuestThreadExecution.cs b/src/SharpEmu.HLE/GuestThreadExecution.cs index 9b99d7373..e97f8decf 100644 --- a/src/SharpEmu.HLE/GuestThreadExecution.cs +++ b/src/SharpEmu.HLE/GuestThreadExecution.cs @@ -107,6 +107,13 @@ bool TryJoinThread( bool WakeBlockedWaiter(ulong threadHandle, IGuestThreadBlockWaiter waiter, string wakeKey) => WakeBlockedThreads(wakeKey) > 0; + /// + /// Lock-free, possibly stale: whether the scheduler thread is executing a slice on a host + /// CPU right now (false while it is blocked, ready or parked). Unknown threads report true. + /// Used by contended locks to stop spinning on an owner that cannot release soon. + /// + bool IsGuestThreadExecuting(ulong threadHandle) => true; + /// /// Reports whether the current guest thread has an exception that waits for an import safe point. /// diff --git a/tests/SharpEmu.Libs.Tests/Cpu/GuestSchedulerWakeStressTests.cs b/tests/SharpEmu.Libs.Tests/Cpu/GuestSchedulerWakeStressTests.cs new file mode 100644 index 000000000..23285a828 --- /dev/null +++ b/tests/SharpEmu.Libs.Tests/Cpu/GuestSchedulerWakeStressTests.cs @@ -0,0 +1,480 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using System.Collections; +using System.Diagnostics; +using System.Reflection; +using SharpEmu.Core.Cpu.Native; +using SharpEmu.HLE; +using SharpEmu.Libs.Kernel; +using Xunit; + +namespace SharpEmu.Libs.Tests.Cpu; + +[CollectionDefinition("GuestSchedulerWakeStress", DisableParallelization = true)] +public sealed class GuestSchedulerWakeStressCollection; + +// Many simulated guest threads run on the real cooperative scheduler (runners, dispatcher, +// park-time re-check, direct wake handoff, timer-driven timeouts) and hammer the real HLE +// primitives with randomized timing. Each slice runs managed code inside a simulated import +// frame instead of native guest code (DirectExecutionBackend.TestGuestSlice). A lost wakeup or +// a deadlock leaves a thread blocked and fails the run at the deadline. +[Collection("GuestSchedulerWakeStress")] +public sealed class GuestSchedulerWakeStressTests +{ + private const ulong Base = 0x2_5000_0000; + private const int MemorySize = 0x40000; + private const ulong HandleSlot = Base + 0x100; + private const ulong NameAddress = Base + 0x200; + private const ulong MutexBase = Base + 0x1000; + private const ulong ScratchBase = Base + 0x8000; + private static readonly TimeSpan Deadline = TimeSpan.FromSeconds(90); + private static readonly int Ok = (int)OrbisGen2Result.ORBIS_GEN2_OK; + private static readonly int TimedOut = (int)OrbisGen2Result.ORBIS_GEN2_ERROR_TIMED_OUT; + + [NativeX64Fact] + public void SemaphoreProducersAndTimedConsumersLoseNoWakeups() + { + using var harness = new Harness(); + var sema = harness.CreateSema(); + const int consumers = 8; + const int perConsumer = 300; + const int producers = 4; + const int perProducer = consumers * perConsumer / producers; + var consumed = 0; + var produced = 0; + for (var i = 0; i < consumers; i++) + { + var timed = i % 2 == 0; + harness.AddThread(perConsumer, (ctx, slot, rng) => + { + ctx[CpuRegister.Rdi] = sema; + ctx[CpuRegister.Rsi] = 1; + ctx[CpuRegister.Rdx] = 0; + if (timed && rng.Next(3) == 0) + { + Assert.True(ctx.TryWriteUInt32(harness.Scratch(slot), (uint)rng.Next(5, 200))); + ctx[CpuRegister.Rdx] = harness.Scratch(slot); + } + return KernelSemaphoreCompatExports.KernelWaitSema(ctx); + }, result => + { + if (result == TimedOut) return false; // retry the same step + Assert.Equal(Ok, result); + Interlocked.Increment(ref consumed); + return true; + }); + } + for (var i = 0; i < producers; i++) + { + harness.AddThread(perProducer, (ctx, _, rng) => + { + Harness.Jitter(rng); + return KernelSemaphoreCompatExports.KernelSignalSema(ctx, sema, 1); + }, result => + { + Assert.Equal(Ok, result); + Interlocked.Increment(ref produced); + return true; + }); + } + + harness.RunToCompletion(); + Assert.Equal(consumers * perConsumer, produced); + Assert.Equal(consumers * perConsumer, consumed); + } + + [NativeX64Fact] + public void BlockingPingPongPairsLoseNoWakeups() + { + using var harness = new Harness(); + const int pairs = 6; + const int rounds = 400; + var completed = 0; + for (var p = 0; p < pairs; p++) + { + var ping = harness.CreateSema(); + var pong = harness.CreateSema(); + // Even steps signal, odd steps wait: every round parks both threads once. + harness.AddThread(rounds * 2, (ctx, _, rng, step) => + { + Harness.Jitter(rng); + return step % 2 == 0 + ? KernelSemaphoreCompatExports.KernelSignalSema(ctx, ping, 1) + : harness.WaitSema(ctx, pong); + }, result => { Assert.Equal(Ok, result); return true; }); + harness.AddThread(rounds * 2, (ctx, _, rng, step) => + { + Harness.Jitter(rng); + return step % 2 == 0 + ? harness.WaitSema(ctx, ping) + : KernelSemaphoreCompatExports.KernelSignalSema(ctx, pong, 1); + }, result => { Assert.Equal(Ok, result); Interlocked.Increment(ref completed); return true; }); + } + + harness.RunToCompletion(); + Assert.Equal(pairs * rounds * 2, completed); + } + + [NativeX64Fact] + public void ContendedMutexCounterStaysExactWhileOwnersPark() + { + using var harness = new Harness(); + const int threads = 8; + const int iterations = 250; + var never = harness.CreateSema(); + var mutex = harness.InitMutex(0); + var counter = 0L; + var owner = 0; + for (var i = 0; i < threads; i++) + { + // Steps: lock, (claim + optional park while holding the mutex), release + unlock. + harness.AddThread(iterations * 3, (ctx, slot, rng, step) => + { + switch (step % 3) + { + case 0: + ctx[CpuRegister.Rdi] = mutex; + return KernelPthreadCompatExports.PthreadMutexLock(ctx); + case 1: + Assert.Equal(0, Interlocked.Exchange(ref owner, slot + 1)); + counter++; + if (rng.Next(8) == 0) + { + // The owner parks inside its critical section: waiters must stop + // spinning on it and park too, and nobody may enter meanwhile. + Assert.True(ctx.TryWriteUInt32(harness.Scratch(slot), (uint)rng.Next(20, 300))); + ctx[CpuRegister.Rdi] = never; + ctx[CpuRegister.Rsi] = 1; + ctx[CpuRegister.Rdx] = harness.Scratch(slot); + return KernelSemaphoreCompatExports.KernelWaitSema(ctx); + } + Harness.Spin(rng.Next(0, 20)); + return TimedOut; + default: + Assert.Equal(slot + 1, Interlocked.Exchange(ref owner, 0)); + ctx[CpuRegister.Rdi] = mutex; + return KernelPthreadCompatExports.PthreadMutexUnlock(ctx); + } + }, result => true); + } + + harness.RunToCompletion(); + Assert.Equal(threads * iterations, Interlocked.Read(ref counter)); + } + + [NativeX64Fact] + public void UltMutexAndQueueWakeOnlyTheirOwnWaiters() + { + using var harness = new Harness(); + var ultMutex = MutexBase + 0x800; + var queue = MutexBase + 0x900; + var setup = new CpuContext(harness.Memory, Generation.Gen5) { FsBase = ScratchBase + 0x7F00 }; + setup[CpuRegister.Rdi] = queue; + setup[CpuRegister.Rsi] = 0; + setup[CpuRegister.Rdx] = 8; + Assert.Equal(0, SharpEmu.Libs.Ult.UltExports.QueueCreate(setup)); + const int lockers = 6; + const int iterations = 200; + const int consumers = 4; + const int perConsumer = 250; + var counter = 0L; + var owner = 0; + var popped = 0L; + for (var i = 0; i < lockers; i++) + { + harness.AddThread(iterations * 3, (ctx, slot, rng, step) => + { + ctx[CpuRegister.Rdi] = ultMutex; + switch (step % 3) + { + case 0: + return (int)(long)SharpEmu.Libs.Ult.UltExports.MutexLock(ctx); + case 1: + Assert.Equal(0, Interlocked.Exchange(ref owner, slot + 1)); + counter++; + Harness.Spin(rng.Next(0, 15)); + return 0; + default: + Assert.Equal(slot + 1, Interlocked.Exchange(ref owner, 0)); + return SharpEmu.Libs.Ult.UltExports.MutexUnlock(ctx); + } + }, result => { Assert.Equal(0, result); return true; }); + } + for (var i = 0; i < consumers; i++) + { + harness.AddThread(perConsumer, (ctx, slot, _, _) => + { + ctx[CpuRegister.Rdi] = queue; + ctx[CpuRegister.Rsi] = harness.Scratch(slot); + return SharpEmu.Libs.Ult.UltExports.QueuePop(ctx); + }, result => { Assert.Equal(0, result); Interlocked.Increment(ref popped); return true; }); + } + for (var i = 0; i < 2; i++) + { + harness.AddThread(consumers * perConsumer / 2, (ctx, slot, rng, step) => + { + Harness.Jitter(rng); + Assert.True(ctx.TryWriteUInt64(harness.Scratch(slot), (ulong)step)); + ctx[CpuRegister.Rdi] = queue; + ctx[CpuRegister.Rsi] = harness.Scratch(slot); + return SharpEmu.Libs.Ult.UltExports.QueuePush(ctx); + }, result => { Assert.Equal(0, result); return true; }); + } + + harness.RunToCompletion(); + Assert.Equal(lockers * iterations, Interlocked.Read(ref counter)); + Assert.Equal(consumers * perConsumer, Interlocked.Read(ref popped)); + setup[CpuRegister.Rdi] = queue; + _ = SharpEmu.Libs.Ult.UltExports.QueueDestroy(setup); + } + + private sealed class Harness : IDisposable + { + private const BindingFlags PrivateInstance = BindingFlags.NonPublic | BindingFlags.Instance; + private static readonly Type BackendType = typeof(DirectExecutionBackend); + private static readonly Type ThreadType = BackendType.GetNestedType("GuestThreadState", BindingFlags.NonPublic)!; + private static readonly Type RunStateType = BackendType.GetNestedType("GuestThreadRunState", BindingFlags.NonPublic)!; + + private readonly FakeCpuMemory _memory = new(Base, MemorySize); + private readonly DirectExecutionBackend _backend; + private readonly object _gate; + private readonly IDictionary _threads; + private readonly IDictionary _lockFreeThreads; + private readonly Dictionary _sims = new(); + private readonly IGuestThreadScheduler? _previousScheduler; + private readonly List _semas = new(); + private ulong _nextHandle = 0x7700; + + internal Harness() + { + var modules = new ModuleManager(); + modules.Freeze(); + _backend = new DirectExecutionBackend(modules); + _gate = BackendType.GetField("_guestThreadGate", PrivateInstance)!.GetValue(_backend)!; + _threads = (IDictionary)BackendType.GetField("_guestThreads", PrivateInstance)!.GetValue(_backend)!; + _lockFreeThreads = (IDictionary)BackendType.GetField("_guestThreadsLockFree", PrivateInstance)!.GetValue(_backend)!; + _backend.TestGuestSlice = Slice; + _previousScheduler = GuestThreadExecution.Scheduler; + GuestThreadExecution.Scheduler = _backend; + } + + internal ulong Scratch(int slot) => ScratchBase + (ulong)slot * 0x100; + + internal FakeCpuMemory Memory => _memory; + + internal uint CreateSema() + { + var ctx = new CpuContext(_memory, Generation.Gen5) { FsBase = ScratchBase + 0x7F00 }; + Assert.True(_memory.TryWrite(NameAddress, "w3-sema\0"u8)); + ctx[CpuRegister.Rdi] = HandleSlot; + ctx[CpuRegister.Rsi] = NameAddress; + ctx[CpuRegister.Rdx] = 1; + ctx[CpuRegister.Rcx] = 0; + ctx[CpuRegister.R8] = 0x7FFFFFFF; + ctx[CpuRegister.R9] = 0; + Assert.Equal(Ok, KernelSemaphoreCompatExports.KernelCreateSema(ctx)); + Assert.True(ctx.TryReadUInt64(HandleSlot, out var handle)); + _semas.Add((uint)handle); + return (uint)handle; + } + + internal ulong InitMutex(int index) + { + var address = MutexBase + (ulong)index * 0x40; + var ctx = new CpuContext(_memory, Generation.Gen5) { FsBase = ScratchBase + 0x7F00 }; + ctx[CpuRegister.Rdi] = address; + ctx[CpuRegister.Rsi] = 0; + Assert.Equal(Ok, KernelPthreadCompatExports.PthreadMutexInit(ctx)); + return address; + } + + internal int WaitSema(CpuContext ctx, uint sema) + { + ctx[CpuRegister.Rdi] = sema; + ctx[CpuRegister.Rsi] = 1; + ctx[CpuRegister.Rdx] = 0; + return KernelSemaphoreCompatExports.KernelWaitSema(ctx); + } + + internal static void Jitter(System.Random rng) + { + switch (rng.Next(16)) + { + case 0: Thread.Yield(); break; + case 1: Spin(rng.Next(1, 50)); break; + case 2: Thread.SpinWait(rng.Next(1, 200)); break; + } + } + + internal static void Spin(int microseconds) + { + var end = Stopwatch.GetTimestamp() + Stopwatch.Frequency * microseconds / 1_000_000; + while (Stopwatch.GetTimestamp() < end) Thread.SpinWait(8); + } + + internal void AddThread(int steps, Func op, Func onResult) => + AddThread(steps, (ctx, slot, rng, _) => op(ctx, slot, rng), onResult); + + internal void AddThread(int steps, Func op, Func onResult) + { + var handle = _nextHandle++; + var slot = _sims.Count; + var sim = new SimThread(steps, op, onResult, new System.Random(unchecked((int)handle * 7919)), slot); + _sims.Add(handle, sim); + var thread = Activator.CreateInstance(ThreadType)!; + ThreadType.GetProperty("ThreadHandle")!.SetValue(thread, handle); + ThreadType.GetProperty("Name")!.SetValue(thread, $"w3-stress-{handle:X}"); + ThreadType.GetProperty("State")!.SetValue(thread, Enum.Parse(RunStateType, "Ready")); + ThreadType.GetProperty("Context")!.SetValue( + thread, + new CpuContext(_memory, Generation.Gen5) { FsBase = Scratch(slot) + 0x80 }); + lock (_gate) + { + _threads.Add(handle, thread); + _lockFreeThreads[handle] = thread; + } + } + + internal void RunToCompletion() + { + lock (_gate) + { + foreach (var thread in _threads.Values) + { + BackendType.GetMethod("EnqueueReadyGuestThreadLocked", PrivateInstance)! + .Invoke(_backend, new object?[] { thread, null }); + } + } + BackendType.GetMethod("StartReadyThreadDispatcher", PrivateInstance)!.Invoke(_backend, null); + + var finished = SpinWait.SpinUntil(AllExited, Deadline); + if (!finished) + { + Assert.Fail("guest threads did not finish (lost wakeup or deadlock): " + DescribeThreads()); + } + foreach (var sim in _sims.Values) + { + if (sim.Failure is { } failure) throw new Xunit.Sdk.XunitException("slice failed: " + failure); + } + } + + private bool AllExited() + { + lock (_gate) + { + foreach (var thread in _threads.Values) + { + var state = ThreadType.GetProperty("State")!.GetValue(thread)!.ToString(); + if (state is "Faulted") + { + return true; // reported by DescribeThreads/Failure below + } + if (state != "Exited" || (bool)ThreadType.GetProperty("ExecutorActive")!.GetValue(thread)!) + { + return false; + } + } + return true; + } + } + + private string DescribeThreads() + { + lock (_gate) + { + return string.Join("; ", _threads.Values.Cast().Select(thread => + { + var handle = (ulong)ThreadType.GetProperty("ThreadHandle")!.GetValue(thread)!; + return $"0x{handle:X} {ThreadType.GetProperty("State")!.GetValue(thread)} " + + $"active={ThreadType.GetProperty("ExecutorActive")!.GetValue(thread)} " + + $"step={_sims[handle].Step}/{_sims[handle].Steps} " + + $"block={ThreadType.GetProperty("BlockReason")!.GetValue(thread)}"; + })); + } + } + + private bool Slice(ulong handle, CpuContext ctx, bool resumed, ulong resumeValue) + { + var sim = _sims[handle]; + try + { + if (resumed && sim.OnResult((int)unchecked((long)resumeValue))) + { + sim.Step++; + } + + while (sim.Step < sim.Steps) + { + var previous = GuestThreadExecution.EnterImportCallFrame( + 0x40_0000UL + (ulong)sim.Step, + Scratch(sim.Slot) + 0x40, + Scratch(sim.Slot) + 0x38); + int result; + bool blocked; + try + { + result = sim.Op(ctx, sim.Slot, sim.Rng, sim.Step); + blocked = GuestThreadExecution.TryConsumeCurrentThreadBlock( + out var reason, out _, out _, out var wakeKey, out var waiter, out var deadline); + if (blocked) + { + // Re-issue the request so the scheduler consumes it as the import + // dispatcher would, with the continuation of this import frame. + Assert.True(GuestThreadExecution.RequestCurrentThreadBlock(ctx, reason, wakeKey, waiter, deadline)); + } + } + finally + { + GuestThreadExecution.RestoreImportCallFrame(previous); + } + + if (blocked) + { + return true; + } + if (sim.OnResult(result)) + { + sim.Step++; + } + } + } + catch (Exception exception) + { + sim.Failure ??= exception.ToString(); + sim.Step = sim.Steps; + } + + return false; + } + + public void Dispose() + { + GuestThreadExecution.Scheduler = _previousScheduler; + BackendType.GetMethod("StopReadyThreadDispatcher", PrivateInstance)!.Invoke(_backend, null); + var ctx = new CpuContext(_memory, Generation.Gen5) { FsBase = ScratchBase + 0x7F00 }; + foreach (var sema in _semas) + { + ctx[CpuRegister.Rdi] = sema; + _ = KernelSemaphoreCompatExports.KernelDeleteSema(ctx); + } + _backend.Dispose(); + } + + private sealed class SimThread( + int steps, + Func op, + Func onResult, + System.Random rng, + int slot) + { + public int Steps { get; } = steps; + public Func Op { get; } = op; + public Func OnResult { get; } = onResult; + public System.Random Rng { get; } = rng; + public int Slot { get; } = slot; + public int Step; + public string? Failure; + } + } +} From bba5d6413bfd22dd80b234c565eaf9e7ebea22e5 Mon Sep 17 00:00:00 2001 From: kuba Date: Mon, 26 Oct 2026 06:17:50 +0100 Subject: [PATCH 20/82] [Pthread] Spin for a mutex grant only while the owner is running, for at most 10 us A queued waiter spun up to 40 us for its grant even when the owning guest thread was parked, which burns a host core for nothing: a parked owner cannot release the mutex within the spin window. The waiter now stops spinning as soon as the scheduler reports the owner is not executing, and the bound is 10 us. Parking early is safe because the park path re-checks the grant under the scheduler gate before the thread blocks. Threadsim e (host threads, no guest scheduler, so only the bound applies), total Mops/s over 5 runs: 4 threads 3.15 -> 2.62 mean, 13 threads 2.88 -> 2.92 mean; single-thread uncontended varies 2.1-4.7 between runs, so this is within noise. Test eboot uncontended mutex pair is unchanged (467-991 ns before, 480-1313 ns after, load-dominated). Switches: SHARPEMU_MUTEX_ADAPTIVE_SPIN=0 restores the 40 us unconditional spin; SHARPEMU_MUTEX_SPIN_US sets the adaptive bound. --- .../Kernel/KernelPthreadCompatExports.cs | 21 +++++++++++++++++-- .../Pthread/PthreadMutexSemanticsTests.cs | 4 ++++ 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/src/SharpEmu.Libs/Kernel/KernelPthreadCompatExports.cs b/src/SharpEmu.Libs/Kernel/KernelPthreadCompatExports.cs index 3fa54ae17..f53b025c5 100644 --- a/src/SharpEmu.Libs/Kernel/KernelPthreadCompatExports.cs +++ b/src/SharpEmu.Libs/Kernel/KernelPthreadCompatExports.cs @@ -2751,7 +2751,16 @@ private static void RecordMutexContention(ulong mutexAddress, PthreadMutexState state.ProfileOwner = MutexContentionProfile.CurrentThreadName(); } - private static readonly long MutexWaiterSpinTicks = Stopwatch.Frequency * 40 / 1_000_000; + // Adaptive spin (default): spin at most SHARPEMU_MUTEX_SPIN_US (default 10 us) and only + // while the owner is executing on a host CPU; an owner that is blocked, ready or parked + // cannot release soon, so the waiter parks at once instead of burning a core. + // SHARPEMU_MUTEX_ADAPTIVE_SPIN=0 restores the fixed 40 us spin without the owner check. + private static readonly bool MutexAdaptiveSpin = + !string.Equals(Environment.GetEnvironmentVariable("SHARPEMU_MUTEX_ADAPTIVE_SPIN"), "0", StringComparison.Ordinal); + + private static readonly long MutexWaiterSpinTicks = Stopwatch.Frequency * (MutexAdaptiveSpin + ? (long.TryParse(Environment.GetEnvironmentVariable("SHARPEMU_MUTEX_SPIN_US"), out var spinUs) && spinUs >= 0 ? spinUs : 10) + : 40) / 1_000_000; // A queued waiter spins briefly for its grant before it parks. Under FIFO handoff the unlock // hands the mutex to the queue head, and parking and waking a guest thread costs far more @@ -2764,6 +2773,7 @@ private static bool SpinForMutexGrant(PthreadMutexState state, PthreadMutexWaite try { var deadline = Stopwatch.GetTimestamp() + MutexWaiterSpinTicks; + var scheduler = MutexAdaptiveSpin ? GuestThreadExecution.Scheduler : null; for (var round = 0; ; round++) { if (Volatile.Read(ref waiter.Granted) != 0) @@ -2771,7 +2781,8 @@ private static bool SpinForMutexGrant(PthreadMutexState state, PthreadMutexWaite return true; } - if (state.OwnerThreadId == 0) + var owner = state.OwnerThreadId; + if (owner == 0) { lock (state.SyncRoot) { @@ -2781,6 +2792,12 @@ private static bool SpinForMutexGrant(PthreadMutexState state, PthreadMutexWaite } } } + else if (scheduler is not null && !scheduler.IsGuestThreadExecuting(owner)) + { + // Re-check the grant once: the owner may have released and handed off + // between the reads above. + return Volatile.Read(ref waiter.Granted) != 0; + } if (Stopwatch.GetTimestamp() >= deadline) { diff --git a/tests/SharpEmu.Libs.Tests/Pthread/PthreadMutexSemanticsTests.cs b/tests/SharpEmu.Libs.Tests/Pthread/PthreadMutexSemanticsTests.cs index 22df45c0c..54132045c 100644 --- a/tests/SharpEmu.Libs.Tests/Pthread/PthreadMutexSemanticsTests.cs +++ b/tests/SharpEmu.Libs.Tests/Pthread/PthreadMutexSemanticsTests.cs @@ -89,6 +89,10 @@ public class ExceptionWaitScheduler : DispatchProxy DeliverException!((CpuContext)args![0]!); return null; } + if (targetMethod?.Name == nameof(IGuestThreadScheduler.IsGuestThreadExecuting)) + { + return true; + } throw new NotSupportedException(targetMethod?.Name); } } From f7252cc42e09c9f7cea7abe25c457f2e98d40d3a Mon Sep 17 00:00:00 2001 From: kuba Date: Mon, 26 Oct 2026 06:19:29 +0100 Subject: [PATCH 21/82] [Cpu] A missed sceKernelPollEventFlag is an expected result, like a missed sceKernelPollSema A poll that finds no matching bits returns ORBIS_GEN2_ERROR_BUSY by design, and a guest that polls an event flag in a loop logged an import-result warning every 10,000th call. sceKernelPollSema's NID was already classified as benign for BUSY; sceKernelPollEventFlag (9lvj5DjHZiA) now is too. --- src/SharpEmu.Core/Cpu/Native/DirectExecutionBackend.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/SharpEmu.Core/Cpu/Native/DirectExecutionBackend.cs b/src/SharpEmu.Core/Cpu/Native/DirectExecutionBackend.cs index 2eb2c45bb..48a347a5c 100644 --- a/src/SharpEmu.Core/Cpu/Native/DirectExecutionBackend.cs +++ b/src/SharpEmu.Core/Cpu/Native/DirectExecutionBackend.cs @@ -220,7 +220,7 @@ private static ImportTraits ClassifyImportTraits(string nid) "K-jXhbt2gn4" or "upoVrzMHFeE" => unchecked((int)OrbisGen2Result.ORBIS_GEN2_ERROR_BUSY), "H2a+IN9TP0E" => unchecked((int)OrbisGen2Result.ORBIS_GEN2_ERROR_TRY_AGAIN), - "12wOHk8ywb0" => unchecked((int)OrbisGen2Result.ORBIS_GEN2_ERROR_BUSY), + "12wOHk8ywb0" or "9lvj5DjHZiA" => unchecked((int)OrbisGen2Result.ORBIS_GEN2_ERROR_BUSY), "PIWqhn9oSxc" => unchecked((int)0x80410123), "yH17Q6NWtVg" => unchecked((int)0x80960007), "D-CzAxQL0XI" => unchecked((int)0x80960009), From 0fce69696b92d4a5da235f405e2325984e57a776 Mon Sep 17 00:00:00 2001 From: kuba Date: Mon, 26 Oct 2026 06:26:56 +0100 Subject: [PATCH 22/82] [GPU] Wait for the write fence after a GPU word's own submission, not the whole command buffer A host read of bytes recorded work still writes waited for that work's submission serial, which is signalled only when its command buffer ends. The coalescer keeps one command buffer open across up to 8 guest submissions, so a word an early submission wrote was waited for together with all the GPU work recorded after it. Where the coalescer keeps the buffer open after a guest submission whose writes touched memory the CPU has waited on before (64 KiB blocks learned from those waits), and no render pass is open, the command buffer now signals a second shared event (a write fence) and the write ranges of that serial take its value. A read whose pending writes in the newest serial all carry a fence commits the buffer if it is still open, then waits for that fence and the serial before it. It wakes through an MTLSharedEventListener notify that uses the completion block, and returns at once if the fence has already been signalled. Labels, unbounded writes and ranges noted after a stamp keep waiting for the serial, and every other user of the write ranges still sees the serial pending. run-bench.sh gword (2 runs each, load 10-17), 8g5 = a word read after a long later submission in the same buffer: x64 8g5 1.05-1.41 ms -> 0.18-0.19 ms (8g6 whole-buffer reference 1.08-1.42 -> 1.32 ms) arm64 8g5 1.06-1.32 ms -> 0.16-0.22 ms (8g6 1.06-1.33 -> 1.00-1.33 ms) 8g2 (single submission, no fence) x64 0.22-0.35 -> 0.20 ms, arm64 0.20-0.21 -> 0.18-0.20 ms 7d/7e unchanged within noise; 8w1-3 PendingSerial over 512 records 0.33-0.40 us either way Test eboot group 5 (gpu.dcb16/dcb1k fence_wait medians, one submission per frame, so no fence is planted and none is expected): default 0.54-0.59 -> 0.57-0.90 ms, no pacing 0.20-0.28 -> 0.16-0.19 ms; both within run-to-run noise. SHARPEMU_METAL_V2_WRITE_FENCES=0 plants no fences and waits for whole command buffers. --- .../V2/MetalCommandStream.cs | 2 + .../SharpEmu.Gpu.Metal/V2/MetalGpuWorker.cs | 44 +++ .../V2/MetalGpuWriteRanges.cs | 77 +++++ .../V2/MetalRenderHost.WriteFences.cs | 309 ++++++++++++++++++ .../SharpEmu.Gpu.Metal/V2/MetalRenderHost.cs | 11 +- .../MetalGpuWriteRangesTests.cs | 39 +++ .../Scenes/GpuWordWaitSceneTests.cs | 240 ++++++++++++++ 7 files changed, 721 insertions(+), 1 deletion(-) create mode 100644 src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.WriteFences.cs create mode 100644 tests/SharpEmu.Gpu.Metal.Tests/Scenes/GpuWordWaitSceneTests.cs diff --git a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalCommandStream.cs b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalCommandStream.cs index cfce55314..f128773e0 100644 --- a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalCommandStream.cs +++ b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalCommandStream.cs @@ -184,6 +184,8 @@ private void FlushStream() return; } + // A commit would have been legal here, so a write fence covers everything recorded. + PlantWriteFenceIfWanted(); Interlocked.Increment(ref _deferredStreamFlushes); } diff --git a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalGpuWorker.cs b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalGpuWorker.cs index 446adfd7a..5b7a85f2c 100644 --- a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalGpuWorker.cs +++ b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalGpuWorker.cs @@ -490,6 +490,50 @@ private void WaitForSignal(ulong target) } } + /// + /// Blocks until holds or the timeout passes, woken by every command-buffer + /// completion and every shared-event notification that runs the completion block. False on timeout. + /// + public bool WaitUntil(Func done, int timeoutMilliseconds) + { + var frequency = System.Diagnostics.Stopwatch.Frequency; + var started = System.Diagnostics.Stopwatch.GetTimestamp(); + var spinDeadline = started + (frequency / 20000); + var deadline = started + (frequency * timeoutMilliseconds / 1000); + while (!done()) + { + if (_stopping) + { + return true; + } + + var now = System.Diagnostics.Stopwatch.GetTimestamp(); + if (now >= deadline) + { + return false; + } + + if (now < spinDeadline) + { + Thread.SpinWait(32); + continue; + } + + lock (_wakeGate) + { + // Inside the lock a wakeup cannot fall between the test and the wait. + if (done() || _stopping) + { + return true; + } + + _ = Monitor.Wait(_wakeGate, _handlerBound ? HandlerWaitMilliseconds : 1); + } + } + + return true; + } + /// /// Blocks any thread until the GPU has signalled the serial or the timeout passes. Only /// the signal: unlike it does not wait for the completions to run, diff --git a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalGpuWriteRanges.cs b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalGpuWriteRanges.cs index ae674f56c..0a5993f08 100644 --- a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalGpuWriteRanges.cs +++ b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalGpuWriteRanges.cs @@ -89,6 +89,10 @@ private struct Record // here: which of two overlapping writes in one submission lands last. public ulong Sequence; public MetalGpuWriteOrigin Origin; + + // The write-fence value signalled after every write this record covers, or zero when + // only the end of its command buffer says it landed (MetalRenderHost.WriteFences). + public ulong Fence; } private Record[] _records = new Record[64]; @@ -177,6 +181,8 @@ public ulong Note( } candidate.Sequence = sequence; + // The new write is not behind the fence the record had. + candidate.Fence = 0; return sequence; } @@ -589,6 +595,76 @@ public ulong PendingSerial(ulong address, ulong size, ulong completed) return pending; } + /// + /// A write fence with this value is encoded after every write recorded so far in + /// : each of its records without a fence takes this one. Labels and + /// unbounded writes keep waiting for the serial. + /// + public void StampFence(ulong serial, ulong fence) + { + BeginMutation(); + try + { + for (var index = 0; index < _count; index++) + { + ref var record = ref _records[index]; + if (record.Serial == serial && record.Fence == 0) + { + record.Fence = fence; + } + } + } + finally + { + EndMutation(); + } + } + + /// + /// The write fence after which every write over the range recorded in + /// (the newest pending serial over it, as returned) has landed, or + /// zero when one of them has no fence: a label, an unbounded write, or a record noted since. + /// Writes of older serials are not covered; the caller waits for those serials. + /// + public ulong WriteFence(ulong address, ulong size, ulong serial) + { + if (size == 0 || serial == 0 || address > ulong.MaxValue - size || _unboundedSerial >= serial) + { + return 0; + } + + var end = address + size; + var fence = 0UL; + var records = RecordsMayCover(address, end) ? _count : 0; + for (var index = 0; index < records; index++) + { + ref var record = ref _records[index]; + if (record.Serial < serial || record.End <= address || record.Start >= end) + { + continue; + } + + if (record.Serial > serial || record.Fence == 0) + { + return 0; + } + + fence = Math.Max(fence, record.Fence); + } + + var labels = LabelPagesMayCover(address, end) ? _labelCount : 0; + for (var index = 0; index < labels; index++) + { + ref var label = ref _labels[index]; + if (label.Serial >= serial && label.End > address && label.Start < end) + { + return 0; + } + } + + return fence; + } + /// /// for each of pending.Length adjacent slices of /// bytes from , as whether it is @@ -978,6 +1054,7 @@ private void FoldOldest() } last.Sequence = Math.Max(last.Sequence, current.Sequence); + last.Fence = 0; last.End = Math.Max(last.End, current.End); folded[^1] = last; diff --git a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.WriteFences.cs b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.WriteFences.cs new file mode 100644 index 000000000..41df347b7 --- /dev/null +++ b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.WriteFences.cs @@ -0,0 +1,309 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using System.Runtime.InteropServices; + +namespace SharpEmu.Gpu.Metal.V2; + +// A host read of guest bytes recorded work still writes waits for the submission serial of +// that work, and a serial is signalled only at the end of its command buffer. The coalescer +// keeps one command buffer open across guest submissions, so a word an early submission +// wrote was waited for together with everything recorded after it: the later submissions' +// whole GPU time, on top of the GPU work queued ahead. +// +// A write fence is a second shared event the command buffer signals between two guest +// submissions. It is planted only where the coalescer chose not to commit - a point where a +// commit was legal, so every write noted so far has its work encoded ahead of the signal - +// only when no render pass is open (ending one would store and reload its attachments), and +// only when a write since the last fence touched guest memory the CPU has waited on before +// (64 KiB blocks learned from those waits). Each live write range of that command buffer +// takes the fence's value (MetalGpuWriteRanges.StampFence); a range noted later starts over. +// +// A read waits for the fence when every pending write over its bytes in the newest pending +// serial carries one, and for the serial before it to complete. The guest sees the same bytes +// it would after the whole buffer: the writes it waits for are exactly the ones over its +// bytes, and the older serial covers every write over them from earlier buffers. Ordering with +// CPU writes is unchanged: pending command-processor and label bytes are still applied by the +// callers, and every other user of the write ranges still sees the serial pending. +// +// SHARPEMU_METAL_V2_WRITE_FENCES=0 plants none and waits for whole command buffers again. +internal sealed partial class MetalRenderHost +{ + private static readonly bool WriteFencesDefault = + Environment.GetEnvironmentVariable("SHARPEMU_METAL_V2_WRITE_FENCES") != "0"; + + private static readonly nint SelNotifyListener = MetalNative.Selector("notifyListener:atValue:block:"); + + private const int ObservedBlockShift = 16; + private const int MaxObservedBlocks = 256; + + // A read wider than this is a table or a buffer, not a polled word or an operand. + private const ulong MaxObservedReadBlocks = 4; + + // How long a fence wait goes before saying, once, that it is still waiting. + private const int WriteFenceStallMilliseconds = 5000; + + private readonly HashSet _observedReadBlocks = []; + private bool _writeFenceWanted; + private nint _writeFenceEvent; + private nint _writeFenceListener; + private bool _writeFenceUnavailable; + private ulong _writeFencePlanted; + private long _writeFencesPlanted; + private long _writeFenceWaits; + private long _writeFenceReads; + private long _writeFenceRenderSkips; + + /// Whether reads may wait for a write fence instead of the whole command buffer. + internal bool WriteFences { get; set; } = WriteFencesDefault; + + /// Tests only: runs with the command buffer just after a fence is planted. + internal Action? WriteFencePlantedHook { get; set; } + + /// Host waits for the GPU since the last PERF report (tests read it). + internal long HostGpuWaits => Interlocked.Read(ref _gpuWaits); + + internal (long Planted, long Waits, long Reads, long RenderSkips) WriteFenceCounters => + (Interlocked.Read(ref _writeFencesPlanted), Interlocked.Read(ref _writeFenceWaits), + Interlocked.Read(ref _writeFenceReads), Interlocked.Read(ref _writeFenceRenderSkips)); + + private string TakeWriteFenceCounters() => + $" write_fences={Interlocked.Exchange(ref _writeFencesPlanted, 0)}" + + $" write_fence_waits={Interlocked.Exchange(ref _writeFenceWaits, 0)}" + + $" write_fence_reads={Interlocked.Exchange(ref _writeFenceReads, 0)}" + + $" write_fence_render_skips={Interlocked.Exchange(ref _writeFenceRenderSkips, 0)}"; + + // A read that had to wait for recorded work: its bytes are worth a fence next time. + private void NoteObservedRead(ulong address, ulong size) + { + if (!WriteFences || _writeFenceUnavailable || size == 0 || address > ulong.MaxValue - size) + { + return; + } + + var first = address >> ObservedBlockShift; + var last = (address + size - 1) >> ObservedBlockShift; + if (last - first >= MaxObservedReadBlocks) + { + return; + } + + if (_observedReadBlocks.Count + (int)(last - first + 1) > MaxObservedBlocks) + { + _observedReadBlocks.Clear(); + } + + for (var block = first; block <= last; block++) + { + _observedReadBlocks.Add(block); + } + } + + // A recorded write: one into a block the CPU has waited on asks for a fence at the next boundary. + private void NoteFencedWrite(ulong address, ulong size) + { + if (_writeFenceWanted || _observedReadBlocks.Count == 0 || !WriteFences || size == 0 || + address > ulong.MaxValue - size) + { + return; + } + + var first = address >> ObservedBlockShift; + var last = (address + size - 1) >> ObservedBlockShift; + if ((ulong)_observedReadBlocks.Count <= last - first) + { + foreach (var block in _observedReadBlocks) + { + if (block >= first && block <= last) + { + _writeFenceWanted = true; + return; + } + } + + return; + } + + for (var block = first; block <= last; block++) + { + if (_observedReadBlocks.Contains(block)) + { + _writeFenceWanted = true; + return; + } + } + } + + /// + /// A guest submission ended and the coalescer keeps the command buffer open: plants a write + /// fence when a write since the last one touched memory the CPU reads. + /// + private void PlantWriteFenceIfWanted() + { + if (!_writeFenceWanted || !WriteFences || _commandBuffer == 0) + { + return; + } + + if (_renderEncoder != 0) + { + // Ending the pass costs more than the narrower wait saves; the want stays for the + // next boundary, and a read before it waits for the whole buffer as before. + Interlocked.Increment(ref _writeFenceRenderSkips); + return; + } + + if (!EnsureWriteFenceEvent()) + { + _writeFenceWanted = false; + return; + } + + // The signal goes between encoders, after the staged stores and label copies the + // encoders owe, which ending them encodes. + EndEncoders(PassEndCause.Other); + if (_renderEncoder != 0 || _computeEncoder != 0) + { + return; + } + + var value = ++_writeFencePlanted; + MetalV2Native.SendEventValue(_commandBuffer, SelEncodeSignalEvent, _writeFenceEvent, value); + _gpuWrites.StampFence(_submissionSerial, value); + _writeFenceWanted = false; + Interlocked.Increment(ref _writeFencesPlanted); + WriteFencePlantedHook?.Invoke(_commandBuffer); + } + + private bool EnsureWriteFenceEvent() + { + if (_writeFenceEvent != 0) + { + return true; + } + + if (_writeFenceUnavailable) + { + return false; + } + + // A fence is signalled in the middle of a command buffer, where no completion handler + // runs; without a listener to wake the waiter it would be found by a timed poll, which + // is slower than waiting for the buffer. + var block = MetalCompletionSignal.Block; + var listenerClass = MetalNative.Class("MTLSharedEventListener"); + if (!_completions.WantsCompletionHandler || block == 0 || listenerClass == 0) + { + _writeFenceUnavailable = true; + return false; + } + + var listener = MetalNative.Send(MetalNative.Send(listenerClass, MetalNative.Selector("alloc")), MetalNative.Selector("init")); + var sharedEvent = MetalNative.Send(_device, SelNewSharedEvent); + if (listener == 0 || sharedEvent == 0) + { + if (listener != 0) + { + MetalNative.SendVoid(listener, SelRelease); + } + + if (sharedEvent != 0) + { + MetalNative.SendVoid(sharedEvent, SelRelease); + } + + _writeFenceUnavailable = true; + return false; + } + + _writeFenceListener = listener; + _writeFenceEvent = sharedEvent; + return true; + } + + private bool WriteFenceReached(ulong value) => + MetalV2Native.SendValue64(_writeFenceEvent, SelSignaledValue) >= value; + + /// + /// Waits for the write fence that covers every pending write over the range in + /// , the newest serial writing it, and for the serial before it. + /// False when no fence covers them all; the caller then waits for the serial. + /// + private bool TryWaitForWriteFence(ulong address, ulong size, ulong pending) + { + if (!WriteFences || _writeFencePlanted == 0 || _writeFenceEvent == 0) + { + return false; + } + + var fence = _gpuWrites.WriteFence(address, size, pending); + if (fence == 0) + { + return false; + } + + // The fence was planted in the serial's own command buffer: it signals once that runs. + if (_commandBuffer != 0 && pending >= _submissionSerial) + { + FlushCommandBuffer(MetalFlushCause.WaitGpu); + } + + var older = pending - 1; + bool Landed() => (WriteFenceReached(fence) && _completions.SignaledValue >= older) || + _completions.SignaledValue >= pending; + if (Landed()) + { + Interlocked.Increment(ref _writeFenceReads); + return true; + } + + using var waitProfile = SharpEmu.Libs.VideoOut.RenderPhaseProfile.MeasureDetail( + SharpEmu.Libs.VideoOut.RenderPhaseProfile.Phase.CommandGpuWait); + var started = System.Diagnostics.Stopwatch.GetTimestamp(); + // One notification for this value; the block is the completion handler's, so it wakes + // the same waiters a finished command buffer does. + SendNotifyListener(_writeFenceEvent, SelNotifyListener, _writeFenceListener, fence, MetalCompletionSignal.Block); + while (!_completions.WaitUntil(Landed, WriteFenceStallMilliseconds)) + { + MetalV2Diagnostics.WarnOnce( + "write-fence-stall", + $"A write-fence wait has not finished in {WriteFenceStallMilliseconds} ms and is still waiting: " + + $"fence={fence} reached={MetalV2Native.SendValue64(_writeFenceEvent, SelSignaledValue)} serial={pending} " + + $"completed={CompletedSerial}."); + } + + var buffer = InFlightBufferFor(pending, out var commitTime); + var signalled = System.Diagnostics.Stopwatch.GetTimestamp() - started; + ReportWait(pending, started, started, buffer, commitTime, new MetalCompletionWait(signalled, 0), + MetalWaitReason.ReadRange, partial: true); + Interlocked.Increment(ref _writeFenceWaits); + Interlocked.Increment(ref _partialWaits); + var completed = CompletedSerial; + _guestMemory.NoteCompleted(completed); + _gpuWrites.Retire(completed); + RetirePendingCopies(completed); + _cpOverlay.Retire(completed); + _labelOverlay.Retire(completed); + InvalidateWordLines(); + return true; + } + + private void DisposeWriteFences() + { + if (_writeFenceEvent != 0) + { + MetalNative.SendVoid(_writeFenceEvent, SelRelease); + _writeFenceEvent = 0; + } + + if (_writeFenceListener != 0) + { + MetalNative.SendVoid(_writeFenceListener, SelRelease); + _writeFenceListener = 0; + } + } + + // -notifyListener:atValue:block: + [LibraryImport("/usr/lib/libobjc.A.dylib", EntryPoint = "objc_msgSend")] + private static partial void SendNotifyListener(nint receiver, nint selector, nint listener, ulong value, nint block); +} diff --git a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.cs b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.cs index 0da67a911..810656f01 100644 --- a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.cs +++ b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.cs @@ -1222,6 +1222,7 @@ or MetalGpuWriteOrigin.ImageWriteback var serial = SubmissionSerial; var sequence = _gpuWrites.Note(address, size, serial, origin); NotePendingCopyDestination(address, size, origin, serial, sequence); + NoteFencedWrite(address, size); } /// @@ -1812,6 +1813,12 @@ private void WaitForGuestBytes(ulong address, ulong size) ReportSmallReadWaitSite(address, size); } + NoteObservedRead(address, size); + if (TryWaitForWriteFence(address, size, pending)) + { + return; + } + if (_commandBuffer != 0 && pending >= _submissionSerial) { FlushAndWaitForGpu(MetalWaitReason.ReadRange); @@ -2468,7 +2475,8 @@ public void ReportCounters() .Append(" data_slots_device=").Append(SharpEmu.ShaderCompiler.Resources.DeviceDataLoads.LearnedSlots) .Append(" data_replans=").Append(SharpEmu.ShaderCompiler.Resources.DeviceDataLoads.Replans) .Append(" resident=").Append(_guestMemory.ResidentBuffers.Length) - .Append(" windows_touched=").Append(_guestMemory.TouchedWindows.Count); + .Append(" windows_touched=").Append(_guestMemory.TouchedWindows.Count) + .Append(TakeWriteFenceCounters()); Console.Error.WriteLine($"[METAL-V2][WAITS]{reasons}"); var bindless = TakeBindlessCost(); if (bindless.Tables != 0) @@ -2694,6 +2702,7 @@ public void Dispose() } _completions.Dispose(); + DisposeWriteFences(); DisposeHostMovie(); DisposeBindlessImages(); DisposePipelines(); diff --git a/tests/SharpEmu.Gpu.Metal.Tests/MetalGpuWriteRangesTests.cs b/tests/SharpEmu.Gpu.Metal.Tests/MetalGpuWriteRangesTests.cs index 752843a68..e778553ef 100644 --- a/tests/SharpEmu.Gpu.Metal.Tests/MetalGpuWriteRangesTests.cs +++ b/tests/SharpEmu.Gpu.Metal.Tests/MetalGpuWriteRangesTests.cs @@ -402,6 +402,45 @@ public void WrittenSinceAgreesWithThePendingSerial(int seed, bool folds) } } + + // A write fence covers a read only when every write over its bytes in the serial it waits + // for was stamped: a write noted after the stamp, a merge into a stamped record, a label, an + // unbounded write or a newer serial over the bytes each sends the read back to the serial. + [Fact] + public void AWriteFenceCoversOnlyWritesStampedBeforeIt() + { + var ranges = new MetalGpuWriteRanges(); + ranges.Note(0x1_0000, 0x100, serial: 4); + ranges.Note(0x10_0000, 0x100, serial: 5); + ranges.Note(0x20_0000, 0x100, serial: 5); + Assert.Equal(0UL, ranges.WriteFence(0x10_0000, 4, serial: 5)); + + ranges.StampFence(serial: 5, fence: 1); + Assert.Equal(1UL, ranges.WriteFence(0x10_0000, 4, serial: 5)); + // The older serial's record is the caller's to wait for, not the fence's. + Assert.Equal(1UL, ranges.WriteFence(0x1_0000, 0x10_0000, serial: 5)); + Assert.Equal(0UL, ranges.WriteFence(0x1_0000, 4, serial: 4)); + + // A later write into a stamped record makes it wait for the next fence. + ranges.Note(0x20_0000, 0x200, serial: 5); + Assert.Equal(0UL, ranges.WriteFence(0x20_0000, 4, serial: 5)); + Assert.Equal(1UL, ranges.WriteFence(0x10_0000, 4, serial: 5)); + ranges.StampFence(serial: 5, fence: 2); + Assert.Equal(2UL, ranges.WriteFence(0x20_0000, 4, serial: 5)); + Assert.Equal(2UL, ranges.WriteFence(0x10_0000, 0x10_0100, serial: 5)); + Assert.Equal(1UL, ranges.WriteFence(0x10_0000, 4, serial: 5)); + + // A newer serial over the bytes, or a label, is not behind the fence. + ranges.Note(0x10_0080, 4, serial: 6); + Assert.Equal(0UL, ranges.WriteFence(0x10_0000, 0x100, serial: 5)); + ranges.Note(0x20_0100, 8, serial: 5, MetalGpuWriteOrigin.EndOfPipeLabel); + ranges.StampFence(serial: 5, fence: 3); + Assert.Equal(0UL, ranges.WriteFence(0x20_0100, 8, serial: 5)); + + ranges.NoteUnbounded(serial: 5); + Assert.Equal(0UL, ranges.WriteFence(0x20_0000, 4, serial: 5)); + } + // The indexes over the records and labels (the bucket dictionary PendingSerial uses, the hashed // counts the snapshot another thread reads) only let a query skip what cannot reach it: both // answer what the unindexed walk of every record and label answers, over narrow and wide writes, diff --git a/tests/SharpEmu.Gpu.Metal.Tests/Scenes/GpuWordWaitSceneTests.cs b/tests/SharpEmu.Gpu.Metal.Tests/Scenes/GpuWordWaitSceneTests.cs new file mode 100644 index 000000000..65baefda3 --- /dev/null +++ b/tests/SharpEmu.Gpu.Metal.Tests/Scenes/GpuWordWaitSceneTests.cs @@ -0,0 +1,240 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using SharpEmu.Gpu.Metal.V2; +using Xunit; +using Xunit.Abstractions; + +namespace SharpEmu.Gpu.Metal.Tests.Scenes; + +/// +/// A host read of a word the GPU writes waits for the work that writes it and nothing recorded +/// after it, and a WAIT_REG_MEM on a label the GPU has yet to write resolves without a host +/// wait. Both hold the GPU on an event only the test signals, so a wait that included the +/// held work could not return: nothing here depends on how fast the GPU is. +/// +public sealed class GpuWordWaitSceneTests(ITestOutputHelper output) : SceneTestBase(output) +{ + private static readonly nint SelNewSharedEvent = MetalNative.Selector("newSharedEvent"); + private static readonly nint SelSetSignaledValue = MetalNative.Selector("setSignaledValue:"); + private static readonly nint SelEncodeWaitForEvent = MetalNative.Selector("encodeWaitForEvent:value:"); + private static readonly nint SelRelease = MetalNative.Selector("release"); + + // A held GPU is released by this deadline whatever happens, so a wrong wait fails the test + // instead of hanging the run. + private const int WatchdogMilliseconds = 30_000; + + private const uint Words = 64; + private const uint LaterGroups = 16; + + [Fact] + public void AWordAnEarlySubmissionWroteIsReadWhileALaterSubmissionInTheSameBufferIsHeld() + { + using var rig = CreateRig(); + if (rig is null) return; + var host = rig.Host; + Assert.True(host.WriteFences, "write fences are on by default"); + var word = rig.Memory.Allocate(Words * 4, 256); + // Apart, so the two submissions' write ranges stay two records rather than merging into one. + var later = rig.Memory.Allocate(Words * 4 * LaterGroups, 0x10000); + var writer = rig.Dispatch(new ScenePm4(), ScenePrograms.StoreGlobalId, Descriptors.Buffer(word, 4, Words, format: 20), 1).ToArray(); + var consumer = rig.Dispatch(new ScenePm4(), ScenePrograms.StoreGlobalId, + Descriptors.Buffer(later, 4, Words * LaterGroups, format: 20), LaterGroups).ToArray(); + + // Both programs are compiled first, so neither submission parks on a pipeline below. + rig.Submit(writer); + rig.Submit(consumer); + + // The first read of the word waits for the whole buffer, and teaches the host that the + // CPU reads it. The GPU is held until shortly after the read starts, so the read does wait. + var learnGate = MetalNative.Send(DeviceOf(host), SelNewSharedEvent); + try + { + MetalV2Native.SendEventValue(host.CommandBuffer, SelEncodeWaitForEvent, learnGate, 1); + RunBoth(rig, writer, consumer); + var signaller = new Thread(() => + { + Thread.Sleep(100); + MetalNative.SendVoid(learnGate, SelSetSignaledValue, (nint)1); + }) { IsBackground = true }; + signaller.Start(); + Assert.Equal(1u ^ 0x5A5A, ReadWord(host, word + 4)); + } + finally + { + MetalNative.SendVoid(learnGate, SelSetSignaledValue, (nint)1); + } + + rig.Drain(); + MetalNative.SendVoid(learnGate, SelRelease); + rig.Memory.Write(word, new byte[Words * 4]); + rig.Memory.Write(later, new byte[Words * 4 * LaterGroups]); + var plantedBefore = host.WriteFenceCounters.Planted; + + // Now the later submission sits behind an event the GPU waits on, after the fence the + // host plants between the two submissions. + var gate = MetalNative.Send(DeviceOf(host), SelNewSharedEvent); + Assert.NotEqual(0, gate); + var released = 0; + using var watchdog = Watchdog(gate, () => Interlocked.Exchange(ref released, 1)); + try + { + host.WriteFencePlantedHook = buffer => MetalV2Native.SendEventValue(buffer, SelEncodeWaitForEvent, gate, 1); + RunBoth(rig, writer, consumer); + host.WriteFencePlantedHook = null; + Assert.Equal(plantedBefore + 1, host.WriteFenceCounters.Planted); + + var value = ReadWord(host, word + 4); + Assert.Equal(0, Volatile.Read(ref released)); + Assert.Equal(1u ^ 0x5A5A, value); + // The held submission has not run: its bytes are still the zeros written above. + Assert.Equal(0u, BitConverter.ToUInt32(rig.Memory.ReadRaw(later + 4 * 5, 4))); + // Answered through the fence: waited for, or already signalled when the read came. + var counters = host.WriteFenceCounters; + Assert.True(counters.Waits + counters.Reads >= 1, $"fence counters {counters}"); + } + finally + { + host.WriteFencePlantedHook = null; + MetalNative.SendVoid(gate, SelSetSignaledValue, (nint)1); + } + + rig.Drain(); + var words = rig.Read(later, (int)(Words * LaterGroups)); + for (var index = 0u; index < words.Length; index++) + { + Assert.Equal(index ^ 0x5A5A, words[index]); + } + + MetalNative.SendVoid(gate, SelRelease); + Assert.Empty(rig.Fatals); + } + + [Fact] + public void WithWriteFencesOffNoFenceIsPlantedAndTheWordStillReadsWritten() + { + using var rig = CreateRig(); + if (rig is null) return; + var host = rig.Host; + host.WriteFences = false; + var word = rig.Memory.Allocate(Words * 4, 256); + var later = rig.Memory.Allocate(Words * 4 * LaterGroups, 256); + var writer = rig.Dispatch(new ScenePm4(), ScenePrograms.StoreGlobalId, Descriptors.Buffer(word, 4, Words, format: 20), 1).ToArray(); + var consumer = rig.Dispatch(new ScenePm4(), ScenePrograms.StoreGlobalId, + Descriptors.Buffer(later, 4, Words * LaterGroups, format: 20), LaterGroups).ToArray(); + rig.Submit(writer); + rig.Submit(consumer); + for (var round = 0; round < 2; round++) + { + rig.Memory.Write(word, new byte[Words * 4]); + RunBoth(rig, writer, consumer); + Assert.Equal(7u ^ 0x5A5A, ReadWord(host, word + 7 * 4)); + rig.Drain(); + } + + Assert.Equal(0, host.WriteFenceCounters.Planted); + Assert.Empty(rig.Fatals); + } + + [Fact] + public void AWaitRegMemOnALabelAnotherQueueHasYetToWriteResolvesWithoutAHostWait() + { + using var rig = CreateRig(); + if (rig is null) return; + var host = rig.Host; + var produced = rig.Memory.Allocate(Words * 4, 256); + var consumed = rig.Memory.Allocate(Words * 4, 256); + var label = rig.Memory.Allocate(8, 8); + const uint LabelValue = 7; + + // Graphics queue: a dispatch, then RELEASE_MEM writes the label at end of pipe (64-bit data). + var producer = rig.Dispatch(new ScenePm4(), ScenePrograms.StoreGlobalId, Descriptors.Buffer(produced, 4, Words, format: 20), 1) + .Packet(0x49, 0x2Fu | (5u << 8), 2u << 29, (uint)label, (uint)(label >> 32), LabelValue, 0, 0) + .ToArray(); + // Compute queue: WAIT_REG_MEM (memory, equal) on the label, then a dispatch of its own. + var waiter = rig.Dispatch( + new ScenePm4().Packet(0x3C, 0x10u | 3u, (uint)label, (uint)(label >> 32), LabelValue, 0xFFFF_FFFFu, 0x10), + ScenePrograms.StoreGlobalId, Descriptors.Buffer(consumed, 4, Words, format: 20), 1) + .ToArray(); + + var gate = MetalNative.Send(DeviceOf(host), SelNewSharedEvent); + Assert.NotEqual(0, gate); + var released = 0; + using var watchdog = Watchdog(gate, () => Interlocked.Exchange(ref released, 1)); + try + { + // The GPU holds before the producer, so the label cannot land until the test says so. + MetalV2Native.SendEventValue(host.CommandBuffer, SelEncodeWaitForEvent, gate, 1); + var waitsBefore = host.HostGpuWaits; + var suspendsBefore = SharpEmu.Libs.Gpu.GpuCommands.GpuCommandInterpreter.OperandSuspends; + rig.Enqueue(producer); + host.RunPendingCommands(); + host.RunCommandStreamSlices(64); + rig.Enqueue(waiter, queue: 0x20); + host.RunPendingCommands(); + host.RunCommandStreamSlices(64); + + Assert.Equal(0, Volatile.Read(ref released)); + // The wait met the pending label: the compute queue ran to its end without a host wait or a park. + Assert.False(host.CommandStream.HasUnblockedPending); + Assert.Equal(waitsBefore, host.HostGpuWaits); + Assert.Equal(suspendsBefore, SharpEmu.Libs.Gpu.GpuCommands.GpuCommandInterpreter.OperandSuspends); + // And nothing has landed yet. + Assert.Equal(0UL, BitConverter.ToUInt64(rig.Memory.ReadRaw(label, 8))); + Assert.Equal(0u, BitConverter.ToUInt32(rig.Memory.ReadRaw(consumed + 4, 4))); + } + finally + { + MetalNative.SendVoid(gate, SelSetSignaledValue, (nint)1); + } + + rig.Drain(); + Assert.Equal((ulong)LabelValue, BitConverter.ToUInt64(rig.Memory.ReadRaw(label, 8))); + var producedWords = rig.Read(produced, (int)Words); + var consumedWords = rig.Read(consumed, (int)Words); + for (var index = 0u; index < Words; index++) + { + Assert.Equal(index ^ 0x5A5A, producedWords[index]); + Assert.Equal(index ^ 0x5A5A, consumedWords[index]); + } + + MetalNative.SendVoid(gate, SelRelease); + Assert.Empty(rig.Fatals); + } + + // Both submissions run in one worker pass, so the coalescer keeps one command buffer open + // across them and commits it when the stream runs dry. + private static void RunBoth(SceneRig rig, uint[] first, uint[] second) + { + rig.Enqueue(first); + rig.Enqueue(second); + rig.Host.RunPendingCommands(); + rig.Host.RunCommandStreamSlices(64); + Assert.False(rig.Host.CommandStream.HasPending); + } + + // Releases a held GPU after WatchdogMilliseconds unless cancelled by disposal. + private static IDisposable Watchdog(nint gate, Action fired) + { + var cancel = new CancellationTokenSource(); + var thread = new Thread(() => + { + if (!cancel.Token.WaitHandle.WaitOne(WatchdogMilliseconds)) + { + fired(); + MetalNative.SendVoid(gate, SelSetSignaledValue, (nint)1); + } + }) { IsBackground = true }; + thread.Start(); + return cancel; + } + + private static uint ReadWord(MetalRenderHost host, ulong address) + { + Span bytes = stackalloc byte[4]; + Assert.True(host.TryReadGuest(address, bytes)); + return BitConverter.ToUInt32(bytes); + } + + private static nint DeviceOf(MetalRenderHost host) => ((IMetalImageCacheOwner)host).Device; +} From 75e9e628ca160e8d60a5fb8ecfb47594b14848bf Mon Sep 17 00:00:00 2001 From: kuba Date: Mon, 26 Oct 2026 07:40:49 +0100 Subject: [PATCH 23/82] [Tests] The GPU-word wait tests stop their watchdog before releasing its gate The watchdog handed back its CancellationTokenSource as the disposable, and disposing a token source does not cancel it: the watchdog thread woke 30 s after its test had finished and signalled a shared event the test had already released. When a later test was still running at that point the test host crashed (2 of 6 device runs). The watchdog is now cancelled and joined on dispose, and each test stops it before releasing the gate. 6 of 6 device runs pass, 1496 tests. --- .../Scenes/GpuWordWaitSceneTests.cs | 23 ++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/tests/SharpEmu.Gpu.Metal.Tests/Scenes/GpuWordWaitSceneTests.cs b/tests/SharpEmu.Gpu.Metal.Tests/Scenes/GpuWordWaitSceneTests.cs index 65baefda3..fe2a615db 100644 --- a/tests/SharpEmu.Gpu.Metal.Tests/Scenes/GpuWordWaitSceneTests.cs +++ b/tests/SharpEmu.Gpu.Metal.Tests/Scenes/GpuWordWaitSceneTests.cs @@ -106,6 +106,7 @@ public void AWordAnEarlySubmissionWroteIsReadWhileALaterSubmissionInTheSameBuffe Assert.Equal(index ^ 0x5A5A, words[index]); } + watchdog.Dispose(); MetalNative.SendVoid(gate, SelRelease); Assert.Empty(rig.Fatals); } @@ -198,6 +199,7 @@ public void AWaitRegMemOnALabelAnotherQueueHasYetToWriteResolvesWithoutAHostWait Assert.Equal(index ^ 0x5A5A, consumedWords[index]); } + watchdog.Dispose(); MetalNative.SendVoid(gate, SelRelease); Assert.Empty(rig.Fatals); } @@ -226,7 +228,26 @@ private static IDisposable Watchdog(nint gate, Action fired) } }) { IsBackground = true }; thread.Start(); - return cancel; + return new WatchdogStop(cancel, thread); + } + + // Disposing a token source does not cancel it: the watchdog is stopped and joined first, so it + // never signals a gate the test has already released. + private sealed class WatchdogStop(CancellationTokenSource cancel, Thread thread) : IDisposable + { + private int _stopped; + + public void Dispose() + { + if (Interlocked.Exchange(ref _stopped, 1) != 0) + { + return; + } + + cancel.Cancel(); + thread.Join(); + cancel.Dispose(); + } } private static uint ReadWord(MetalRenderHost host, ulong address) From 720a1db8125c1731229bbe59421cde34ace7737a Mon Sep 17 00:00:00 2001 From: kuba Date: Mon, 26 Oct 2026 08:51:23 +0100 Subject: [PATCH 24/82] [CLI] The guest's main thread runs on a plain managed thread again; SHARPEMU_EMULATION_THREAD_QOS=1 makes it user-interactive Starting the emulation thread natively at the user-interactive QoS class made every timed wait the guest's main thread makes overshoot about 20% more. Synthetic test eboot, 2 runs each, same bundle: usleep(1000) 1.265 -> 1.51 ms, usleep(16000) 18.07 -> 19.7-21.9 ms, sched_yield 322 -> 383 us, flip-only frame p99 18.5 -> 24 ms; SHARPEMU_USLEEP_NAP_LEGACY=1 did not change it, running the thread as a managed thread did. The nap policy's gains stay (back-to-back usleep(100) 320 -> 129 us). The GPU submission thread keeps its class. --- src/SharpEmu.CLI/Program.cs | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/SharpEmu.CLI/Program.cs b/src/SharpEmu.CLI/Program.cs index f127cdcbb..7b3205e8e 100644 --- a/src/SharpEmu.CLI/Program.cs +++ b/src/SharpEmu.CLI/Program.cs @@ -159,9 +159,15 @@ void RunEmulation() // Guest pthreads are managed threads and do not inherit it (measured: a managed // child of a user-interactive thread reports no class), which is what keeps a // title with thirty worker threads from outranking the presenter. - var emulationQos = SharpEmu.HLE.Host.HostQosThread.TryStart( - "SharpEmu Emulation", SharpEmu.HLE.Host.HostThreadQos.UserInteractive, 32 * 1024 * 1024, - RunEmulation, out _); + // Opt-in (SHARPEMU_EMULATION_THREAD_QOS=1): on a user-interactive thread every guest + // sleep overshoots ~20% more (synthetic test eboot: usleep(1000) 1.27 -> 1.51 ms, + // usleep(16000) 18.1 -> 20-22 ms, sched_yield 322 -> 383 us), which outweighs anything + // the class was measured to give. + var emulationQos = string.Equals(Environment.GetEnvironmentVariable("SHARPEMU_EMULATION_THREAD_QOS"), "1", StringComparison.Ordinal) + ? SharpEmu.HLE.Host.HostQosThread.TryStart( + "SharpEmu Emulation", SharpEmu.HLE.Host.HostThreadQos.UserInteractive, 32 * 1024 * 1024, + RunEmulation, out _) + : null; Thread? emulation = null; if (emulationQos is null) { From e457cc4d517091ddac92ccd4b0075cb5b572880b Mon Sep 17 00:00:00 2001 From: kuba Date: Mon, 26 Oct 2026 09:11:43 +0100 Subject: [PATCH 25/82] [GPU] A shader identity memo whose memory cannot compare in place falls back to the plain hash instead of re-adding every draw ShaderIdentity.Compute with a memo (63ad96d5f) treated "the memory cannot compare in place" like "the code changed": TryGet missed, and the miss path copied the code into fresh arrays, allocated a new XxHash3, and wrote the concurrent map, on every draw of every shader without a declared hash. Its comment promised the plain path for such memory. The per-draw microbench's SceneMemory has no TryCompare and its programs carry no declared hash, so the program-resolve thread paid that miss path each draw (sampled: ShaderIdentity.Compute and ConcurrentDictionary.TryAddInternal lock enter/exit) and the draw thread spun waiting for it. TryGet now reports whether the compare ran; when it could not, Compute takes the plain thread-local hash as before W4a. Memories that compare in place (TrackedCpuMemory, PhysicalVirtualMemory, RemoteGuestAddressSpace) keep the memo. Hashes are unchanged, so no switch. Per-draw microbench, ns/draw, unfixed tip 720a1db81 -> this commit, interleaved, two runs each (pre-W4a a63c92cb2 in brackets): arm64 8c 12,731 / 12,644 -> 3,916 / 3,922 [3,996 / 4,215] 8e 8,882 / 5,319 -> 3,576 / 3,697 [3,730 / 3,820] 8f 14,170 / 14,119 -> 5,006 / 4,429 [4,094 / 4,450] 8a 15,786 / 15,004 -> 14,728 / 14,558 [11,342 / 17,228] x64 8c 25,054 / 24,342 -> 7,539 / 6,928 [7,098 / 7,210] 8e 13,699 / 12,498 -> 7,693 / 6,325 [7,530 / 6,496] 8f 27,155 / 27,003 -> 6,820 / 7,456 [8,381 / 9,821] 8a 20,187 / 21,402 -> 7,321 / 21,076 [7,604 / 21,737] 8a runs first and is bimodal on every build; with a 8 s soak per scenario it is 4.0 us here (the unfixed tip soaked at ~13 us on every 8x row). --- .../Gpu/Pipelines/ShaderIdentity.cs | 20 +++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/src/SharpEmu.Libs/Gpu/Pipelines/ShaderIdentity.cs b/src/SharpEmu.Libs/Gpu/Pipelines/ShaderIdentity.cs index 96af51e90..ca72eeb61 100644 --- a/src/SharpEmu.Libs/Gpu/Pipelines/ShaderIdentity.cs +++ b/src/SharpEmu.Libs/Gpu/Pipelines/ShaderIdentity.cs @@ -101,12 +101,14 @@ public static ulong Compute(ICpuMemory memory, ulong codeAddress, ReadOnlySpan<( return declaredHash; } - if (memo.TryGet(memory, codeAddress, ranges, out var remembered)) + if (memo.TryGet(memory, codeAddress, ranges, out var remembered, out var compared)) { return remembered; } - if (!memo.CanKeep(ranges)) + // A memory that cannot compare in place would miss every draw and pay for a copy, a fresh hasher + // and a map write each time; it takes the plain path, as before the memo. + if (!compared || !memo.CanKeep(ranges)) { return Compute(memory, codeAddress, ranges, label); } @@ -171,9 +173,13 @@ internal bool CanKeep(ReadOnlySpan<(ulong Address, uint SizeBytes)> ranges) return total <= MaxEntryBytes; } - internal bool TryGet(ICpuMemory memory, ulong codeAddress, ReadOnlySpan<(ulong Address, uint SizeBytes)> ranges, out ulong hash) + // compared is false when the remembered entry matches the ranges but the memory could not compare + // them in place (no in-place compare, or unmapped); it is true otherwise. + internal bool TryGet(ICpuMemory memory, ulong codeAddress, ReadOnlySpan<(ulong Address, uint SizeBytes)> ranges, out ulong hash, + out bool compared) { hash = 0; + compared = true; if (!_entries.TryGetValue(codeAddress, out var entry) || entry.Ranges.Length != ranges.Length) { return false; @@ -189,7 +195,13 @@ internal bool TryGet(ICpuMemory memory, ulong codeAddress, ReadOnlySpan<(ulong A for (var index = 0; index < ranges.Length; index++) { - if (!memory.TryCompare(ranges[index].Address, entry.Code[index], out var equal) || !equal) + if (!memory.TryCompare(ranges[index].Address, entry.Code[index], out var equal)) + { + compared = false; + return false; + } + + if (!equal) { return false; } From 94ada0577548afc3bb2e70de56ab507227ee2619 Mon Sep 17 00:00:00 2001 From: kuba Date: Mon, 26 Oct 2026 11:04:54 +0100 Subject: [PATCH 26/82] [Agc] Wait for the frame boundary after releasing the submission gate sceAgcSuspendPoint held CommandSubmissionGate through SubmitDone, whose wait lasts until the worker retires the previous boundary. Every DCB, ACB and multi-DCB submit from another guest thread blocked behind it. That also deadlocks when work before the boundary waits on a label written by a compute submit that another thread has not made yet. IGuestGpuBackend.BeginSubmitDone now splits the call. The ordering half runs under the gate: it queues the boundary, marks the graphics pipe done and resets the geometry capture in submit order. The wait runs after the gate is released. CommandStreamQueue gives each boundary a ticket, so a boundary queued behind another does not lengthen the first one's wait. Metal and Vulkan return the queue's wait. Headless runs inline and has no wait. The split backend sends a FrameBoundary ring record behind the submissions and waits with a new WaitFrameBoundary call (protocol 4). Logged DS runs (141 runs, 6570 two-second windows) show the wait is short there: 983 ms in total, at most 50.7 ms in one window. It is long when the worker falls behind, as in worker-bound titles or during first-use pipeline compiles. Synthetic two-thread test with the worker 250 ms behind (AgcSuspendPointGateTests): the other thread's ACB submit took 256.8 ms with the gate held and 5.9 ms (cold) with the split. SHARPEMU_SUBMIT_DONE_SPLIT=0 holds the gate through the wait again. --- docs/macos/split-protocol.md | 2 + src/SharpEmu.GpuRemote/GpuRemoteProtocol.cs | 11 +- src/SharpEmu.GpuRemote/GpuSplitMessages.cs | 15 ++ src/SharpEmu.GpuSplit/RemoteGpuBackend.cs | 28 +++ src/SharpEmu.GpuSplit/SplitGpuHost.cs | 32 +++ .../Agc/AgcExports.Submissions.cs | 17 +- .../Gpu/GpuCommands/CommandStreamQueue.cs | 32 ++- .../Gpu/HeadlessCommandStreamBackend.cs | 7 + src/SharpEmu.Libs/Gpu/IGuestGpuBackend.cs | 9 + .../SharpEmu.Gpu.Metal/V2/MetalV2Backend.cs | 26 +++ .../VulkanVideoPresenter.CommandStream.cs | 21 ++ .../VulkanGuestGpuBackend.cs | 3 + .../SplitInProcessTests.cs | 26 +++ .../SplitTestSupport.cs | 15 ++ .../Agc/AgcSuspendPointGateTests.cs | 218 ++++++++++++++++++ 15 files changed, 457 insertions(+), 5 deletions(-) create mode 100644 tests/SharpEmu.Libs.Tests/Agc/AgcSuspendPointGateTests.cs diff --git a/docs/macos/split-protocol.md b/docs/macos/split-protocol.md index efdf8e1cd..76544bbbc 100644 --- a/docs/macos/split-protocol.md +++ b/docs/macos/split-protocol.md @@ -56,6 +56,7 @@ fused program. | `HostFrame` | width, height, total, offset + piece | `Submit` of a CPU frame (splash, video), in pieces of 256 KiB written back to back. | | `GamepadOutput` | rumble, trigger rumble, adaptive triggers, lightbar | To the pad, which lives with B's window. | | `VideoOptions` | window options | Options configured after the start. | +| `FrameBoundary` | ticket | The ordering half of a split `SubmitDone` (`BeginSubmitDone`): B queues the boundary behind the submissions before it and keeps its wait for `WaitFrameBoundary`. A sends it under its submission gate and waits after releasing it. | ### A -> B, synchronous (`SyncCalls`) @@ -65,6 +66,7 @@ many, so they come after A's earlier records. | Opcode | Request | Response | Semantics | |---|---|---|---| | `SubmitDone` (barrier) | - | IdleOutcome | B's `SubmitDone`: returns at the frame boundary. With A's session shutdown requested A sends `RequestClose` and answers `Cancelled` itself. | +| `WaitFrameBoundary` (barrier) | ticket | IdleOutcome | The waiting half of a split `SubmitDone`: runs the wait B kept for the `FrameBoundary` record of that ticket. | | `BracketBegin` | kind, barrier (AfterPendingCommandStreams only) | bracket id | `GuestGpuMemory.RunMappingChange` / `Unregister` in A (kind MappingChange: the relay's `TryRunAfterPendingWork`) and `RunAfterPendingCommandStreams` (kind AfterPendingCommandStreams). B parks its GPU worker (`WorkerBracket`: B's own `RunMappingChange` / `RunAfterPendingCommandStreams` hand the worker a park loop); A then runs the work itself, as the GPU queue thread (`RemoteGpuQueueRelay.IsGpuQueueThread`). | | `BracketEnd` | bracket id, viewLogSequence | - | B syncs its replica that far on the parked worker and lets it go. | | `Fault` | kind, address, size | handled, restoration version | From A's signal handler (`RemoteImageStore.MarkCpuWrite`, allocation-free). B runs its stores' fault path, then waits until A applied every protection B requested so far: the guest retries the access as soon as this answers. Every write fault crosses (the page faulted because B protected it). When no store handled it, B answers its guard's write-restoration version of the page: a watch removed before the fault reached the store. A retries the store once per version when the page is writable (one process's `TryRetryReleasedWrite`, from B's guard). | diff --git a/src/SharpEmu.GpuRemote/GpuRemoteProtocol.cs b/src/SharpEmu.GpuRemote/GpuRemoteProtocol.cs index 3ccb5f053..2cf293d8c 100644 --- a/src/SharpEmu.GpuRemote/GpuRemoteProtocol.cs +++ b/src/SharpEmu.GpuRemote/GpuRemoteProtocol.cs @@ -10,8 +10,9 @@ public static class GpuRemoteProtocol { /// Bumped on any change to the region layout or a message format. /// 2: the B → A call table, the application block and the split's messages (GpuSplitMessages.cs). - /// 3: . - public const uint Version = 3; + /// 3: . + /// 4: and . + public const uint Version = 4; } /// Record types on the two rings. 0 is never used; is reserved. @@ -93,6 +94,9 @@ public enum GpuRemoteMessage : uint /// A → B: , the guest driver's tessellation memory. TessellationRings = 25, + + /// A → B: , the ordering half of a split SubmitDone. + FrameBoundary = 26, } /// Opcodes of the A → B synchronous call table. @@ -127,6 +131,9 @@ public enum GpuRemoteSyncOpcode : uint /// → . Query = 10, + + /// → : the waiting half of a split SubmitDone. + WaitFrameBoundary = 11, } /// Opcodes of the B → A synchronous call table: what B's GPU code asks of A's video-out state. diff --git a/src/SharpEmu.GpuRemote/GpuSplitMessages.cs b/src/SharpEmu.GpuRemote/GpuSplitMessages.cs index 318b7c704..e0f99722a 100644 --- a/src/SharpEmu.GpuRemote/GpuSplitMessages.cs +++ b/src/SharpEmu.GpuRemote/GpuSplitMessages.cs @@ -376,6 +376,21 @@ public struct BarrierRequest public ulong Barrier; } +[StructLayout(LayoutKind.Sequential)] +public struct FrameBoundaryMessage +{ + /// A's number for the boundary; the matching names it. + public ulong Ticket; +} + +[StructLayout(LayoutKind.Sequential)] +public struct FrameBoundaryWaitRequest +{ + /// Split records A had written once the boundary was on the ring. + public ulong Barrier; + public ulong Ticket; +} + [StructLayout(LayoutKind.Sequential)] public struct IdleOutcomeResponse { diff --git a/src/SharpEmu.GpuSplit/RemoteGpuBackend.cs b/src/SharpEmu.GpuSplit/RemoteGpuBackend.cs index 125ce0cfa..cfdbfd1ee 100644 --- a/src/SharpEmu.GpuSplit/RemoteGpuBackend.cs +++ b/src/SharpEmu.GpuSplit/RemoteGpuBackend.cs @@ -149,6 +149,34 @@ public IdleOutcome SubmitDone(ICpuMemory memory) : IdleOutcome.Failed; } + // The boundary goes on the ring behind the submissions already sent, so a submission another + // thread sends after it is ordered after it in the GPU process too. Only the wait is a call. + public IdleOutcome BeginSubmitDone(ICpuMemory memory, out Func? wait) + { + wait = null; + if (HostSessionControl.IsShutdownRequested || !_session.IsAlive) + { + return SubmitDone(memory); + } + + var message = new FrameBoundaryMessage { Ticket = (ulong)Interlocked.Increment(ref _frameBoundaryTickets) }; + if (!_session.Writer.Write(GpuRemoteMessage.FrameBoundary, in message)) + { + return _session.IsAlive ? IdleOutcome.Failed : IdleOutcome.Cancelled; + } + + var request = new FrameBoundaryWaitRequest { Barrier = _session.Writer.Written, Ticket = message.Ticket }; + wait = () => + !_session.IsAlive ? IdleOutcome.Cancelled : + _session.Call(GpuRemoteSyncOpcode.WaitFrameBoundary, request, + out IdleOutcomeResponse response, SplitGpuSession.LongCallNanoseconds) + ? (IdleOutcome)response.Outcome + : IdleOutcome.Failed; + return IdleOutcome.Completed; + } + + private long _frameBoundaryTickets; + public void RunAfterPendingCommandStreams(Action work) { ArgumentNullException.ThrowIfNull(work); diff --git a/src/SharpEmu.GpuSplit/SplitGpuHost.cs b/src/SharpEmu.GpuSplit/SplitGpuHost.cs index d7bc222c0..2ec6e749a 100644 --- a/src/SharpEmu.GpuSplit/SplitGpuHost.cs +++ b/src/SharpEmu.GpuSplit/SplitGpuHost.cs @@ -103,6 +103,8 @@ public sealed class SplitGpuHost : IGpuHostService, IProtectionRequests, IDispos private SplitSharedState? _state; private readonly object _barrierGate = new(); private readonly ConcurrentDictionary _brackets = new(); + // Frame boundaries the ring thread queued, by A's ticket, until A's wait call takes them. + private readonly ConcurrentDictionary? Wait)> _frameBoundaries = new(); private SplitGuestMemory? _guestMemory; private RemoteGuestAddressSpace? _space; private GuestGpuMemory? _memory; @@ -258,6 +260,20 @@ private void Dispatch(GpuRemoteMessage type, ReadOnlySpan payload) _options.ConfigureVideo?.Invoke(SplitConvert.FromWire(GpuSplitWire.Read(payload))); break; + case GpuRemoteMessage.FrameBoundary: + { + // In ring order, behind the submissions A sent before it; A's wait call collects it. + // A session shutting down leaves the whole SubmitDone (its worker teardown) to the call thread. + var message = GpuSplitWire.Read(payload); + var space = _space!; + Func? wait = () => backend.SubmitDone(space); + var outcome = HostSessionControl.IsShutdownRequested + ? IdleOutcome.Completed + : backend.BeginSubmitDone(space, out wait); + _frameBoundaries[message.Ticket] = (outcome, wait); + break; + } + default: Report($"Ignoring split record type {type} ({payload.Length} bytes)."); break; @@ -441,6 +457,22 @@ public int HandleCall(uint opcode, ReadOnlySpan request, Span respon return GpuSplitWire.Write(response, new IdleOutcomeResponse { Outcome = (int)outcome }); } + case GpuRemoteSyncOpcode.WaitFrameBoundary: + { + var call = GpuSplitWire.Read(request); + if (!WaitForRecords(call.Barrier, LongWait) || + !_frameBoundaries.TryRemove(call.Ticket, out var boundary)) + { + status = GpuSplitStatus.Failed; + return 0; + } + + var outcome = boundary.Outcome == IdleOutcome.Completed && boundary.Wait is { } wait + ? wait() + : boundary.Outcome; + return GpuSplitWire.Write(response, new IdleOutcomeResponse { Outcome = (int)outcome }); + } + case GpuRemoteSyncOpcode.BracketBegin: return BeginBracket(GpuSplitWire.Read(request), response, out status); diff --git a/src/SharpEmu.Libs/Agc/AgcExports.Submissions.cs b/src/SharpEmu.Libs/Agc/AgcExports.Submissions.cs index b0cecebd7..2003671f1 100644 --- a/src/SharpEmu.Libs/Agc/AgcExports.Submissions.cs +++ b/src/SharpEmu.Libs/Agc/AgcExports.Submissions.cs @@ -210,10 +210,13 @@ public static int SuspendPoint(CpuContext ctx) TraceAgc("agc.suspend_point"); // The frame boundary: off the worker it waits for every accepted submission first. var gpuState = _submittedGpuStates.GetValue(CanonicalMemory(ctx.Memory), static _ => new SubmittedGpuState()); + Func? wait = null; lock (gpuState.CommandSubmissionGate) { SplitCrossingStats.Count(SplitCrossing.SubmitDone); - var outcome = GuestGpu.Current.SubmitDone(ctx.Memory); + var outcome = SplitSubmitDoneWait + ? GuestGpu.Current.BeginSubmitDone(ctx.Memory, out wait) + : GuestGpu.Current.SubmitDone(ctx.Memory); if (outcome != IdleOutcome.Completed) { TraceAgc($"agc.suspend_point_incomplete outcome={outcome}"); @@ -230,10 +233,22 @@ public static int SuspendPoint(CpuContext ctx) capture.IndirectArgsAddress = 0; } + // The boundary and the capture reset are in submit order now; another thread's DCB or ACB + // may go in behind them while this one waits for the worker. + if (wait is not null && wait() is var waited && waited != IdleOutcome.Completed) + { + TraceAgc($"agc.suspend_point_incomplete outcome={waited}"); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_CANCELED); + } + ctx[CpuRegister.Rax] = 0; return (int)OrbisGen2Result.ORBIS_GEN2_OK; } + // SHARPEMU_SUBMIT_DONE_SPLIT=0 holds CommandSubmissionGate through the frame-boundary wait again, + // so every other submitter blocks while the guest waits on the worker. + internal static bool SplitSubmitDoneWait = Environment.GetEnvironmentVariable("SHARPEMU_SUBMIT_DONE_SPLIT") != "0"; + // Synthetic label for an uncatalogued NID (the Unknown* convention); the NID is authoritative. #pragma warning disable SHEM006 [SysAbiExport( diff --git a/src/SharpEmu.Libs/Gpu/GpuCommands/CommandStreamQueue.cs b/src/SharpEmu.Libs/Gpu/GpuCommands/CommandStreamQueue.cs index 195317376..4bbb9f44d 100644 --- a/src/SharpEmu.Libs/Gpu/GpuCommands/CommandStreamQueue.cs +++ b/src/SharpEmu.Libs/Gpu/GpuCommands/CommandStreamQueue.cs @@ -92,6 +92,10 @@ private static int ReadWatchInterval() private ulong _lastCompletedGpuTick; private int _doneCount; private int _pendingBoundaries; + // Boundaries queued by BeginDone and boundaries the worker finished or dropped; both only grow, + // and boundaries retire in queue order, so a ticket's wait is ticket - _boundariesRetired. + private long _boundaryTickets; + private long _boundariesRetired; private IdleOutcome _outcome = IdleOutcome.Completed; private Thread? _processingThread; private CommandSubmissionPreparation? _activePreparation; @@ -507,6 +511,16 @@ public bool TryEnqueueControlBarrier(Action work, Action cancel) public IdleOutcome Done() { + var outcome = BeginDone(out var ticket); + return outcome == IdleOutcome.Completed && ticket != 0 ? WaitDone(ticket) : outcome; + } + + // The ordering half of Done: marks the graphics pipe done and queues the frame boundary behind + // the accepted submissions. A nonzero ticket is the boundary for WaitDone; zero means there is + // nothing to wait for (the worker's own call, or a shutdown drain that already ran). + public IdleOutcome BeginDone(out long ticket) + { + ticket = 0; if (_processingThread == Thread.CurrentThread) { lock (_gate) @@ -539,11 +553,22 @@ public IdleOutcome Done() } _pendingBoundaries++; + ticket = ++_boundaryTickets; EnqueueLocked(new CommandSubmission(CommandSubmissionKind.FrameBoundary, 0, 0, 0, 0, null)); + return IdleOutcome.Completed; + } + } + + // The waiting half of Done: blocks while more than the allowed boundaries up to and including + // this ticket's are still ahead of the worker. A boundary queued after it does not extend the wait. + public IdleOutcome WaitDone(long ticket) + { + lock (_gate) + { var dsSuspend = DsTrace.Gpu && DsTrace.Armed; - var dsPending = _pendingBoundaries; + var dsPending = ticket - _boundariesRetired; var dsStarted = dsSuspend ? System.Diagnostics.Stopwatch.GetTimestamp() : 0L; - while (_outcome == IdleOutcome.Completed && _pendingBoundaries > _boundariesInFlight) + while (_outcome == IdleOutcome.Completed && ticket - _boundariesRetired > _boundariesInFlight) { WaitLocked(Timeout.Infinite); } @@ -1511,6 +1536,7 @@ submission.Kind is not (CommandSubmissionKind.FlipPreparation or CommandSubmissi if (submission!.Kind == CommandSubmissionKind.FrameBoundary) { _pendingBoundaries--; + _boundariesRetired++; _doneCount++; } @@ -1737,6 +1763,7 @@ private void CancelBlocked() if (dropped.Kind == CommandSubmissionKind.FrameBoundary) { _pendingBoundaries--; + _boundariesRetired++; } } @@ -1968,5 +1995,6 @@ private void DropAllLocked() _submissionCount = 0; _pendingBoundaries = 0; + _boundariesRetired = _boundaryTickets; } } diff --git a/src/SharpEmu.Libs/Gpu/HeadlessCommandStreamBackend.cs b/src/SharpEmu.Libs/Gpu/HeadlessCommandStreamBackend.cs index 98b011b4e..ff4d9bd5e 100644 --- a/src/SharpEmu.Libs/Gpu/HeadlessCommandStreamBackend.cs +++ b/src/SharpEmu.Libs/Gpu/HeadlessCommandStreamBackend.cs @@ -34,6 +34,13 @@ public void SubmitCommandStream(ICpuMemory memory, uint queue, ulong address, ui public IdleOutcome SubmitDone(ICpuMemory memory) => _streams(memory).Done(); + // The inline stream runs the boundary on the submitting thread: nothing is left to wait for. + public IdleOutcome BeginSubmitDone(ICpuMemory memory, out Func? wait) + { + wait = null; + return SubmitDone(memory); + } + public void RunAfterPendingCommandStreams(Action work) { ArgumentNullException.ThrowIfNull(work); diff --git a/src/SharpEmu.Libs/Gpu/IGuestGpuBackend.cs b/src/SharpEmu.Libs/Gpu/IGuestGpuBackend.cs index 51c46e224..cdc8f6895 100644 --- a/src/SharpEmu.Libs/Gpu/IGuestGpuBackend.cs +++ b/src/SharpEmu.Libs/Gpu/IGuestGpuBackend.cs @@ -71,6 +71,15 @@ bool TrySubmitGuestImage( // Marks the frame boundary; off the worker it first waits for the accepted submissions. IdleOutcome SubmitDone(ICpuMemory memory); + // SubmitDone in two halves. This one orders the frame boundary among the accepted submissions and + // runs under the caller's submission gate; the wait it hands back, when not null, is the + // boundaries-in-flight back-pressure and runs after the caller has released the gate. + IdleOutcome BeginSubmitDone(ICpuMemory memory, out Func? wait) + { + wait = null; + return SubmitDone(memory); + } + void RunAfterPendingCommandStreams(Action work) { work(); diff --git a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalV2Backend.cs b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalV2Backend.cs index fab86c708..3e5d54f36 100644 --- a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalV2Backend.cs +++ b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalV2Backend.cs @@ -155,6 +155,32 @@ public IdleOutcome SubmitDone(ICpuMemory memory) return outcome; } + public IdleOutcome BeginSubmitDone(ICpuMemory memory, out Func? wait) + { + wait = null; + if (HostSessionControl.IsShutdownRequested || !MetalV2Presenter.IsStarted) + { + return SubmitDone(memory); + } + + var started = System.Diagnostics.Stopwatch.GetTimestamp(); + var commandStream = MetalV2Presenter.Host.CommandStream; + var outcome = commandStream.BeginDone(out var ticket); + if (outcome != IdleOutcome.Completed || ticket == 0) + { + ReportGuestWait(System.Diagnostics.Stopwatch.GetTimestamp() - started); + return outcome; + } + + wait = () => + { + var waited = commandStream.WaitDone(ticket); + ReportGuestWait(System.Diagnostics.Stopwatch.GetTimestamp() - started); + return waited; + }; + return outcome; + } + private static long _flipSubmissions; private static long _flipsRefused; private static long _doneCalls; diff --git a/src/graphics/SharpEmu.Gpu.Vulkan/Presentation/VulkanVideoPresenter.CommandStream.cs b/src/graphics/SharpEmu.Gpu.Vulkan/Presentation/VulkanVideoPresenter.CommandStream.cs index 2ac4a2036..9871caf25 100644 --- a/src/graphics/SharpEmu.Gpu.Vulkan/Presentation/VulkanVideoPresenter.CommandStream.cs +++ b/src/graphics/SharpEmu.Gpu.Vulkan/Presentation/VulkanVideoPresenter.CommandStream.cs @@ -73,6 +73,27 @@ public static IdleOutcome SubmitDone(ICpuMemory memory) => ? testStream.Done() : IdleOutcome.Completed; + // SubmitDone split for IGuestGpuBackend.BeginSubmitDone: the live presenter's queue hands back its wait. + public static IdleOutcome BeginSubmitDone(ICpuMemory memory, out Func? wait) + { + wait = null; + if (Volatile.Read(ref _presenterStartupFailure) is not null || + HostSessionControl.IsShutdownRequested || Volatile.Read(ref _closed) || Volatile.Read(ref _presenterCloseRequested) || + !TryGetActivePresenter(out var presenter)) + { + return SubmitDone(memory); + } + + var commandStream = presenter.CommandStream; + var outcome = commandStream.BeginDone(out var ticket); + if (outcome == IdleOutcome.Completed && ticket != 0) + { + wait = () => commandStream.WaitDone(ticket); + } + + return outcome; + } + public static void RunAfterPendingCommandStreams(Action work) { ArgumentNullException.ThrowIfNull(work); diff --git a/src/graphics/SharpEmu.Gpu.Vulkan/VulkanGuestGpuBackend.cs b/src/graphics/SharpEmu.Gpu.Vulkan/VulkanGuestGpuBackend.cs index a2ea2567d..c68c304af 100644 --- a/src/graphics/SharpEmu.Gpu.Vulkan/VulkanGuestGpuBackend.cs +++ b/src/graphics/SharpEmu.Gpu.Vulkan/VulkanGuestGpuBackend.cs @@ -69,6 +69,9 @@ public void SubmitCommandStream(ICpuMemory memory, uint queue, ulong address, ui public IdleOutcome SubmitDone(ICpuMemory memory) => VulkanVideoPresenter.SubmitDone(memory); + public IdleOutcome BeginSubmitDone(ICpuMemory memory, out Func? wait) => + VulkanVideoPresenter.BeginSubmitDone(memory, out wait); + public void RunAfterPendingCommandStreams(Action work) => VulkanVideoPresenter.RunAfterPendingCommandStreams(work); diff --git a/tests/SharpEmu.GpuSplit.Tests/SplitInProcessTests.cs b/tests/SharpEmu.GpuSplit.Tests/SplitInProcessTests.cs index 5e8f93105..9119163e3 100644 --- a/tests/SharpEmu.GpuSplit.Tests/SplitInProcessTests.cs +++ b/tests/SharpEmu.GpuSplit.Tests/SplitInProcessTests.cs @@ -160,6 +160,32 @@ public void AFrameBoundaryWaitsForTheSubmissionsBeforeIt() Assert.True(split.Host.GuestMemory!.Replica.ReplayedSequence >= split.Guest.Table.LogSequence); } + [MacOSFact] + public void ASplitFrameBoundaryIsOrderedOnTheRingAndWaitsOutsideIt() + { + using var split = new InProcessSplit(); + var backend = split.Session.Backend; + backend.SubmitCommandStream(null!, queue: 0, address: 0x1000, dwordCount: 16, submissionId: 1, null); + backend.SubmitCommandStream(null!, queue: 0x20, address: 0x2000, dwordCount: 16, submissionId: 2, null); + split.Backend.BoundaryRelease.Reset(); + + // The ordering half only writes the record; the GPU side queues it behind both submissions. + Assert.Equal(IdleOutcome.Completed, backend.BeginSubmitDone(null!, out var wait)); + Assert.NotNull(wait); + var waiter = Task.Run(wait!); + + // A submission sent while the boundary is still waited on reaches the GPU side after it. + backend.SubmitCommandStream(null!, queue: 0x20, address: 0x3000, dwordCount: 16, submissionId: 3, null); + Wait.Until(() => split.Backend.Submissions.Count == 3, "the submission behind the boundary"); + Assert.Contains("boundary 2", split.Backend.Calls); + Assert.False(waiter.IsCompleted); + + split.Backend.BoundaryRelease.Set(); + Assert.True(waiter.Wait(TimeSpan.FromSeconds(10))); + Assert.Equal(IdleOutcome.Completed, waiter.Result); + Assert.Contains("boundary-waited", split.Backend.Calls); + } + [MacOSFact] public void ExportFlipsAreAcceptedOrRefusedBack() { diff --git a/tests/SharpEmu.GpuSplit.Tests/SplitTestSupport.cs b/tests/SharpEmu.GpuSplit.Tests/SplitTestSupport.cs index 9bc54bbd0..0c072bc1d 100644 --- a/tests/SharpEmu.GpuSplit.Tests/SplitTestSupport.cs +++ b/tests/SharpEmu.GpuSplit.Tests/SplitTestSupport.cs @@ -269,6 +269,9 @@ internal sealed class RecordingBackend : IGuestGpuBackend public int SubmissionDelayMilliseconds { get; set; } + // Holds the waiting half of a split SubmitDone until set. + public ManualResetEventSlim BoundaryRelease { get; } = new(true); + public string BackendName => "Recording"; public void EnsureStarted(uint width, uint height) => Calls.Enqueue($"start {width}x{height}"); @@ -317,6 +320,18 @@ public IdleOutcome SubmitDone(ICpuMemory memory) return IdleOutcome.Completed; } + public IdleOutcome BeginSubmitDone(ICpuMemory memory, out Func? wait) + { + Calls.Enqueue($"boundary {Submissions.Count}"); + wait = () => + { + BoundaryRelease.Wait(); + Calls.Enqueue("boundary-waited"); + return IdleOutcome.Completed; + }; + return IdleOutcome.Completed; + } + public void RunAfterPendingCommandStreams(Action work) { Calls.Enqueue("after-pending"); diff --git a/tests/SharpEmu.Libs.Tests/Agc/AgcSuspendPointGateTests.cs b/tests/SharpEmu.Libs.Tests/Agc/AgcSuspendPointGateTests.cs new file mode 100644 index 000000000..f4802a69c --- /dev/null +++ b/tests/SharpEmu.Libs.Tests/Agc/AgcSuspendPointGateTests.cs @@ -0,0 +1,218 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later +using System.Buffers.Binary; +using System.Diagnostics; +using System.Reflection; +using SharpEmu.HLE; +using SharpEmu.Libs.Agc; +using SharpEmu.Libs.Gpu; +using SharpEmu.Libs.Gpu.GpuCommands; +using SharpEmu.Libs.Tests.Gpu.GpuCommands; +using SharpEmu.Libs.Tests.Gpu.Scheduling; +using SharpEmu.ShaderCompiler; +using Xunit; +using Xunit.Abstractions; + +namespace SharpEmu.Libs.Tests.Agc; + +// A guest render thread blocked in sceAgcSuspendPoint (the worker one boundary behind) and a second +// guest thread submitting: the frame-boundary wait no longer holds CommandSubmissionGate. +[Collection(SchedulingStateCollection.Name)] +public sealed class AgcSuspendPointGateTests(ITestOutputHelper output) +{ + private const ulong Base = 0x10000000; + private const ulong FirstDcb = Base + 0x100, SecondDcb = Base + 0x200, Acb = Base + 0x280; + private const ulong FirstPacket = Base + 0x300, SecondPacket = Base + 0x340, AcbPacket = Base + 0x380; + private static readonly TimeSpan WorkerLag = TimeSpan.FromMilliseconds(250); + + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task ASubmitFromAnotherThreadDoesNotWaitForTheFrameBoundary(bool split) + { + var (memory, host, queue) = Setup(); + using var scope = new BackendScope(new QueueBackend(queue)); + var previous = AgcExports.SplitSubmitDoneWait; + AgcExports.SplitSubmitDoneWait = split; + try + { + // Frame 1, then frame 2's boundary: the worker has run nothing, so the second blocks. + Assert.Equal(0, SubmitDcb(memory, FirstPacket)); + Assert.Equal(0, SuspendPoint(memory)); + Assert.Equal(0, SubmitDcb(memory, FirstPacket)); + var render = Task.Run(() => SuspendPoint(memory)); + await Task.WhenAny(render, Task.Delay(100)); + Assert.False(render.IsCompleted); + + // Another guest thread's async compute submit, measured while the boundary waits. + var watch = Stopwatch.StartNew(); + var compute = Task.Run(() => { var result = SubmitAcb(memory); watch.Stop(); return result; }); + await Task.WhenAny(compute, Task.Delay(WorkerLag)); + Assert.Equal(split, compute.IsCompleted); + + Drain(queue, render); + Assert.Equal(0, await render.WaitAsync(TimeSpan.FromSeconds(10))); + Assert.Equal(0, await compute.WaitAsync(TimeSpan.FromSeconds(10))); + output.WriteLine(FormattableString.Invariant( + $"split={split} acb_submit_latency_ms={watch.Elapsed.TotalMilliseconds:F3} worker_lag_ms={WorkerLag.TotalMilliseconds}")); + if (!split) + { + Assert.True(watch.Elapsed >= WorkerLag * 0.8, $"the gated submit returned after {watch.Elapsed}"); + } + } + finally + { + AgcExports.SplitSubmitDoneWait = previous; + } + } + + // The capture reset stays where the boundary is in submit order: a DCB submitted while the + // boundary is waited on keeps the registers it shadowed. + [Fact] + public async Task ADcbSubmittedDuringTheWaitKeepsItsGeometryCapture() + { + var (memory, _, queue) = Setup(); + using var scope = new BackendScope(new QueueBackend(queue)); + Assert.Equal(0, SubmitDcb(memory, FirstPacket)); + Assert.Equal(0, SuspendPoint(memory)); + Assert.Equal(0, SubmitDcb(memory, FirstPacket)); + var render = Task.Run(() => SuspendPoint(memory)); + await Task.WhenAny(render, Task.Delay(100)); + Assert.False(render.IsCompleted); + + Assert.Equal(0, await Task.Run(() => SubmitDcb(memory, SecondPacket)).WaitAsync(TimeSpan.FromSeconds(10))); + Assert.False(render.IsCompleted); + Drain(queue, render); + Assert.Equal(0, await render.WaitAsync(TimeSpan.FromSeconds(10))); + Assert.Equal(new Dictionary { [0x49] = 2 }, AgcExports.GetGeometryCaptureRegistersForTests(memory)); + } + + [Fact] + public async Task ConcurrentBoundariesEachWaitOnlyForTheirOwn() + { + var queue = new CommandStreamQueue(new RecordingCommandStreamHost(), boundariesInFlight: 1); + Assert.Equal(IdleOutcome.Completed, queue.BeginDone(out var first)); + Assert.Equal(IdleOutcome.Completed, queue.BeginDone(out var second)); + Assert.Equal(IdleOutcome.Completed, queue.BeginDone(out var third)); + Assert.Equal(IdleOutcome.Completed, queue.WaitDone(first)); + + var secondWait = Task.Run(() => queue.WaitDone(second)); + var thirdWait = Task.Run(() => queue.WaitDone(third)); + await Task.WhenAny(secondWait, Task.Delay(100)); + Assert.False(secondWait.IsCompleted); + + // The first boundary retiring is enough for the second; the third still has two ahead. + Assert.Equal(SliceResult.Completed, queue.ProcessOne()); + Assert.Equal(IdleOutcome.Completed, await secondWait.WaitAsync(TimeSpan.FromSeconds(10))); + await Task.WhenAny(thirdWait, Task.Delay(100)); + Assert.False(thirdWait.IsCompleted); + Assert.Equal(SliceResult.Completed, queue.ProcessOne()); + Assert.Equal(IdleOutcome.Completed, await thirdWait.WaitAsync(TimeSpan.FromSeconds(10))); + } + + private static (FakeCpuMemory Memory, RecordingCommandStreamHost Host, CommandStreamQueue Queue) Setup() + { + var memory = new FakeCpuMemory(Base, 0x1000); + var host = new RecordingCommandStreamHost(); + // Two DCBs that each set one shader register, and a NOP compute buffer. + uint[] first = [0xC0010000u | (0x76u << 8), 0x48u, 1u]; + uint[] second = [0xC0010000u | (0x76u << 8), 0x49u, 2u]; + uint[] nop = [0xC0001000u, 0u]; + Write(memory, host, FirstDcb, first); + Write(memory, host, SecondDcb, second); + Write(memory, host, Acb, nop); + WritePacket(memory, FirstPacket, FirstDcb, (uint)first.Length); + WritePacket(memory, SecondPacket, SecondDcb, (uint)second.Length); + WritePacket(memory, AcbPacket, Acb, (uint)nop.Length); + return (memory, host, new CommandStreamQueue(host, boundariesInFlight: 1)); + } + + // The worker, late: runs everything queued until the guest's boundary wait returns. + private static void Drain(CommandStreamQueue queue, Task render) + { + var watch = Stopwatch.StartNew(); + while (!render.IsCompleted && watch.Elapsed < TimeSpan.FromSeconds(10)) + { + if (queue.ProcessOne() is SliceResult.NoWork or SliceResult.AllBlocked) + { + Thread.Sleep(1); + } + } + } + + private static int SubmitDcb(FakeCpuMemory memory, ulong packet) + { + var context = new CpuContext(memory, Generation.Gen5); + context[CpuRegister.Rdi] = packet; + return AgcExports.DriverSubmitDcb(context); + } + + private static int SubmitAcb(FakeCpuMemory memory) + { + var context = new CpuContext(memory, Generation.Gen5); + context[CpuRegister.Rdi] = 0x20; + context[CpuRegister.Rsi] = AcbPacket; + return AgcExports.DriverSubmitAcb(context); + } + + private static int SuspendPoint(FakeCpuMemory memory) => AgcExports.SuspendPoint(new CpuContext(memory, Generation.Gen5)); + + private static void Write(FakeCpuMemory memory, RecordingCommandStreamHost host, ulong address, uint[] words) + { + Span bytes = stackalloc byte[4]; + for (var index = 0; index < words.Length; index++) + { + BinaryPrimitives.WriteUInt32LittleEndian(bytes, words[index]); + Assert.True(memory.TryWrite(address + (ulong)index * 4, bytes)); + } + + host.WriteWords(address, words); + } + + private static void WritePacket(FakeCpuMemory memory, ulong packet, ulong address, uint dwords) + { + Span bytes = stackalloc byte[12]; + BinaryPrimitives.WriteUInt64LittleEndian(bytes, address); + BinaryPrimitives.WriteUInt32LittleEndian(bytes[8..], dwords); + Assert.True(memory.TryWrite(packet, bytes)); + } + + private sealed class BackendScope : IDisposable + { + private readonly FieldInfo _field = typeof(GuestGpu).GetField("_instance", BindingFlags.Static | BindingFlags.NonPublic)!; + private readonly object? _previous; + public BackendScope(IGuestGpuBackend backend) { _previous = _field.GetValue(null); _field.SetValue(null, backend); } + public void Dispose() => _field.SetValue(null, _previous); + } + + // A host whose queue is run by the test as a worker that is behind. + private sealed class QueueBackend(CommandStreamQueue queue) : IGuestGpuBackend + { + public string BackendName => "suspend point gate"; + public bool ConsumesSubmittedGeometrySnapshots => true; + public void SubmitCommandStream(ICpuMemory memory, uint queueId, ulong address, uint dwordCount, ulong submissionId, object? geometrySnapshots) + { + if (queueId == 0) queue.EnqueueGraphics(address, dwordCount, submissionId, geometrySnapshots); + else queue.EnqueueCompute(queueId, address, dwordCount, submissionId, geometrySnapshots); + } + public IdleOutcome SubmitDone(ICpuMemory memory) => queue.Done(); + public IdleOutcome BeginSubmitDone(ICpuMemory memory, out Func? wait) + { + var outcome = queue.BeginDone(out var ticket); + wait = ticket != 0 ? () => queue.WaitDone(ticket) : null; + return outcome; + } + public void EnsureStarted(uint width, uint height) => throw new NotSupportedException(); + public bool TryCompileProgram(ShaderCompileRequest request, out IGuestCompiledShader? shader, out string error) => throw new NotSupportedException(); + public IGuestCompiledShader GetDepthOnlyFragmentShader() => throw new NotSupportedException(); + public void HideSplashScreen() => throw new NotSupportedException(); + public void Submit(byte[] pixels, uint width, uint height) => throw new NotSupportedException(); + public void SubmitGuestDraw(GuestDrawKind kind, uint width, uint height) => throw new NotSupportedException(); + public bool TrySubmitGuestImage(int handle, int index, ulong address, uint width, uint height, uint pitch, ulong id) => throw new NotSupportedException(); + public void RegisterKnownDisplayBuffer(ulong address, uint format) => throw new NotSupportedException(); + public bool IsGpuGuestImageAvailable(ulong address, uint format, uint type) => false; + public void CountShaderCompilation() => throw new NotSupportedException(); + public (long Draws, double DrawMs, long Pipelines, long ShaderCompilations) ReadAndResetPerfCounters() => throw new NotSupportedException(); + public void RequestClose() => throw new NotSupportedException(); + } +} From 6a660e89fb7d52924998eac63db42f1a1a6b0497 Mon Sep 17 00:00:00 2001 From: kuba Date: Fri, 23 Oct 2026 21:45:48 +0200 Subject: [PATCH 27/82] [GPU] Fuse Metal texture alias transitions and discard dead contents Ported from ds-fps-b2-alias-transitions 55722acdf, which was never merged or measured in a game. Two images over the same guest bytes hand their texels over with a blit (the same texel mapping, through raw integer views when the formats differ) or one gather pass (dense one-level 2D surfaces of 4, 8 or 16 bytes a texel, any tiling), instead of a retile out to guest memory and a detile back. An alias preserved for a pass that then provably overwrites the whole image is discarded rather than published, and so is GPU ownership under an exact managed store of the whole allocation. A partial host read publishes only its covering run. Port: the TBDR report on this base has no frame count, so the alias counters print totals per report rather than per frame. The switch is now a cache property, so tests can compare both ways in one process. Measured (MetalImageTransferTests.AliasTransitionGpuTime, M-series, best of 5, GPU time): - 1920x1080 RGBA16F RT -> RG32UI view (blit, padded): 0.38 ms vs 0.80 ms round trip. - 1920x1088 RGBA16F 64KB_S -> RT (gather): 0.61 ms vs 0.93 ms. - 2048x1024 RGBA8 64KB_S -> 1024x1024 RGBA16F RT (gather): 0.28 ms vs 0.61 ms. MetalImageAliasTests.ATiledAliasOfAGpuWrittenImageFusesOnTheDeviceOrRoundTrips, a 256x256 linear image and a 64 KiB-tiled alias of it: storage images go from 1 writeback + 1 upload to one fused pass (0.27 -> 0.02 ms). Colour targets are created compressible, without the shader-write and format-view usages the views and the gather need, so a pair whose source or destination is a render target still round-trips (alias_transition_refused=format-or-usage); only same-format blits fuse there. SHARPEMU_METAL_V2_ALIAS_TRANSITIONS=0 turns the new paths off. --- .../V2/MetalImageCache.AliasTransitions.cs | 100 +++++++++ .../V2/MetalImageCache.CpuStores.cs | 4 + .../V2/MetalImageCache.DeferredRefresh.cs | 13 +- .../SharpEmu.Gpu.Metal/V2/MetalImageCache.cs | 44 +++- .../V2/MetalImageTransfer.Aliases.cs | 172 +++++++++++++++ .../SharpEmu.Gpu.Metal/V2/MetalRenderHost.cs | 1 + .../V2/MetalTiler.Aliases.cs | 107 +++++++++ .../SharpEmu.Gpu.Metal/V2/MetalTiler.cs | 2 +- .../V2/MetalTilerShaders.cs | 2 +- .../SharpEmu.Gpu.Metal/V2/MetalV2Contracts.cs | 14 ++ .../MetalImageAliasTests.Transitions.cs | 170 +++++++++++++++ .../MetalImageAliasTests.cs | 2 +- ...etalImageTransferTests.AliasTransitions.cs | 205 ++++++++++++++++++ .../MetalImageTransferTests.Timing.cs | 59 +++++ 14 files changed, 883 insertions(+), 12 deletions(-) create mode 100644 src/graphics/SharpEmu.Gpu.Metal/V2/MetalImageCache.AliasTransitions.cs create mode 100644 src/graphics/SharpEmu.Gpu.Metal/V2/MetalImageTransfer.Aliases.cs create mode 100644 src/graphics/SharpEmu.Gpu.Metal/V2/MetalTiler.Aliases.cs create mode 100644 tests/SharpEmu.Gpu.Metal.Tests/MetalImageAliasTests.Transitions.cs create mode 100644 tests/SharpEmu.Gpu.Metal.Tests/MetalImageTransferTests.AliasTransitions.cs diff --git a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalImageCache.AliasTransitions.cs b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalImageCache.AliasTransitions.cs new file mode 100644 index 000000000..831c3455a --- /dev/null +++ b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalImageCache.AliasTransitions.cs @@ -0,0 +1,100 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using System.Globalization; +using SharpEmu.HLE.GpuMemory; +using SharpEmu.Libs.Gpu.Buffers; +using SharpEmu.Libs.Gpu.Images; + +namespace SharpEmu.Gpu.Metal.V2; + +internal sealed partial class MetalImageCache +{ + private static readonly bool AliasTransitionsDefault = + Environment.GetEnvironmentVariable("SHARPEMU_METAL_V2_ALIAS_TRANSITIONS") != "0"; + + /// Whether alias transitions fuse on the device and discard dead contents. + internal bool AliasTransitionsEnabled { get; set; } = AliasTransitionsDefault; + + private long _aliasFused, _aliasFusedBytes, _aliasDiscarded, _aliasDiscardedBytes; + private long _aliasRoundTrip, _aliasRoundTripBytes; + private long _aliasCpuReadRuns, _aliasCpuReadBytes; + private readonly Dictionary _aliasTransitionRefusals = new(StringComparer.Ordinal); + + private void RefuseAliasTransition(string reason) => + _aliasTransitionRefusals[reason] = _aliasTransitionRefusals.GetValueOrDefault(reason) + 1; + + internal (long Fused, long FusedBytes, long Discarded, long DiscardedBytes, long RoundTrip, long RoundTripBytes) + TakeAliasTransitions() => + (Interlocked.Exchange(ref _aliasFused, 0), Interlocked.Exchange(ref _aliasFusedBytes, 0), + Interlocked.Exchange(ref _aliasDiscarded, 0), Interlocked.Exchange(ref _aliasDiscardedBytes, 0), + Interlocked.Exchange(ref _aliasRoundTrip, 0), Interlocked.Exchange(ref _aliasRoundTripBytes, 0)); + + internal string TakeAliasTransitionText() + { + using var held = _lock.Hold(); + var counts = TakeAliasTransitions(); + string Mb(double bytes) => (bytes / 1048576.0).ToString("F2", CultureInfo.InvariantCulture); + var refusals = string.Join(',', _aliasTransitionRefusals.OrderBy(p => p.Key).Select(p => $"{p.Key}:{p.Value}")); + _aliasTransitionRefusals.Clear(); + var readRuns = Interlocked.Exchange(ref _aliasCpuReadRuns, 0); + var readBytes = Interlocked.Exchange(ref _aliasCpuReadBytes, 0); + return $"alias_transitions=[fused:{counts.Fused},discarded:{counts.Discarded},round-trip:{counts.RoundTrip}] " + + $"alias_mib=[fused:{Mb(counts.FusedBytes)},discarded:{Mb(counts.DiscardedBytes)},round-trip:{Mb(counts.RoundTripBytes)}] " + + $"alias_transition_refused=[{refusals}] alias_cpu_read_runs={readRuns} " + + $"alias_cpu_read_requested_mib={Mb(readBytes)}"; + } + + // The pass replaces every visible texel of this image. Refuse when an alias owns any + // byte outside it: partial ownership remains with the established run machinery. + private bool TryDiscardOverwrittenAliases(ResourceSlotIdentifier identifier) + { + var image = _slots[identifier]; + var span = image.Description.Data; + foreach (var otherId in FindImagesInRange(span.Address, span.Size, pageOverlap: false)) + { + if (otherId == identifier || _slots.TryGet(otherId) is not { } other) continue; + var data = other.Description.Data; + if (other.Description.HasStencil || data.Address < span.Address || + data.Size > span.Size || data.Address - span.Address > span.Size - data.Size || + !_transfer.CanDiscardAliasForOverwrite(image.Description, other.Description)) + { + RefuseAliasTransition("discard-partial-owner"); + return false; + } + } + NoteGpuWrittenImageLocked(image); + foreach (var otherId in FindImagesInRange(span.Address, span.Size, pageOverlap: false)) + { + if (otherId == identifier || _slots.TryGet(otherId) is not { } other) continue; + if (other.IsGpuModified) + { + _aliasDiscarded++; + _aliasDiscardedBytes += (long)other.Description.Data.Size; + } + other.ClearGpuModified(); + other.MarkCpuDirty(MetalImageDirtyCause.AliasWrite); + } + return true; + } + + // Exact managed stores replace the full allocation, including padding and stencil. + // The caller has already waited for in-flight writebacks and holds the cache lock + // through invalidation. Native faults cannot make this coverage claim. + private void DiscardFullyOverwrittenByCpuLocked(ulong address, ulong size) + { + foreach (var id in FindImagesInRange(address, size, pageOverlap: false)) + { + if (_slots.TryGet(id) is not { IsGpuModified: true } image) continue; + bool Covers(GuestSpan span) => span.Address >= address && span.Size <= size && + span.Address - address <= size - span.Size; + if (!Covers(image.Description.Data) || + (image.Description.HasStencil && !Covers(image.Description.Stencil))) continue; + image.ClearGpuModified(); + image.MarkCpuDirty(MetalImageDirtyCause.CpuWrite); + _aliasDiscarded++; + _aliasDiscardedBytes += (long)(image.Description.Data.Size + + (image.Description.HasStencil ? image.Description.Stencil.Size : 0)); + } + } +} diff --git a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalImageCache.CpuStores.cs b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalImageCache.CpuStores.cs index e03174548..724305434 100644 --- a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalImageCache.CpuStores.cs +++ b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalImageCache.CpuStores.cs @@ -187,6 +187,10 @@ private static StoreRequest[] CreateStoreRequests() // otherwise for every image on their pages, as a fault does. private StoreFallback? PublishBeforeCpuStoreLocked(ulong address, ulong size, bool managedWrite, bool exact) { + if (AliasTransitionsEnabled && managedWrite && exact && WritebackFenceEnabled) + { + DiscardFullyOverwrittenByCpuLocked(address, size); + } if (!PublishBeforeStore || !WritebackFenceEnabled || !HasGpuWrittenImageLocked(address, size, exact)) { return null; diff --git a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalImageCache.DeferredRefresh.cs b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalImageCache.DeferredRefresh.cs index dc87f3539..79501784d 100644 --- a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalImageCache.DeferredRefresh.cs +++ b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalImageCache.DeferredRefresh.cs @@ -2,6 +2,7 @@ // SPDX-License-Identifier: GPL-2.0-or-later using SharpEmu.Libs.Gpu.Abstractions; +using SharpEmu.Libs.Gpu.Buffers; using SharpEmu.Libs.Gpu.Images; namespace SharpEmu.Gpu.Metal.V2; @@ -28,7 +29,8 @@ internal sealed partial class MetalImageCache // The view a pass will name the attachment by, the image, and the GPU-write marking // the acquire left for later: a colour target, or a depth target and whether it // writes only the stencil plane. - private readonly record struct DeferredRefresh(nint View, MetalCachedImage Image, bool Depth, bool StencilOnly); + private readonly record struct DeferredRefresh(nint View, ResourceSlotIdentifier Identifier, + MetalCachedImage Image, bool Depth, bool StencilOnly, bool AliasesPending); private readonly List _deferredRefreshes = []; @@ -56,9 +58,10 @@ private static bool CanDeferRefresh(MetalCachedImage image) => image.Texture != 0 && !ImageDescription.IsEmptyRange(image.Description.Data); // Caller holds the lock. - private void DeferRefreshLocked(nint view, MetalCachedImage image, bool depth, bool stencilOnly) + private void DeferRefreshLocked(nint view, ResourceSlotIdentifier identifier, MetalCachedImage image, + bool depth, bool stencilOnly, bool aliasesPending) { - _deferredRefreshes.Add(new DeferredRefresh(view, image, depth, stencilOnly)); + _deferredRefreshes.Add(new DeferredRefresh(view, identifier, image, depth, stencilOnly, aliasesPending)); Volatile.Write(ref _deferredRefreshCount, _deferredRefreshes.Count); } @@ -243,6 +246,10 @@ private static bool CoversWholeImage(in DeferredRefresh entry, bool stencilOverw private void CompleteDeferredLocked(in DeferredRefresh entry, bool overwritten) { var image = entry.Image; + if (entry.AliasesPending && (!overwritten || !TryDiscardOverwrittenAliases(entry.Identifier))) + { + SynchronizeAliases(entry.Identifier, takesGpuOwnership: true); + } if (overwritten && image.IsCpuDirty) { Interlocked.Increment(ref _refreshesSkipped); diff --git a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalImageCache.cs b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalImageCache.cs index 2c7b4cee5..da082c7e8 100644 --- a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalImageCache.cs +++ b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalImageCache.cs @@ -609,6 +609,8 @@ private void SynchronizeAliases( continue; } + _aliasRoundTrip++; + _aliasRoundTripBytes += (long)sharedSize; var (dirtyStart, dirtyEnd) = image.OverlapWithin(sharedStart, sharedSize); var declined = MetalAliasFallbackReason.None; if (takesGpuOwnership && TryHandOverAliasRun(other, otherStart, otherEnd, out declined)) @@ -1381,7 +1383,10 @@ public nint AcquireColorTargetView( PromoteLinearView(image); EnsureViewUsageLocked(image, in request.View); WatchImage(imageIdentifier); - SynchronizeAliases(imageIdentifier, takesGpuOwnership: true); + var deferAliases = deferRefresh && AliasTransitionsEnabled && CanDeferRefresh(image) && + !image.Description.HasStencil && !image.Description.IsBlock && !image.Description.IsVolume && + image.Description.Resources.Levels == 1 && image.Description.TransferLayers == 1; + if (!deferAliases) SynchronizeAliases(imageIdentifier, takesGpuOwnership: true); // The pass about to begin may clear or overwrite all of it; BeginRendering decides // whether the upload is worth making (MetalImageCache.DeferredRefresh). var defer = deferRefresh && CanDeferRefresh(image); @@ -1410,7 +1415,7 @@ public nint AcquireColorTargetView( { image.NoteUse(_owner.CurrentSerial); var deferredView = image.GetOrCreateView(request.View); - DeferRefreshLocked(deferredView, image, depth: false, stencilOnly: false); + DeferRefreshLocked(deferredView, imageIdentifier, image, depth: false, stencilOnly: false, deferAliases); return deferredView; } @@ -1497,7 +1502,7 @@ public nint AcquireDepthTargetView( { image.NoteUse(_owner.CurrentSerial); var deferredView = image.GetOrCreateView(request.View); - DeferRefreshLocked(deferredView, image, depth: true, stencilOnly); + DeferRefreshLocked(deferredView, imageIdentifier, image, depth: true, stencilOnly, aliasesPending: false); return deferredView; } @@ -1632,6 +1637,20 @@ public void FlushGpuWrites(ulong address, ulong size, MetalWritebackCause cause) { if (_slots.TryGet(imageIdentifier) is { } image && image.GpuOverlaps(address, size)) { + var (start, end) = image.OverlapWithin(address, size); + if (AliasTransitionsEnabled && cause == MetalWritebackCause.HostRead && + !image.Description.HasStencil && !image.IsCpuDirty && image.FirstLevel == 0 && + image.Sparse is null && (start != 0 || end != image.Description.Data.Size)) + { + if (image.IsPublished(start, end)) continue; + if (TryWriteBackRun(image, start, end, cause, out var refusal)) + { + _aliasCpuReadRuns++; + _aliasCpuReadBytes += (long)(end - start); + continue; + } + RefuseAliasTransition($"cpu-read-{refusal}"); + } WriteBackToGuest(image, cause); } } @@ -1761,6 +1780,7 @@ private bool TryPublishAliasOnDevice(MetalCachedImage image, MetalCachedImage ot image.Description.Data.Address != other.Description.Data.Address || image.Description.Data.Size != other.Description.Data.Size) { + RefuseAliasTransition("range-or-owner"); return false; } @@ -1776,10 +1796,22 @@ private bool TryPublishAliasOnDevice(MetalCachedImage image, MetalCachedImage ot EnsureSparseBacked(image); var context = _owner.BeginTransfer(MetalTransferCause.Writeback, other.Texture, image.Texture); - if (!_transfer.TryEncodeTextureToTexture( - context, in other.Description, other.Texture, in image.Description, image.Texture)) + var refusal = "disabled"; + var fused = AliasTransitionsEnabled && _transfer.TryEncodeFusedAlias( + context, in other.Description, other.Texture, in image.Description, image.Texture, out refusal); + if (fused) { - return false; + _aliasFused++; + _aliasFusedBytes += (long)image.Description.Data.Size; + } + else + { + RefuseAliasTransition(refusal); + if (!_transfer.TryEncodeTextureToTexture( + context, in other.Description, other.Texture, in image.Description, image.Texture)) + { + return false; + } } if (context.WantsEncodedLabels) diff --git a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalImageTransfer.Aliases.cs b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalImageTransfer.Aliases.cs new file mode 100644 index 000000000..78cc1db9f --- /dev/null +++ b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalImageTransfer.Aliases.cs @@ -0,0 +1,172 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using SharpEmu.Libs.Gpu.Images; + +namespace SharpEmu.Gpu.Metal.V2; + +internal sealed partial class MetalImageTransfer +{ + public bool HasDenseAliasCoverage(in ImageDescription description) => + !description.IsDepth && !description.IsBlock && !description.HasStencil && + description.Samples == 1 && description.Resources.Levels == 1 && + TryDenseAliasSurface(description, PlanFor(description), out _); + + public bool CanDiscardAliasForOverwrite(in ImageDescription target, in ImageDescription source) + { + if (HasDenseAliasCoverage(target)) return true; + if (target.IsDepth || source.IsDepth || target.HasStencil || source.HasStencil || + target.IsVolume || source.IsVolume || target.IsBlock || source.IsBlock || + target.Resources.Levels != 1 || source.Resources.Levels != 1 || + target.TransferLayers != 1 || source.TransferLayers != 1 || + target.BytesPerBlock != source.BytesPerBlock || target.Samples != 1 || source.Samples != 1) + return false; + if (source.Data.Address == target.Data.Address && source.Data.Size == target.Data.Size && + SameTransferGrid(source, target) && source.TileMode == target.TileMode && source.Pitch == target.Pitch) + return true; + var from = PlanFor(source); + var to = PlanFor(target); + if (from.Tiles.Count != 1 || to.Tiles.Count != 1 || !from.Tiled || !to.Tiled) return false; + var a = from.Tiles[0]; + var b = to.Tiles[0]; + if (a.Tail || b.Tail || a.TiledOffset != 0 || b.TiledOffset != 0 || + a.SurfaceZ != 0 || b.SurfaceZ != 0 || a.Kind != b.Kind || from.ElementBytes != to.ElementBytes || + !TileGeometry.TryGetBlockLayout(a.Kind, from.ElementBytes, out var block) || block.BlockDepth != 1 || + source.Data.Address < target.Data.Address) return false; + var offset = source.Data.Address - target.Data.Address; + if (offset % block.BlockSize != 0) return false; + var sourceColumns = ((a.TiledWidth != 0 ? a.TiledWidth : a.Pitch) + block.BlockWidth - 1) / block.BlockWidth; + var targetColumns = ((b.TiledWidth != 0 ? b.TiledWidth : b.Pitch) + block.BlockWidth - 1) / block.BlockWidth; + if (sourceColumns == 0 || targetColumns == 0) return false; + // A full destination block kills every byte a smaller/padded source block + // could own. Padding and incomplete destination edge blocks never qualify. + for (uint y = 0; y < a.Height; y += block.BlockHeight) + for (uint x = 0; x < a.Width; x += block.BlockWidth) + { + var index = offset / block.BlockSize + (ulong)(y / block.BlockHeight) * sourceColumns + x / block.BlockWidth; + var tx = (index % targetColumns) * block.BlockWidth; + var ty = (index / targetColumns) * block.BlockHeight; + if (tx + block.BlockWidth > b.Width || ty + block.BlockHeight > b.Height) return false; + } + return true; + } + + public bool TryEncodeFusedAlias(IMetalSubmissionContext context, + in ImageDescription sourceDescription, nint sourceTexture, + in ImageDescription targetDescription, nint targetTexture, out string refusal) + { + refusal = "shape"; + if (_disposed || context.CommandBuffer == 0 || sourceTexture == 0 || targetTexture == 0 || + sourceTexture == targetTexture || sourceDescription.IsDepth || targetDescription.IsDepth || + sourceDescription.HasStencil || targetDescription.HasStencil || + sourceDescription.IsBlock || targetDescription.IsBlock || + sourceDescription.IsVolume || targetDescription.IsVolume || + sourceDescription.Samples != 1 || targetDescription.Samples != 1 || + sourceDescription.Data.Size != targetDescription.Data.Size || + sourceDescription.Resources.Levels != 1 || targetDescription.Resources.Levels != 1) + return false; + + var source = PlanFor(sourceDescription); + var target = PlanFor(targetDescription); + // Matching texel mappings can use a blit, including compression-friendly render + // targets: do not add shader-write/view usage just to make a transfer cheaper. + if (!source.Swap && !target.Swap && source.ElementBytes == target.ElementBytes && + SameTransferGrid(sourceDescription, targetDescription) && + sourceDescription.TileMode == targetDescription.TileMode && + sourceDescription.Pitch == targetDescription.Pitch && + sourceDescription.MipLayout[0].Equals(targetDescription.MipLayout[0])) + { + if (source.Format == target.Format) + { + EncodeWholeTextureCopy(context, sourceTexture, targetTexture); + refusal = "none"; + return true; + } + // Reinterpreting views make a bitwise blit possible even for padded images; + // both sides address the same guest texel, so padding is never observed. + var fromView = AliasView(sourceTexture, source.Format, source.ElementBytes, writing: false, sourceDescription.TransferLayers); + if (fromView != 0) + { + try + { + var toView = AliasView(targetTexture, target.Format, target.ElementBytes, writing: false, targetDescription.TransferLayers); + if (toView != 0) + { + try { EncodeWholeTextureCopy(context, fromView, toView); } + finally { MetalNative.SendVoid(toView, SelReleaseView); } + refusal = "none"; + return true; + } + } + finally { MetalNative.SendVoid(fromView, SelReleaseView); } + } + } + + refusal = "padding-or-subresources"; + if (!TryDenseAliasSurface(sourceDescription, source, out var from) || + !TryDenseAliasSurface(targetDescription, target, out var to)) return false; + refusal = "format-or-usage"; + var sourceView = AliasView(sourceTexture, source.Format, source.ElementBytes, writing: false); + if (sourceView == 0) return false; + try + { + var targetView = AliasView(targetTexture, target.Format, target.ElementBytes, writing: true); + if (targetView == 0) return false; + try + { + var compute = _tiler.BeginCompute(context); + _tiler.EncodeAlias(compute, sourceView, targetView, from, to); + MetalTiler.EndEncoding(compute); + refusal = "none"; + return true; + } + finally { MetalNative.SendVoid(targetView, SelReleaseView); } + } + finally { MetalNative.SendVoid(sourceView, SelReleaseView); } + } + + private static bool TryDenseAliasSurface(in ImageDescription description, TransferPlan plan, + out MetalAliasSurface surface) + { + surface = default; + var width = description.Extent.Width; + var height = description.Extent.Height; + if (plan.Swap || plan.Volume || plan.DepthStencil || description.TransferLayers != 1 || + plan.ElementBytes is not (4 or 8 or 16) || width == 0 || height == 0 || + plan.TexelWidth != 1 || plan.TexelHeight != 1 || + description.Data.Size != (ulong)width * height * plan.ElementBytes || + description.Data.Size > uint.MaxValue || description.Pitch != width || + description.MipLayout[0].Offset != 0) return false; + TileBlockKind? kind = null; + if (plan.Tiled) + { + if (plan.Tiles.Count != 1) return false; + var tile = plan.Tiles[0]; + if (tile.Tail || tile.BaseY != 0 || tile.SurfaceZ != 0 || tile.TiledOffset != 0 || + tile.Width != width || tile.Height != height || tile.Depth != 1 || + tile.TiledSize != description.Data.Size || + (tile.TiledWidth != 0 && tile.TiledWidth != width) || + (tile.TiledHeight != 0 && tile.TiledHeight != height) || + !TileGeometry.TryGetBlockLayout(tile.Kind, plan.ElementBytes, out var block) || + block.BlockDepth != 1 || width % block.BlockWidth != 0 || height % block.BlockHeight != 0) + return false; + kind = tile.Kind; + } + surface = new MetalAliasSurface(width, height, plan.ElementBytes, kind); + return true; + } + + private static nint AliasView(nint texture, MtlPixelFormat format, uint bytes, bool writing, uint layers = 1) + { + var raw = TextureTileViewFormat(format); + var usage = (MtlTextureUsage)(uint)MetalV2Native.SendValue(texture, SelUsage); + var needed = writing ? MtlTextureUsage.ShaderWrite : MtlTextureUsage.ShaderRead; + var type = (MtlTextureType)(uint)MetalV2Native.SendValue(texture, SelTextureType); + if (raw == MtlPixelFormat.Invalid || ViewElementBytes(raw) != bytes || + (usage & needed) == 0 || (raw != format && (usage & MtlTextureUsage.PixelFormatView) == 0) || + type is not (MtlTextureType.Type2D or MtlTextureType.Type2DArray)) return 0; + return MetalV2Native.SendNewTextureView(texture, SelNewTextureView, (nuint)raw, + (nuint)MtlTextureType.Type2DArray, + new NsRange { Location = 0, Length = 1 }, new NsRange { Location = 0, Length = layers }); + } +} diff --git a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.cs b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.cs index 810656f01..b95bf3886 100644 --- a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.cs +++ b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.cs @@ -2542,6 +2542,7 @@ public void ReportCounters() var (subsetJoins, subsetJoinsSampled) = TakeSubsetJoins(); Console.Error.WriteLine( $"[METAL-V2][TBDR] passes={passes} empty={emptyPasses} returned={returnedPasses} " + + _imageCache.TakeAliasTransitionText() + " " + $"load_mb={(loadBytes / (1024.0 * 1024.0)).ToString("F1", culture)} " + $"store_mb={((storeBytes - discardedBytes) / (1024.0 * 1024.0)).ToString("F1", culture)} " + $"discarded_mb={(discardedBytes / (1024.0 * 1024.0)).ToString("F1", culture)} " + diff --git a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalTiler.Aliases.cs b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalTiler.Aliases.cs new file mode 100644 index 000000000..c3a62a11b --- /dev/null +++ b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalTiler.Aliases.cs @@ -0,0 +1,107 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using System.Numerics; +using System.Text; +using SharpEmu.Libs.Gpu.Images; +using SharpEmu.Libs.Gpu.Scheduling; + +namespace SharpEmu.Gpu.Metal.V2; + +// Dense surfaces only: every byte maps to a texel, so the direct gather never needs +// guest padding that a texture cannot hold. Layouts are constants in a cached pipeline. +internal readonly record struct MetalAliasSurface(uint Width, uint Height, uint Bytes, TileBlockKind? Kind); + +internal static partial class MetalTilerShaders +{ + // Invert the tile's linear XOR transform. Each result mask reconstructs one x/y + // coordinate bit as parity(offset & mask). Derive from the shared CPU tile oracle, + // rather than maintaining a second inverse table by hand. + internal static (uint[] X, uint[] Y) AliasInverseMasks(in TileBlockLayout block) + { + var xs = new uint[BitOperations.Log2(block.BlockWidth)]; + var ys = new uint[BitOperations.Log2(block.BlockHeight)]; + var basis = 0; + for (uint y = 0; y < block.BlockHeight; y++) + for (uint x = 0; x < block.BlockWidth; x++) + { + if (!TileGeometry.TryGetBlockOffset(block, x, y, 0, out var offset)) + throw GpuFatal.Fatal("An alias tile has no invertible address mapping."); + if (!BitOperations.IsPow2(offset)) continue; + basis++; + for (var bit = 0; bit < xs.Length; bit++) + if ((x & (1u << bit)) != 0) xs[bit] |= offset; + for (var bit = 0; bit < ys.Length; bit++) + if ((y & (1u << bit)) != 0) ys[bit] |= offset; + } + if (basis != xs.Length + ys.Length) + throw GpuFatal.Fatal("An alias tile's address transform is not a bijection."); + return (xs, ys); + } + + internal static string CreateAliasLibrary(MetalAliasSurface source, MetalAliasSurface target) + { + var text = new StringBuilder("#include \nusing namespace metal;\n"); + AppendAliasAddress(text, "source", source); + AppendAliasAddress(text, "target", target); + text.Append("kernel void alias_copy(texture2d_array source [[texture(0)]],\n" + + " texture2d_array target [[texture(1)]], uint2 gid [[thread_position_in_grid]]) {\n"); + text.Append($" if (gid.x >= {target.Width}u || gid.y >= {target.Height}u) return;\n uint4 value = uint4(0);\n"); + // Gather raw dwords even when the two formats have different texel sizes. No + // floating-point/normalized conversion, filtering, swizzle or sRGB decode. + for (uint word = 0; word < target.Bytes / 4; word++) + { + text.Append($" {{ uint address = target_address(gid) + {word * 4}u;\n" + + $" uint2 xy = source_coords(address);\n" + + $" value[{word}] = source.read(xy, 0)[(address % {source.Bytes}u) / 4u]; }}\n"); + } + text.Append(" target.write(value, gid, 0);\n}\n"); + return text.ToString(); + } + + private static void AppendAliasAddress(StringBuilder text, string name, MetalAliasSurface surface) + { + var rowBytes = surface.Width * surface.Bytes; + if (surface.Kind is not { } kind) + { + text.Append($"uint {name}_address(uint2 p) {{ return p.y * {rowBytes}u + p.x * {surface.Bytes}u; }}\n" + + $"uint2 {name}_coords(uint a) {{ return uint2((a % {rowBytes}u) / {surface.Bytes}u, a / {rowBytes}u); }}\n"); + return; + } + TileGeometry.TryGetBlockLayout(kind, surface.Bytes, out var block); + var columns = surface.Width / block.BlockWidth; + text.Append($"uint {name}_offset(uint sx, uint sy) {{ uint sz = 0u; return {BlockOffset(kind, surface.Bytes)}; }}\n"); + text.Append($"uint {name}_address(uint2 p) {{ return ((p.y / {block.BlockHeight}u) * {columns}u + p.x / {block.BlockWidth}u) * {block.BlockSize}u + {name}_offset(p.x, p.y); }}\n"); + text.Append($"uint2 {name}_coords(uint a) {{ uint b = a / {block.BlockSize}u;\n" + + $" uint2 p = uint2((b % {columns}u) * {block.BlockWidth}u, (b / {columns}u) * {block.BlockHeight}u);\n" + + $" uint o = (a % {block.BlockSize}u) ^ {name}_offset(p.x, p.y);\n"); + var inverse = AliasInverseMasks(block); + for (var bit = 0; bit < inverse.X.Length; bit++) + text.Append($" p.x |= (popcount(o & {inverse.X[bit]}u) & 1u) << {bit}u;\n"); + for (var bit = 0; bit < inverse.Y.Length; bit++) + text.Append($" p.y |= (popcount(o & {inverse.Y[bit]}u) & 1u) << {bit}u;\n"); + text.Append(" return p; }\n"); + } +} + +internal sealed unsafe partial class MetalTiler +{ + public void EncodeAlias(nint encoder, nint sourceView, nint targetView, + MetalAliasSurface source, MetalAliasSurface target) + { + var key = $"alias:{source}:{target}"; + if (!_conversionPipelines.TryGetValue(key, out var pipeline)) + { + var library = CompileLibrary(MetalTilerShaders.CreateAliasLibrary(source, target), key); + try { pipeline = CreatePipeline(library, "alias_copy", 0, null, key); } + finally { Release(library); } + _conversionPipelines.Add(key, pipeline); + } + MetalNative.SendVoid(encoder, SelSetPipeline, pipeline); + MetalV2Native.SendSetResourceAtIndex(encoder, SelSetTexture, sourceView, 0); + MetalV2Native.SendSetResourceAtIndex(encoder, SelSetTexture, targetView, 1); + MetalNative.SendDispatch(encoder, SelDispatch, + new MtlSize { Width = (target.Width + 15) / 16, Height = (target.Height + 15) / 16, Depth = 1 }, + new MtlSize { Width = 16, Height = 16, Depth = 1 }); + } +} diff --git a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalTiler.cs b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalTiler.cs index 0250e5ab8..9231c24bf 100644 --- a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalTiler.cs +++ b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalTiler.cs @@ -72,7 +72,7 @@ internal struct MetalRepackArguments // Runs the tiling, detiling, depth conversion, channel swap and row repack compute // passes, and owns the staging buffers they stage through. The Metal twin of the Vulkan // backend's GpuTiler: same arguments, same per-element addressing, no descriptor sets. -internal sealed unsafe class MetalTiler : IDisposable +internal sealed unsafe partial class MetalTiler : IDisposable { private const uint KindCount = 10; private const uint ElementSizeCount = 5; diff --git a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalTilerShaders.cs b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalTilerShaders.cs index ee88ec500..4de6efb43 100644 --- a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalTilerShaders.cs +++ b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalTilerShaders.cs @@ -25,7 +25,7 @@ internal enum MetalTileDirection // element sizes cost one newFunctionWithName:constantValues: each. Emitting every kind // into a single library would compile ~1200 lines of MSL before the first transfer; a // library per element size would be a hundred compiles instead. -internal static class MetalTilerShaders +internal static partial class MetalTilerShaders { // The 14 dwords every block-copy kernel reads, and the subset the conversions use: // the 13 the Vulkan tiler shares, and then baseY for a transfer cut down to a run of diff --git a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalV2Contracts.cs b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalV2Contracts.cs index 4609b6182..464933417 100644 --- a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalV2Contracts.cs +++ b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalV2Contracts.cs @@ -155,6 +155,20 @@ bool TryEncodeTextureToTexture(IMetalSubmissionContext context, in ImageDescription sourceDescription, nint sourceTexture, in ImageDescription targetDescription, nint targetTexture); + bool HasDenseAliasCoverage(in ImageDescription description) => false; + + bool CanDiscardAliasForOverwrite(in ImageDescription target, in ImageDescription source) => false; + + // A single texture-to-texture pass, without any intermediate buffer. A refusal + // encodes nothing and leaves the conservative publication path available. + bool TryEncodeFusedAlias(IMetalSubmissionContext context, + in ImageDescription sourceDescription, nint sourceTexture, + in ImageDescription targetDescription, nint targetTexture, out string refusal) + { + refusal = "unsupported-transfer"; + return false; + } + // Whether the depth image's texture, sampled in its own format, reads what a texture of // the colour description over the same guest bytes would (MetalImageTransfer). False // keeps the two as separate textures. diff --git a/tests/SharpEmu.Gpu.Metal.Tests/MetalImageAliasTests.Transitions.cs b/tests/SharpEmu.Gpu.Metal.Tests/MetalImageAliasTests.Transitions.cs new file mode 100644 index 000000000..d008f62f2 --- /dev/null +++ b/tests/SharpEmu.Gpu.Metal.Tests/MetalImageAliasTests.Transitions.cs @@ -0,0 +1,170 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using SharpEmu.Gpu.Metal.V2; +using SharpEmu.HLE.GpuMemory; +using SharpEmu.Libs.Gpu.Images; +using Xunit; + +namespace SharpEmu.Gpu.Metal.Tests; + +public sealed unsafe partial class MetalImageAliasTests +{ + [Theory] + [InlineData(true)] + [InlineData(false)] + public void ADeferredAliasDiscardsOnlyAfterAWholeOverwriteProof(bool overwrite) + { + if (!RequireDevice()) return; + using var rig = new Rig(Device.Value); + var (source, gpu, stale) = GpuWrittenLarge(rig); + var targetRequest = Rig.Request(ImageAddress, 128, 32, ImageRole.ColorTarget); + var target = rig.Cache.FindImage(ref targetRequest); + Assert.NotEqual(source, rig.Cache.GetImage(target)); + var writebacks = rig.Cache.Counters.Writebacks; + var view = rig.Cache.AcquireColorTargetView(target, targetRequest, deferRefresh: true); + Assert.True(rig.Cache.HasDeferredRefreshes); + Assert.True(source.IsGpuModified); + Assert.Equal(writebacks, rig.Cache.Counters.Writebacks); + rig.Cache.ResolveDeferredRefresh(view, overwritten: overwrite); + rig.Harness.Submit(); + rig.Harness.Begin(); + Assert.False(source.IsGpuModified); + var transitions = rig.Cache.TakeAliasTransitions(); + if (overwrite) + { + Assert.Equal((1L, (long)LargeSize), (transitions.Discarded, transitions.DiscardedBytes)); + Assert.Equal(writebacks, rig.Cache.Counters.Writebacks); + Assert.Equal(stale, rig.ReadGuest(ImageAddress, LargeSize)); + // Simulate the proven full draw with an actual GPU upload to the target. + var drawn = MetalTileOracle.Pattern((int)LargeSize, 0xA2); + var image = rig.Cache.GetImage(target); + rig.Harness.Transfer.EncodeGuestToTexture(rig.Harness, image.Description, SubresourceRange.First, + new MetalGuestRange(rig.Harness.Buffer(drawn), 0, LargeSize, 0), default, image.Texture, 0, LargeSize); + rig.Cache.FlushGpuWrites(ImageAddress, LargeSize, MetalWritebackCause.HostRead); + rig.Harness.Submit(); + rig.Harness.Begin(); + Assert.Equal(drawn, rig.ReadGuest(ImageAddress, LargeSize)); + } + else + { + Assert.Equal(0, transitions.Discarded); + rig.Cache.FlushGpuWrites(ImageAddress, LargeSize, MetalWritebackCause.HostRead); + rig.Harness.Submit(); + rig.Harness.Begin(); + Assert.Equal(gpu, rig.ReadGuest(ImageAddress, LargeSize)); + } + } + + [Fact] + public void APartialCpuReadPublishesOnlyItsTransferableRowsAndKeepsTheRestOnDevice() + { + if (!RequireDevice()) return; + using var rig = new Rig(Device.Value); + var (image, gpu, stale) = GpuWrittenLarge(rig); + const ulong row = LargeSide * 4; + rig.Cache.FlushGpuWrites(ImageAddress + row, row, MetalWritebackCause.HostRead); + rig.Harness.Submit(); + rig.Harness.Begin(); + Assert.True(image.IsGpuModified); + Assert.True(image.IsPublished(row, row * 2)); + var expected = stale.ToArray(); + gpu.AsSpan((int)row, (int)row).CopyTo(expected.AsSpan((int)row)); + Assert.Equal(expected, rig.ReadGuest(ImageAddress, LargeSize)); + var writebacks = rig.Cache.Counters.Writebacks; + rig.Cache.FlushGpuWrites(ImageAddress + row, row, MetalWritebackCause.HostRead); + Assert.Equal(writebacks, rig.Cache.Counters.Writebacks); + rig.Cache.FlushGpuWrites(ImageAddress, LargeSize, MetalWritebackCause.HostRead); + rig.Harness.Submit(); + rig.Harness.Begin(); + Assert.Equal(gpu, rig.ReadGuest(ImageAddress, LargeSize)); + } + + [Fact] + public void ACompleteManagedOverwriteDiscardsWithoutPublishingAndReloadsExactBytes() + { + if (!RequireDevice()) return; + using var rig = new Rig(Device.Value); + var (image, _, stale) = GpuWrittenLarge(rig); + var writebacks = rig.Cache.Counters.Writebacks; + Assert.True(((IGuestImageStore)rig.Cache).MarkPreciseCpuWrite(ImageAddress, LargeSize)); + var replacement = MetalTileOracle.Pattern((int)LargeSize, 0xB9); + Assert.Equal(stale, rig.ReadGuest(ImageAddress, LargeSize)); + rig.WriteGuest(ImageAddress, replacement); + ((IGuestImageStore)rig.Cache).MarkCpuWriteLanded(ImageAddress, LargeSize); + Assert.False(image.IsGpuModified); + Assert.Equal(writebacks, rig.Cache.Counters.Writebacks); + Assert.Equal(1, rig.Cache.TakeAliasTransitions().Discarded); + rig.Acquire(ImageAddress, LargeSide, ImageRole.ColorTarget); + rig.Cache.FlushGpuWrites(ImageAddress, LargeSize, MetalWritebackCause.HostRead); + rig.Harness.Submit(); + rig.Harness.Begin(); + Assert.Equal(replacement, rig.ReadGuest(ImageAddress, LargeSize)); + } + + // A GPU-written linear image and a 64 KiB-tiled image over exactly its bytes, both bound + // for shader stores: with alias transitions on, the tiled one takes the linear one's + // texels in one device pass; off, through a writeback of the linear one and an upload of + // the tiled one. It holds the same guest bytes either way. Colour targets are created + // compressible, without the shader-write and format-view usages the pass needs, and + // keep the round trip. Prints the transfer counts and the GPU time. + [Theory] + [InlineData(true, true)] + [InlineData(false, true)] + [InlineData(true, false)] + [InlineData(false, false)] + public void ATiledAliasOfAGpuWrittenImageFusesOnTheDeviceOrRoundTrips(bool fused, bool storage) + { + if (!RequireDevice()) return; + using var rig = new Rig(Device.Value); + rig.Cache.AliasTransitionsEnabled = fused; + const uint side = 256; + const ulong size = side * side * 4; + var role = storage ? ImageRole.StorageImage : ImageRole.ColorTarget; + var written = MetalTileOracle.Pattern((int)size, 0x3D); + rig.WriteGuest(ImageAddress, written); + var linear = Rig.Request(ImageAddress, side, ImageRole.ColorTarget); + linear = new ImageRequest(linear.Description, linear.View, role); + var source = rig.Cache.GetImage(rig.Acquire(linear)); + source.MarkGpuModified(); + rig.Harness.Submit(); + rig.Harness.Begin(); + rig.WriteGuest(ImageAddress, MetalTileOracle.Pattern((int)size, 0xC3)); + _ = rig.Cache.TakeAliasTransitionText(); + + var tiled = Rig.TiledRequest(ImageAddress, side); + Assert.Equal(size, tiled.Description.Data.Size); + tiled = new ImageRequest(tiled.Description, tiled.View, role); + var (_, uploads, writebacks, _) = rig.Cache.Counters; + var buffer = rig.Harness.CommandBuffer; + MetalNative.SendVoid(buffer, MetalNative.Selector("retain")); + var alias = rig.Cache.GetImage(rig.Acquire(tiled)); + rig.Harness.Submit(); + var gpu = MetalNative.SendDouble(buffer, MetalNative.Selector("GPUEndTime")) - + MetalNative.SendDouble(buffer, MetalNative.Selector("GPUStartTime")); + MetalNative.SendVoid(buffer, MetalNative.Selector("release")); + rig.Harness.Begin(); + var counts = rig.Cache.Counters; + var text = rig.Cache.TakeAliasTransitionText(); + output.WriteLine($"alias transitions {(fused ? "on" : "off")}, {role}: uploads={counts.Uploads - uploads} " + + $"writebacks={counts.Writebacks - writebacks} gpu={gpu * 1000:F3} ms {text}"); + if (fused && storage) + { + Assert.Contains("fused:1,", text); + Assert.Equal((0L, 0L), (counts.Uploads - uploads, counts.Writebacks - writebacks)); + } + else + { + Assert.Contains("fused:0,", text); + Assert.Equal((1L, 1L), (counts.Uploads - uploads, counts.Writebacks - writebacks)); + } + + // The tiled image's texels, retiled to guest bytes, are the bytes the linear one held. + var scratch = rig.Harness.Buffer(size); + Assert.True(rig.Harness.Transfer.EncodeTextureToGuest(rig.Harness, alias.Description, SubresourceRange.First, + alias.Texture, new MetalGuestRange(scratch, 0, size, 0), default, 0, size)); + rig.Harness.Submit(); + rig.Harness.Begin(); + Assert.Equal(written, rig.Harness.Read(scratch, 0, size)); + } +} diff --git a/tests/SharpEmu.Gpu.Metal.Tests/MetalImageAliasTests.cs b/tests/SharpEmu.Gpu.Metal.Tests/MetalImageAliasTests.cs index df23d2740..4e73d4312 100644 --- a/tests/SharpEmu.Gpu.Metal.Tests/MetalImageAliasTests.cs +++ b/tests/SharpEmu.Gpu.Metal.Tests/MetalImageAliasTests.cs @@ -21,7 +21,7 @@ namespace SharpEmu.Gpu.Metal.Tests; /// target re-uploaded because a smaller surface overlapping its tail was drawn to. /// Device work is behind the exclusive device gate. /// -public sealed unsafe class MetalImageAliasTests(ITestOutputHelper output) +public sealed unsafe partial class MetalImageAliasTests(ITestOutputHelper output) { private const ulong GuestBase = 0x2_0000_0000; private const ulong ImageAddress = GuestBase + 0x10000; diff --git a/tests/SharpEmu.Gpu.Metal.Tests/MetalImageTransferTests.AliasTransitions.cs b/tests/SharpEmu.Gpu.Metal.Tests/MetalImageTransferTests.AliasTransitions.cs new file mode 100644 index 000000000..f0dcbc2e2 --- /dev/null +++ b/tests/SharpEmu.Gpu.Metal.Tests/MetalImageTransferTests.AliasTransitions.cs @@ -0,0 +1,205 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using System.Numerics; +using SharpEmu.Gpu.Metal.V2; +using SharpEmu.Libs.Gpu.Abstractions; +using SharpEmu.Libs.Gpu.Images; +using Xunit; + +namespace SharpEmu.Gpu.Metal.Tests; + +public sealed partial class MetalImageTransferTests +{ + [Fact] + public void AliasInverseReconstructsEveryTexelOfEveryDenseTwoDimensionalTile() + { + var cases = 0; + foreach (var kind in Enum.GetValues()) + foreach (uint bytes in new uint[] { 4, 8, 16 }) + { + if (!TileGeometry.TryGetBlockLayout(kind, bytes, out var block) || block.BlockDepth != 1) continue; + var masks = MetalTilerShaders.AliasInverseMasks(block); + for (uint y = 0; y < block.BlockHeight; y++) + for (uint x = 0; x < block.BlockWidth; x++) + { + Assert.True(TileGeometry.TryGetBlockOffset(block, x, y, 0, out var offset)); + uint actualX = 0, actualY = 0; + for (var bit = 0; bit < masks.X.Length; bit++) + actualX |= (uint)(BitOperations.PopCount(offset & masks.X[bit]) & 1) << bit; + for (var bit = 0; bit < masks.Y.Length; bit++) + actualY |= (uint)(BitOperations.PopCount(offset & masks.Y[bit]) & 1) << bit; + Assert.Equal((x, y), (actualX, actualY)); + } + cases++; + } + Assert.True(cases >= 18); + output.WriteLine($"Exhaustive inverse tile cases: {cases}"); + } + + [Fact] + public void OverwriteCoverageRequiresFullDestinationTilesOrTheSameTexelMapping() + { + if (!RequireDevice()) return; + using var harness = new MetalTransferHarness(Device.Value); + var target = AliasDescription(GuestTileMode.RenderTarget, 1920, 1080, 8); + var small = AliasDescription(GuestTileMode.RenderTarget, 64, 64, 8); + Assert.True(harness.Transfer.CanDiscardAliasForOverwrite(target, small)); + small.Data = small.Data with { Address = target.Data.Address + target.Data.Size - small.Data.Size }; + Assert.False(harness.Transfer.CanDiscardAliasForOverwrite(target, small)); + var same = target; + same.PixelFormat = GpuFormat.R32G32Uint; + Assert.True(harness.Transfer.CanDiscardAliasForOverwrite(target, same)); + same.Resources = new SubresourceCount(2, 1); + Assert.False(harness.Transfer.CanDiscardAliasForOverwrite(target, same)); + } + + [Theory] + [InlineData(GuestTileMode.Linear, GuestTileMode.Linear, 4u, 4u, false)] + [InlineData(GuestTileMode.Standard64KB, GuestTileMode.Standard64KB, 4u, 4u, false)] + [InlineData(GuestTileMode.Standard64KB, GuestTileMode.RenderTarget, 4u, 4u, false)] + [InlineData(GuestTileMode.RenderTarget, GuestTileMode.Standard64KB, 8u, 8u, false)] + [InlineData(GuestTileMode.RenderTarget, GuestTileMode.Linear, 16u, 4u, true)] + [InlineData(GuestTileMode.Linear, GuestTileMode.RenderTarget, 4u, 16u, true)] + [InlineData(GuestTileMode.Standard64KB, GuestTileMode.Depth, 4u, 8u, true)] + [InlineData(GuestTileMode.Depth, GuestTileMode.RenderTarget, 8u, 4u, true)] + [InlineData(GuestTileMode.Standard4KB, GuestTileMode.Standard64KB, 4u, 4u, true)] + public void FusedFormatSizeAndTilingAliasesMatchTheGuestRoundTrip( + GuestTileMode sourceTile, GuestTileMode targetTile, uint sourceBytes, uint targetBytes, bool reshape) + { + if (!RequireDevice()) return; + using var harness = new MetalTransferHarness(Device.Value); + var source = AliasDescription(sourceTile, 512, 512, sourceBytes); + var width = reshape ? 1024u : 512u; + var target = AliasDescription(targetTile, width, 512u * 512 * sourceBytes / (width * targetBytes), targetBytes); + // Different channel interpretations with exactly the same dword bits. + if (sourceBytes == targetBytes) target.PixelFormat = targetBytes switch + { + 4 => GpuFormat.R32Uint, + 8 => GpuFormat.R32G32Uint, + _ => GpuFormat.R32G32B32A32Uint, + }; + Assert.Equal(source.Data.Size, target.Data.Size); + Assert.True(harness.Transfer.HasDenseAliasCoverage(source)); + Assert.True(harness.Transfer.HasDenseAliasCoverage(target)); + var guest = BuildGuest(source, 103); + var sourceTexture = harness.Texture(harness.Transfer.ShapeFor(source)); + var actualTexture = harness.Texture(harness.Transfer.ShapeFor(target)); + var expectedTexture = harness.Texture(harness.Transfer.ShapeFor(target)); + var input = harness.Buffer(guest); + var published = harness.Buffer(source.Data.Size); + harness.Begin(); + Assert.True(harness.Transfer.EncodeGuestToTexture(harness, source, SubresourceRange.First, + new MetalGuestRange(input, 0, source.Data.Size, 0), default, sourceTexture, 0, source.Data.Size)); + harness.Submit(); + var stagingBefore = harness.Tiler.StagingBytes; + harness.Begin(); + Assert.True(harness.Transfer.TryEncodeFusedAlias(harness, source, sourceTexture, + target, actualTexture, out var refusal), refusal); + harness.Submit(); + Assert.Equal(stagingBefore, harness.Tiler.StagingBytes); + harness.Begin(); + Assert.True(harness.Transfer.EncodeTextureToGuest(harness, source, SubresourceRange.First, sourceTexture, + new MetalGuestRange(published, 0, source.Data.Size, 0), default, 0, source.Data.Size)); + harness.Submit(); + harness.Begin(); + Assert.True(harness.Transfer.EncodeGuestToTexture(harness, target, SubresourceRange.First, + new MetalGuestRange(published, 0, target.Data.Size, 0), default, expectedTexture, 0, target.Data.Size)); + harness.Submit(); + Assert.Equal(ReadTexels(harness, target, expectedTexture), ReadTexels(harness, target, actualTexture)); + Assert.Equal(guest, WriteBack(harness, target, actualTexture)); + output.WriteLine($"{sourceTile}/{sourceBytes} -> {targetTile}/{targetBytes}, {width}x{target.Extent.Height}: {guest.Length} bytes exact, no staging"); + } + + [Fact] + public void AFusedColorArrayAliasPreservesEverySlice() + { + if (!RequireDevice()) return; + using var harness = new MetalTransferHarness(Device.Value); + var source = Color2D(GuestPixelFormat.Bits16_16_16_16Float, GpuFormat.R16G16B16A16Sfloat, + 512, 128, 1, 3, GuestTileMode.Standard64KB); + var target = source; + target.PixelFormat = GpuFormat.R32G32Uint; + var bytes = BuildGuest(source, 119); + var from = harness.Texture(harness.Transfer.ShapeFor(source)); + var to = harness.Texture(harness.Transfer.ShapeFor(target)); + var range = new SubresourceRange(0, 1, 0, 3); + harness.Begin(); + harness.Transfer.EncodeGuestToTexture(harness, source, range, + new MetalGuestRange(harness.Buffer(bytes), 0, source.Data.Size, 0), default, from, 0, source.Data.Size); + harness.Submit(); + harness.Begin(); + Assert.True(harness.Transfer.TryEncodeFusedAlias(harness, source, from, target, to, out var refusal), refusal); + harness.Submit(); + Assert.Equal(ReadTexels(harness, source, from), ReadTexels(harness, target, to)); + Assert.Equal(bytes, WriteBack(harness, target, to)); + } + + [Fact] + public void MatchingCompressibleColorTargetsUseABlitWithoutUsagePromotion() + { + if (!RequireDevice()) return; + using var harness = new MetalTransferHarness(Device.Value); + var description = AliasDescription(GuestTileMode.RenderTarget, 512, 512, 8); + var shape = harness.Transfer.ShapeFor(description) with + { Usage = (ulong)(MtlTextureUsage.ShaderRead | MtlTextureUsage.RenderTarget) }; + var source = harness.Texture(shape); + var target = harness.Texture(shape); + var bytes = BuildGuest(description, 117); + harness.Begin(); + harness.Transfer.EncodeGuestToTexture(harness, description, SubresourceRange.First, + new MetalGuestRange(harness.Buffer(bytes), 0, description.Data.Size, 0), default, source, 0, description.Data.Size); + harness.Submit(); + var staging = harness.Tiler.StagingBytes; + harness.Begin(); + Assert.True(harness.Transfer.TryEncodeFusedAlias(harness, description, source, description, target, out var refusal), refusal); + harness.Submit(); + Assert.Equal(staging, harness.Tiler.StagingBytes); + Assert.Equal(bytes, WriteBack(harness, description, target)); + } + + [Fact] + public void PaddedAliasesRefuseFusionAndTheirRoundTripPreservesPixels() + { + if (!RequireDevice()) return; + using var harness = new MetalTransferHarness(Device.Value); + var source = Color2D(GuestPixelFormat.Bits8_8_8_8UNorm, GpuFormat.R8G8B8A8Unorm, + 97, 33, 1, 1, GuestTileMode.Standard64KB); + var target = source; + target.PixelFormat = GpuFormat.R8G8B8A8Uint; + var sourceTexture = harness.Texture(harness.Transfer.ShapeFor(source)); + var targetShape = harness.Transfer.ShapeFor(target) with + { Usage = (ulong)(MtlTextureUsage.ShaderRead | MtlTextureUsage.RenderTarget) }; + var targetTexture = harness.Texture(targetShape); + var bytes = BuildGuest(source, 123); + harness.Begin(); + harness.Transfer.EncodeGuestToTexture(harness, source, SubresourceRange.First, + new MetalGuestRange(harness.Buffer(bytes), 0, source.Data.Size, 0), default, sourceTexture, 0, source.Data.Size); + harness.Submit(); + var staging = harness.Tiler.StagingBytes; + harness.Begin(); + Assert.False(harness.Transfer.TryEncodeFusedAlias(harness, source, sourceTexture, target, targetTexture, out var refusal)); + Assert.Equal("padding-or-subresources", refusal); + Assert.Equal(staging, harness.Tiler.StagingBytes); + harness.Submit(); + var published = WriteBack(harness, source, sourceTexture); + harness.Begin(); + harness.Transfer.EncodeGuestToTexture(harness, target, SubresourceRange.First, + new MetalGuestRange(harness.Buffer(published), 0, target.Data.Size, 0), default, targetTexture, 0, target.Data.Size); + harness.Submit(); + Assert.Equal(ReadTexels(harness, source, sourceTexture), ReadTexels(harness, target, targetTexture)); + } + + private static ImageDescription AliasDescription(GuestTileMode tile, uint width, uint height, uint bytes) => + Color2D(bytes switch + { + 4 => GuestPixelFormat.Bits8_8_8_8UNorm, + 8 => GuestPixelFormat.Bits16_16_16_16Float, + _ => GuestPixelFormat.Bits32_32_32_32Float, + }, bytes switch + { + 4 => GpuFormat.R8G8B8A8Unorm, + 8 => GpuFormat.R16G16B16A16Sfloat, + _ => GpuFormat.R32G32B32A32Sfloat, + }, width, height, 1, 1, tile); +} diff --git a/tests/SharpEmu.Gpu.Metal.Tests/MetalImageTransferTests.Timing.cs b/tests/SharpEmu.Gpu.Metal.Tests/MetalImageTransferTests.Timing.cs index a94fe5cb7..eadf40cb8 100644 --- a/tests/SharpEmu.Gpu.Metal.Tests/MetalImageTransferTests.Timing.cs +++ b/tests/SharpEmu.Gpu.Metal.Tests/MetalImageTransferTests.Timing.cs @@ -61,3 +61,62 @@ public void TransferGpuTime(string label, uint which) output.WriteLine($"TIMING {label}: bytes=0x{size:x} upload={bestUp * 1000:F3} ms writeback={bestDown * 1000:F3} ms"); } } + +// GPU time of an alias transition fused on the device against the guest-memory round trip +// it replaces (a retile of the source out, a detile into the alias), for the shapes the +// fused path takes. Printed, with the fused result checked against the round trip's. +public sealed partial class MetalImageTransferTests +{ + [Theory] + [InlineData("rgba16f 1920x1080 RT -> rg32ui view (blit)", 0u)] + [InlineData("rgba16f 1920x1088 64KB_S -> RT (gather)", 1u)] + [InlineData("rgba8 2048x1024 64KB_S -> rgba16f 1024x1024 RT (gather)", 2u)] + public void AliasTransitionGpuTime(string label, uint which) + { + if (!RequireDevice()) return; + using var harness = new MetalTransferHarness(Device.Value); + var (source, target) = which switch + { + 0 => (Color2D(GuestPixelFormat.Bits16_16_16_16Float, GpuFormat.R16G16B16A16Sfloat, 1920, 1080, 1, 1, GuestTileMode.RenderTarget), + Color2D(GuestPixelFormat.Bits16_16_16_16Float, GpuFormat.R32G32Uint, 1920, 1080, 1, 1, GuestTileMode.RenderTarget)), + 1 => (AliasDescription(GuestTileMode.Standard64KB, 1920, 1088, 8), AliasDescription(GuestTileMode.RenderTarget, 1920, 1088, 8)), + _ => (AliasDescription(GuestTileMode.Standard64KB, 2048, 1024, 4), AliasDescription(GuestTileMode.RenderTarget, 1024, 1024, 8)), + }; + Assert.Equal(source.Data.Size, target.Data.Size); + var size = source.Data.Size; + var guest = harness.Buffer(BuildGuest(source, 41)); + var from = harness.Texture(harness.Transfer.ShapeFor(source)); + var fused = harness.Texture(harness.Transfer.ShapeFor(target)); + var roundTrip = harness.Texture(harness.Transfer.ShapeFor(target)); + harness.Begin(); + Assert.True(harness.Transfer.EncodeGuestToTexture(harness, source, SubresourceRange.First, + new MetalGuestRange(guest, 0, size, 0), default, from, 0, size)); + harness.Submit(); + var published = harness.Buffer(size); + double bestFused = double.MaxValue, bestRoundTrip = double.MaxValue; + for (var rep = 0; rep < 5; rep++) + { + harness.Begin(); + var buffer = harness.CommandBuffer; + Assert.True(harness.Transfer.TryEncodeFusedAlias(harness, source, from, target, fused, out var refusal), refusal); + MetalNative.SendVoid(buffer, MetalNative.Selector("retain")); + harness.Submit(); + bestFused = Math.Min(bestFused, MetalNative.SendDouble(buffer, SelGpuEndTime) - MetalNative.SendDouble(buffer, SelGpuStartTime)); + MetalNative.SendVoid(buffer, MetalNative.Selector("release")); + + harness.Begin(); + buffer = harness.CommandBuffer; + Assert.True(harness.Transfer.EncodeTextureToGuest(harness, source, SubresourceRange.First, from, + new MetalGuestRange(published, 0, size, 0), default, 0, size)); + Assert.True(harness.Transfer.EncodeGuestToTexture(harness, target, SubresourceRange.First, + new MetalGuestRange(published, 0, size, 0), default, roundTrip, 0, size)); + MetalNative.SendVoid(buffer, MetalNative.Selector("retain")); + harness.Submit(); + bestRoundTrip = Math.Min(bestRoundTrip, MetalNative.SendDouble(buffer, SelGpuEndTime) - MetalNative.SendDouble(buffer, SelGpuStartTime)); + MetalNative.SendVoid(buffer, MetalNative.Selector("release")); + } + + Assert.Equal(ReadTexels(harness, target, roundTrip), ReadTexels(harness, target, fused)); + output.WriteLine($"TIMING alias {label}: bytes=0x{size:x} fused={bestFused * 1000:F3} ms round-trip={bestRoundTrip * 1000:F3} ms"); + } +} From 18323b71870897ea657939d621a1bacfbde83a2b Mon Sep 17 00:00:00 2001 From: kuba Date: Mon, 26 Oct 2026 16:20:08 +0100 Subject: [PATCH 28/82] [GPU] Integrate audited runtime and materialization performance fixes --- docs/macos/split-protocol.md | 10 +- docs/performance/combined-audit.md | 281 +++++++++++++++ docs/performance/shader-serial.runsettings | 10 + .../Cpu/Native/DirectExecutionBackend.cs | 32 +- src/SharpEmu.GpuHost/GpuHostMain.cs | 12 +- src/SharpEmu.GpuRemote/GpuProcessClient.cs | 30 +- src/SharpEmu.GpuRemote/GpuRemoteProtocol.cs | 6 +- src/SharpEmu.GpuRemote/GpuSplitMessages.cs | 6 + .../PendingWritebackPages.cs | 180 ++++++++++ .../RemoteGuestAddressSpace.cs | 5 +- src/SharpEmu.GpuSplit/RemoteImageStore.cs | 16 +- src/SharpEmu.GpuSplit/RemoteVideoOutBridge.cs | 26 +- src/SharpEmu.GpuSplit/SplitGpuHost.cs | 61 +++- src/SharpEmu.GpuSplit/SplitGpuSession.cs | 18 +- src/SharpEmu.GpuSplit/SplitSharedState.cs | 55 ++- src/SharpEmu.GpuSplit/WorkerBracket.cs | 79 +++- .../GpuMemory/IPendingWritebackPages.cs | 36 ++ src/SharpEmu.HLE/GpuMemory/RegionLock.cs | 4 + .../GpuMemory/SplitCrossingStats.cs | 3 + .../GuestMemory/GuestMirrorTranslator.cs | 94 ++++- src/SharpEmu.HLE/Host/HostHelperThread.cs | 53 +++ .../Agc/AgcExports.CommandStream.cs | 3 + src/SharpEmu.Libs/Audio/AjmExports.Jobs.cs | 11 +- src/SharpEmu.Libs/Audio/AjmExports.cs | 91 +++-- .../AjmTraceInterpolatedStringHandler.cs | 39 ++ src/SharpEmu.Libs/Audio/Atrac9DecodeState.cs | 41 ++- src/SharpEmu.Libs/Audio/AudioOut2Exports.cs | 9 +- src/SharpEmu.Libs/CxxAbiExports.cs | 8 +- .../GpuCommands/Registers/ContextRegisters.cs | 11 +- src/SharpEmu.Libs/Gpu/IGuestCompiledShader.cs | 4 + .../Gpu/Pipelines/PixelStageInputResolver.cs | 22 +- .../Gpu/Pipelines/ShaderProgramCache.cs | 5 +- .../Gpu/Rendering/ParallelProgramPrefetch.cs | 7 +- .../Gpu/Rendering/RenderExecutor.Resources.cs | 5 + .../Gpu/Rendering/RenderExecutor.cs | 16 +- .../Gpu/Rendering/ShaderStageResources.cs | 3 + src/SharpEmu.Libs/Gpu/Scheduling/GpuFatal.cs | 6 + .../KernelEventQueueCompatExports.Kevent.cs | 29 +- .../Kernel/KernelEventQueueCompatExports.cs | 96 ++++- .../Kernel/KernelMemoryCompatExports.cs | 5 +- src/SharpEmu.Libs/Pad/HostWindowInput.cs | 39 +- src/SharpEmu.Libs/PlayGo/PlayGoExports.cs | 18 +- .../VideoOut/VideoOutBackendServices.cs | 12 +- .../Gen5MslCodegenOptions.cs | 45 ++- .../Gen5MslShader.cs | 5 + .../Gen5MslTranslator.Alu.cs | 199 ++++++++++- .../Gen5MslTranslator.Pixel.cs | 6 +- .../Gen5MslTranslator.Resources.cs | 79 +++- .../Gen5MslTranslator.cs | 59 ++- .../Templates/prelude.msl | 26 ++ .../Resources/ResourceMaterializationCache.cs | 330 +++++++++++++++-- .../ResourceMaterializationProfile.cs | 3 +- .../MetalCompiledGuestShader.cs | 2 + .../V2/MetalBinaryArchive.cs | 15 +- .../SharpEmu.Gpu.Metal/V2/MetalCachedImage.cs | 85 ++++- .../V2/MetalCommandStream.cs | 27 +- .../V2/MetalCompileThread.cs | 117 ++++++ .../SharpEmu.Gpu.Metal/V2/MetalGpuWorker.cs | 22 +- .../V2/MetalImageCache.AliasTransitions.cs | 5 +- .../V2/MetalImageCache.ConsumerRuns.cs | 235 ++++++++++++ .../V2/MetalImageCache.PendingWritebacks.cs | 123 +++++++ .../V2/MetalImageCache.Tracking.cs | 2 + .../V2/MetalImageCache.WritebackFence.cs | 1 + .../SharpEmu.Gpu.Metal/V2/MetalImageCache.cs | 36 +- .../V2/MetalImageTransfer.Aliases.cs | 5 +- .../V2/MetalImageTransfer.cs | 160 ++++++--- .../V2/MetalLibraryCompilePool.cs | 7 +- .../V2/MetalLibraryPrewarm.cs | 37 +- .../V2/MetalPipelineCompiler.cs | 13 +- .../V2/MetalRenderHost.Descriptors.cs | 22 +- .../V2/MetalRenderHost.Pipelines.cs | 9 +- .../V2/MetalRenderHost.Rendering.cs | 32 +- .../V2/MetalRenderHost.StagedStorage.cs | 63 +++- .../V2/MetalRenderHost.WriteFences.cs | 31 +- .../SharpEmu.Gpu.Metal/V2/MetalRenderHost.cs | 65 +++- .../V2/MetalShaderPrewarm.cs | 75 ++-- .../V2/MetalStagedStorage.cs | 15 +- .../V2/MetalStaticPrecompile.cs | 67 ++-- .../SharpEmu.Gpu.Metal/V2/MetalTileViews.cs | 165 +++++++++ .../SharpEmu.Gpu.Metal/V2/MetalTiler.cs | 44 ++- .../V2/MetalTilerShaders.cs | 34 +- .../V2/MetalV2Diagnostics.cs | 19 + .../VulkanVideoPresenter.CommandStream.cs | 2 +- .../MetalCommandBufferPathLeakTests.cs | 22 +- .../MetalShaderObjectCostTests.cs | 35 +- .../NativeHeap.cs | 135 +++++++ .../MetalBinaryArchiveTests.cs | 22 ++ .../MetalCompileThreadTests.cs | 52 +++ .../MetalEndOfPipeOrderTests.cs | 72 +++- .../MetalGuestViewTests.cs | 58 ++- ...MetalImageBudgetTests.PendingWritebacks.cs | 104 ++++++ .../MetalImageTransferTests.Harness.cs | 5 + .../MetalImageTransferTests.TextureTiles.cs | 89 ++++- .../MetalImageTransferTests.Timing.cs | 96 ++++- .../MetalLossyTextureTests.cs | 21 +- .../MetalStagedStorageTests.cs | 62 ++++ .../Scenes/BarrierSceneTests.cs | 99 +++++ .../Scenes/ConsumerRunSceneTests.cs | 148 ++++++++ .../Scenes/EarlyDepthSceneTests.cs | 93 +++++ .../Scenes/GpuWordNoWaitSceneTests.cs | 283 +++++++++++++++ .../Scenes/GpuWordWaitSceneTests.cs | 2 + .../GpuRemoteChannelTests.cs | 14 + .../SplitTransportTests.cs | 1 + .../InputDeliveryBenchmarks.cs | 55 +++ .../RemoteGuestAddressSpaceTests.cs | 58 +++ .../SplitInProcessTests.cs | 49 ++- .../SplitPendingWritebackTests.cs | 276 ++++++++++++++ .../WorkerBracketTests.cs | 76 ++++ .../Audio/AjmTraceHandlerTests.cs | 47 +++ .../Cpu/ImportResultLoggingTests.cs | 14 +- .../Cpu/Sse4aPosixSignalRecoveryTests.cs | 55 +-- .../PixelDepthExportResolverTests.cs | 47 +++ .../Gpu/Rendering/RenderExecutorDrawTests.cs | 48 +++ .../Gpu/Split/SplitHookTests.cs | 4 +- .../GuestThreadQosPolicyTests.cs | 25 ++ .../Kernel/KernelEventQueueSignalSkipTests.cs | 338 ++++++++++++++++++ .../Loader/RosettaVectorStorePatchTests.cs | 12 +- .../Memory/GuestMemory/GuestViewTableTests.cs | 73 ++++ .../Pad/HostWindowInputTests.cs | 44 +++ .../Gen5MslMrtzExportTests.cs | 28 +- .../Goldens/exec-store-layout.msl | 28 +- .../Goldens/fmac-layout.msl | 28 +- .../Goldens/lds-layout.msl | 28 +- .../Goldens/loop-layout.msl | 28 +- .../Goldens/muls-layout.msl | 28 +- .../Goldens/pixel-layout.msl | 26 ++ .../Goldens/typed-load-layout.msl | 28 +- .../Goldens/typed-store-layout.msl | 28 +- .../Goldens/vertex-layout.msl | 26 ++ .../MetalNative.Benchmark.cs | 119 +++++- .../MslLoadWordsTests.cs | 27 +- .../MslNativeMinMaxSweepTests.cs | 249 +++++++++++++ .../MslTranslatorOverheadBenchTests.cs | 316 ++++++++++++++++ .../ResourceMaterializationCacheTests.cs | 52 +++ .../ResourceMaterializationContentKeyTests.cs | 193 ++++++++++ .../Benches.Materialization.cs | 205 +++++++++++ .../Descriptors.cs | 65 ++++ .../Harness.cs | 107 ++++++ .../NativeGuestMemory.cs | 43 +++ .../Program.cs | 12 + .../README.md | 38 ++ .../ResourceTestProgram.cs | 231 ++++++++++++ ...SharpEmu.Tools.MaterializationBench.csproj | 14 + 143 files changed, 7782 insertions(+), 583 deletions(-) create mode 100644 docs/performance/combined-audit.md create mode 100644 docs/performance/shader-serial.runsettings create mode 100644 src/SharpEmu.GpuSplit/PendingWritebackPages.cs create mode 100644 src/SharpEmu.HLE/GpuMemory/IPendingWritebackPages.cs create mode 100644 src/SharpEmu.HLE/Host/HostHelperThread.cs create mode 100644 src/SharpEmu.Libs/Audio/AjmTraceInterpolatedStringHandler.cs create mode 100644 src/graphics/SharpEmu.Gpu.Metal/V2/MetalCompileThread.cs create mode 100644 src/graphics/SharpEmu.Gpu.Metal/V2/MetalImageCache.ConsumerRuns.cs create mode 100644 src/graphics/SharpEmu.Gpu.Metal/V2/MetalImageCache.PendingWritebacks.cs create mode 100644 src/graphics/SharpEmu.Gpu.Metal/V2/MetalTileViews.cs create mode 100644 tests/SharpEmu.Gpu.Metal.LeakTests/NativeHeap.cs create mode 100644 tests/SharpEmu.Gpu.Metal.Tests/MetalCompileThreadTests.cs create mode 100644 tests/SharpEmu.Gpu.Metal.Tests/MetalImageBudgetTests.PendingWritebacks.cs create mode 100644 tests/SharpEmu.Gpu.Metal.Tests/Scenes/BarrierSceneTests.cs create mode 100644 tests/SharpEmu.Gpu.Metal.Tests/Scenes/ConsumerRunSceneTests.cs create mode 100644 tests/SharpEmu.Gpu.Metal.Tests/Scenes/EarlyDepthSceneTests.cs create mode 100644 tests/SharpEmu.Gpu.Metal.Tests/Scenes/GpuWordNoWaitSceneTests.cs create mode 100644 tests/SharpEmu.GpuSplit.Tests/InputDeliveryBenchmarks.cs create mode 100644 tests/SharpEmu.GpuSplit.Tests/SplitPendingWritebackTests.cs create mode 100644 tests/SharpEmu.GpuSplit.Tests/WorkerBracketTests.cs create mode 100644 tests/SharpEmu.Libs.Tests/Audio/AjmTraceHandlerTests.cs create mode 100644 tests/SharpEmu.Libs.Tests/Kernel/KernelEventQueueSignalSkipTests.cs create mode 100644 tests/SharpEmu.ShaderCompiler.Metal.Tests/MslNativeMinMaxSweepTests.cs create mode 100644 tests/SharpEmu.ShaderCompiler.Metal.Tests/MslTranslatorOverheadBenchTests.cs create mode 100644 tools/SharpEmu.Tools.MaterializationBench/Benches.Materialization.cs create mode 100644 tools/SharpEmu.Tools.MaterializationBench/Descriptors.cs create mode 100644 tools/SharpEmu.Tools.MaterializationBench/Harness.cs create mode 100644 tools/SharpEmu.Tools.MaterializationBench/NativeGuestMemory.cs create mode 100644 tools/SharpEmu.Tools.MaterializationBench/Program.cs create mode 100644 tools/SharpEmu.Tools.MaterializationBench/README.md create mode 100644 tools/SharpEmu.Tools.MaterializationBench/ResourceTestProgram.cs create mode 100644 tools/SharpEmu.Tools.MaterializationBench/SharpEmu.Tools.MaterializationBench.csproj diff --git a/docs/macos/split-protocol.md b/docs/macos/split-protocol.md index 76544bbbc..756732fcc 100644 --- a/docs/macos/split-protocol.md +++ b/docs/macos/split-protocol.md @@ -1,6 +1,6 @@ # Two-process split: wiring contract (Phase 2) -Status: implemented behind the flag, 2026-10-24 (SharpEmu.GpuSplit; transport protocol 2). Design: `SPLIT-PLAN.md` +Status: implemented behind the flag, 2026-10-24 (SharpEmu.GpuSplit; transport protocol 7). Design: `SPLIT-PLAN.md` (repo-external draft), Phase 0 rates: commit `fc963f02`, shared memory: `docs/macos/split-shared-memory.md`, transport: `src/SharpEmu.GpuRemote`. @@ -31,7 +31,11 @@ page watches 100-2600/s (batch); submissions 30 and 200-350/s; SubmitDone/flips All records are little-endian blittable structs (`GpuSplitMessages.cs`) on the transport rings; synchronous calls use `SyncCallTable` both ways: A → B (`SyncCalls`, signal-safe on the A side) and B → A (`ReverseSyncCalls`, ordinary B -threads). Add new kinds at the end of the enums; bump `GpuRemoteProtocol.Version` (now 2). +threads). Add new kinds at the end of the enums; bump `GpuRemoteProtocol.Version` (now 7). + +Revision 7 combines frame-boundary messages, input-change/display reservations and +the pending-writeback page summary. Independent revisions 4, 5 and 6 are rejected +on attach because their shared layouts differ. Both processes must use revision 7. ### Start @@ -70,7 +74,7 @@ many, so they come after A's earlier records. | `BracketBegin` | kind, barrier (AfterPendingCommandStreams only) | bracket id | `GuestGpuMemory.RunMappingChange` / `Unregister` in A (kind MappingChange: the relay's `TryRunAfterPendingWork`) and `RunAfterPendingCommandStreams` (kind AfterPendingCommandStreams). B parks its GPU worker (`WorkerBracket`: B's own `RunMappingChange` / `RunAfterPendingCommandStreams` hand the worker a park loop); A then runs the work itself, as the GPU queue thread (`RemoteGpuQueueRelay.IsGpuQueueThread`). | | `BracketEnd` | bracket id, viewLogSequence | - | B syncs its replica that far on the parked worker and lets it go. | | `Fault` | kind, address, size | handled, restoration version | From A's signal handler (`RemoteImageStore.MarkCpuWrite`, allocation-free). B runs its stores' fault path, then waits until A applied every protection B requested so far: the guest retries the access as soon as this answers. Every write fault crosses (the page faulted because B protected it). When no store handled it, B answers its guard's write-restoration version of the page: a watch removed before the fault reached the store. A retries the store once per version when the page is writable (one process's `TryRetryReleasedWrite`, from B's guard). | -| `CpuWrite` | address, size | handled | A managed write before it lands, when the summary marks it: B's `MarkCpuWrite` (the cache publishes a GPU-written image before the write lands over it). From A's reverse-call threads and from inside a bracket it is sent as a `CpuWrite` record instead (there B's worker cannot take the writeback: the in-process cache's rule for a store from the worker). No barrier. | +| `CpuWrite` | address, size | handled | A managed write before it lands, when both the interest and pending-writeback summaries mark it (all interested pages count as pending until B publishes the pending summary live): B's `MarkCpuWrite` (the cache publishes a GPU-written image before the write lands over it). From A's reverse-call threads and from inside a bracket it is sent as a `CpuWrite` record instead (there B's worker cannot take the writeback: the in-process cache's rule for a store from the worker). No barrier. | | `SpanRegister` / `SpanProtect` / `SpanUnregister` | address, size, protection, bracket id | (handled) | B's `GuestGpuMemory` follows A's spans and permission ledger (`IGuestSpanObserver`, `RemoteImageStore.Unregister`). B's page guard owns the host protections of every span: A's `Register` / `NoteProtected` apply none of their own when the call succeeds (A's watch-less guard would hand back, for a moment, rights B's watches took); on a failed call A applies the plain guest protection. An unregistration inside a bracket runs on B's parked worker. B waits for the protections it requested before it answers, so they land before A unmaps, remaps or lets the guest go on. | | `Query` | which | value | Performance counters, HDR output support, mesh pipelines. | diff --git a/docs/performance/combined-audit.md b/docs/performance/combined-audit.md new file mode 100644 index 000000000..7278b8f4e --- /dev/null +++ b/docs/performance/combined-audit.md @@ -0,0 +1,281 @@ + + +# Combined synthetic performance audit + +This integrates the round-one audit (`cde04b713e60de7ab02dfa092e8f91d086df0c83`) +and the valid round-two changes. Validation uses fabricated guest programs, +synthetic descriptors and real Metal device tests. No commercial game was run. +Microkernel, byte-work and microbenchmark results do not establish game performance. + +## Integration and correctness + +The combined wire protocol is revision **7**. Independent revisions 4 (frame boundary), +5 (input/display reservation) and 6 (pending writeback pages) are incompatible and +rejected. Both processes use the same combined shared-state layout. Compatibility +and pinned x64-to-arm64 smoke tests cover the boundary. + +Host completion work stays pending while its callback executes. Waiting through a +committed serial includes every callback at that serial, including a second callback +whose publication has not finished. Monitor waits and producer ordering are retained. +GPU-written image claims survive retirement and transfer to in-flight writeback claims. +The pending bitmap is cleared only when the image cache publishes completed ownership. + +The six former Libs failures are corrected without removing their behavioral checks: + +- Import result sampling constructs the current private import entry and invokes the + current `in ImportStubEntry` API; every occurrence/result assertion remains. +- Fabricated POSIX frames use a fully initialized backend with process handler + installation disabled. The fixture owns/disposes it and restores the previous + registration. Register, ABI, instruction-pointer and extended-vector checks remain. +- Two vector stores can share one grouped trampoline. The fixture tests that form + and a branch-separated two-trampoline form. Both must expand two stores, use no + red zone, report no failed sites and preserve every source/destination byte. + +The scattered-view fixture aligns backing inside its own reserved mapping. The lossy +footprint fixture uses a texture larger than Metal's minimum allocation page. Neither +production backing refusals nor strict footprint/data assertions were relaxed. +The pending-writeback fake-store fixture now drains prior CPU-write records before +its direct render calls. Real GPU renders enter the ordered worker stream; the +fixture's out-of-stream render could otherwise be invalidated by an older record. +The in-flight blocking assertion, exact bytes and zero-lost-bytes checks remain. + +The combined leak suite exposed an allocator-counter defect that also reproduces +as seven failures on unchanged `cde04b713`. On this translated macOS 27.2 host, +`malloc_zone_statistics` reported 9,137 more blocks for 3,000 label flushes while +independent `heap` enumeration fell from 85,055 to 84,982 live nodes; live command +buffers grew from one to two, rather than by thousands. One aggregate counter also +wrapped through 32 bits. Leak tests now enumerate the allocator's actual in-use +ranges. The original per-flush and release-fraction assertions/thresholds remain. +A separate isolated-zone regression verifies exactly 32 allocated blocks and their +complete release. This is a measurement correction, not a claim that a production +leak was fixed. + +## Cold materialization + +Descriptor evaluation and flattened-table evaluation usually record two ordered runs. +The recorder now merges those runs in linear time instead of sorting the complete +list. Already ordered reads stay untouched; irregular graphs retain the general sort. +The merge keeps every read, clean requirement and descriptor/table flag. Entry range +construction, fresh validation, descriptor-content keys, bounds, bias, user-data and +mapping checks are unchanged. Retained snapshots remain immutable. + +After actual generation/variant evictions and 32 consecutive misses, a bounded +512-tag admission history declines retention for novel contents. Every draw still +performs a full walk. Recurring contents require a second, fully recorded walk +before normal admission and fresh validation can produce a hit. Tag collisions can +only add work, never authorize a hit. A hit resets the miss streak. Set +`SHARPEMU_RESOURCE_CACHE_ADMISSION=0` to disable this policy. + +The preliminary walk owns its user-data array and keeps its outputs local. A failed +recorded rewalk leaves the caller's original snapshot and specialization intact; +a successful rewalk publishes only its own results. Regression tests cover both +failure atomicity and reuse of the caller's input array, in addition to cold scans, +recurrence, resident unreadability, replacement mappings and retained snapshots. + +An instrumented diagnostic run recorded 361,272 merged lists, zero already ordered +lists and zero general-sort fallbacks for the synthetic materialization workload. +Sorting/merging accounted for 121.372 ms of 479.539 ms in entry building in that run. +Those instrumented timings are diagnostic, not the unprofiled A/B result. + +The standalone harness is `tools/SharpEmu.Tools.MaterializationBench`. It measures the +same 8-image/2-sampler/4-buffer workloads for uncached walks, repeated hits, distinct +cold descriptors, stale descriptors at the same address, repeated pointers and moving +rings. The general tests also compare reordered read shapes with fresh walks and +check descriptor/bounds changes and unreadable resident memory. + +## Workstream dispositions + +| Workstream | Integrated or rejected | +|---|---| +| K1 | Disabled continuation-trace guards integrated. Temporary ELF survey field/harness omitted; speculative spin-policy changes rejected. | +| K2 | Content-key/pointer improvements and regressions integrated, with the additional linear read merge. Fresh-read safety is retained; see A/B results below. | +| K3 | No additional GC/tiering policy change: no general benefit established. | +| K4 | Event-driven input and one-use display reservation reuse; owner and intervening-slot invalidation tests retained. Input benchmark is explicit opt-in. | +| K5 | Existing frame-boundary commit and gate tests integrated; the gate is released before the boundary wait. | +| K6 | Lock-free image lookup rejected without a reader-lifetime protocol; no recovered source to integrate. | +| K7 | GPU-pristine staging snapshots and exact-byte/concurrent-CPU-write regressions retained with their kill switch. | +| K8 | Compile-thread QoS and asynchronous preparation retained; synthetic throughput is a priority tradeoff, not a universal gain. Command-thread suspension rejected. | +| K9 | Early-depth decoding, side-effect handling and device regressions retained. | +| K10 | Exact store/barrier tracking plus completion-publication correction and held-second-callback regression retained. No blanket cross-pass elision. | +| K11 | Further fence/depth-store elision rejected without producer/consumer proof. | +| K12 | Existing alias-transition commit plus narrow consumer runs, GPU-write CPU-dirty marking and four exact-data device regressions retained. | +| K13 | Owned cached texture views retained. Concurrent tiling remains opt-in: representative GPU writebacks were slower; serial is the default. Alias property reads use the shared texture-info helper. | +| K14 | Write-fence/prefetch work and aligned scattered-view fixtures retained; fatal/refusal behavior unchanged. | +| K15 | Historical group8 binary/evidence artifacts remain outside the PR. No unsupported latency claim or new runtime launch. | +| K16 | Parallel planning rejected without immutable descriptor/mapping lifetime proof; no recovered source to integrate. | +| K17 | Mapping-generation-aware translator and helper-thread work retained. Zero-allocation fixture measures before assertions. Single-view latency and per-translator storage remain tradeoffs. | +| K18 | Pending-writeback summary and real retirement/fenced-completion tests retained in combined protocol 7. Watch-arm mprotect overlap rejected. | +| K19 | Waiter registration, bounded lost-wake regression, audio/environment/disabled-formatting work retained. Decoder-result pooling and shorter unnotified polls rejected. | +| K20 | Disabled formatting subset is in K1. Continuation pooling and ABI conversion rejected without ownership/crash proof. Snapshot cloning retained. | +| K21 | Raw dword loads, min/max and reciprocal work, boundary/wrap parity tests and strict precise-sqrt regression retained. Contraction remains opt-in. Threadgroup hints/table padding are configurable experiments without an accepted speedup claim. No captured-corpus replay or wave-barrier skip claim. | + +## Measurements and validation + +The historical compiler controls used the same benchmark executable/dependencies, +x64 .NET 10.0.3 on an Apple Silicon host, 5 s warmup and seven 200 ms samples. +The sequence was main/cde/R2/merge/merge/R2/cde/main. Each cell below contains both +run medians, in ns/op, for distinct cold descriptors (3c). Main is +`42eec4aa76b85e3e03d6e12a4bb29adb7a1a56fa`; cde is the round-one revision above. +R2 is the recovered compiler before the linear merge; merge predates admission. + +| Compiler | Tiering disabled | Tiered/PGO | +|---|---:|---:| +| Historical main | 10,152.5 / 10,650.1 | 5,646.6 / 6,714.4 | +| Round one (cde) | 9,470.3 / 11,532.4 | 6,415.2 / 6,810.3 | +| Recovered R2 | 9,580.9 / 10,153.5 | 6,665.2 / 6,588.7 | +| Linear merge | 10,158.7 / 14,378.2 | 6,957.1 / 12,913.1 | + +All four retained 8,264 B/op with tiering disabled, 8,160 B/op tiered. The merge +alone did not establish a timing win. The earlier R2 repeated-pointer improvement +and cold regression therefore remain qualified; this PR does not claim all cold +latency regressions against historical main are resolved. + +Other recovered measurements are narrower comparisons: K17 strided/fragmented +translation removed about 45-47 B/op, while its single warm-view case moved from +4.3-4.5 to 4.9-5.0 ns and its inline storage costs about 40 KiB per translator. +K13 cached serial texture views reduced selected CPU transfer costs; concurrent +GPU writeback regressed representative depth/RGBA8/R8 shapes and stays opt-in. +K21 grouped loads/minmax/reciprocal microkernels improved selected GPU timestamps, +with precise sqrt retained; those are not whole-shader or game measurements. +K8 Utility QoS reduces background compile throughput and has no established +foreground win. K4's two transport halves in one process reduced synthetic input +p50 from 3.1-3.5 to about 0.1 ms; this is not guest draw latency. + +Reproduce the final policy comparison with the checked-in benchmark, holding the +executable, architecture and dependencies fixed: + +```sh +~/.dotnet-x64/dotnet build tools/SharpEmu.Tools.MaterializationBench -c Release -m:2 -p:UseSharedCompilation=false -nr:false +# Run 0,1,1,0 in separate processes, then repeat with DOTNET_TieredCompilation=1. +SHARPEMU_RESOURCE_CACHE_ADMISSION=0 DOTNET_TieredCompilation=0 DOTNET_TieredPGO=1 PA_WARMUP_MS=5000 PA_RUNS=7 PA_RUN_MS=200 ~/.dotnet-x64/dotnet artifacts/bin/Release/net10.0/SharpEmu.Tools.MaterializationBench.dll +``` + +Validation command forms (build/test commands ran sequentially under the shared +machine lock; the device-required flag was confined to Metal suites): + +```sh +~/.dotnet-x64/dotnet build SharpEmu.slnx -c Release -m:2 -p:UseSharedCompilation=false -nr:false +~/.dotnet-x64/dotnet build tools/SharpEmu.Tools.GpuProcessSmoke -r osx-x64 -c Release -m:2 +~/.dotnet-x64/dotnet test tests/PROJECT -c Release -m:2 -p:UseSharedCompilation=false -nr:false --blame-hang-timeout 120s --logger trx +SHARPEMU_TEST_REQUIRE_DEVICE=1 ~/.dotnet-x64/dotnet test tests/SharpEmu.ShaderCompiler.Metal.Tests -c Release -m:2 --settings docs/performance/shader-serial.runsettings --blame-hang-timeout 120s +SHARPEMU_TEST_REQUIRE_DEVICE=1 ~/.dotnet-x64/dotnet test tests/SharpEmu.Gpu.Metal.Tests -c Release -m:2 --blame-hang-timeout 120s +reuse lint +python3 -m unittest discover -s tests/scripts -v +# Also run each scripts/macos/test-*.py with python3 and -v. +``` + +Raw logs, TRX, controls, source/assembly hashes and interrupted diagnostics remain +local audit evidence, excluded from the PR. The tables below summarize final +source-bound results; historical/interrupted runs are not counted as final passes. + + +The opt-in native-memory pass pins x64, .NET 10.0.3 and 16,384-byte kernel pages. +Set `SHARPEMU_TEST_REQUIRE_MAPPED_COPY_NATIVE`, `SHARPEMU_TEST_REQUIRE_MAPPED_PUBLICATION_NATIVE`, +`SHARPEMU_TEST_REQUIRE_MAPPED_COPY_PROTECTION_NATIVE`, `SHARPEMU_TEST_REQUIRE_BACKING_NATIVE`, +`SHARPEMU_TEST_REQUIRE_OWNED_NATIVE`, `SHARPEMU_TEST_REQUIRE_GUEST_MIRROR_PROCESS` and +`SHARPEMU_TEST_REQUIRE_PVM_MAPPED_COPY_NATIVE` to `1`, with +`SHARPEMU_TEST_EXPECT_ARCH=X64`, `SHARPEMU_TEST_EXPECT_FRAMEWORK='.NET 10.0.3'` and +`SHARPEMU_TEST_EXPECT_KERNEL_PAGE=16384`, then run Host.Memory and Core.Memory separately. +The final runs pass 517 and 7 tests respectively, with zero skips. All six guest +replica process variants complete, including full/split-size shm and chunked +backings, remapping and exact shared-byte checks. The earlier 120 s aborted run +and failed stack collection remain local evidence: no reproducible production +fault was localized, so this is a successful rerun, not a claimed stall fix. + + +| Final project run | Passed | Failed | Skipped | +|---|---:|---:|---:| +| Libs | 8,943 | 0 | 5 | +| ShaderCompiler | 2,167 | 0 | 1 | +| ShaderCompiler.Metal (device required, serial) | 3,296 | 0 | 0 | +| Gpu.Metal (device required) | 1,573 | 0 | 7 | +| Gpu.Metal.LeakTests (device required) | 19 | 0 | 0 | +| GpuRemote (including pinned x64/arm64 smoke) | 56 | 0 | 0 | +| GpuSplit (after fixture ordering correction) | 34 | 0 | 0 | +| Application | 80 | 0 | 0 | +| Fonts | 13 | 0 | 0 | +| SourceGenerators | 36 | 0 | 0 | +| Host.Memory (native enabled) | 517 | 0 | 0 | +| Core.Memory (native enabled) | 7 | 0 | 0 | +| **Total, one final run per project** | **16,741** | **0** | **13** | + +The five Libs skips need unavailable native FFmpeg plugins; ShaderCompiler's skip +needs an unavailable captured fused-program fixture. Metal's seven existing skips +cover depth/stencil, shadow precision, HZB, line-width and packed-float cases; no +new skip was added. The default Host.Memory run separately passed 486 with six +opt-in skips; default Core.Memory ran no cases and skipped its native theory. +Those are not counted again in the total. Split's original 33-pass/1-fail run is +retained; the corrected full run and five independent repeats of the affected +blocking case pass without changing its 100 ms/10 s assertions. + +The Release solution build completed with zero errors and 347 warnings (including +existing platform/volatile-field analyzer warnings). The pinned osx-x64 smoke build +completed with zero errors and warnings. The final project runs in the table exited zero; +the earlier aborted native run and initial Split failure are separate evidence. + +Final policy comparison: A=admission disabled, B=enabled, in ABBA order. +Each median cell retains both process medians (ns/op); p95 cells show the range +of the two reported per-process p95 samples. These are tails of seven run samples, +not per-operation latency percentiles. B/op is measured on the benchmark thread; +GC is forced between samples, not controlled during a sample. No instrumented +phase profiling was enabled. + +| Mode / workload | A medians ns | B medians ns | A / B p95 ranges ns | A / B bytes/op | +|---|---:|---:|---:|---:| +| tieroff: 3a uncached walk | 4,852.5 / 3,992.3 | 3,805.0 / 6,371.7 | 4,107.3-6,109.8 / 4,097.1-11,755.5 | 3,152.0 / 3,152.0 | +| tieroff: 3b warm hit | 244.0 / 254.9 | 657.7 / 337.0 | 308.7-311.8 / 404.5-822.9 | 64.0 / 64.0 | +| tieroff: 3c cold distinct | 17,693.7 / 17,421.2 | 12,938.9 / 11,394.1 | 20,498.7-31,232.0 / 14,505.8-14,541.2 | 8,264.0 / 6,379.9-6,382.6 | +| tieroff: 3d stale same address + writes | 30,318.0 / 19,327.3 | 15,891.0 / 11,926.6 | 20,554.5-49,633.8 / 13,029.8-18,005.8 | 8,392.2-8,392.3 / 7,032.6-7,035.4 | +| tieroff: 3f repeated pointers | 568.6 / 816.0 | 555.8 / 435.1 | 624.2-1,461.0 / 481.3-648.2 | 64.0 / 64.0 | +| tieroff: 3g moving ring + writes | 1,757.0 / 905.4 | 1,003.9 / 853.2 | 1,067.5-2,184.9 / 993.1-1,022.0 | 856.0 / 856.0 | +| tieroff: 3e write-only control | 243.3 / 223.3 | 184.0 / 165.2 | 255.5-353.6 / 187.3-203.4 | 688.0 / 688.0 | +| tiered: 3a uncached walk | 2,112.1 / 2,802.2 | 2,570.7 / 2,030.8 | 2,306.1-3,574.2 / 2,256.5-3,211.5 | 3,152.0 / 3,152.0 | +| tiered: 3b warm hit | 178.0 / 531.4 | 154.4 / 168.9 | 204.5-1,342.4 / 177.4-189.1 | 64.0 / 64.0 | +| tiered: 3c cold distinct | 9,735.4 / 40,677.9 | 4,209.6 / 4,339.9 | 11,797.7-61,512.0 / 4,308.5-4,370.5 | 8,160.0 / 6,378.9-6,384.2 | +| tiered: 3d stale same address + writes | 11,998.3 / 25,968.5 | 5,689.3 / 5,803.2 | 14,114.2-48,384.9 / 6,639.0-8,241.3 | 8,392.2 / 7,032.6-7,037.8 | +| tiered: 3f repeated pointers | 312.0 / 948.0 | 270.1 / 225.8 | 359.5-1,239.4 / 237.2-280.3 | 64.0 / 64.0 | +| tiered: 3g moving ring + writes | 672.0 / 3,886.5 | 700.4 / 507.5 | 726.6-15,023.1 / 585.1-784.2 | 856.0 / 856.0 | +| tiered: 3e write-only control | 141.6 / 687.8 | 125.1 / 113.3 | 153.5-1,320.9 / 116.5-135.4 | 688.0 / 688.0 | + +The consistent result is lower cold/stale allocation. Warm hits and the repeated +pointer/ring workloads retain their allocations and steady-state hits. Host load +and the uncached/write-only controls drift substantially; the latency numbers +are qualified synthetic observations, not proof of a general timing win over +historical main/cde. Admission deliberately adds a second walk for recurring +contents during a saturated miss streak. Fresh-walk regression comparisons, not +benchmark hit counters alone, establish descriptor/readability/snapshot correctness. + + +REUSE 3.3 lint passes for all 2,787 files, with zero missing/invalid licenses or +read errors. Python/macOS script checks finish 52 passed and three existing +`task_for_pid` permission skips (passwordless sudo is unavailable); no assertion +was removed or new skip introduced. Staged and unstaged whitespace checks pass. + +Final validation and benchmark source trees (Git tree IDs, so the code remains +identifiable independently of this report's later commit): + +- `src`: `8ff3e09deb2a43995515ad2da4ea6c4f5abaff17` +- `tests`: `4bfe34bfe4a09626ce885f6a7405d0c90ce43076` +- `tools/SharpEmu.Tools.MaterializationBench`: `008a7ca713e6ca860699d7e908552e70dffd5038` + +The final benchmark uses the same x64 .NET 10.0.3 host and executable in all eight +arms. Its SHA256 is `f9dfe18115e593cd9f7e30ff2b58572254572eb6ff9cbb4e2141111f963773bc`; +the compiler assembly SHA256 is `88c2a3f45f50435215a507fc418a93f46e5fa45ea6d7665aaee7cf7be1091b0b`. + + +The first additional admission-disabled ShaderCompiler run stalled after 170 passes +and aborted without assertion failures. Its native sample, unresponsive diagnostic +IPC and failed hang-dump collection are retained locally; only its verified owned +test host was terminated after SIGTERM failed to stop it. No production change is +claimed to fix that unlocalized intermittent test-host stall. A separate retry uses +the same assertions, detailed logging, the 120 s inactivity bound and an outer +180 s process-group bound. This is separate from the enabled final run above. + +The detailed parallel retry also reached its outer 180 s bound. The subsequent +full admission-disabled run with `docs/performance/shader-serial.runsettings` +passed 2,167 / failed 0 / skipped 1 in 12.4 s. No assertions, explicit concurrency +inside tests or thresholds changed. Parallel-suite stall diagnosis remains open; +it is not counted as a resolved regression or substituted into the successful +project totals. The final incremental Release solution build passes with zero +warnings and errors on the exact source trees listed above. diff --git a/docs/performance/shader-serial.runsettings b/docs/performance/shader-serial.runsettings new file mode 100644 index 000000000..e85e2d6c8 --- /dev/null +++ b/docs/performance/shader-serial.runsettings @@ -0,0 +1,10 @@ + + + + false + 1 + true + 60 + + diff --git a/src/SharpEmu.Core/Cpu/Native/DirectExecutionBackend.cs b/src/SharpEmu.Core/Cpu/Native/DirectExecutionBackend.cs index 48a347a5c..73a1bb852 100644 --- a/src/SharpEmu.Core/Cpu/Native/DirectExecutionBackend.cs +++ b/src/SharpEmu.Core/Cpu/Native/DirectExecutionBackend.cs @@ -1367,7 +1367,12 @@ private static void RestoreActiveExecutionThread( _activeGuestThreadYieldReason = previousYieldReason; } - public unsafe DirectExecutionBackend(IModuleManager moduleManager) + public DirectExecutionBackend(IModuleManager moduleManager) : this(moduleManager, installFaultHandling: true) + { + } + + // Fabricated signal-frame tests need initialized recovery state without installing process-wide handlers. + internal unsafe DirectExecutionBackend(IModuleManager moduleManager, bool installFaultHandling) { if (RuntimeInformation.ProcessArchitecture != Architecture.X64) { @@ -1430,9 +1435,9 @@ public unsafe DirectExecutionBackend(IModuleManager moduleManager) Console.Error.WriteLine( "[LOADER][WARN] Worker abort stub unavailable; TBB execute-fault recover will use host_exit"); } - SetupExceptionHandler(); + if (installFaultHandling) SetupExceptionHandler(); // Cover the Astro TBB spawn storm (often 8–12 concurrent tbb_thead). - PrewarmNativeGuestWorkers(Math.Max(NativeWorkerMaxConcurrent, 4)); + if (installFaultHandling) PrewarmNativeGuestWorkers(Math.Max(NativeWorkerMaxConcurrent, 4)); } NativeCpuSessionStatistics INativeCpuSessionStatisticsProvider.LastSessionStatistics => @@ -5079,8 +5084,11 @@ void RunContinuation() var previousLastError = LastError; try { - TraceGuestContext( - $"continuation-enter reason={reason} managed={Environment.CurrentManagedThreadId} guest=0x{GuestThreadExecution.CurrentGuestThreadHandle:X16} fiber=0x{GuestThreadExecution.CurrentFiberAddress:X16} captured_guest=0x{currentGuestThreadHandle:X16} captured_fiber=0x{currentFiberAddress:X16} restore_guest={restoreGuestThread} restore_fiber={restoreFiber}"); + if (_logGuestContext) + { + TraceGuestContext( + $"continuation-enter reason={reason} managed={Environment.CurrentManagedThreadId} guest=0x{GuestThreadExecution.CurrentGuestThreadHandle:X16} fiber=0x{GuestThreadExecution.CurrentFiberAddress:X16} captured_guest=0x{currentGuestThreadHandle:X16} captured_fiber=0x{currentFiberAddress:X16} restore_guest={restoreGuestThread} restore_fiber={restoreFiber}"); + } LastError = null; exitReason = ExecuteGuestContinuationEntry( context, @@ -5099,8 +5107,11 @@ void RunContinuation() finally { _activeGuestThreadState = previousGuestThreadState; - TraceGuestContext( - $"continuation-exit reason={reason} managed={Environment.CurrentManagedThreadId} guest=0x{GuestThreadExecution.CurrentGuestThreadHandle:X16} fiber=0x{GuestThreadExecution.CurrentFiberAddress:X16} exit={exitReason}"); + if (_logGuestContext) + { + TraceGuestContext( + $"continuation-exit reason={reason} managed={Environment.CurrentManagedThreadId} guest=0x{GuestThreadExecution.CurrentGuestThreadHandle:X16} fiber=0x{GuestThreadExecution.CurrentFiberAddress:X16} exit={exitReason}"); + } LastError = previousLastError; if (restoreFiber) { @@ -5136,8 +5147,11 @@ void RunContinuation() } else if (runner is not null) { - TraceGuestContext( - $"continuation-inline reason={reason} managed={Environment.CurrentManagedThreadId} guest=0x{currentGuestThreadHandle:X16} fiber=0x{currentFiberAddress:X16}"); + if (_logGuestContext) + { + TraceGuestContext( + $"continuation-inline reason={reason} managed={Environment.CurrentManagedThreadId} guest=0x{currentGuestThreadHandle:X16} fiber=0x{currentFiberAddress:X16}"); + } RunContinuation(); } else diff --git a/src/SharpEmu.GpuHost/GpuHostMain.cs b/src/SharpEmu.GpuHost/GpuHostMain.cs index 04b6869bc..2011d2fb5 100644 --- a/src/SharpEmu.GpuHost/GpuHostMain.cs +++ b/src/SharpEmu.GpuHost/GpuHostMain.cs @@ -100,9 +100,12 @@ public static int Main(string[] args) // Calls that wait for the GPU (a frame boundary, a fault's writeback, a bracket) each // hold a thread; enough of them that a write fault never queues behind those. client.SyncThreadCount = 8; + // The ring and the calls are on the GPU worker's path (and on the guest's): they start + // at its class, which a thread CoreCLR creates cannot take later. + client.StartThread = static (name, stackBytes, body) => SharpEmu.HLE.Host.HostHelperThread.Start(name, stackBytes, body).Join; var exitCode = GpuProcessClient.ExitClean; - var ring = new Thread(() => + var ring = SharpEmu.HLE.Host.HostHelperThread.Start("SharpEmu GPU host ring", 4 * 1024 * 1024, () => { try { @@ -113,12 +116,7 @@ public static int Main(string[] args) // Ends the pump below, after asking the window to close. HostMainThread.Shutdown(); } - }, 4 * 1024 * 1024) - { - IsBackground = true, - Name = "SharpEmu GPU host ring", - }; - ring.Start(); + }); HostMainThread.Pump(); _ = ring.Join(TimeSpan.FromSeconds(5)); return exitCode; diff --git a/src/SharpEmu.GpuRemote/GpuProcessClient.cs b/src/SharpEmu.GpuRemote/GpuProcessClient.cs index 2b8a3a17c..a6f6170cf 100644 --- a/src/SharpEmu.GpuRemote/GpuProcessClient.cs +++ b/src/SharpEmu.GpuRemote/GpuProcessClient.cs @@ -35,7 +35,7 @@ public sealed class GpuProcessClient : IDisposable private readonly int _parentProcessId; private readonly ManualResetEventSlim _parentExited = new(false); - private readonly List _syncThreads = new(); + private readonly List> _syncThreads = new(); private volatile bool _stopRequested; private int _disposed; @@ -66,6 +66,13 @@ private GpuProcessClient(GpuRemoteChannel channel, int parentProcessId, GpuPeerI /// Threads serving the A → B call table; read by . public int SyncThreadCount { get; set; } = 4; + /// + /// Starts a call thread (name, stack bytes, body) and returns its join; read by . + /// Null starts managed threads. The host sets one that starts them at a QoS class, which this + /// assembly cannot reach. + /// + public Func>? StartThread { get; set; } + /// Taken around every write to the B → A ring (see the type remarks). public object OutboundLock { get; } = new(); @@ -145,12 +152,15 @@ public int Run() { for (var index = 0; index < Math.Max(1, SyncThreadCount); index++) { - var thread = new Thread(ServeSyncCalls, 1024 * 1024) + var name = $"SharpEmu GPU host sync calls {index}"; + if (StartThread is { } start) { - IsBackground = true, - Name = $"SharpEmu GPU host sync calls {index}", - }; - _syncThreads.Add(thread); + _syncThreads.Add(start(name, 1024 * 1024, ServeSyncCalls)); + continue; + } + + var thread = new Thread(ServeSyncCalls, 1024 * 1024) { IsBackground = true, Name = name }; + _syncThreads.Add(thread.Join); thread.Start(); } @@ -167,9 +177,9 @@ public int Run() { _stopRequested = true; Channel.SyncCalls.WakeAll(); - foreach (var thread in _syncThreads) + foreach (var join in _syncThreads) { - _ = thread.Join(TimeSpan.FromSeconds(2)); + _ = join(TimeSpan.FromSeconds(2)); } Service?.Stop(); @@ -192,9 +202,9 @@ public void Dispose() _stopRequested = true; Channel.WakeAll(); - foreach (var thread in _syncThreads) + foreach (var join in _syncThreads) { - _ = thread.Join(TimeSpan.FromSeconds(2)); + _ = join(TimeSpan.FromSeconds(2)); } Channel.Dispose(); diff --git a/src/SharpEmu.GpuRemote/GpuRemoteProtocol.cs b/src/SharpEmu.GpuRemote/GpuRemoteProtocol.cs index 2cf293d8c..fda4b01d4 100644 --- a/src/SharpEmu.GpuRemote/GpuRemoteProtocol.cs +++ b/src/SharpEmu.GpuRemote/GpuRemoteProtocol.cs @@ -11,8 +11,10 @@ public static class GpuRemoteProtocol /// Bumped on any change to the region layout or a message format. /// 2: the B → A call table, the application block and the split's messages (GpuSplitMessages.cs). /// 3: . - /// 4: and . - public const uint Version = 4; + /// 4: frame-boundary messages. 5: input-change and display reservations. + /// 6: pending-writeback pages. 7: combined frame, input and writeback layout. + public const uint Version = 7; + } /// Record types on the two rings. 0 is never used; is reserved. diff --git a/src/SharpEmu.GpuRemote/GpuSplitMessages.cs b/src/SharpEmu.GpuRemote/GpuSplitMessages.cs index e0f99722a..d9eccfd68 100644 --- a/src/SharpEmu.GpuRemote/GpuSplitMessages.cs +++ b/src/SharpEmu.GpuRemote/GpuSplitMessages.cs @@ -504,6 +504,12 @@ public struct ReserveFlipResponse /// A's IsHdrOutputRequested after the reservation. public uint HdrRequested; public ulong RequestId; + + /// + /// The display buffer the flip names, read with the reservation so the GPU process does not + /// ask for it in a second call ( is zero when none). + /// + public DisplayBufferResponse Buffer; } [StructLayout(LayoutKind.Sequential)] diff --git a/src/SharpEmu.GpuSplit/PendingWritebackPages.cs b/src/SharpEmu.GpuSplit/PendingWritebackPages.cs new file mode 100644 index 000000000..a974b235c --- /dev/null +++ b/src/SharpEmu.GpuSplit/PendingWritebackPages.cs @@ -0,0 +1,180 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using System.Runtime.CompilerServices; +using SharpEmu.HLE.GpuMemory; + +namespace SharpEmu.GpuSplit; + +/// +/// B's pending-writeback summary (): a claim count per 4 KiB page, +/// with the page's shared bit set while the count is above zero. +/// +/// +/// Counts live in 1 GiB chunks made on first use (1 MiB each), so a claim costs one add per page; +/// a render target of a few thousand pages is written back many times a frame. A page whose count +/// drops to zero keeps its bit until , which clears it only if the count is +/// still zero then. Everything happens under one lock, which nests inside the image cache's. +/// SHARPEMU_SPLIT_WRITEBACK_SUMMARY=0 keeps no summary: A then calls for every watched write, as before. +/// +internal sealed class PendingWritebackPages : IPendingWritebackPages +{ + public static readonly bool Enabled = + Environment.GetEnvironmentVariable("SHARPEMU_SPLIT_WRITEBACK_SUMMARY") != "0"; + + private const int PageShift = SplitSharedState.PageShift; + private const int ChunkPageShift = 18; + private const ulong ChunkPageMask = (1UL << ChunkPageShift) - 1; + + private readonly SplitSharedState _state; + private readonly object _gate = new(); + private readonly int[]?[] _chunks = new int[(TrackerLayout.SpaceBytes >> PageShift) >> ChunkPageShift][]; + // Page runs whose count reached zero since the last publish (first page, page count). + private readonly List<(ulong First, ulong Count)> _released = []; + private int _releasedRuns; + + public PendingWritebackPages(SplitSharedState state) => _state = state; + + public void Activate() => _state.SetPendingLive(); + + public void Add(ulong address, ulong size) + { + if (!TryPages(address, size, out var first, out var count)) + { + return; + } + + lock (_gate) + { + var word = first >> 6; + ulong mask = 0; + for (var page = first; page < first + count; page++) + { + if ((page >> 6) != word) + { + if (mask != 0) _state.SetPendingBits(word, mask); + word = page >> 6; + mask = 0; + } + + ref var slot = ref Count(page, create: true); + if (slot++ == 0) + { + mask |= 1UL << (int)(page & 63); + } + } + + if (mask != 0) _state.SetPendingBits(word, mask); + } + } + + public void Remove(ulong address, ulong size) + { + if (!TryPages(address, size, out var first, out var count)) + { + return; + } + + lock (_gate) + { + var reachedZero = false; + for (var page = first; page < first + count; page++) + { + ref var slot = ref Count(page, create: false); + if (Unsafe.IsNullRef(ref slot) || slot == 0) + { + continue; + } + + if (--slot == 0) + { + reachedZero = true; + } + } + + if (reachedZero) + { + _released.Add((first, count)); + Volatile.Write(ref _releasedRuns, _released.Count); + } + } + } + + public void Publish() + { + if (Volatile.Read(ref _releasedRuns) == 0) + { + return; + } + + lock (_gate) + { + foreach (var (first, count) in _released) + { + var word = first >> 6; + ulong mask = 0; + for (var page = first; page < first + count; page++) + { + if ((page >> 6) != word) + { + if (mask != 0) _state.ClearPendingBits(word, mask); + word = page >> 6; + mask = 0; + } + + ref var slot = ref Count(page, create: false); + if (Unsafe.IsNullRef(ref slot) || slot == 0) + { + mask |= 1UL << (int)(page & 63); + } + } + + if (mask != 0) _state.ClearPendingBits(word, mask); + } + + _released.Clear(); + Volatile.Write(ref _releasedRuns, 0); + } + } + + /// The claims on the page holding (tests). + internal int ClaimsAt(ulong address) + { + lock (_gate) + { + ref var slot = ref Count(address >> PageShift, create: false); + return Unsafe.IsNullRef(ref slot) ? 0 : slot; + } + } + + private ref int Count(ulong page, bool create) + { + var index = page >> ChunkPageShift; + var chunk = _chunks[index]; + if (chunk is null) + { + if (!create) + { + return ref Unsafe.NullRef(); + } + + chunk = _chunks[index] = new int[1 << ChunkPageShift]; + } + + return ref chunk[page & ChunkPageMask]; + } + + private static bool TryPages(ulong address, ulong size, out ulong first, out ulong count) + { + first = count = 0; + if (size == 0 || address >= TrackerLayout.SpaceBytes) + { + return false; + } + + var end = size > TrackerLayout.SpaceBytes - address ? TrackerLayout.SpaceBytes : address + size; + first = address >> PageShift; + count = ((end - 1) >> PageShift) - first + 1; + return true; + } +} diff --git a/src/SharpEmu.GpuSplit/RemoteGuestAddressSpace.cs b/src/SharpEmu.GpuSplit/RemoteGuestAddressSpace.cs index ee2068ae7..2b7097d82 100644 --- a/src/SharpEmu.GpuSplit/RemoteGuestAddressSpace.cs +++ b/src/SharpEmu.GpuSplit/RemoteGuestAddressSpace.cs @@ -56,7 +56,7 @@ internal interface IInterestSummary /// internal sealed unsafe class RemoteGuestAddressSpace : ICpuMemory, IGuestBackedSpace, IGuestAddressSpace, IGuestDerivedProtectionAddressSpace, IPageWatchObserver, - IProtectionBatchSpace + IProtectionBatchSpace, IPendingWritebackSource { private const int MaxMissReports = 16; @@ -83,6 +83,9 @@ public RemoteGuestAddressSpace(HostBackingObject mirror, GuestMirrorTranslator t _summary = summary; } + /// The pending-writeback summary the GPU caches keep for the guest process (null: none is kept). + public IPendingWritebackPages? PendingWritebacks { get; set; } + /// Accesses to memory the backing does not hold. public long PrivateMemoryMisses => Interlocked.Read(ref _misses); diff --git a/src/SharpEmu.GpuSplit/RemoteImageStore.cs b/src/SharpEmu.GpuSplit/RemoteImageStore.cs index 5bb9049d3..13c3eb309 100644 --- a/src/SharpEmu.GpuSplit/RemoteImageStore.cs +++ b/src/SharpEmu.GpuSplit/RemoteImageStore.cs @@ -22,8 +22,11 @@ namespace SharpEmu.GpuSplit; /// /// Managed writes are filtered by the GPU-interest summary first; GTA V makes over a /// million a second and the caches watch a few pages. One about to land -/// () is a call, because a GPU-written image under it must reach -/// guest memory before the write lands over it. From a thread that is answering the GPU process +/// () is a call when the pending-writeback summary has a page +/// under it, because a GPU-written image there must reach guest memory, or a writeback recorded +/// there must land, before the write lands over it. Under no such page the call would only +/// invalidate, which the write's ordered record does as well, so it is sent as a record +/// (, "Pending-writeback summary"). From a thread that is answering the GPU process /// or that runs inside a GPU worker bracket it is sent as a landed write instead: there the GPU /// worker cannot take the image's writeback, which is the in-process cache's own rule for a store /// from the worker. One that has landed () is a record, handled @@ -117,6 +120,15 @@ public bool MarkPreciseCpuWrite(ulong address, ulong size) return SendLanded(address, size); } + // Nothing GPU-written to put under the write: the record (ahead of every later submission) + // invalidates as the call would. Sent now, as well as after the write lands, for a caller + // that reports only the write about to land. + if (!_state.MayPendWriteback(address, size)) + { + SplitCrossingStats.Count(SplitCrossing.ManagedWriteRecord); + return SendLanded(address, size); + } + // No barrier: in one process a submission this thread enqueued before may still wait for // the worker as the write is published, so the GPU caches need nothing more from the ring // here, and a managed write does not wait for a ring thread held by the shader prewarm. diff --git a/src/SharpEmu.GpuSplit/RemoteVideoOutBridge.cs b/src/SharpEmu.GpuSplit/RemoteVideoOutBridge.cs index 80b43d1c4..517144961 100644 --- a/src/SharpEmu.GpuSplit/RemoteVideoOutBridge.cs +++ b/src/SharpEmu.GpuSplit/RemoteVideoOutBridge.cs @@ -85,7 +85,10 @@ public bool IsFlipPresentationPending(ulong requestId) => public void NotifyPresentationWindowClosed() => host.SendNotice(GpuSplitNotice.PresentationWindowClosed); + // Only a flip that names no buffer (-1) is released here; every other flip answers false without + // the guest's help, so it is not worth a round trip. public bool ReleaseNoBufferFlip(int bufferIndex, ulong requestId) => + bufferIndex == -1 && host.CallGuest(GpuRemoteReverseOpcode.ReleaseNoBufferFlip, new ReleaseNoBufferRequest { BufferIndex = bufferIndex, RequestId = requestId }, out ValueResponse answer) && answer.Value != 0; @@ -95,12 +98,28 @@ public bool ReleaseNoBufferFlip(int bufferIndex, ulong requestId) => public void SetSelectedGpuName(string gpuName) => host.SendNotice(GpuSplitNotice.SelectedGpuName, gpuName); // The trace reads guest memory and video-out state of the guest process; not carried yet. - public void TraceGpuFlip(int handle, int bufferIndex, int flipMode, long flipArg, ulong address, int flipEventCount) + public void TraceGpuFlip(int handle, int bufferIndex, int flipMode, long flipArg, ulong address, ulong requestId) { } + private readonly record struct ReservedBuffer(RemoteVideoOutBridge? Owner, int Handle, int BufferIndex, DisplayBufferResponse Wire); + + // One reservation's buffer, consumed by the next lookup on this thread. Only the same + // bridge and slot may use it; an intervening lookup discards it and asks the guest. + [ThreadStatic] + private static ReservedBuffer t_reservedBuffer; + public bool TryGetDisplayBufferInfo(int handle, int bufferIndex, out VideoOutExports.DisplayBufferInfo info) { + var reserved = t_reservedBuffer; + t_reservedBuffer = default; + if (ReferenceEquals(reserved.Owner, this) && reserved.Wire.Found != 0 && + reserved.Handle == handle && reserved.BufferIndex == bufferIndex) + { + info = SplitConvert.FromWire(reserved.Wire); + return true; + } + info = default; if (!host.CallGuest(GpuRemoteReverseOpcode.TryGetDisplayBuffer, new FlipSlotRequest { Handle = handle, BufferIndex = bufferIndex }, out DisplayBufferResponse answer) || answer.Found == 0) @@ -123,6 +142,11 @@ public int TryReserveFlipRequest(int handle, int bufferIndex, int flipMode, long _hdrRequested = answer.HdrRequested != 0; requestId = answer.RequestId; + // The display buffer this flip names came with the answer; the capture that follows on this + // thread takes it from here (see TryGetDisplayBufferInfo). + t_reservedBuffer = answer.Result == 0 && answer.Buffer.Found != 0 + ? new ReservedBuffer(this, handle, bufferIndex, answer.Buffer) + : default; return answer.Result; } diff --git a/src/SharpEmu.GpuSplit/SplitGpuHost.cs b/src/SharpEmu.GpuSplit/SplitGpuHost.cs index 2ec6e749a..43a875b65 100644 --- a/src/SharpEmu.GpuSplit/SplitGpuHost.cs +++ b/src/SharpEmu.GpuSplit/SplitGpuHost.cs @@ -2,6 +2,7 @@ // SPDX-License-Identifier: GPL-2.0-or-later using System.Collections.Concurrent; +using System.Diagnostics; using System.Runtime.CompilerServices; using System.Runtime.Versioning; using System.Text; @@ -47,6 +48,13 @@ public sealed record SplitGpuHostOptions /// How often the window's input () is checked for changes; zero sends none. public TimeSpan InputPollInterval { get; init; } = TimeSpan.FromMilliseconds(4); + + /// + /// The input thread waits for a change instead of polling every + /// (which then only spaces consecutive messages). SHARPEMU_SPLIT_INPUT_POLL=1 turns it off. + /// + public bool InputWakeOnChange { get; init; } = + Environment.GetEnvironmentVariable("SHARPEMU_SPLIT_INPUT_POLL") != "1"; } /// What the backend is composed over in the GPU process. @@ -307,7 +315,10 @@ private void Start(ReadOnlySpan payload) _ = _guestMemory.Replica.Sync(start.ViewLogSequence); _space = new RemoteGuestAddressSpace( _guestMemory.Mirror, _guestMemory.Translator, _guestMemory.Replica, _guestMemory.ChunkBytes, - _guestMemory.ObjectBytes, this, _state); + _guestMemory.ObjectBytes, this, _state) + { + PendingWritebacks = PendingWritebackPages.Enabled ? new PendingWritebackPages(_state) : null, + }; _memory = new GuestGpuMemory(_space); _bridge = new RemoteVideoOutBridge(this, titleId, start.HdrRequested != 0); if (_options.InstallGlobals) @@ -757,7 +768,9 @@ internal bool CallGuest(GpuRemoteReverseOpcode opcode, in T where TResponse : unmanaged { response = default; - if (!CallGuest(opcode, GpuSplitWire.Bytes(in request), out var answer, out var length)) + // The answer's size is the response type's, so it fits the stack: no buffer per call. + Span answer = stackalloc byte[Unsafe.SizeOf()]; + if (!CallGuest(opcode, GpuSplitWire.Bytes(in request), answer, out var length)) { return false; } @@ -775,6 +788,18 @@ internal bool CallGuest(GpuRemoteReverseOpcode opcode, in T internal bool CallGuest(GpuRemoteReverseOpcode opcode, ReadOnlySpan request, out byte[] response, out int length) { response = Array.Empty(); + var buffer = new byte[_channel.ReverseSyncCalls.PayloadCapacity]; + if (!CallGuest(opcode, request, buffer, out length)) + { + return false; + } + + response = buffer; + return true; + } + + private bool CallGuest(GpuRemoteReverseOpcode opcode, ReadOnlySpan request, Span response, out int length) + { length = 0; if (t_onRingThread) { @@ -782,8 +807,7 @@ internal bool CallGuest(GpuRemoteReverseOpcode opcode, ReadOnlySpan reques return false; } - var buffer = new byte[_channel.ReverseSyncCalls.PayloadCapacity]; - var result = _channel.ReverseSyncCalls.Call((uint)opcode, request, buffer, out length, out var code, ReverseTimeoutNanoseconds); + var result = _channel.ReverseSyncCalls.Call((uint)opcode, request, response, out length, out var code, ReverseTimeoutNanoseconds); if (result != SyncCallStatus.Completed || code != GpuSplitStatus.Ok) { if (result != SyncCallStatus.PeerExited) @@ -794,7 +818,6 @@ internal bool CallGuest(GpuRemoteReverseOpcode opcode, ReadOnlySpan reques return false; } - response = buffer; return true; } @@ -829,6 +852,8 @@ private void StartReplicaFollower() private const int ReplicaFollowMilliseconds = 100; + private static readonly TimeSpan IdleCheck = TimeSpan.FromMilliseconds(250); + private void StartInputPoll() { if (_options.InputPollInterval <= TimeSpan.Zero) @@ -843,18 +868,42 @@ private void StartInputPoll() // The window lives here; the guest process's pad and keyboard exports read its state there. private unsafe void PollInput() { + // The thread sleeps on the input state's change signal instead of waking every interval to + // look: a change goes out at once when the last message is at least one interval old and + // otherwise at the interval's end (the old poll's rate and worst case, coalescing a burst of + // events into one message), and an idle window costs a wake-up per IdleCheck, only to see + // whether the peer is still there. SHARPEMU_SPLIT_INPUT_POLL=1 keeps the fixed-interval poll. var interval = _options.InputPollInterval; var sent = long.MinValue; + var lastSend = Stopwatch.GetTimestamp() - Stopwatch.Frequency; Span keys = stackalloc int[InputMessage.MaxKeys]; while (!_stopping && _channel.IsPeerAlive) { - Thread.Sleep(interval); + if (_options.InputWakeOnChange) + { + if (!HostWindowInput.WaitForChange(sent, IdleCheck)) + { + continue; + } + + var sinceLast = Stopwatch.GetElapsedTime(lastSend); + if (sinceLast < interval) + { + Thread.Sleep(interval - sinceLast); + } + } + else + { + Thread.Sleep(interval); + } + var version = HostWindowInput.Version; if (version == sent) { continue; } + lastSend = Stopwatch.GetTimestamp(); sent = version; var count = HostWindowInput.CopyState(keys, out var focused, out var gamepad, out var name); var message = new InputMessage diff --git a/src/SharpEmu.GpuSplit/SplitGpuSession.cs b/src/SharpEmu.GpuSplit/SplitGpuSession.cs index f30fc9b00..96e0a09a7 100644 --- a/src/SharpEmu.GpuSplit/SplitGpuSession.cs +++ b/src/SharpEmu.GpuSplit/SplitGpuSession.cs @@ -58,7 +58,7 @@ public sealed class SplitGpuSession : IDisposable private readonly IDisposable? _process; private readonly IDisposable? _memoryServer; private readonly BlockingCollection _events = new(); - private readonly List _threads = new(); + private readonly List _threads = new(); private readonly ManualResetEventSlim _ready = new(false); private int _readyStatus = GpuSplitStatus.NotReady; private string? _readyError; @@ -560,6 +560,9 @@ internal int HandleReverseCall(uint opcode, ReadOnlySpan request, Span + _threads.Add(SharpEmu.HLE.Host.HostHelperThread.Start(name, 1024 * 1024, start)); public void Dispose() { @@ -738,7 +740,7 @@ public void Dispose() // The process's teardown unmaps the region: these threads must be off it first. foreach (var thread in _threads) { - if (thread != Thread.CurrentThread) + if (!thread.IsCurrentThread) { _ = thread.Join(TimeSpan.FromSeconds(2)); } diff --git a/src/SharpEmu.GpuSplit/SplitSharedState.cs b/src/SharpEmu.GpuSplit/SplitSharedState.cs index d631ceb73..6c0d9d7f0 100644 --- a/src/SharpEmu.GpuSplit/SplitSharedState.cs +++ b/src/SharpEmu.GpuSplit/SplitSharedState.cs @@ -27,18 +27,36 @@ namespace SharpEmu.GpuSplit; /// barrier: a managed write in A either sees the bit (and is reported) or lands before B reads /// the bytes the watch covers. /// -/// Layout: the counter at 0 ( bytes), the summary at -/// , words (32 MiB of address space in the -/// region, touched only where watches are). +/// Pending-writeback summary. One bit per 4 KiB page, set while B holds bytes for the +/// page that guest memory does not have yet: an image the GPU wrote and has not written back, or a +/// recorded writeback whose submission has not completed (). +/// Those are the only cases in which B's answer to a managed write's +/// call does anything before the write lands (it writes the image back, or waits for the +/// writeback); everything else the call does, B's handling of the write's ordered record does as +/// well. So A calls only when a bit under the write is set, and otherwise sends the record. A bit +/// is set before the state it stands for exists and cleared only once B's image cache, as it lets +/// its lock go, finds the state gone: a clear bit at any moment means a call made then would have +/// found nothing to write back or wait for. Until B's cache keeps the summary +/// ( is 0) every page counts as pending. +/// +/// Layout: the counter at 0 ( bytes), the pending +/// summary's live word at , the interest summary at +/// and the pending summary at , +/// words each (32 MiB of address space in the region apiece, touched only +/// where watches and GPU-written images are). /// public sealed unsafe class SplitSharedState : IInterestSummary { public const int PageShift = 12; + public const int PendingLiveOffset = SharedCounter.BlockSize; public const int SummaryOffset = 4096; public const int SummaryWords = (int)(TrackerLayout.SpaceBytes >> PageShift >> 6); - public const int Bytes = SummaryOffset + SummaryWords * sizeof(ulong); + public const int PendingOffset = SummaryOffset + SummaryWords * sizeof(ulong); + public const int Bytes = PendingOffset + SummaryWords * sizeof(ulong); private readonly ulong* _summary; + private readonly ulong* _pending; + private readonly long* _pendingLive; public SplitSharedState(GpuRemoteChannel channel) { @@ -51,6 +69,8 @@ public SplitSharedState(GpuRemoteChannel channel) ProtectApplied = SharedCounter.InApplicationBlock(channel, 0); _summary = (ulong*)(channel.Application + SummaryOffset); + _pending = (ulong*)(channel.Application + PendingOffset); + _pendingLive = (long*)(channel.Application + PendingLiveOffset); } /// The layout of a region that holds this state. @@ -60,7 +80,30 @@ public SplitSharedState(GpuRemoteChannel channel) public SharedCounter ProtectApplied { get; } /// A: whether any 4 KiB page under [address, address + size) carries a watch. - public bool MayInterest(ulong address, ulong size) + public bool MayInterest(ulong address, ulong size) => AnyPage(_summary, address, size); + + /// Whether B keeps the pending-writeback summary (until it does, every page counts as pending). + public bool PendingLive => Volatile.Read(ref *_pendingLive) != 0; + + /// + /// A: whether B may hold bytes for a page under [address, address + size) that guest memory + /// lacks, so a managed write there has to wait for B before it lands. + /// + public bool MayPendWriteback(ulong address, ulong size) => !PendingLive || AnyPage(_pending, address, size); + + /// B: its image cache keeps the pending summary from now on. + public void SetPendingLive() => Interlocked.Exchange(ref *_pendingLive, 1); + + /// B: sets the pending bits in of summary word (a full barrier). + public void SetPendingBits(ulong word, ulong mask) => Interlocked.Or(ref _pending[word], mask); + + /// B: clears the pending bits in of summary word . + public void ClearPendingBits(ulong word, ulong mask) => Interlocked.And(ref _pending[word], ~mask); + + /// The pending bits of summary word . + public ulong PendingBits(ulong word) => Volatile.Read(ref _pending[word]); + + private static bool AnyPage(ulong* bitmap, ulong address, ulong size) { if (size == 0 || address >= TrackerLayout.SpaceBytes) { @@ -73,7 +116,7 @@ public bool MayInterest(ulong address, ulong size) while (page <= last) { var word = page >> 6; - var bits = Volatile.Read(ref _summary[word]); + var bits = Volatile.Read(ref bitmap[word]); var wordLast = Math.Min(last, (word << 6) | 63); if (bits != 0) { diff --git a/src/SharpEmu.GpuSplit/WorkerBracket.cs b/src/SharpEmu.GpuSplit/WorkerBracket.cs index f8f31e452..f987f9c00 100644 --- a/src/SharpEmu.GpuSplit/WorkerBracket.cs +++ b/src/SharpEmu.GpuSplit/WorkerBracket.cs @@ -1,6 +1,8 @@ // Copyright (C) 2026 SharpEmu Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later +using System.Collections.Concurrent; + namespace SharpEmu.GpuSplit; /// @@ -44,7 +46,7 @@ public WorkerBracket(ulong id, Func alive) /// public void Start(Action runParked) { - new Thread(() => + BracketRunner.Run(release => { try { @@ -63,13 +65,11 @@ public void Start(Action runParked) } _parked.Set(); + // Idle before the end is seen: the guest's next bracket finds this thread. + release(); _finished.Set(); } - }) - { - IsBackground = true, - Name = $"SharpEmu GPU split bracket {Id}", - }.Start(); + }); } /// Waits until the worker is parked (or the bracket ended without it). @@ -194,6 +194,73 @@ private void Park() } } + /// + /// The threads that hand brackets to the worker and block there, parked between brackets: a + /// mapping change is two calls and a replica sync, and a thread made for each one added its + /// creation and first wake to all three. One that is busy is never taken, so brackets that + /// overlap still each have a thread, as before. + /// + internal sealed class BracketRunner + { + private static readonly ConcurrentStack Idle = new(); + private static int _created; + + private readonly SemaphoreSlim _go = new(0); + private readonly Action _release; + private Action? _work; + private bool _released; + + private BracketRunner() => _release = Release; + + /// Runner threads made so far (diagnostics and tests). + internal static int Created => Volatile.Read(ref _created); + + /// + /// Runs on an idle runner, or on a new one when none is idle. The + /// work calls the action it gets once it no longer needs the thread (the runner is idle + /// from then on); one that returns without calling it is released on return. + /// + public static void Run(Action work) + { + if (!Idle.TryPop(out var runner)) + { + runner = new BracketRunner(); + var index = Interlocked.Increment(ref _created); + runner._work = work; + runner._go.Release(); + _ = SharpEmu.HLE.Host.HostHelperThread.Start($"SharpEmu GPU split bracket {index}", 1024 * 1024, runner.Loop); + return; + } + + // Taken off the stack: nothing else hands this runner work until it releases itself. + runner._work = work; + runner._go.Release(); + } + + private void Release() + { + if (!_released) + { + _released = true; + Idle.Push(this); + } + } + + // Start's work catches its own exceptions. + private void Loop() + { + while (true) + { + _go.Wait(); + var work = _work!; + _work = null; + _released = false; + work(_release); + Release(); + } + } + } + private bool WaitWhileAlive(ManualResetEventSlim signal, TimeSpan timeout) { var deadline = Environment.TickCount64 + (long)timeout.TotalMilliseconds; diff --git a/src/SharpEmu.HLE/GpuMemory/IPendingWritebackPages.cs b/src/SharpEmu.HLE/GpuMemory/IPendingWritebackPages.cs new file mode 100644 index 000000000..8e5ae6a92 --- /dev/null +++ b/src/SharpEmu.HLE/GpuMemory/IPendingWritebackPages.cs @@ -0,0 +1,36 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +namespace SharpEmu.HLE.GpuMemory; + +/// +/// The GPU side's summary of the guest pages a managed write must wait on: pages under an image +/// the GPU wrote and has not written back, and pages a recorded writeback has not reached yet. +/// The split's guest process reads it to make a managed write's blocking call only where the GPU +/// side has something to put under the write first. +/// +/// +/// A page counts every claim on it. marks its pages at once, as a full barrier, +/// so the caller makes the claim visible before the state it stands for. +/// only lowers the counts: a page whose count reached zero is unmarked by the next +/// , which the owner calls where its state is whole (the GPU process's image +/// cache does so as it lets its lock go), so a claim dropped and replaced inside one operation +/// (an image written back: its GPU claim ends, its writeback's begins) never shows a gap. +/// +public interface IPendingWritebackPages +{ + /// The caller keeps the summary from now on. Until then every page counts as pending. + void Activate(); + + void Add(ulong address, ulong size); + + void Remove(ulong address, ulong size); + + void Publish(); +} + +/// A guest address space that carries a summary. +public interface IPendingWritebackSource +{ + IPendingWritebackPages? PendingWritebacks { get; } +} diff --git a/src/SharpEmu.HLE/GpuMemory/RegionLock.cs b/src/SharpEmu.HLE/GpuMemory/RegionLock.cs index 9bcefda3b..e4b1ecce4 100644 --- a/src/SharpEmu.HLE/GpuMemory/RegionLock.cs +++ b/src/SharpEmu.HLE/GpuMemory/RegionLock.cs @@ -37,6 +37,9 @@ public Held(RegionLock regionLock) public Held Hold() => new(this); + /// Runs at the end of every hold, with the lock still held. Null on every lock but the split GPU process's image cache. + public Action? BeforeRelease { get; set; } + /// True when the calling thread is the one holding the lock. public bool IsHeldByCurrentThread => Volatile.Read(ref _owner) == Environment.CurrentManagedThreadId; @@ -83,6 +86,7 @@ public void Exit() return; } + BeforeRelease?.Invoke(); // Copy measurements before release; the next owner can then replace the lock's fields. var holdOperation = _holdOperation; var holdTicks = GpuMemoryAccessProfile.Enabled ? Stopwatch.GetTimestamp() - _holdStarted : 0; diff --git a/src/SharpEmu.HLE/GpuMemory/SplitCrossingStats.cs b/src/SharpEmu.HLE/GpuMemory/SplitCrossingStats.cs index c68abd584..f9399852b 100644 --- a/src/SharpEmu.HLE/GpuMemory/SplitCrossingStats.cs +++ b/src/SharpEmu.HLE/GpuMemory/SplitCrossingStats.cs @@ -44,6 +44,9 @@ public enum SplitCrossing ManagedWriteInterest, ManagedWriteCall, ProtectWait, + // Split only. Watched managed writes with no pending writeback under them, sent as a record + // instead of the call. + ManagedWriteRecord, } public static class SplitCrossingStats diff --git a/src/SharpEmu.HLE/GuestMemory/GuestMirrorTranslator.cs b/src/SharpEmu.HLE/GuestMemory/GuestMirrorTranslator.cs index c2e085899..3443a72e3 100644 --- a/src/SharpEmu.HLE/GuestMemory/GuestMirrorTranslator.cs +++ b/src/SharpEmu.HLE/GuestMemory/GuestMirrorTranslator.cs @@ -16,10 +16,13 @@ namespace SharpEmu.HLE.GuestMemory; /// /// /// A hit costs one load of the table's mapping generation and one cached range check. Each of the -/// slots holds one immutable range, so threads share the cache without a -/// lock; a slot is good while the mapping generation it was read under is current, and protection -/// changes do not move that generation. A miss reads the table under its sequence and caches the -/// whole run of entries around the address whose addresses and offsets both continue. +/// slots, picked by a hash of the address's 64 KiB block, holds one range +/// under a sequence of its own, so threads share the cache without a lock and a miss allocates +/// nothing; a slot is good while the mapping generation it was read under is current, and +/// protection changes do not move that generation. Behind the slots each thread keeps the last +/// range it read from the table, which serves the other blocks of a range the thread walks through. +/// A miss reads the table under its sequence and caches the whole run of entries around the +/// address whose addresses and offsets both continue. /// /// /// @@ -33,26 +36,45 @@ namespace SharpEmu.HLE.GuestMemory; /// public sealed unsafe class GuestMirrorTranslator { - public const int CacheSlots = 64; + public const int CacheSlots = 1 << CacheBits; - // Slots are picked by the 64 KiB block of the address: the guest's own mapping granule. + private const int CacheBits = 10; + + // Slots are picked by a hash of the 64 KiB block of the address, the guest's own mapping + // granule: buffers placed at a power-of-two stride would otherwise all land in one slot. private const int CacheShift = 16; private readonly GuestViewTableHeader* _header; private readonly GuestViewTableEntry* _entries; private readonly int _capacity; - private readonly CachedRange?[] _cache = new CachedRange?[CacheSlots]; + private readonly Slot[] _cache = new Slot[CacheSlots]; private long _hits; private long _misses; - private sealed class CachedRange(ulong start, ulong end, ulong offset, ulong generation) + // The last range the thread read from the table, for whichever translator it read it from. + [ThreadStatic] + private static LastRange t_last; + + // One cached range under a sequence: odd while a writer fills it, so a reader that sees the + // same even sequence before and after its loads read one writer's range. An empty slot + // (End 0) holds no address. + private struct Slot { - public readonly ulong Start = start; - public readonly ulong End = end; - public readonly ulong Offset = offset; - public readonly ulong Generation = generation; + public long Sequence; + public ulong Start; + public ulong End; + public ulong Offset; + public ulong Generation; } + private struct LastRange + { + public GuestMirrorTranslator? Owner; + public CachedRange Range; + } + + private readonly record struct CachedRange(ulong Start, ulong End, ulong Offset, ulong Generation); + /// /// A translator over a mapped view table (often read-only) and a mirror of the backing it /// describes. Both mappings must outlive the translator. @@ -157,17 +179,34 @@ public bool TryGetEntry(ulong address, out GuestViewEntry entry) => private bool TryGetRange(ulong address, out CachedRange range) { var generation = Volatile.Read(ref _header->MappingGeneration); - ref var slot = ref _cache[(int)((address >> CacheShift) & (CacheSlots - 1))]; - var cached = Volatile.Read(ref slot); - if (cached is not null && cached.Generation == generation && address >= cached.Start && address < cached.End) + ref var slot = ref _cache[SlotIndex(address)]; + var sequence = Volatile.Read(ref slot.Sequence); + if ((sequence & 1) == 0) + { + // Acquire loads: none moves above the sequence read, and the second read stays below them. + range = new CachedRange(Volatile.Read(ref slot.Start), Volatile.Read(ref slot.End), + Volatile.Read(ref slot.Offset), Volatile.Read(ref slot.Generation)); + if (Volatile.Read(ref slot.Sequence) == sequence && range.Generation == generation && + address >= range.Start && address < range.End) + { + if (CountCacheAccesses) _hits++; + return true; + } + } + + ref var last = ref t_last; + if (ReferenceEquals(last.Owner, this) && last.Range.Generation == generation && + address >= last.Range.Start && address < last.Range.End) { if (CountCacheAccesses) _hits++; - range = cached; + range = last.Range; + // The slot answers the next address of this block without the thread's lookup. + Store(ref slot, range); return true; } if (CountCacheAccesses) _misses++; - range = null!; + range = default; if (!GuestViewTableReader.TryFind(_header, _entries, _capacity, address, joinRun: true, out var entry, out var readGeneration)) return false; // A run reaching past the mirror is cut at its end; an address past it has no translation. @@ -175,7 +214,26 @@ private bool TryGetRange(ulong address, out CachedRange range) var size = Math.Min(entry.Size, MirrorBytes - entry.Offset); if (address - entry.Address >= size) return false; range = new CachedRange(entry.Address, entry.Address + size, entry.Offset, readGeneration); - Volatile.Write(ref slot, range); + last.Owner = this; + last.Range = range; + Store(ref slot, range); return true; } + + private static int SlotIndex(ulong address) => + (int)(((address >> CacheShift) * 0x9E3779B97F4A7C15UL) >> (64 - CacheBits)); + + // A writer that finds the slot being filled leaves it to the other one. + private static void Store(ref Slot slot, in CachedRange range) + { + var sequence = Volatile.Read(ref slot.Sequence); + if ((sequence & 1) != 0 || Interlocked.CompareExchange(ref slot.Sequence, sequence + 1, sequence) != sequence) + return; + // The exchange is a full barrier: these stores are not seen before the odd sequence. + slot.Start = range.Start; + slot.End = range.End; + slot.Offset = range.Offset; + slot.Generation = range.Generation; + Volatile.Write(ref slot.Sequence, sequence + 2); + } } diff --git a/src/SharpEmu.HLE/Host/HostHelperThread.cs b/src/SharpEmu.HLE/Host/HostHelperThread.cs new file mode 100644 index 000000000..174d23292 --- /dev/null +++ b/src/SharpEmu.HLE/Host/HostHelperThread.cs @@ -0,0 +1,53 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +namespace SharpEmu.HLE.Host; + +/// +/// A helper thread a frame-deadline thread waits on (the GPU worker's program resolver, the +/// completion thread, the split's ring, call and inbound threads): started natively at a QoS class +/// through , since a thread CoreCLR creates cannot take one later, and +/// as a managed background thread where that fails or is switched off. +/// +/// +/// Left at the default class these threads queue behind a title's spinning job threads in the +/// other process while the worker waits on them. SHARPEMU_SPLIT_THREAD_QOS=0 starts them all as +/// managed threads, as before. +/// +public sealed class HostHelperThread +{ + public static readonly bool QosEnabled = + !string.Equals(Environment.GetEnvironmentVariable("SHARPEMU_SPLIT_THREAD_QOS"), "0", StringComparison.Ordinal); + + private readonly HostQosThread? _native; + private readonly Thread? _managed; + + private HostHelperThread(HostQosThread? native, Thread? managed) + { + _native = native; + _managed = managed; + } + + /// Whether the thread runs at the class it asked for. + public bool HasQos => _native is not null; + + public bool IsCurrentThread => _native?.IsCurrentThread ?? ReferenceEquals(Thread.CurrentThread, _managed); + + /// Waits for the body to return. + public bool Join(TimeSpan timeout) => + _native?.Join((int)Math.Min(timeout.TotalMilliseconds, int.MaxValue)) ?? _managed!.Join(timeout); + + public static HostHelperThread Start(string name, int stackBytes, Action body, + HostThreadQos qos = HostThreadQos.UserInteractive) + { + ArgumentNullException.ThrowIfNull(body); + if (QosEnabled && HostQosThread.TryStart(name, qos, stackBytes, body, out _) is { } native) + { + return new HostHelperThread(native, null); + } + + var managed = new Thread(() => body(), stackBytes) { IsBackground = true, Name = name }; + managed.Start(); + return new HostHelperThread(null, managed); + } +} diff --git a/src/SharpEmu.Libs/Agc/AgcExports.CommandStream.cs b/src/SharpEmu.Libs/Agc/AgcExports.CommandStream.cs index 5ef079e58..b5151e668 100644 --- a/src/SharpEmu.Libs/Agc/AgcExports.CommandStream.cs +++ b/src/SharpEmu.Libs/Agc/AgcExports.CommandStream.cs @@ -647,6 +647,9 @@ public void ReleaseShaderWrites() _executor?.ReleaseShaderWrites(); } + /// See ; read where the draws run. + public bool HasUnorderedStoresInOpenPass => _executor?.HasUnorderedStoresInOpenPass ?? true; + public void Dispatch(ulong submitId, uint endX, uint endY, uint endZ, uint dispatchInitiator, ulong indirectArgumentsAddress = 0) { var state = RequireCurrent(); diff --git a/src/SharpEmu.Libs/Audio/AjmExports.Jobs.cs b/src/SharpEmu.Libs/Audio/AjmExports.Jobs.cs index 1bfb84707..c76eba0da 100644 --- a/src/SharpEmu.Libs/Audio/AjmExports.Jobs.cs +++ b/src/SharpEmu.Libs/Audio/AjmExports.Jobs.cs @@ -83,10 +83,13 @@ private static int AjmBatchJobControlCore(CpuContext ctx) WriteBasicResult(ctx, outputAddress, status); } - Trace( - $"batch_job_control instance=0x{instanceId:X8} flags=0x{flags:X} " + - $"in=0x{inputAddress:X16}+0x{inputSize:X} out=0x{outputAddress:X16}+0x{outputSize:X} " + - $"status=0x{status:X8}"); + if (TraceEnabled) + { + Trace( + $"batch_job_control instance=0x{instanceId:X8} flags=0x{flags:X} " + + $"in=0x{inputAddress:X16}+0x{inputSize:X} out=0x{outputAddress:X16}+0x{outputSize:X} " + + $"status=0x{status:X8}"); + } return ctx.SetReturn(0); } diff --git a/src/SharpEmu.Libs/Audio/AjmExports.cs b/src/SharpEmu.Libs/Audio/AjmExports.cs index fd52f6226..16c054bb1 100644 --- a/src/SharpEmu.Libs/Audio/AjmExports.cs +++ b/src/SharpEmu.Libs/Audio/AjmExports.cs @@ -121,7 +121,7 @@ public static int AjmInitialize(CpuContext ctx) } Contexts[contextId] = new AjmContextState(); - if (string.Equals(Environment.GetEnvironmentVariable("SHARPEMU_LOG_AJM"), "1", StringComparison.Ordinal)) + if (TraceEnabled) { Console.Error.WriteLine( $"[LOADER][TRACE] ajm.initialize reserved_or_options=0x{reservedOrOptions:X16} " + @@ -239,7 +239,7 @@ public static int AjmModuleRegister(CpuContext ctx) } } - if (string.Equals(Environment.GetEnvironmentVariable("SHARPEMU_LOG_AJM"), "1", StringComparison.Ordinal)) + if (TraceEnabled) { Console.Error.WriteLine( $"[LOADER][TRACE] ajm.module_register context={contextId} codec={codecType} reserved={reserved}"); @@ -631,9 +631,12 @@ public static int AjmBatchJobSetGaplessDecode(CpuContext ctx) } WriteBasicResult(ctx, resultAddress, status); - Trace( - $"batch_job_set_gapless_decode instance=0x{instanceId:X8} " + - $"gapless=0x{gaplessAddress:X16} reset={reset} status=0x{status:X8}"); + if (TraceEnabled) + { + Trace( + $"batch_job_set_gapless_decode instance=0x{instanceId:X8} " + + $"gapless=0x{gaplessAddress:X16} reset={reset} status=0x{status:X8}"); + } return ctx.SetReturn(0); } @@ -862,9 +865,12 @@ public static int AjmBatchStart(CpuContext ctx) return ctx.SetReturn(OrbisAjmErrorInvalidParameter); } - Trace( - $"batch_start context={contextId} info=0x{infoAddress:X16} " + - $"priority={priority} batch={batchId} error=0x{errorAddress:X16}"); + if (TraceEnabled) + { + Trace( + $"batch_start context={contextId} info=0x{infoAddress:X16} " + + $"priority={priority} batch={batchId} error=0x{errorAddress:X16}"); + } return ctx.SetReturn(0); } @@ -878,10 +884,13 @@ public static int AjmBatchWait(CpuContext ctx) // Batches complete synchronously in Start; Wait is a no-op success. var errorAddress = ctx[CpuRegister.Rcx]; ClearAjmBatchError(ctx, errorAddress); - Trace( - $"batch_wait context={unchecked((uint)ctx[CpuRegister.Rdi])} " + - $"batch={unchecked((uint)ctx[CpuRegister.Rsi])} " + - $"timeout={unchecked((uint)ctx[CpuRegister.Rdx])}"); + if (TraceEnabled) + { + Trace( + $"batch_wait context={unchecked((uint)ctx[CpuRegister.Rdi])} " + + $"batch={unchecked((uint)ctx[CpuRegister.Rsi])} " + + $"timeout={unchecked((uint)ctx[CpuRegister.Rdx])}"); + } return ctx.SetReturn(0); } @@ -892,9 +901,12 @@ public static int AjmBatchWait(CpuContext ctx) LibraryName = "libSceAjm")] public static int AjmBatchCancel(CpuContext ctx) { - Trace( - $"batch_cancel context={unchecked((uint)ctx[CpuRegister.Rdi])} " + - $"batch={unchecked((uint)ctx[CpuRegister.Rsi])}"); + if (TraceEnabled) + { + Trace( + $"batch_cancel context={unchecked((uint)ctx[CpuRegister.Rdi])} " + + $"batch={unchecked((uint)ctx[CpuRegister.Rsi])}"); + } return ctx.SetReturn(0); } @@ -963,10 +975,13 @@ private static int AjmBatchJobDecodeCore(CpuContext ctx, bool multipleFrames) } WriteDecodeStreamResult(ctx, resultAddress, result, multipleFrames); - Trace( - $"batch_job_decode instance=0x{instanceId:X8} in=0x{inputAddress:X16}+0x{inputSize:X} " + - $"out=0x{outputAddress:X16}+0x{outputSize:X} consumed={result.InputConsumed} " + - $"produced={result.OutputWritten} frames={result.Frames} status=0x{result.Status:X8}"); + if (TraceEnabled) + { + Trace( + $"batch_job_decode instance=0x{instanceId:X8} in=0x{inputAddress:X16}+0x{inputSize:X} " + + $"out=0x{outputAddress:X16}+0x{outputSize:X} consumed={result.InputConsumed} " + + $"produced={result.OutputWritten} frames={result.Frames} status=0x{result.Status:X8}"); + } return ctx.SetReturn(0); } @@ -1033,11 +1048,14 @@ private static int AppendRunJob( out var instance, out var config); WriteRunSideband(ctx, sidebandAddress, sidebandSize, flags, result, instance, config); - Trace( - $"batch_job_run{(split ? "_split" : string.Empty)} instance=0x{instanceId:X8} flags=0x{flags:X} " + - $"in=0x{inputAddress:X16}#{inputCountOrSize} out=0x{outputAddress:X16}#{outputCountOrSize} " + - $"sideband=0x{sidebandAddress:X16}+0x{sidebandSize:X} consumed={result.InputConsumed} " + - $"produced={result.OutputWritten} frames={result.Frames} status=0x{result.Status:X8}"); + if (TraceEnabled) + { + Trace( + $"batch_job_run{(split ? "_split" : string.Empty)} instance=0x{instanceId:X8} flags=0x{flags:X} " + + $"in=0x{inputAddress:X16}#{inputCountOrSize} out=0x{outputAddress:X16}#{outputCountOrSize} " + + $"sideband=0x{sidebandAddress:X16}+0x{sidebandSize:X} consumed={result.InputConsumed} " + + $"produced={result.OutputWritten} frames={result.Frames} status=0x{result.Status:X8}"); + } TraceBufferDescriptors(ctx, split, "in", inputAddress, inputCountOrSize); TraceBufferDescriptors(ctx, split, "out", outputAddress, outputCountOrSize); return ctx.SetReturn(0); @@ -1117,7 +1135,7 @@ private static void TraceBufferDescriptors( address == 0 || countOrSize == 0 || countOrSize > MaxBufferDescriptors || - !string.Equals(Environment.GetEnvironmentVariable("SHARPEMU_LOG_AJM"), "1", StringComparison.Ordinal)) + !TraceEnabled) { return; } @@ -1137,9 +1155,12 @@ private static void TraceBufferDescriptors( var bufferAddress = BinaryPrimitives.ReadUInt64LittleEndian(descriptor); var bufferSize = BinaryPrimitives.ReadUInt64LittleEndian(descriptor[8..]); var readable = bufferAddress != 0 && bufferSize != 0 && ctx.Memory.TryRead(bufferAddress, head); - Trace( - $"buffer[{label}][{index}] address=0x{bufferAddress:X16} size=0x{bufferSize:X} " + - $"head={(readable ? Convert.ToHexString(head) : "")}"); + if (TraceEnabled) + { + Trace( + $"buffer[{label}][{index}] address=0x{bufferAddress:X16} size=0x{bufferSize:X} " + + $"head={(readable ? Convert.ToHexString(head) : "")}"); + } } } @@ -1619,11 +1640,23 @@ private static bool TryWriteUInt64(CpuContext ctx, ulong address, ulong value) return ctx.Memory.TryWrite(address, buffer); } + // SHARPEMU_LOG_AJM=1 enables the ajm.* traces; read once, not per call. + internal static readonly bool TraceEnabled = string.Equals(Environment.GetEnvironmentVariable("SHARPEMU_LOG_AJM"), "1", StringComparison.Ordinal); + private static void Trace(string message) { - if (string.Equals(Environment.GetEnvironmentVariable("SHARPEMU_LOG_AJM"), "1", StringComparison.Ordinal)) + if (TraceEnabled) { Console.Error.WriteLine($"[LOADER][TRACE] ajm.{message}"); } } + + // A plain interpolated message binds here and is only formatted when tracing is on. + private static void Trace(AjmTraceInterpolatedStringHandler message) + { + if (message.Enabled) + { + Console.Error.WriteLine($"[LOADER][TRACE] ajm.{message.ToStringAndClear()}"); + } + } } diff --git a/src/SharpEmu.Libs/Audio/AjmTraceInterpolatedStringHandler.cs b/src/SharpEmu.Libs/Audio/AjmTraceInterpolatedStringHandler.cs new file mode 100644 index 000000000..7b90ea2e6 --- /dev/null +++ b/src/SharpEmu.Libs/Audio/AjmTraceInterpolatedStringHandler.cs @@ -0,0 +1,39 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using System.Runtime.CompilerServices; + +namespace SharpEmu.Libs.Audio; + +// Lets AjmExports.Trace($"...") skip building the message (and evaluating its holes) when +// SHARPEMU_LOG_AJM is off. Every AJM call traces, so an eagerly formatted string is a per-call allocation. +[InterpolatedStringHandler] +internal ref struct AjmTraceInterpolatedStringHandler +{ + private DefaultInterpolatedStringHandler _inner; + private readonly bool _enabled; + + public AjmTraceInterpolatedStringHandler(int literalLength, int formattedCount, out bool shouldAppend) + { + _enabled = AjmExports.TraceEnabled; + shouldAppend = _enabled; + _inner = _enabled ? new DefaultInterpolatedStringHandler(literalLength, formattedCount) : default; + } + + public void AppendLiteral(string value) => _inner.AppendLiteral(value); + + public void AppendFormatted(T value) => _inner.AppendFormatted(value); + + public void AppendFormatted(T value, string? format) => _inner.AppendFormatted(value, format); + + public void AppendFormatted(T value, int alignment) => _inner.AppendFormatted(value, alignment); + + public void AppendFormatted(T value, int alignment, string? format) => + _inner.AppendFormatted(value, alignment, format); + + public void AppendFormatted(string? value) => _inner.AppendFormatted(value); + + internal bool Enabled => _enabled; + + internal string ToStringAndClear() => _enabled ? _inner.ToStringAndClear() : string.Empty; +} diff --git a/src/SharpEmu.Libs/Audio/Atrac9DecodeState.cs b/src/SharpEmu.Libs/Audio/Atrac9DecodeState.cs index c57a1887c..cc1bc817f 100644 --- a/src/SharpEmu.Libs/Audio/Atrac9DecodeState.cs +++ b/src/SharpEmu.Libs/Audio/Atrac9DecodeState.cs @@ -145,11 +145,14 @@ public bool TryInitialize(ReadOnlySpan configData) _totalDecodedSamples = 0; _containerHeaderLength = 0; _gapless = _gaplessInit; - Trace( - $"initialized config={Convert.ToHexString(normalizedConfig)} channels={config.ChannelCount} " + - $"rate={config.SampleRate} frame_samples={config.FrameSamples} " + - $"superframe_samples={config.SuperframeSamples} superframe_bytes={config.SuperframeBytes} " + - $"frames_per_superframe={config.FramesPerSuperframe}"); + if (TraceEnabled) + { + Trace( + $"initialized config={Convert.ToHexString(normalizedConfig)} channels={config.ChannelCount} " + + $"rate={config.SampleRate} frame_samples={config.FrameSamples} " + + $"superframe_samples={config.SuperframeSamples} superframe_bytes={config.SuperframeBytes} " + + $"frames_per_superframe={config.FramesPerSuperframe}"); + } return true; } catch (Exception exception) when ( @@ -291,11 +294,14 @@ public Atrac9DecodeResult Decode( catch (Exception exception) when ( exception is ArgumentException or InvalidDataException or InvalidOperationException or IndexOutOfRangeException) { - Trace( - $"decode_failed superframe_bytes={config.SuperframeBytes} " + - $"config={Convert.ToHexString(_configData ?? [])} " + - $"head={Convert.ToHexString(_compressed.AsSpan(0, Math.Min(16, _compressed.Length)))} " + - $"error={exception.GetType().Name}: {exception.Message}"); + if (TraceEnabled) + { + Trace( + $"decode_failed superframe_bytes={config.SuperframeBytes} " + + $"config={Convert.ToHexString(_configData ?? [])} " + + $"head={Convert.ToHexString(_compressed.AsSpan(0, Math.Min(16, _compressed.Length)))} " + + $"error={exception.GetType().Name}: {exception.Message}"); + } _compressedLength = 0; return new Atrac9DecodeResult( status | ResultInvalidData | ResultCodecError, @@ -361,7 +367,10 @@ private ContainerScan ScanContainerHeader(ReadOnlySpan input, out int inpu // as raw superframes rather than swallowing it forever. The // scratch is dropped without advancing the caller's cursor, so // the bytes are still decoded normally. - Trace($"container_scan_gave_up bytes={totalLength}"); + if (TraceEnabled) + { + Trace($"container_scan_gave_up bytes={totalLength}"); + } _containerHeader = null; _containerHeaderLength = 0; return ContainerScan.NotContainer; @@ -372,7 +381,10 @@ private ContainerScan ScanContainerHeader(ReadOnlySpan input, out int inpu } inputHeaderBytes = dataOffset - previousLength; - Trace($"container_header_skipped bytes={dataOffset} from_this_input={inputHeaderBytes}"); + if (TraceEnabled) + { + Trace($"container_header_skipped bytes={dataOffset} from_this_input={inputHeaderBytes}"); + } return ContainerScan.Found; } @@ -469,9 +481,12 @@ private static void WriteInterleaved( } } + // SHARPEMU_LOG_AJM=1 enables the ajm.at9.* traces; read once, not per call. + private static readonly bool TraceEnabled = string.Equals(Environment.GetEnvironmentVariable("SHARPEMU_LOG_AJM"), "1", StringComparison.Ordinal); + private static void Trace(string message) { - if (string.Equals(Environment.GetEnvironmentVariable("SHARPEMU_LOG_AJM"), "1", StringComparison.Ordinal)) + if (TraceEnabled) { Console.Error.WriteLine($"[LOADER][TRACE] ajm.at9.{message}"); } diff --git a/src/SharpEmu.Libs/Audio/AudioOut2Exports.cs b/src/SharpEmu.Libs/Audio/AudioOut2Exports.cs index acd484a6f..9a12e2676 100644 --- a/src/SharpEmu.Libs/Audio/AudioOut2Exports.cs +++ b/src/SharpEmu.Libs/Audio/AudioOut2Exports.cs @@ -126,6 +126,12 @@ private sealed class ContextState private readonly object _paceGate = new(); private long _nextAdvanceTimestamp; + // Serialises the grains of this context (they share the ports' spare PCM buffers and must reach + // the host stream in order). Submitting holds it across the host stream's pacing wait, which can + // sleep up to a quarter of a second for ring space, so it is per context: one global gate made + // two contexts on two threads wait out each other's sleeps. + public readonly object SubmitGate = new(); + public ContextState(ulong handle, uint frequency, uint grainSamples, uint queueDepth, IHostAudioStream? backend) { Handle = handle; @@ -294,7 +300,6 @@ public PortState( private static string PrimaryBackendName = "none"; private static string SecondaryBackendName = "none"; private static ulong PrimaryContextHandle; - private static readonly object HostSubmitGate = new(); private static Func? _streamFactoryForTests; private static int _chatEnabled; @@ -1573,7 +1578,7 @@ private static bool TrySubmitContextAudio(CpuContext ctx, ContextState context) return false; } - lock (HostSubmitGate) + lock (context.SubmitGate) { var mix = ArrayPool.Shared.Rent(frames * 2); var output = ArrayPool.Shared.Rent(frames * AudioPcmConversion.OutputFrameSize); diff --git a/src/SharpEmu.Libs/CxxAbiExports.cs b/src/SharpEmu.Libs/CxxAbiExports.cs index 1d174dc00..fab20ee57 100644 --- a/src/SharpEmu.Libs/CxxAbiExports.cs +++ b/src/SharpEmu.Libs/CxxAbiExports.cs @@ -197,9 +197,13 @@ private static bool TryWriteGuardState(CpuContext ctx, ulong guardPtr, ulong sta return ctx.TryWriteUInt64(guardPtr, newWord); } + // SHARPEMU_LOG_GUARDS=1 traces the __cxa_guard_* calls; read once, not per call. + private static readonly bool LogGuards = + string.Equals(Environment.GetEnvironmentVariable("SHARPEMU_LOG_GUARDS"), "1", StringComparison.Ordinal); + private static void LogGuardState(CpuContext ctx, string op, ulong guardPtr, bool initialized, bool inProgress) { - if (!string.Equals(Environment.GetEnvironmentVariable("SHARPEMU_LOG_GUARDS"), "1", StringComparison.Ordinal)) + if (!LogGuards) { return; } @@ -211,7 +215,7 @@ private static void LogGuardState(CpuContext ctx, string op, ulong guardPtr, boo private static void LogGuardResult(string op, ulong guardPtr, int result, bool initialized, bool inProgress, int ownerThreadId) { - if (!string.Equals(Environment.GetEnvironmentVariable("SHARPEMU_LOG_GUARDS"), "1", StringComparison.Ordinal)) + if (!LogGuards) { return; } diff --git a/src/SharpEmu.Libs/Gpu/GpuCommands/Registers/ContextRegisters.cs b/src/SharpEmu.Libs/Gpu/GpuCommands/Registers/ContextRegisters.cs index 705658b2f..9f7ffe8fa 100644 --- a/src/SharpEmu.Libs/Gpu/GpuCommands/Registers/ContextRegisters.cs +++ b/src/SharpEmu.Libs/Gpu/GpuCommands/Registers/ContextRegisters.cs @@ -261,22 +261,27 @@ public struct DepthShaderControlRegisters public bool DepthExportEnable; public bool StencilExportEnable; public bool MaskExportEnable; - public bool DualExportEnable; + // EXEC_ON_HIER_FAIL (bit 9) and EXEC_ON_NOOP (bit 10): the shader still runs for a fragment the + // early depth test or HiZ rejects, so its memory writes happen. + public bool ExecuteOnHierFail; public bool ExecuteOnNoop; public bool AlphaToMaskDisable; + // DEPTH_BEFORE_SHADER (bit 12): the depth test is forced ahead of the shader ([earlydepthstencil]). + public bool DepthBeforeShader; public static DepthShaderControlRegisters Decode(uint value) => new() { - RemainingBits = value & 0xFFFF_908Cu, + RemainingBits = value & 0xFFFF_808Cu, ConservativeDepthExport = (byte)RegisterField.Get(value, 13, 0x3), DepthExportOrder = (byte)RegisterField.Get(value, 4, 0x3), KillEnable = RegisterField.Bit(value, 6), DepthExportEnable = RegisterField.Bit(value, 0), StencilExportEnable = RegisterField.Bit(value, 1), MaskExportEnable = RegisterField.Bit(value, 8), - DualExportEnable = RegisterField.Bit(value, 9), + ExecuteOnHierFail = RegisterField.Bit(value, 9), ExecuteOnNoop = RegisterField.Bit(value, 10), AlphaToMaskDisable = RegisterField.Bit(value, 11), + DepthBeforeShader = RegisterField.Bit(value, 12), }; } diff --git a/src/SharpEmu.Libs/Gpu/IGuestCompiledShader.cs b/src/SharpEmu.Libs/Gpu/IGuestCompiledShader.cs index 5bb1d4a9f..155e31474 100644 --- a/src/SharpEmu.Libs/Gpu/IGuestCompiledShader.cs +++ b/src/SharpEmu.Libs/Gpu/IGuestCompiledShader.cs @@ -14,6 +14,10 @@ public interface IGuestCompiledShader byte[] Payload { get; } uint? InputLocationMask => null; + /// The words the flattened resource table bound with this module must have: a + /// host binds a shorter table zero-padded to it. + uint FlattenedTableMinimumWords => 0; + /// File extension for diagnostics dumps of ("spv", /// "msl", ...), so dumps stay honestly labeled whatever the backend. string PayloadFileExtension { get; } diff --git a/src/SharpEmu.Libs/Gpu/Pipelines/PixelStageInputResolver.cs b/src/SharpEmu.Libs/Gpu/Pipelines/PixelStageInputResolver.cs index 6fc49799a..9239fe80d 100644 --- a/src/SharpEmu.Libs/Gpu/Pipelines/PixelStageInputResolver.cs +++ b/src/SharpEmu.Libs/Gpu/Pipelines/PixelStageInputResolver.cs @@ -160,14 +160,26 @@ public static PixelInputInfo Resolve( DepthExportFormat = shaderInterface.DepthExportFormat is var zFormat and > 0 and <= 9 ? (SharpEmu.ShaderCompiler.Gen5ColorExportFormat)zFormat : null, - // Z_ORDER 0 is LATE_Z; EARLY_Z_THEN_LATE_Z, RE_Z and EARLY_Z_THEN_RE_Z (1..3) all test - // before the shader. A shader that kills or exports depth, stencil or coverage is - // decided after it ran, whatever the order says. - EarlyDepth = control.DepthExportOrder != 0 && !control.KillEnable && !control.DepthExportEnable && - !control.StencilExportEnable && !control.MaskExportEnable, + EarlyDepth = TestsDepthBeforeShader(control), ExecuteOnNoop = control.ExecuteOnNoop, }; } + + // Whether the pixel function may declare [[early_fragment_tests]]: the guest tests depth before + // the shader AND skips the shader for a fragment that test rejects. That only changes what a + // shader with memory writes does (Metal already tests early for one without), and then it + // matters: the stores of a rejected fragment must happen exactly when the guest's do. + // - DEPTH_BEFORE_SHADER forces the test ahead of the shader whatever the other bits say + // (Mesa RADV sets it with [earlydepthstencil] next to EXEC_ON_* = writes_memory). + // - Z_ORDER 1..3 (EARLY_Z_THEN_LATE_Z, RE_Z, EARLY_Z_THEN_RE_Z) test early too, but EXEC_ON_HIER_FAIL + // and EXEC_ON_NOOP make the shader run for a rejected fragment, and then its stores happen: + // those shaders stay on Metal's late order. + // - A shader that kills or exports depth, stencil or coverage is decided after it ran, and Metal + // rejects depth or stencil outputs with the attribute. + internal static bool TestsDepthBeforeShader(in DepthShaderControlRegisters control) => + !control.KillEnable && !control.DepthExportEnable && !control.StencilExportEnable && !control.MaskExportEnable && + (control.DepthBeforeShader || + (control.DepthExportOrder != 0 && !control.ExecuteOnHierFail && !control.ExecuteOnNoop)); } // The arrays of the last pixel input resolved. Nothing writes a resolved input's arrays, and diff --git a/src/SharpEmu.Libs/Gpu/Pipelines/ShaderProgramCache.cs b/src/SharpEmu.Libs/Gpu/Pipelines/ShaderProgramCache.cs index f89a5f351..27933df80 100644 --- a/src/SharpEmu.Libs/Gpu/Pipelines/ShaderProgramCache.cs +++ b/src/SharpEmu.Libs/Gpu/Pipelines/ShaderProgramCache.cs @@ -1583,7 +1583,7 @@ private ProgramPermutation CompilePermutation( CompiledShaderDump.Write(source.Label, source.Address, source.Hash, compiled, program); var id = ++_nextProgramId; var module = _host.CreateShaderModule(compiled, source.Stage, source.Hash, id); - var info = CreateProgramInfo(source, entry, resources, layout, request, compiled.InputLocationMask); + var info = CreateProgramInfo(source, entry, resources, layout, request, compiled.InputLocationMask, compiled.FlattenedTableMinimumWords); if (source.Stage != ShaderStage.Compute && _host.GraphicsPrewarm is { } graphicsPrewarm && _codeCaptures.TryGetValue((source.Hash, source.CodeSize, source.ContinuationAddressOffset), out var graphicsCapture)) @@ -2026,7 +2026,7 @@ private static ShaderProgramInfo CreateProgramInfo( ProgramSourceEntry entry, SpecializedResourceInfo resources, BindingLayout layout, - ShaderCompileRequest request, uint? inputLocationMask) + ShaderCompileRequest request, uint? inputLocationMask, uint flattenedTableMinimumWords = 0) { var info = resources.Info; var buffers = new BufferResourceInfo[info.Buffers.Count]; @@ -2091,6 +2091,7 @@ private static ShaderProgramInfo CreateProgramInfo( Resources = resources, Bindings = layout, CallRecordCallees = request.CallRecordCallees, + FlattenedTableMinimumWords = flattenedTableMinimumWords, }; } } diff --git a/src/SharpEmu.Libs/Gpu/Rendering/ParallelProgramPrefetch.cs b/src/SharpEmu.Libs/Gpu/Rendering/ParallelProgramPrefetch.cs index a7f1d13ad..a654314b2 100644 --- a/src/SharpEmu.Libs/Gpu/Rendering/ParallelProgramPrefetch.cs +++ b/src/SharpEmu.Libs/Gpu/Rendering/ParallelProgramPrefetch.cs @@ -34,7 +34,7 @@ public sealed class Job(RegisterBanks banks, long gate, bool indexed = false) private readonly IConcurrentResolveHost _concurrent; private readonly object _lock = new(); private readonly Queue _queue = new(); - private readonly Thread _thread; + private readonly SharpEmu.HLE.Host.HostHelperThread _thread; private int _outstanding; private long _finished; private long _executedGate; @@ -52,8 +52,9 @@ public ParallelProgramPrefetch(RenderExecutor executor, IRenderHost host, IConcu _host = host; _fatal = host.Fatal; _concurrent = concurrent; - _thread = new Thread(Run) { IsBackground = true, Name = "SharpEmu program resolve" }; - _thread.Start(); + // The worker waits on this thread per draw: it runs at the worker's class, with the + // worker's stack, since it resolves what the worker would resolve itself. + _thread = SharpEmu.HLE.Host.HostHelperThread.Start("SharpEmu program resolve", 16 * 1024 * 1024, Run); } // The job of the draw the render thread runs now; set around each deferred draw. diff --git a/src/SharpEmu.Libs/Gpu/Rendering/RenderExecutor.Resources.cs b/src/SharpEmu.Libs/Gpu/Rendering/RenderExecutor.Resources.cs index 60a03da0c..4e1239369 100644 --- a/src/SharpEmu.Libs/Gpu/Rendering/RenderExecutor.Resources.cs +++ b/src/SharpEmu.Libs/Gpu/Rendering/RenderExecutor.Resources.cs @@ -754,6 +754,11 @@ private string RecordDraw( { writeStages |= meshInput is null ? GpuPipelineStage.VertexShaderBit : GpuPipelineStage.MeshShaderBitExt; } + else if (vertexProgram.WritesDeviceAddresses || WritesStorageImage(vertexProgram)) + { + _vertexStoresUnordered = true; + _vertexStoresPass = _host.RenderPassSerial; + } if (state.PixelActive && DrawWritesMemory(pixelInput.Stage)) { diff --git a/src/SharpEmu.Libs/Gpu/Rendering/RenderExecutor.cs b/src/SharpEmu.Libs/Gpu/Rendering/RenderExecutor.cs index 129758d80..588e2857f 100644 --- a/src/SharpEmu.Libs/Gpu/Rendering/RenderExecutor.cs +++ b/src/SharpEmu.Libs/Gpu/Rendering/RenderExecutor.cs @@ -95,6 +95,20 @@ private struct ColorTargetStates private bool _fragmentWritesPending; private long _fragmentWritesPass; + // A draw stored to memory from its vertex stage through a device address or a storage image, + // which no barrier after the draw orders, in the pass with this serial. + private bool _vertexStoresUnordered; + private long _vertexStoresPass; + + /// + /// Whether the open render pass holds shader stores no barrier has ordered yet: fragment stores + /// whose pass end is deferred, or vertex stores the draw's own barrier does not cover. A guest + /// cache flush or invalidate in the pass has nothing else to make visible to later draws. + /// + public bool HasUnorderedStoresInOpenPass => + (_fragmentWritesPending && _host.RenderPassSerial == _fragmentWritesPass) || + (_vertexStoresUnordered && _host.RenderPassSerial == _vertexStoresPass); + // The buffer ranges those deferred stores may have written, or untracked when one of the // storing draws wrote through a device address or a storage image. private readonly List<(ulong Start, ulong End)> _fragmentWriteRanges = new(); @@ -1025,7 +1039,7 @@ private static bool PixelShaderHasDepthOrCoverageSideEffects(ShaderInterfaceRegi var control = shaderInterface.DepthShaderControl; // The export format describes data; it does not enable shader execution. return control.KillEnable || control.DepthExportEnable || control.StencilExportEnable || - control.MaskExportEnable || control.DualExportEnable || control.ExecuteOnNoop; + control.MaskExportEnable || control.ExecuteOnHierFail || control.ExecuteOnNoop; } private static bool HasActivePixelShader(RegisterBanks banks) diff --git a/src/SharpEmu.Libs/Gpu/Rendering/ShaderStageResources.cs b/src/SharpEmu.Libs/Gpu/Rendering/ShaderStageResources.cs index ddc00abb9..0941203f2 100644 --- a/src/SharpEmu.Libs/Gpu/Rendering/ShaderStageResources.cs +++ b/src/SharpEmu.Libs/Gpu/Rendering/ShaderStageResources.cs @@ -78,6 +78,9 @@ public class ShaderProgramInfo public SpecializedResourceInfo? Resources { get; init; } public BindingLayout? Bindings { get; init; } + // The words a host pads the flattened resource table it binds to (IGuestCompiledShader). + public uint FlattenedTableMinimumWords { get; init; } + // What the host resolves for each record of each callee, by callee slot, when the program reads its records' // descriptors at run time (ShaderCompileRequest.CallRecordCallees). public IReadOnlyList CallRecordCallees { get; init; } = []; diff --git a/src/SharpEmu.Libs/Gpu/Scheduling/GpuFatal.cs b/src/SharpEmu.Libs/Gpu/Scheduling/GpuFatal.cs index 8a1bacb71..b6627c238 100644 --- a/src/SharpEmu.Libs/Gpu/Scheduling/GpuFatal.cs +++ b/src/SharpEmu.Libs/Gpu/Scheduling/GpuFatal.cs @@ -24,6 +24,12 @@ public static class GpuFatal public sealed class AbandonedWorkException(string message) : Exception(message); + /// + /// Whether this thread is doing work ahead of need (): a backend + /// may abandon it rather than block, for example rather than wait for the GPU. + /// + public static bool IsSpeculative => ThrowOnFatal; + /// /// Runs work done ahead of need on any thread with set: a fatal /// condition inside it abandons the work instead of ending the process, and whoever needs the diff --git a/src/SharpEmu.Libs/Kernel/KernelEventQueueCompatExports.Kevent.cs b/src/SharpEmu.Libs/Kernel/KernelEventQueueCompatExports.Kevent.cs index ebfd6dde3..bdcd72936 100644 --- a/src/SharpEmu.Libs/Kernel/KernelEventQueueCompatExports.Kevent.cs +++ b/src/SharpEmu.Libs/Kernel/KernelEventQueueCompatExports.Kevent.cs @@ -178,7 +178,11 @@ public static int PosixKevent(CpuContext ctx) return PosixFail(ctx, PosixEfault); } - RefreshDescriptorKnotes(state); + if (Volatile.Read(ref state.DescriptorKnotes) != 0) + { + RefreshDescriptorKnotes(state); + } + var delivered = DequeueEvents(ctx, state, eventList, eventCapacity); if (delivered > 0 || poll) { @@ -186,25 +190,28 @@ public static int PosixKevent(CpuContext ctx) return delivered; } + var waiter = new EqueueWaiter + { + Ctx = ctx, + State = state, + EventsAddress = eventList, + EventCapacity = eventCapacity, + OutCountAddress = 0, + WaiterId = Interlocked.Increment(ref _nextEventQueueWaiterId), + PosixResult = true, + }; + waiter.Arm(); if (GuestThreadExecution.RequestCurrentThreadBlock( ctx, "kevent", state.WakeKey, - new EqueueWaiter - { - Ctx = ctx, - State = state, - EventsAddress = eventList, - EventCapacity = eventCapacity, - OutCountAddress = 0, - WaiterId = Interlocked.Increment(ref _nextEventQueueWaiterId), - PosixResult = true, - }, + waiter, deadline)) { return 0; } + waiter.Disarm(); var outcome = HostWait(state, deadline, eventCapacity, out var events, out var count); return outcome switch { diff --git a/src/SharpEmu.Libs/Kernel/KernelEventQueueCompatExports.cs b/src/SharpEmu.Libs/Kernel/KernelEventQueueCompatExports.cs index e61b2b6f1..5e38de0b1 100644 --- a/src/SharpEmu.Libs/Kernel/KernelEventQueueCompatExports.cs +++ b/src/SharpEmu.Libs/Kernel/KernelEventQueueCompatExports.cs @@ -110,6 +110,15 @@ private sealed class Kqueue public readonly KnoteList Active = new(); public bool Deleted; public int Descriptor = -1; + + // Guest waiters (sceKernelWaitEqueue / kevent) that have asked to block on this queue and have + // not resumed yet. Signal skips the scheduler scan while it is zero; see Signal for why that + // cannot lose a wake. + public int BlockedWaiters; + + // Read/write-filter knotes in Knotes, mutated under Gate. A wait or poll only has descriptor + // readiness to refresh when this is non-zero. + public int DescriptorKnotes; } private struct Occurrence @@ -280,8 +289,31 @@ private enum WaitCompletion // kevent(2) reports the count as its return value and failures through errno. public bool PosixResult { get; init; } + private int _armed; + + // Counts this waiter against its queue until it resumes. Call before asking the scheduler to + // block: the increment must precede the scheduler's post-park TryWake check (see Signal). + public void Arm() + { + if (Interlocked.Exchange(ref _armed, 1) == 0) + { + Interlocked.Increment(ref State.BlockedWaiters); + } + } + + // Idempotent; a waiter that never resumes (thread torn down) only leaves the count high, which + // keeps the scan on for that queue and is never unsafe. + public void Disarm() + { + if (Interlocked.Exchange(ref _armed, 0) == 1) + { + Interlocked.Decrement(ref State.BlockedWaiters); + } + } + public int Resume() { + Disarm(); KernelQueuedEvent[]? reservedEvents; int reservedCount; WaitCompletion completion; @@ -836,7 +868,11 @@ public static int KernelWaitEqueue(CpuContext ctx) return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; } - RefreshDescriptorKnotes(state); + if (Volatile.Read(ref state.DescriptorKnotes) != 0) + { + RefreshDescriptorKnotes(state); + } + var deliveredCount = DequeueEvents(ctx, state, eventsAddress, eventCapacity); if (outCountAddress != 0 && !TryWriteUInt32(ctx, outCountAddress, (uint)deliveredCount)) { @@ -866,19 +902,21 @@ public static int KernelWaitEqueue(CpuContext ctx) var deadline = timeoutAddress == 0 ? 0 : GuestThreadExecution.ComputeDeadlineTimestamp(TimeSpan.FromMicroseconds(timeoutUsec)); + var waiter = new EqueueWaiter + { + Ctx = ctx, + State = state, + EventsAddress = eventsAddress, + EventCapacity = eventCapacity, + OutCountAddress = outCountAddress, + WaiterId = waiterId, + }; + waiter.Arm(); if (GuestThreadExecution.RequestCurrentThreadBlock( ctx, "sceKernelWaitEqueue", state.WakeKey, - new EqueueWaiter - { - Ctx = ctx, - State = state, - EventsAddress = eventsAddress, - EventCapacity = eventCapacity, - OutCountAddress = outCountAddress, - WaiterId = waiterId, - }, + waiter, deadline)) { // The scheduler re-checks the waiter once the thread is parked, so an @@ -896,6 +934,7 @@ public static int KernelWaitEqueue(CpuContext ctx) } // A host thread parks on the queue monitor; every activation pulses it. + waiter.Disarm(); var outcome = HostWait(state, deadline, eventCapacity, out var events, out var count); if (outcome == HostWaitOutcome.Deleted) { @@ -1070,6 +1109,10 @@ private static void DropKnoteLocked(Kqueue state, Knote knote) ReferenceEquals(current, knote)) { state.Knotes.Remove((knote.Ident, knote.Filter)); + if (knote.Filter is KernelEventFilterRead or KernelEventFilterWrite) + { + state.DescriptorKnotes--; + } } } @@ -1248,6 +1291,21 @@ private static bool TryReserveEvents( } /// Wakes every waiter of the queue: guest threads by wake key. + /// + /// The scheduler scan visits every guest thread under its gate, so it is skipped when no guest + /// waiter is registered. That is a store-buffering (Dekker) pair and cannot lose a wake: + /// + /// Waiter: BlockedWaiters++ (Interlocked, a full fence) happens before it asks to + /// block, and the scheduler re-checks after parking the thread, + /// which reads the queue under Gate. + /// Signaller: the event is made visible under Gate before this call, then the full + /// fence below, then the read of BlockedWaiters. + /// + /// If this read sees zero, it precedes the waiter's increment in the fence order, so the event + /// was already published when the waiter's re-check took Gate and the re-check finds it + /// (the waiter is readied directly, no wake needed). If it sees non-zero, the scan runs as before. + /// A stale non-zero count only costs an unneeded scan. + /// private static void Signal(Kqueue state, string? detail = null) { if (_logEqueue) @@ -1258,9 +1316,18 @@ private static void Signal(Kqueue state, string? detail = null) $"generation={state.Generation}" + (detail is null ? string.Empty : $" {detail}")); } + Interlocked.MemoryBarrier(); + if (Volatile.Read(ref state.BlockedWaiters) == 0) + { + return; + } + _ = GuestThreadExecution.Scheduler?.WakeBlockedThreads(state.WakeKey); } + internal static int BlockedWaiterCountForTest(ulong handle) => + TryGetLiveEventQueue(handle, out var state) ? Volatile.Read(ref state.BlockedWaiters) : -1; + // ---- Registration API used by the other libraries ---- public static bool RegisterEvent( @@ -1337,6 +1404,11 @@ private static Knote AttachOrUpdateLocked( } state.Knotes[key] = knote; + if (filter is KernelEventFilterRead or KernelEventFilterWrite) + { + state.DescriptorKnotes++; + } + created = true; return knote; } @@ -1440,6 +1512,10 @@ public static bool EnqueueEvent(ulong handle, KernelQueuedEvent queuedEvent) { knote = new Knote(state, queuedEvent.Ident, queuedEvent.Filter) { Transient = true }; state.Knotes[key] = knote; + if (queuedEvent.Filter is KernelEventFilterRead or KernelEventFilterWrite) + { + state.DescriptorKnotes++; + } } knote.Injected = queuedEvent; diff --git a/src/SharpEmu.Libs/Kernel/KernelMemoryCompatExports.cs b/src/SharpEmu.Libs/Kernel/KernelMemoryCompatExports.cs index 1b52be4ae..29e0d95d5 100644 --- a/src/SharpEmu.Libs/Kernel/KernelMemoryCompatExports.cs +++ b/src/SharpEmu.Libs/Kernel/KernelMemoryCompatExports.cs @@ -36,6 +36,9 @@ public static partial class KernelMemoryCompatExports // SHARPEMU_MEMSET_INPLACE=0 sends every memset through the staged chunk loop again. private static readonly bool InPlaceMemsetEnabled = !string.Equals(Environment.GetEnvironmentVariable("SHARPEMU_MEMSET_INPLACE"), "0", StringComparison.Ordinal); + // SHARPEMU_LOG_LIBC_ALLOC=1 traces every malloc/calloc/realloc/free; read once, not per call. + private static readonly bool LogLibcAllocations = + string.Equals(Environment.GetEnvironmentVariable("SHARPEMU_LOG_LIBC_ALLOC"), "1", StringComparison.Ordinal); private static readonly GuestSpanFill FillWithByte = static (value, destination) => { destination.Fill(value); @@ -4712,7 +4715,7 @@ private static void TraceLibcAllocation( ulong resultAddress = 0, int? errorCode = null) { - if (!string.Equals(Environment.GetEnvironmentVariable("SHARPEMU_LOG_LIBC_ALLOC"), "1", StringComparison.Ordinal)) + if (!LogLibcAllocations) { return; } diff --git a/src/SharpEmu.Libs/Pad/HostWindowInput.cs b/src/SharpEmu.Libs/Pad/HostWindowInput.cs index 3cf66fcde..5db2ae456 100644 --- a/src/SharpEmu.Libs/Pad/HostWindowInput.cs +++ b/src/SharpEmu.Libs/Pad/HostWindowInput.cs @@ -21,6 +21,31 @@ public static class HostWindowInput /// Moves on every change of the state below; a reader polls it to find changes. public static long Version => Interlocked.Read(ref _version); + // Under Gate: moves the version and wakes a WaitForChange. + private static void Bump() + { + _version++; + Monitor.PulseAll(Gate); + } + + /// + /// Blocks until differs from or + /// passes; true when it changed. A change is seen as soon as it is made, not at the next poll. + /// + public static bool WaitForChange(long seen, TimeSpan timeout) + { + lock (Gate) + { + if (_version != seen) + { + return true; + } + + _ = Monitor.Wait(Gate, timeout); + return _version != seen; + } + } + /// /// The whole state at once, for the two-process split's GPU process, which owns the window and /// sends it to the guest process: the pressed keys (as many as fit), whether the window has @@ -59,7 +84,7 @@ public static void SetKeys(ReadOnlySpan keys) PressedKeys.Add(key); } - _version++; + Bump(); } } @@ -70,7 +95,7 @@ public static void Connect(IHostGamepadOutput? gamepadOutput = null) _focused = true; _gamepadOutput = gamepadOutput; PressedKeys.Clear(); - _version++; + Bump(); } HostWindowInputSource.Set(Source); @@ -86,7 +111,7 @@ public static void Disconnect() _gamepadState = default; _gamepadOutput = null; PressedKeys.Clear(); - _version++; + Bump(); } HostWindowInputSource.Clear(Source); @@ -102,7 +127,7 @@ public static void SetFocused(bool focused) PressedKeys.Clear(); } - _version++; + Bump(); } } @@ -119,7 +144,7 @@ public static void SetKey(int virtualKey, bool down) PressedKeys.Remove(virtualKey); } - _version++; + Bump(); } } @@ -130,7 +155,7 @@ public static void SetGamepad(string? name, HostGamepadState state) _gamepadConnected = state.Connected; _gamepadName = name; _gamepadState = state; - _version++; + Bump(); } } @@ -141,7 +166,7 @@ public static void ClearGamepad() _gamepadConnected = false; _gamepadName = null; _gamepadState = default; - _version++; + Bump(); } } diff --git a/src/SharpEmu.Libs/PlayGo/PlayGoExports.cs b/src/SharpEmu.Libs/PlayGo/PlayGoExports.cs index 2b048ae3b..ade265821 100644 --- a/src/SharpEmu.Libs/PlayGo/PlayGoExports.cs +++ b/src/SharpEmu.Libs/PlayGo/PlayGoExports.cs @@ -489,7 +489,11 @@ public static int PlayGoGetLocus(CpuContext ctx) return OrbisPlayGoErrorBadSize; } - var loci = new byte[numberOfEntries]; + // numberOfEntries is at most MaxPlayGoQueryEntries; the usual query is a handful of chunks, + // so keep those on the stack and fall back to the heap for a larger one. + Span lociStorage = numberOfEntries <= 256 ? stackalloc byte[256] : new byte[numberOfEntries]; + var loci = lociStorage[..(int)numberOfEntries]; + loci.Clear(); for (uint i = 0; i < numberOfEntries; i++) { if (!ctx.TryReadUInt16(chunkIds + (i * sizeof(ushort)), out var chunkId)) @@ -516,13 +520,13 @@ public static int PlayGoGetLocus(CpuContext ctx) // Titles rely on this as an enumeration terminator: Monster Truck // scans ids 0,1,2,... until BAD_CHUNK_ID, and answering OK for every // id makes that scan wrap the ushort range and spin forever. - loci[i] = PlayGoLocusNotDownloaded; + loci[(int)i] = PlayGoLocusNotDownloaded; return ctx.Memory.TryWrite(outLoci, loci) ? OrbisPlayGoErrorBadChunkId : (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; } - loci[i] = PlayGoLocusLocalFast; + loci[(int)i] = PlayGoLocusLocalFast; } TracePlayGoLocus(ctx, numberOfEntries, chunkIds, outLoci); @@ -990,9 +994,13 @@ private static void AddChunkIds(string xml, Regex pattern, HashSet chunk } } + // SHARPEMU_LOG_PLAYGO=1 enables the playgo.* traces; read once, not per call. + private static readonly bool LogPlayGo = + string.Equals(Environment.GetEnvironmentVariable("SHARPEMU_LOG_PLAYGO"), "1", StringComparison.Ordinal); + private static void TracePlayGo(string message) { - if (string.Equals(Environment.GetEnvironmentVariable("SHARPEMU_LOG_PLAYGO"), "1", StringComparison.Ordinal)) + if (LogPlayGo) { Console.Error.WriteLine($"[LOADER][TRACE] playgo.{message}"); } @@ -1000,7 +1008,7 @@ private static void TracePlayGo(string message) private static void TracePlayGoLocus(CpuContext ctx, uint entries, ulong chunkIds, ulong outLoci) { - if (!string.Equals(Environment.GetEnvironmentVariable("SHARPEMU_LOG_PLAYGO"), "1", StringComparison.Ordinal)) + if (!LogPlayGo) { return; } diff --git a/src/SharpEmu.Libs/VideoOut/VideoOutBackendServices.cs b/src/SharpEmu.Libs/VideoOut/VideoOutBackendServices.cs index a11c1ffd0..3b3dda0c5 100644 --- a/src/SharpEmu.Libs/VideoOut/VideoOutBackendServices.cs +++ b/src/SharpEmu.Libs/VideoOut/VideoOutBackendServices.cs @@ -136,12 +136,16 @@ public static void SetSelectedGpuName(string gpuName) VideoOutExports.SetSelectedGpuName(gpuName); } - public static void TraceGpuFlip(ICpuMemory memory, int handle, int bufferIndex, int flipMode, long flipArg, ulong address, int flipEventCount) + // Takes the request id rather than its event count: under the split the count is a round trip + // into the guest process, and the remote bridge traces nothing, so only the in-process path + // reads it (a local lookup). + public static void TraceGpuFlip(ICpuMemory memory, int handle, int bufferIndex, int flipMode, long flipArg, ulong address, ulong requestId) { if (Bridge is { } bridge) - bridge.TraceGpuFlip(handle, bufferIndex, flipMode, flipArg, address, flipEventCount); + bridge.TraceGpuFlip(handle, bufferIndex, flipMode, flipArg, address, requestId); else - VideoOutExports.TraceGpuFlip(memory, handle, bufferIndex, flipMode, flipArg, address, flipEventCount); + VideoOutExports.TraceGpuFlip( + memory, handle, bufferIndex, flipMode, flipArg, address, VideoOutExports.GetFlipEventCount(requestId)); } public static bool TryGetDisplayBufferInfo(int handle, int bufferIndex, out DisplayBufferInfo info) @@ -210,7 +214,7 @@ public interface IVideoOutBackendBridge // The trace reads guest memory through the caller's memory in one process; across the // split it gets the flip's numbers only. - void TraceGpuFlip(int handle, int bufferIndex, int flipMode, long flipArg, ulong address, int flipEventCount); + void TraceGpuFlip(int handle, int bufferIndex, int flipMode, long flipArg, ulong address, ulong requestId); bool TryGetDisplayBufferInfo(int handle, int bufferIndex, out DisplayBufferInfo info); diff --git a/src/SharpEmu.ShaderCompiler.Metal/Gen5MslCodegenOptions.cs b/src/SharpEmu.ShaderCompiler.Metal/Gen5MslCodegenOptions.cs index 29b0e15f1..bd73b4910 100644 --- a/src/SharpEmu.ShaderCompiler.Metal/Gen5MslCodegenOptions.cs +++ b/src/SharpEmu.ShaderCompiler.Metal/Gen5MslCodegenOptions.cs @@ -87,6 +87,35 @@ namespace SharpEmu.ShaderCompiler.Metal; /// only tests depth before the shader on its own for a function that writes no memory, so a shader /// with buffer or image stores ran for fragments the guest hardware would have rejected first. /// SHARPEMU_MSL_EARLYZ=0 leaves the choice to Metal. +/// Read the dwords of an S_BUFFER_LOAD/S_LOAD_DWORDXn from a bound +/// buffer, and of a BUFFER_LOAD_DWORDXn with a contiguous address, in runs of up to four behind one +/// bounds check (sharpemu_load_dwords) instead of one checked load per dword. A run that crosses +/// the end of the binding still reads each dword alone, so nothing a shader sees changes. +/// SHARPEMU_MSL_VECLOAD=0 loads dword by dword. +/// Declare a compute kernel's workgroup size as +/// [[max_total_threads_per_threadgroup(N)]], so the compiler budgets registers for the N +/// threads the dispatch launches rather than the device's 1024. Only a size the dispatch is +/// known to use (more than one thread) is declared. SHARPEMU_MSL_TGHINT=0 leaves it +/// out. +/// Lower V_MIN/MAX/MIN3/MAX3/MED3_F32 outside IEEE mode to float +/// compares and selects on the operand bits, where that gives the bit-level helpers' answer for +/// every input: against an inline constant, and between two registers while FLOAT_MODE flushes +/// f32 input denormals. Everything else keeps the helpers. SHARPEMU_MSL_FMINMAX=0 keeps +/// the helpers everywhere. +/// Read a flattened-table word at a slot fixed at translation +/// time without the bounds select: the shader reports the highest such slot +/// () and the host zero-pads the table it +/// uploads to it, so a slot past the evaluated table still reads 0. +/// SHARPEMU_MSL_TABLEPAD=0 keeps the select. +/// Lower V_RCP/V_RSQ_F32 to the fast:: forms, retaining +/// input flush and special results (0, infinity, negative, NaN). Device sweeps check a +/// one-ULP budget against the precise forms. V_SQRT_F32 stays precise because fast::sqrt +/// exceeded that budget. SHARPEMU_MSL_FASTRCP=0 keeps the precise forms. +/// Leave out #pragma clang fp contract(off) in pixel and +/// compute programs, so the relaxed math mode may fuse a multiply and an add. V_MAD_F32 rounds +/// the product on the guest, so a fused result can differ by an ULP; vertex and mesh programs, +/// which feed positions, keep the pragma either way. Off by default; +/// SHARPEMU_MSL_CONTRACT=1 turns it on. public sealed record Gen5MslCodegenOptions( bool StructuredControlFlow = true, bool RegisterAllocation = true, @@ -104,7 +133,13 @@ public sealed record Gen5MslCodegenOptions( bool PersistentWaitFences = true, bool PersistentRecordLocks = true, bool DeviceAddressSpans = true, - bool EarlyFragmentTests = true) + bool EarlyFragmentTests = true, + bool VectorDwordLoads = true, + bool ThreadgroupSizeHint = true, + bool NativeFloatMinMax = true, + bool ConstantTableSlots = true, + bool FastReciprocals = true, + bool ContractPixelCompute = false) { public static Gen5MslCodegenOptions FromEnvironment { get; } = new( StructuredControlFlow: Environment.GetEnvironmentVariable("SHARPEMU_MSL_STRUCTURED") != "0", @@ -125,5 +160,11 @@ public sealed record Gen5MslCodegenOptions( PersistentWaitFences: Environment.GetEnvironmentVariable("SHARPEMU_MSL_PERSISTENT_FENCES") != "0", PersistentRecordLocks: Environment.GetEnvironmentVariable("SHARPEMU_MSL_PERSISTENT_RECORDS") != "0", DeviceAddressSpans: Environment.GetEnvironmentVariable("SHARPEMU_MSL_DEVICESPAN") != "0", - EarlyFragmentTests: Environment.GetEnvironmentVariable("SHARPEMU_MSL_EARLYZ") != "0"); + EarlyFragmentTests: Environment.GetEnvironmentVariable("SHARPEMU_MSL_EARLYZ") != "0", + VectorDwordLoads: Environment.GetEnvironmentVariable("SHARPEMU_MSL_VECLOAD") != "0", + ThreadgroupSizeHint: Environment.GetEnvironmentVariable("SHARPEMU_MSL_TGHINT") != "0", + NativeFloatMinMax: Environment.GetEnvironmentVariable("SHARPEMU_MSL_FMINMAX") != "0", + ConstantTableSlots: Environment.GetEnvironmentVariable("SHARPEMU_MSL_TABLEPAD") != "0", + FastReciprocals: Environment.GetEnvironmentVariable("SHARPEMU_MSL_FASTRCP") != "0", + ContractPixelCompute: Environment.GetEnvironmentVariable("SHARPEMU_MSL_CONTRACT") == "1"); } diff --git a/src/SharpEmu.ShaderCompiler.Metal/Gen5MslShader.cs b/src/SharpEmu.ShaderCompiler.Metal/Gen5MslShader.cs index eadf317ce..913c3040b 100644 --- a/src/SharpEmu.ShaderCompiler.Metal/Gen5MslShader.cs +++ b/src/SharpEmu.ShaderCompiler.Metal/Gen5MslShader.cs @@ -203,4 +203,9 @@ public sealed record Gen5MslShader( /// role the source is a post-tessellation vertex function, and the host's pipeline takes its /// partition, winding and maximum factor from here. public Gen5TessellationLaunch? Tessellation { get; init; } + + /// The words the flattened resource table bound to this program must have: the + /// source reads the fixed slots below this without a bounds check, so a host binds a + /// shorter table zero-padded to it (see ). + public uint FlattenedTableMinimumWords { get; init; } } diff --git a/src/SharpEmu.ShaderCompiler.Metal/Gen5MslTranslator.Alu.cs b/src/SharpEmu.ShaderCompiler.Metal/Gen5MslTranslator.Alu.cs index e2aa86f3a..8eb4c423b 100644 --- a/src/SharpEmu.ShaderCompiler.Metal/Gen5MslTranslator.Alu.cs +++ b/src/SharpEmu.ShaderCompiler.Metal/Gen5MslTranslator.Alu.cs @@ -291,8 +291,10 @@ private bool TryEmitVectorValue( "VTruncF32" => FloatResult(instruction, $"trunc({F(instruction, 0)})"), "VRndneF32" => FloatResult(instruction, $"rint({F(instruction, 0)})"), "VFractF32" => EmitFractF32(instruction), + // fast::sqrt exceeded the one-ULP budget on input 0x5E606496. Keep + // square root precise even when the reciprocal fast paths are enabled. "VSqrtF32" => InvalidNanF32Result(instruction, "sqrt"), - "VRsqF32" => InvalidNanF32Result(instruction, "rsqrt"), + "VRsqF32" => InvalidNanF32Result(instruction, _options.FastReciprocals ? "fast::rsqrt" : "rsqrt"), "VRcpF16" => Float16Reciprocal(instruction, destination), "VSqrtF16" => Float16SquareRoot(instruction, destination), "VLogF16" => Float16Logarithm(instruction, destination), @@ -310,7 +312,11 @@ private bool TryEmitVectorValue( $"rsqrt({F16(instruction, 0)})"), // F32 denormal inputs reach rcp/rsq/sqrt/log/exp as a signed zero, as // on the guest; the IEEE-flag reciprocal keeps the unflushed input. - "VRcpF32" => FloatResult(instruction, $"(1.0f / sharpemu_flush_f32_denorm({F(instruction, 0)}))"), + "VRcpF32" => FloatResult( + instruction, + _options.FastReciprocals + ? $"fast::divide(1.0f, sharpemu_flush_f32_denorm({F(instruction, 0)}))" + : $"(1.0f / sharpemu_flush_f32_denorm({F(instruction, 0)}))"), "VRcpIflagF32" => FloatResult(instruction, $"(1.0f / {F(instruction, 0)})"), "VLogF32" => InvalidNanF32Result(instruction, "fast::log2"), "VExpF32" => FloatResult(instruction, $"fast::exp2(sharpemu_flush_f32_denorm({F(instruction, 0)}))"), @@ -3028,6 +3034,12 @@ private string MinMaxF32Result(Gen5ShaderInstruction instruction) { var ieee = _request.IeeeMode ? "true" : "false"; string Bits(int index) => FlushInputDenormalF32(AsUInt(F(instruction, index))); + if (_options.NativeFloatMinMax && !_request.IeeeMode && TryNativeMinMaxF32(instruction, Bits) is { } native) + { + var nativeResult = Temp("uint", native); + return HasFloatOutputModifiers(instruction) ? FloatResult(instruction, AsFloat(nativeResult)) : nativeResult; + } + var bits = instruction.Opcode switch { "VMinF32" => $"sharpemu_min_f32({Bits(0)}, {Bits(1)}, {ieee})", @@ -3040,6 +3052,189 @@ private string MinMaxF32Result(Gen5ShaderInstruction instruction) return HasFloatOutputModifiers(instruction) ? FloatResult(instruction, AsFloat(result)) : result; } + /// V_MIN/MAX/MIN3/MAX3/MED3_F32 outside IEEE mode without the bit-level helpers, or + /// null where that is not provably the helpers' answer (see + /// ): + /// * against a non-NaN inline constant c, the answer is x exactly when x is a number that + /// orders above (max) or below (min) c, and c otherwise. The x that do are one or two + /// ranges of bit patterns, so the select tests the bits: no float compare, which a + /// denormal x could not survive; + /// * between two registers while FLOAT_MODE flushes f32 input denormals, fmax/fmin give + /// the number when one operand is NaN, and only a NaN src0 (answer src1) and two + /// operands that compare equal (the same bits, or +0 and -0, where the AND or OR of + /// the bits is the ordered answer) need the bits; + /// * V_MED3 with two constants lo < hi, hi not a zero, is hi above hi, x between them, + /// and lo below lo or for a NaN x (MIN3 then answers lo). + private string? TryNativeMinMaxF32(Gen5ShaderInstruction instruction, Func bits) + { + var count = instruction.Opcode is "VMinF32" or "VMaxF32" ? 2 : 3; + if (instruction.Sources.Count < count) + { + return null; + } + + var constants = new uint?[count]; + for (var index = 0; index < count; index++) + { + constants[index] = TryConstantF32Operand(instruction, index, out var value) ? value : null; + } + + var flushed = (_request.FloatMode & 0x10u) == 0; + string? Pair(string? left, uint? leftConstant, string? right, uint? rightConstant, bool max) + { + if (rightConstant is { } c) + { + return leftConstant is null ? MinMaxAgainstConstant(left!, c, max) : null; + } + + if (leftConstant is { } d) + { + return MinMaxAgainstConstant(right!, d, max); + } + + if (!flushed) + { + return null; + } + + var a = Temp("uint", left!); + var b = Temp("uint", right!); + var fa = AsFloat(a); + var fb = AsFloat(b); + return $"(isnan({fa}) ? {b} : ({fa} == {fb} ? ({a} {(max ? "&" : "|")} {b}) : as_type({(max ? "fmax" : "fmin")}({fa}, {fb}))))"; + } + + string? Operand(int index) => constants[index] is null ? bits(index) : null; + switch (instruction.Opcode) + { + case "VMinF32": + case "VMaxF32": + return Pair(Operand(0), constants[0], Operand(1), constants[1], instruction.Opcode == "VMaxF32"); + case "VMin3F32": + case "VMax3F32": + { + var max = instruction.Opcode == "VMax3F32"; + if (constants[0] is not null && constants[1] is not null) + { + return null; + } + + var first = Pair(Operand(0), constants[0], Operand(1), constants[1], max); + return first is null ? null : Pair(Temp("uint", first), null, Operand(2), constants[2], max); + } + + case "VMed3F32": + { + var variable = -1; + var known = new List(); + for (var index = 0; index < 3; index++) + { + if (constants[index] is { } value) + { + known.Add(value); + } + else + { + variable = index; + } + } + + if (known.Count != 2) + { + return null; + } + + var (lo, hi) = OrderKeyF32(known[0]) < OrderKeyF32(known[1]) ? (known[0], known[1]) : (known[1], known[0]); + if (OrderKeyF32(lo) == OrderKeyF32(hi) || (hi & 0x7FFFFFFFu) == 0) + { + return null; + } + + var x = Temp("uint", bits(variable)); + return $"({BitRangesTest(x, AboveF32(hi))} ? {FormatUInt(hi)} : ({BitRangesTest(x, AboveF32(lo))} ? {x} : {FormatUInt(lo)}))"; + } + } + + return null; + } + + /// Min or max of and the non-NaN constant + /// , as the ISA orders them. + private string MinMaxAgainstConstant(string value, uint constant, bool max) + { + var x = Temp("uint", value); + return $"({BitRangesTest(x, max ? AboveF32(constant) : BelowF32(constant))} ? {x} : {FormatUInt(constant)})"; + } + + /// An operand that is an inline constant or literal, as the f32 bits the + /// instruction reads: its abs and neg applied, and a denormal flushed to a zero of its + /// sign while FLOAT_MODE flushes f32 input denormals. False for a NaN, and for an SDWA + /// operand, whose selects the bits would have to follow. + private bool TryConstantF32Operand(Gen5ShaderInstruction instruction, int index, out uint bits) + { + bits = 0; + if (instruction.Control is not (null or Gen5Vop3Control)) + { + return false; + } + + var operand = instruction.Sources[index]; + if (operand.Kind == Gen5OperandKind.LiteralConstant) + { + bits = operand.Value; + } + else if (operand.Kind != Gen5OperandKind.EncodedConstant || operand.Value is 251 or 252 or 253 || + !Gen5InlineConstants.TryDecode(operand.Value, out bits)) + { + return false; + } + + var (absolute, negate) = SourceModifierMasks(instruction); + if ((absolute & (1u << index)) != 0) + { + bits &= 0x7FFFFFFFu; + } + + if ((negate & (1u << index)) != 0) + { + bits ^= 0x80000000u; + } + + if ((_request.FloatMode & 0x10u) == 0 && (bits & 0x7F800000u) == 0) + { + bits &= 0x80000000u; + } + + return (bits & 0x7FFFFFFFu) <= 0x7F800000u; + } + + /// The ISA's f32 order as an unsigned key (sharpemu_order_f32): -0 below +0. + private static uint OrderKeyF32(uint bits) => (bits & 0x80000000u) != 0 ? ~bits : bits | 0x80000000u; + + /// The non-NaN bit patterns that order above the non-NaN : + /// the positive numbers past it, or, for a negative c, every positive number and the + /// negative ones of smaller magnitude. + private static (uint Low, uint High)[] AboveF32(uint c) => (c & 0x80000000u) == 0 + ? c == 0x7F800000u ? [] : [(c + 1, 0x7F800000u)] + : c == 0x80000000u ? [(0u, 0x7F800000u)] : [(0u, 0x7F800000u), (0x80000000u, c - 1)]; + + /// The non-NaN bit patterns that order below the non-NaN . + private static (uint Low, uint High)[] BelowF32(uint c) => (c & 0x80000000u) != 0 + ? c == 0xFF800000u ? [] : [(c + 1, 0xFF800000u)] + : c == 0u ? [(0x80000000u, 0xFF800000u)] : [(0x80000000u, 0xFF800000u), (0u, c - 1)]; + + private static string BitRangesTest(string value, (uint Low, uint High)[] ranges) + { + if (ranges.Length == 0) + { + return "false"; + } + + return string.Join(" || ", ranges.Select(range => range.Low == 0 + ? $"{value} <= {FormatUInt(range.High)}" + : $"({value} - {FormatUInt(range.Low)}) <= {FormatUInt(range.High - range.Low)}")); + } + /// The f32 bits of an operand read by a bit-exact path, with a denormal flushed /// to a zero of its sign when FLOAT_MODE flushes f32 input denormals (bit 4 clear), as /// the float paths get from Metal. diff --git a/src/SharpEmu.ShaderCompiler.Metal/Gen5MslTranslator.Pixel.cs b/src/SharpEmu.ShaderCompiler.Metal/Gen5MslTranslator.Pixel.cs index a42c4ec48..1a8ce951b 100644 --- a/src/SharpEmu.ShaderCompiler.Metal/Gen5MslTranslator.Pixel.cs +++ b/src/SharpEmu.ShaderCompiler.Metal/Gen5MslTranslator.Pixel.cs @@ -206,7 +206,9 @@ private bool EmitImageOperation( var readsMip = !isStorage && imageClass.SupportsMipLevels && !imageClass.Multisampled; var outside = new List(); var mip = readsMip ? Temp("uint", mipLevel) : null; - if (mip is not null) + // Mip 0 is inside every texture that has one, so a load without a mip operand + // skips the query (a null texture reads zero at any mip either way). + if (mip is not null && mipLevel != "0u") { outside.Add($"{mip} >= {texture}.get_num_mip_levels()"); } @@ -252,7 +254,7 @@ private bool EmitImageOperation( } else if (mip is not null) { - arguments += $", ({away} ? 0u : {mip})"; + arguments += away is null ? $", {mip}" : $", ({away} ? 0u : {mip})"; } sampled = Temp($"vec<{kind}, 4>", $"{texture}.read({arguments})"); diff --git a/src/SharpEmu.ShaderCompiler.Metal/Gen5MslTranslator.Resources.cs b/src/SharpEmu.ShaderCompiler.Metal/Gen5MslTranslator.Resources.cs index 9c05bc5b2..59812916f 100644 --- a/src/SharpEmu.ShaderCompiler.Metal/Gen5MslTranslator.Resources.cs +++ b/src/SharpEmu.ShaderCompiler.Metal/Gen5MslTranslator.Resources.cs @@ -567,8 +567,26 @@ private string ComputeThreadLimit(uint component) return (ShaderDataDword(layout.ShaderBaseDword), ShaderDataDword(layout.ShaderBaseDword + 1)); } - private string LoadFlattenedWord(string slot) => - $"({slot} < {ResourcesName}.flattened_table_words ? {ResourcesName}.flattened_table[{slot}] : 0u)"; + private string LoadFlattenedWord(string slot) + { + // A slot fixed here is read without the select: the host pads the table it binds to + // FlattenedTableMinimumWords with zeros, which is what the select would have read. + if (_options.ConstantTableSlots && slot.Length > 1 && slot[^1] == 'u' && + uint.TryParse(slot.AsSpan(0, slot.Length - 1), NumberStyles.None, CultureInfo.InvariantCulture, out var fixedSlot) && + fixedSlot < MaxPaddedFlattenedTableWords) + { + _flattenedTableMinimumWords = Math.Max(_flattenedTableMinimumWords, fixedSlot + 1); + return $"{ResourcesName}.flattened_table[{slot}]"; + } + + return $"({slot} < {ResourcesName}.flattened_table_words ? {ResourcesName}.flattened_table[{slot}] : 0u)"; + } + + /// A fixed slot past this keeps the select, so no shader asks the host for a + /// table of more than 256 KB. + private const uint MaxPaddedFlattenedTableWords = 1u << 16; + + private uint _flattenedTableMinimumWords; private static string FormatULong(ulong value) => $"0x{value.ToString("X", CultureInfo.InvariantCulture)}ul"; @@ -1276,6 +1294,47 @@ private bool TryEmitLayoutScalarMemory(Gen5ShaderInstruction instruction, Gen5Sc string? deviceAddress = null; string? deviceSpan = null; string? deviceSpanValid = null; + string? scalarRun = null; + + // The run of up to four dwords, starting at a multiple of four, that the dword at + // `component` loads with: every dword of it a plain read of the same bound buffer. + bool TryScalarBufferRun(int component, out (int First, int Count) run) + { + run = default; + if (!_options.VectorDwordLoads || !_options.RawDwordAccess || control.Glc) + { + return false; + } + + var first = component & ~3; + var count = Math.Min(4, instruction.Destinations.Count - first); + if (count < 2) + { + return false; + } + + uint? binding = null; + for (var index = first; index < first + count; index++) + { + if (!request.Memory.TryGetIndex(instruction.Pc, (uint)index, out var memory)) + { + return false; + } + + var record = request.Memory[memory]; + if (record.PlanningOnly || record.Kind != MemoryResourceKind.ScalarBuffer || record.DeviceDescriptor || + record.Resource == MemoryAccessInfo.NoResource || (binding is { } known && known != record.Resource)) + { + return false; + } + + binding = record.Resource; + } + + run = (first, count); + return true; + } + for (var component = 0; component < instruction.Destinations.Count; component++) { var destination = instruction.Destinations[component]; @@ -1338,6 +1397,22 @@ private bool TryEmitLayoutScalarMemory(Gen5ShaderInstruction instruction, Gen5Sc deviceDescriptor ??= DeviceDescriptorBounds(instruction.Sources[0].Value); value = DeviceDescriptorWord(deviceDescriptor.Value, descriptorOffset, control.Glc); } + else if (entry.Kind == MemoryResourceKind.ScalarBuffer && + TryScalarBufferRun(component, out var run)) + { + // A run of the load's dwords from one bound buffer, behind one bounds check + // (Gen5MslCodegenOptions.VectorDwordLoads); every dword is read before the + // first destination store, as the hardware does. + if (run.First == component) + { + var bindingIndex = (int)entry.Resource; + var byteOffset = unchecked((uint)control.ImmediateOffsetBytes + ((uint)component * sizeof(uint))); + var address = Temp("uint", ApplyByteBias(bindingIndex, $"({dynamicOffset} + {FormatUInt(byteOffset)})")); + scalarRun = Temp("uint4", LoadDwords(bindingIndex, address, (uint)run.Count)); + } + + value = Temp("uint", $"{scalarRun}[{component - run.First}]"); + } else if (entry.Kind == MemoryResourceKind.ScalarBuffer) { var bindingIndex = (int)entry.Resource; diff --git a/src/SharpEmu.ShaderCompiler.Metal/Gen5MslTranslator.cs b/src/SharpEmu.ShaderCompiler.Metal/Gen5MslTranslator.cs index 9848cf80f..b3034b2f2 100644 --- a/src/SharpEmu.ShaderCompiler.Metal/Gen5MslTranslator.cs +++ b/src/SharpEmu.ShaderCompiler.Metal/Gen5MslTranslator.cs @@ -648,6 +648,7 @@ public bool TryCompile(out Gen5MslShader shader, out string error) ColorOutputMask = _stage == Gen5MslStage.Pixel ? ColorOutputLocations(_pixelOutputBindings) : 0, PersistentThreads = IsPersistentThreadsKernel(), Tessellation = _request.Tessellation, + FlattenedTableMinimumWords = _flattenedTableMinimumWords, }; return true; } @@ -729,6 +730,19 @@ private static void CloseParameterList(StringBuilder source) _ => "gen5_cs", }; + /// The workgroup size of a compute kernel as its thread limit + /// (). The host dispatches the + /// threadgroup size the shader reports (), + /// except for a one-thread size, which it may replace with the dispatch's, so that one + /// is not declared. + private string ThreadgroupSizeAttribute() + { + var threads = (ulong)_localSizeX * _localSizeY * _localSizeZ; + return _options.ThreadgroupSizeHint && _stage == Gen5MslStage.Compute && threads is > 1 and <= 1024 + ? $"[[max_total_threads_per_threadgroup({threads})]] " + : string.Empty; + } + private void EmitModule(StringBuilder source, IReadOnlyList blocks) { source.AppendLine("// Generated by SharpEmu Gen5MslTranslator."); @@ -741,8 +755,15 @@ private void EmitModule(StringBuilder source, IReadOnlyList blocks) // first. Letting the compiler contract a*b+c makes one program compute a value a // different way than another program that spells it the same, which a depth // EQUAL pass against a prepass cannot survive. - source.AppendLine("#pragma clang fp contract(off)"); - source.AppendLine(); + // Gen5MslCodegenOptions.ContractPixelCompute lifts it where no position or depth + // comes out of the program. + if (!(_options.ContractPixelCompute && + ((_stage == Gen5MslStage.Compute && _request.Tessellation is null) || + (_stage == Gen5MslStage.Pixel && !_pixelDepthExport)))) + { + source.AppendLine("#pragma clang fp contract(off)"); + source.AppendLine(); + } // Uniforms: dispatch bounds plus per-buffer byte lengths. Metal has // no OpArrayLength equivalent, so buffer extents travel with the @@ -895,7 +916,7 @@ private void EmitModule(StringBuilder source, IReadOnlyList blocks) } else if (_stage != Gen5MslStage.Mesh) { - source.AppendLine($"kernel void {EntryPointName}("); + source.AppendLine($"{ThreadgroupSizeAttribute()}kernel void {EntryPointName}("); } { @@ -4085,6 +4106,33 @@ string DwordAddress(uint index) => return true; } + if (!glc && dwordCount > 1 && byteAddress.Contiguous && _options.VectorDwordLoads && _options.RawDwordAccess) + { + // Every dword is loaded before the first destination is written, as the + // hardware reads its address operands once. + var runs = new List(); + for (uint first = 0; first < dwordCount; first += 4) + { + var load = LoadDwords(bindingIndex, DwordAddress(first), Math.Min(4u, dwordCount - first)); + if (_execKnownFull) + { + runs.Add(Temp("uint4", load)); + continue; + } + + var run = Temp("uint4", "uint4(0u)"); + Line($"if (exec) {{ {run} = {load}; }}"); + runs.Add(run); + } + + for (uint index = 0; index < dwordCount; index++) + { + StoreVector(destinationVectorRegister + index, $"{runs[(int)(index / 4)]}[{index % 4}]"); + } + + return true; + } + for (uint index = 0; index < dwordCount; index++) { var address = DwordAddress(index); @@ -4160,6 +4208,11 @@ private string LoadDword(int bindingIndex, string byteAddress) => ? $"sharpemu_load_dword(b{bindingIndex}, {BufferBytes(bindingIndex)}, {byteAddress})" : LoadWord(bindingIndex, byteAddress); + /// A run of one to four raw dwords from on, as a + /// uint4 (see ). + private string LoadDwords(int bindingIndex, string byteAddress, uint count) => + $"sharpemu_load_dwords<{count}>(b{bindingIndex}, {BufferBytes(bindingIndex)}, {byteAddress})"; + private string StoreDwordStatement(int bindingIndex, string byteAddress, string value) => _options.RawDwordAccess ? $"sharpemu_store_dword(b{bindingIndex}, {BufferBytes(bindingIndex)}, {byteAddress}, {value});" diff --git a/src/SharpEmu.ShaderCompiler.Metal/Templates/prelude.msl b/src/SharpEmu.ShaderCompiler.Metal/Templates/prelude.msl index 6881e9750..ab9396da3 100644 --- a/src/SharpEmu.ShaderCompiler.Metal/Templates/prelude.msl +++ b/src/SharpEmu.ShaderCompiler.Metal/Templates/prelude.msl @@ -7,6 +7,32 @@ static inline uint sharpemu_load_dword(device uint* b, uint bytes, uint addr) return index < (bytes >> 2) ? b[index] : 0u; } +// Up to four consecutive raw dwords of an S_BUFFER_LOAD/BUFFER_LOAD_DWORDXn, each what +// sharpemu_load_dword reads at addr + 4 * i. A run that lies inside the binding is one +// bounds check and adjacent loads; one that crosses the end reads dword by dword. +template +static inline uint4 sharpemu_load_dwords(device uint* b, uint bytes, uint addr) +{ + uint4 result = uint4(0u); + uint index = addr >> 2; + uint words = bytes >> 2; + if (index < words && (uint)N <= words - index) + { + for (int i = 0; i < N; i++) + { + result[i] = b[index + (uint)i]; + } + } + else + { + for (int i = 0; i < N; i++) + { + result[i] = sharpemu_load_dword(b, bytes, addr + (uint)(4 * i)); + } + } + return result; +} + // A dword a format access may straddle, assembled from the two it spans. Only the // format path needs this: a descriptor's stride is a byte count, so a component can // start at any byte. diff --git a/src/SharpEmu.ShaderCompiler/Resources/ResourceMaterializationCache.cs b/src/SharpEmu.ShaderCompiler/Resources/ResourceMaterializationCache.cs index 33113dbdf..1de30ad55 100644 --- a/src/SharpEmu.ShaderCompiler/Resources/ResourceMaterializationCache.cs +++ b/src/SharpEmu.ShaderCompiler/Resources/ResourceMaterializationCache.cs @@ -33,6 +33,49 @@ public sealed partial class ResourceMaterializationCache // _generationCapacity keys or at _generationByteBudget estimated retained bytes, // whichever comes first: a count alone let entries with large bindless tables or long // read lists hold hundreds of MB. Dropping the old generation only costs rebuilds. + // A scan of distinct textures can miss forever: retaining a full validation entry + // for every draw then costs more than the walk. Admission is only a retention hint; + // it can decline to store a successful walk, never authorize a hit or skip a read. + private static readonly Action NoteTablePhase = inTable => ReadingTable = inTable; + private const int ColdScanThreshold = 32; + private static readonly bool ColdAdmissionEnabled = + Environment.GetEnvironmentVariable("SHARPEMU_RESOURCE_CACHE_ADMISSION") != "0"; + private int _consecutiveMisses; + private bool _cacheSaturated; + private uint[]? _admissionTags; + public long ColdBypasses { get; private set; } + + private bool DeferColdEntry(ResourceSnapshot snapshot) + { + if (!ColdAdmissionEnabled || !_cacheSaturated) return false; + var hash = new HashCode(); + foreach (var image in snapshot.Images) + { + hash.Add(image.Length); + foreach (var word in image) hash.Add(word); + } + foreach (var sampler in snapshot.Samplers) + { + hash.Add(sampler.Length); + foreach (var word in sampler) hash.Add(word); + } + foreach (var buffer in snapshot.Buffers) + { + hash.Add(buffer.Length); + // Memory-base learning may patch the V# address. Do not let admission + // starve those proofs when the buffer moves on every draw. + for (var index = 1; index < buffer.Length; index++) + hash.Add(index == 1 ? buffer[index] & 0xFFFF0000u : buffer[index]); + } + var tag = (uint)hash.ToHashCode(); + if (tag == 0) tag = 1; + var tags = _admissionTags ??= new uint[512]; + var slot = tag & (uint)(tags.Length - 1); + var repeated = tags[slot] == tag; + tags[slot] = tag; + return _consecutiveMisses >= ColdScanThreshold && !repeated; + } + private readonly int _generationCapacity; private readonly long _generationByteBudget; private Dictionary _young = new(); @@ -123,20 +166,28 @@ public void ForgetRecent() if (!_hasPrint || EntryPrint(built) is not { } print) return null; var key = MixPrint(structure, print); - _recentByPrint.TryGetValue(key, out var previous); + Entry? previous; if (_recentByPrint.Count >= RecentCapacity) { + _recentByPrint.TryGetValue(key, out previous); _recentByPrint.Clear(); RecentBytes -= _recentByPrintBytes; _recentByPrintBytes = 0; + _recentByPrint[key] = built; } - else if (previous is not null) + else { - RecentBytes -= previous.EstimatedBytes; - _recentByPrintBytes -= previous.EstimatedBytes; + // One lookup: a miss whose print is new, the usual case, pays for one hash probe. + ref var slot = ref System.Runtime.InteropServices.CollectionsMarshal.GetValueRefOrAddDefault(_recentByPrint, key, out _); + previous = slot; + slot = built; + if (previous is not null) + { + RecentBytes -= previous.EstimatedBytes; + _recentByPrintBytes -= previous.EstimatedBytes; + } } - _recentByPrint[key] = built; RecentBytes += built.EstimatedBytes; _recentByPrintBytes += built.EstimatedBytes; return previous; @@ -573,12 +624,33 @@ private long[][] PrintOffsetsOf(ShaderResourcePlan plan) private static ulong MixPrint(ulong print, ulong value) => System.Numerics.BitOperations.RotateLeft((print ^ value) * 0x9E37_79B9_7F4A_7C15UL, 29); + // Folds the words into the print in four independent lanes: one dependent chain of up to + // PrintWords multiplies was most of what a print cost. + private static ulong FoldPrint(ulong print, ReadOnlySpan words) + { + ulong first = print, second = print ^ 0x1357_9BDF_2468_ACE1UL, third = print ^ 0x0F1E_2D3C_4B5A_6979UL, + fourth = print ^ 0x7A69_5847_3625_1403UL; + var index = 0; + for (; index + 4 <= words.Length; index += 4) + { + first = MixPrint(first, words[index]); + second = MixPrint(second, words[index + 1]); + third = MixPrint(third, words[index + 2]); + fourth = MixPrint(fourth, words[index + 3]); + } + + for (; index < words.Length; index++) + first = MixPrint(first, words[index]); + return MixPrint(MixPrint(MixPrint(first, second), third), fourth); + } + // The print of the descriptors the entry read through every candidate not rejected: what the // print of the same object drawn again is, wherever its table then lies. Null when none has one. private ulong? EntryPrint(Entry entry) { var print = PrintSeed; var any = false; + Span words = stackalloc ulong[PrintWords]; for (var index = 0; index < _candidates.Length; index++) { var offsets = _printOffsets[index]; @@ -590,26 +662,54 @@ private static ulong MixPrint(ulong print, ulong value) => var addressBase = RawBase(entry.UserData, candidate, entry.UserData[candidate.UserData]); any = true; print = MixPrint(print, (ulong)index); - foreach (var offset in offsets) + var found = words[..offsets.Length]; + if (!TryRecordedRun(entry, addressBase, offsets, found)) { - print = TryFindWord(entry, unchecked(addressBase + (ulong)offset), out var at) - ? MixPrint(print, System.Buffers.Binary.BinaryPrimitives.ReadUInt32LittleEndian(entry.Bytes.AsSpan(at, sizeof(uint)))) - : MixPrint(print, MissingWord); + for (var word = 0; word < offsets.Length; word++) + { + found[word] = TryFindWord(entry, unchecked(addressBase + (ulong)offsets[word]), out var at) + ? System.Buffers.Binary.BinaryPrimitives.ReadUInt32LittleEndian(entry.Bytes.AsSpan(at, sizeof(uint))) + : MissingWord; + } } + + print = FoldPrint(print, found); } return any ? print : null; } + // The words at the offsets when one recorded range holds them all, as a descriptor table's + // usually does: one search for the run instead of one per word, with the same words found. + private static bool TryRecordedRun(Entry entry, ulong addressBase, long[] offsets, Span words) + { + var first = unchecked(addressBase + (ulong)offsets[0]); + if (!TryFindRange(entry, first, out var range, out var at)) + return false; + var last = unchecked(first + (ulong)(offsets[^1] - offsets[0])); + if (last < first || last - entry.RangeAddresses[range] + sizeof(uint) > (ulong)entry.RangeLengths[range]) + return false; + // A later range that starts inside the run would be where the search finds its words + // from there on (overlapping ranges, which only unaligned reads make). + if (range + 1 < entry.RangeAddresses.Length && entry.RangeAddresses[range + 1] <= last) + return false; + var bytes = entry.Bytes.AsSpan(at); + for (var word = 0; word < offsets.Length; word++) + words[word] = System.Buffers.Binary.BinaryPrimitives.ReadUInt32LittleEndian(bytes[(int)(offsets[word] - offsets[0])..]); + return true; + } + // The print of the descriptors behind this draw's rebased pointers, read where they lie now; // zero when no rebased candidate has one. Words that cannot be read print as missing, the - // same for the lookup and for the entry stored after it. + // same for the lookup and for the entry stored after it. Remembered for TryRecallPrint. public ulong MemoryPrint(IReadOnlyList userData, ResidentGuestBytesReader reader, ref byte[] scratch) { if (!_hasPrint || Active.Length == 0) return 0; var print = PrintSeed; + var bases = PrintSeed; var any = false; + Span words = stackalloc ulong[PrintWords]; for (var index = 0; index < _candidates.Length; index++) { var offsets = _printOffsets[index]; @@ -621,21 +721,62 @@ public ulong MemoryPrint(IReadOnlyList userData, ResidentGuestBytesReader var addressBase = RawBase(userData, candidate, userData[candidate.UserData]); any = true; print = MixPrint(print, (ulong)index); + bases = MixPrint(bases, addressBase); var length = (int)(offsets[^1] - offsets[0]) + sizeof(uint); if (scratch.Length < length) scratch = new byte[Math.Max(length, scratch.Length * 2)]; var bytes = scratch.AsSpan(0, length); var start = (long)addressBase + offsets[0]; var read = start >= 0 && reader((ulong)start, bytes, false); - foreach (var offset in offsets) + var found = words[..offsets.Length]; + for (var word = 0; word < offsets.Length; word++) { - print = read - ? MixPrint(print, System.Buffers.Binary.BinaryPrimitives.ReadUInt32LittleEndian(bytes[(int)(offset - offsets[0])..])) - : MixPrint(print, MissingWord); + found[word] = read + ? System.Buffers.Binary.BinaryPrimitives.ReadUInt32LittleEndian(bytes[(int)(offsets[word] - offsets[0])..]) + : MissingWord; } + + print = FoldPrint(print, found); + } + + if (!any) + return 0; + (_printMemo ??= new (ulong, ulong[]?, ulong)[PrintMemoSlots])[MemoSlot(bases)] = (bases, Active, print); + return print; + } + + // The print each recent set of rebased pointers gave (MemoryPrint), so a hit through the same + // pointers finds its key without reading the descriptors here and again in Validate. A recalled + // print only guesses the key: Entry.Matches and Validate still decide the hit, and when there + // is none the draw reads the print again. Allocated once a pointer with a print is rebased. + private const int PrintMemoSlots = 32; + private (ulong Bases, ulong[]? Active, ulong Print)[]? _printMemo; + + private static int MemoSlot(ulong bases) => (int)(bases >> 59); + + public bool TryRecallPrint(IReadOnlyList userData, out ulong print) + { + print = 0; + if (_printMemo is not { } memo || Active.Length == 0) + return false; + var bases = PrintSeed; + var any = false; + for (var index = 0; index < _candidates.Length; index++) + { + var candidate = _candidates[index]; + if (_printOffsets[index].Length == 0 || !IsUsed(Active, candidate.UserData)) + continue; + if (candidate.UserData >= userData.Count || candidate.High >= userData.Count) + return false; + bases = MixPrint(bases, RawBase(userData, candidate, userData[candidate.UserData])); + any = true; } - return any ? print : 0; + ref readonly var slot = ref memo[MemoSlot(bases)]; + if (!any || slot.Bases != bases || !ReferenceEquals(slot.Active, Active)) + return false; + print = slot.Print; + return true; } // The window [start, start + size) of reads a candidate's base covers in this entry. @@ -1261,10 +1402,13 @@ public bool Materialize( ref ResourceSpecialization specialization, out ResourceMaterializationFailure failure) { + var missesBefore = Misses; var timing = new MaterializationTiming(MissDiagnosticsEnabled); try { - return MaterializeCore(plan, inputs, residentReader, ref snapshot, ref specialization, out failure, ref timing); + var success = MaterializeCore(plan, inputs, residentReader, ref snapshot, ref specialization, out failure, ref timing); + if (success && Misses == missesBefore) _consecutiveMisses = 0; + return success; } finally { @@ -1294,7 +1438,19 @@ private bool MaterializeCore( drawKey = DrawKeyOf(plan, inputs); _learnedUserData.TryGetValue(drawKey, out learned); rebase = RebaseOf(drawKey, plan); - key = KeyFor(drawKey, inputs, learned, rebase, residentReader); + key = KeyOf(drawKey, inputs.UserData, learned, rebase.Active); + } + + if (rebase.TryRecallPrint(inputs.UserData, out var recalled) && + TryRecalledHit(WithPrint(key, recalled), plan, inputs, residentReader, rebase, ref snapshot, ref specialization)) + { + failure = default; + return true; + } + + using (ResourceMaterializationProfile.Measure(ResourceMaterializationProfile.Phase.CacheKey)) + { + key = WithPrint(key, rebase.MemoryPrint(inputs.UserData, residentReader, ref _printScratch)); } bool found; @@ -1492,8 +1648,48 @@ private bool MaterializeCore( timing.ChangePhase(MaterializationPhase.Materialization); Misses++; + if (_consecutiveMisses < ColdScanThreshold) _consecutiveMisses++; RecordMiss(plan, key, found); Interlocked.Increment(ref _totalMisses); + if (ColdAdmissionEnabled && _cacheSaturated && _consecutiveMisses >= ColdScanThreshold) + { + // Scan misses need no validation record unless their contents recur. A + // recurring result is walked again with recording below before admission; + // only that fresh, fully recorded walk can ever become a cached entry. + var walking = new ResourceRuntimeInputs + { + // Match the recorder's owned user-data snapshot even when retention is bypassed. + UserData = inputs.UserData.ToArray(), + ShaderBase = inputs.ShaderBase, + ReadMemory = inputs.ReadMemory, + ReadCleanMemory = inputs.ReadCleanMemory, + ReadResidentMemory = inputs.ReadResidentMemory, + ReadsClean = inputs.ReadsClean, + DataWritePending = inputs.DataWritePending, + ComputeState = inputs.ComputeState, + TablePhase = NoteTablePhase, + }; + // Keep the caller's outputs intact if the required recorded rewalk fails. + var walkedSnapshot = snapshot; + var walkedSpecialization = specialization; + try + { + if (!ResourceMaterializer.Materialize(plan, walking, ref walkedSnapshot, ref walkedSpecialization, out failure)) + return false; + } + finally + { + ReadingTable = false; + } + if (DeferColdEntry(walkedSnapshot)) + { + snapshot = walkedSnapshot; + specialization = walkedSpecialization; + ColdBypasses++; + return true; + } + } + var recorder = TakeRecorder(); try { @@ -1508,7 +1704,7 @@ private bool MaterializeCore( ReadCleanMemory = recorder.Wrap(inputs.ReadCleanMemory, clean: true), ReadResidentMemory = recorder.WrapResident(inputs.ReadResidentMemory), ReadsClean = inputs.ReadsClean, - DataWritePending = inputs.DataWritePending, + DataWritePending = inputs.DataWritePending, ComputeState = inputs.ComputeState, TablePhase = recorder.TablePhase, }; @@ -1524,6 +1720,12 @@ private bool MaterializeCore( return true; } + if (DeferColdEntry(snapshot)) + { + ColdBypasses++; + return true; + } + timing.ChangePhase(MaterializationPhase.CacheStore); Entry built; using (ResourceMaterializationProfile.Measure(ResourceMaterializationProfile.Phase.EntryBuild)) @@ -1556,6 +1758,7 @@ private bool MaterializeCore( { if (depth++ >= MaxVariants) { + _cacheSaturated = true; variant.Next = null; break; } @@ -1791,11 +1994,46 @@ private static ulong KeyOf(ulong drawKey, IReadOnlyList userData, ulong[]? // now (AddressRebase.MemoryPrint): objects drawn through moving tables keep apart, and the same // object finds its entry wherever its table lies this time. private ulong KeyFor(ulong drawKey, ResourceRuntimeInputs inputs, ulong[]? learned, AddressRebase rebase, - ResidentGuestBytesReader residentReader) + ResidentGuestBytesReader residentReader) => + WithPrint(KeyOf(drawKey, inputs.UserData, learned, rebase.Active), + rebase.MemoryPrint(inputs.UserData, residentReader, ref _printScratch)); + + private static ulong WithPrint(ulong key, ulong print) => print == 0 ? key : key ^ (print * 0xC2B2_AE3D_27D4_EB4FUL); + + // A hit under the key a recalled print gives (AddressRebase.TryRecallPrint), decided by the + // checks every hit passes. Anything short of a hit leaves the draw to the full path, which + // reads the print again and finds, refreshes or rebuilds the entry under the key it gives. + private bool TryRecalledHit(ulong key, ShaderResourcePlan plan, ResourceRuntimeInputs inputs, + ResidentGuestBytesReader residentReader, AddressRebase rebase, ref ResourceSnapshot snapshot, + ref ResourceSpecialization specialization) { - var key = KeyOf(drawKey, inputs.UserData, learned, rebase.Active); - var print = rebase.MemoryPrint(inputs.UserData, residentReader, ref _printScratch); - return print == 0 ? key : key ^ (print * 0xC2B2_AE3D_27D4_EB4FUL); + Entry cached; + using (ResourceMaterializationProfile.Measure(ResourceMaterializationProfile.Phase.CacheFind)) + { + if (!TryFind(key, plan, inputs, rebase.Active, out cached)) + return false; + } + + for (Entry? previous = null, variant = cached; variant is not null; previous = variant, variant = variant.Next) + { + if (!variant.Matches(plan, inputs, rebase.Active) || !rebase.AppliesTo(variant) || + !Validate(variant, residentReader, rebase, inputs.UserData, false, out _, out _, out _, inputs.CompareResidentMemory)) + continue; + if (previous is not null) + { + previous.Next = variant.Next; + variant.Next = cached; + Store(key, variant); + } + + Hits++; + Interlocked.Increment(ref _totalHits); + snapshot = HitSnapshot(variant, inputs.UserData, rebase); + specialization = variant.Specialization; + return true; + } + + return false; } // SHARPEMU_RESOURCE_CONTENT_KEY=0: no content print, so a rebased pointer's key holds every @@ -1918,6 +2156,7 @@ private void Store(ulong key, Entry entry) } else if (_young.Count >= _generationCapacity || (_young.Count != 0 && _youngBytes + bytes > _generationByteBudget)) { + _cacheSaturated |= _old.Count != 0; CountDiagnostic(Diagnostic.GenerationRotation); CountDiagnostic(Diagnostic.GenerationEntriesDropped, _old.Count); RecordEvictions(_old.Keys); @@ -2601,6 +2840,7 @@ public void Reset() // tables alive for the lifetime of the renderer. if (_reads.Capacity > 16384) _reads.Capacity = 0; + if (_sortScratch.Length > 16384) _sortScratch = []; _reader = null; _cleanReader = null; _residentReader = null; @@ -2724,16 +2964,50 @@ private bool ReadResident(ulong address, Span destination, bool clean) private byte[] _scratchBytes = []; private int[] _scratchBaseWords = []; + private (ulong Address, uint Word, bool Clean, bool Table)[] _sortScratch = []; + + // Descriptor evaluation followed by table evaluation usually records two ordered runs. + // Merge those runs in linear time; irregular graphs still use the general sort. + private void SortReads() + { + var split = -1; + for (var index = 1; index < _reads.Count; index++) + { + if (_reads[index - 1].Address <= _reads[index].Address) continue; + if (split >= 0) + { + ResourceMaterializationProfile.CountRead(ResourceMaterializationProfile.ReadCounter.SortedReadLists); + _reads.Sort(static (left, right) => left.Address.CompareTo(right.Address)); + return; + } + split = index; + } + if (split < 0) + { + ResourceMaterializationProfile.CountRead(ResourceMaterializationProfile.ReadCounter.OrderedReadLists); + return; + } + + ResourceMaterializationProfile.CountRead(ResourceMaterializationProfile.ReadCounter.MergedReadLists); + var reads = System.Runtime.InteropServices.CollectionsMarshal.AsSpan(_reads); + if (_sortScratch.Length < reads.Length) + _sortScratch = new (ulong, uint, bool, bool)[Math.Max(reads.Length, _sortScratch.Length * 2)]; + reads.CopyTo(_sortScratch); + int left = 0, right = split, output = 0; + while (left < split && right < reads.Length) + reads[output++] = _sortScratch[left].Address <= _sortScratch[right].Address + ? _sortScratch[left++] : _sortScratch[right++]; + _sortScratch.AsSpan(left, split - left).CopyTo(reads[output..]); + output += split - left; + _sortScratch.AsSpan(right, reads.Length - right).CopyTo(reads[output..]); + } + public Entry Build(ShaderResourcePlan plan, ResourceRuntimeInputs inputs, ResourceSnapshot snapshot, ResourceSpecialization specialization, IReadOnlyList memoryBases, ulong[]? usedUserData = null, Entry? like = null) { - // Reads mostly arrive in address order; sorting a sorted list was ~5% of a draw-heavy render thread. - var ordered = true; - for (var index = 1; ordered && index < _reads.Count; index++) - ordered = _reads[index - 1].Address <= _reads[index].Address; - if (!ordered) - _reads.Sort((left, right) => left.Address.CompareTo(right.Address)); + using (ResourceMaterializationProfile.Measure(ResourceMaterializationProfile.Phase.RecorderSort)) + SortReads(); // Count unique words and contiguous ranges first, then fill the // retained arrays directly. Temporary lists previously duplicated // every recorded byte and range on each cache miss. diff --git a/src/SharpEmu.ShaderCompiler/Resources/ResourceMaterializationProfile.cs b/src/SharpEmu.ShaderCompiler/Resources/ResourceMaterializationProfile.cs index 942dbad80..70210e9af 100644 --- a/src/SharpEmu.ShaderCompiler/Resources/ResourceMaterializationProfile.cs +++ b/src/SharpEmu.ShaderCompiler/Resources/ResourceMaterializationProfile.cs @@ -26,6 +26,7 @@ public enum Phase CacheFind, RecorderWrap, EntryBuild, + RecorderSort, // The three parts of the snapshot walk, which is where a miss spends almost all of // its time: the branch conditions that decide which descriptor sources are live, the // descriptor words themselves, and the flattened table, one graph value per word. @@ -68,7 +69,7 @@ public enum Phase internal enum ReadCounter { GroupAttempt, GroupIneligible, GroupRefused, GroupSucceeded, GroupWords, - ScalarAddress, ScalarBuffer, Count + ScalarAddress, ScalarBuffer, OrderedReadLists, MergedReadLists, SortedReadLists, Count } private static readonly long[] _readCounts = new long[(int)ReadCounter.Count]; diff --git a/src/graphics/SharpEmu.Gpu.Metal/MetalCompiledGuestShader.cs b/src/graphics/SharpEmu.Gpu.Metal/MetalCompiledGuestShader.cs index 425f77342..4654ddb24 100644 --- a/src/graphics/SharpEmu.Gpu.Metal/MetalCompiledGuestShader.cs +++ b/src/graphics/SharpEmu.Gpu.Metal/MetalCompiledGuestShader.cs @@ -48,6 +48,8 @@ internal sealed class MetalCompiledGuestShader(Gen5MslShader shader, ShaderCompi public string PayloadFileExtension => "msl"; + public uint FlattenedTableMinimumWords => Shader.FlattenedTableMinimumWords; + /// XxHash3 of the UTF-8 , computed once and kept. internal ulong PayloadHash { diff --git a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalBinaryArchive.cs b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalBinaryArchive.cs index 7f09c2f63..92d64f461 100644 --- a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalBinaryArchive.cs +++ b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalBinaryArchive.cs @@ -136,7 +136,7 @@ private sealed class ResidentSegment(nint archive, long bytes) private readonly ConcurrentDictionary _known = new(); private readonly ConcurrentDictionary _pending = new(); private readonly BlockingCollection _jobs = new(MaxQueuedJobs); - private readonly Thread _writer; + private readonly MetalCompileThread _writer; private readonly ManualResetEventSlim _loaded = new(false); private readonly object _archivesLock = new(); private readonly object _trimLock = new(); @@ -189,12 +189,9 @@ private MetalBinaryArchive(nint device, string root, string translatorStamp, lon _directory = Path.Combine(root, Hex64(XxHash3.HashToUInt64(MemoryMarshal.AsBytes(DeviceStamp(device).AsSpan())))); _libraries = Path.Combine(_directory, "libraries"); _lastFlushTicks = Stopwatch.GetTimestamp(); - _writer = new Thread(RunWriter) - { - IsBackground = true, - Priority = ThreadPriority.BelowNormal, - Name = "SharpEmu Metal v2 binary archive", - }; + // Pipelines asked for before the segments are listed miss them, so the listing runs at the + // class of the compiles that wait; the writes after it at Utility (RunWriter). + _writer = MetalCompileThread.Create("SharpEmu Metal v2 binary archive", SharpEmu.HLE.Host.HostThreadQos.UserInitiated, RunWriter); } /// The process's archive, once the presenter started it, for this device. @@ -474,6 +471,9 @@ public void CaptureCompute(UInt128 key, nint function) /// Whether the library file of is on disk (or being written). public bool HasLibrary(UInt128 key) => key != default && (_known.ContainsKey(key) || _pending.ContainsKey(key)); + /// Whether the library file of is written (a compute's holds its pipeline's binary too). + public bool IsStored(UInt128 key) => key != default && _known.ContainsKey(key); + /// /// Stores the library file of from a pipeline that is never made: /// the functions of added to an archive of their own and @@ -711,6 +711,7 @@ private void RunWriter() _loaded.Set(); } + MetalCompileThread.ApplyToCurrent(SharpEmu.HLE.Host.HostThreadQos.Utility); try { while (!_jobs.IsCompleted) diff --git a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalCachedImage.cs b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalCachedImage.cs index 2ee2381a6..7abcbac2a 100644 --- a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalCachedImage.cs +++ b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalCachedImage.cs @@ -109,6 +109,29 @@ internal sealed class MetalCachedImage : IDisposable /// public MetalImageEpochs? Epochs { get; set; } + /// + /// The split GPU process's pending-writeback summary (MetalImageCache.PendingWritebacks), which + /// holds this image's pages while it is GPU-modified. Set while the image is in the cache; + /// setting it moves the claim, and null drops it. + /// + public IPendingWritebackPages? PendingWritebacks + { + get => _pendingWritebacks; + set + { + ReleasePendingWriteback(); + _pendingWritebacks = value; + if (_gpuModified) + { + ClaimPendingWriteback(); + } + } + } + + private IPendingWritebackPages? _pendingWritebacks; + private ulong _pendingAddress; + private ulong _pendingSize; + /// /// Moves wherever this image bumps a count of (and where the cache changes /// its metadata): a memo of a binding of this image alone is stale once it moved. @@ -250,7 +273,25 @@ public bool HoldsStencilFrom(MetalCachedImage source) => public MetalTextureShape Shape { get; private set; } - public nint Texture { get; private set; } + // Every texture this image refers to is counted in MetalTileViews, which keeps the + // tiler's integer view of it until the last image lets go of the texture. + public nint Texture + { + get => _texture; + private set + { + if (_texture == value) + { + return; + } + + MetalTileViews.Reference(value); + MetalTileViews.Unreference(_texture); + _texture = value; + } + } + + private nint _texture; /// /// The guest mip level the texture's level 0 holds. Zero for the whole chain; above zero @@ -664,6 +705,8 @@ public void MarkGpuModified(bool depthPlaneChanged = true) var wasModified = _gpuModified; var wasDirty = _cpuDirty; var wasPublished = PublishedEnd > PublishedStart; + // Claimed before the image is GPU-modified: the guest process never sees it so unclaimed. + ClaimPendingWriteback(); _gpuModified = true; _cpuDirty = false; DirtyStart = 0; @@ -730,6 +773,7 @@ public void ClearGpuModified() { var wasModified = _gpuModified; _gpuModified = false; + ReleasePendingWriteback(); PublishedStart = 0; PublishedEnd = 0; if (wasModified) @@ -754,6 +798,29 @@ public void InvalidateCpuWrite(ulong address, ulong size) } } + private void ClaimPendingWriteback() + { + if (_pendingSize != 0 || _pendingWritebacks is not { } pending || ImageDescription.IsEmptyRange(Description.Data)) + { + return; + } + + _pendingAddress = Description.Data.Address; + _pendingSize = Description.Data.Size; + pending.Add(_pendingAddress, _pendingSize); + } + + private void ReleasePendingWriteback() + { + if (_pendingSize == 0) + { + return; + } + + _pendingWritebacks!.Remove(_pendingAddress, _pendingSize); + _pendingSize = 0; + } + /// /// A guest byte range intersected with this image, as offsets into the image. Empty /// when the range does not reach it. @@ -1072,12 +1139,15 @@ public nint GetOrCreateView(in ImageViewDescription requested, out ulong resourc public void ReplaceTexture(nint texture, in MetalTextureShape shape) { ReleaseViews(); - if (Texture != 0) + // The old texture's tiler view goes before the texture does, so a texture made + // later at the same handle never finds it. + var old = Texture; + Texture = texture; + if (old != 0) { - MetalNative.SendVoid(Texture, SelRelease); + MetalNative.SendVoid(old, SelRelease); } - Texture = texture; Shape = shape; Sparse = null; NoteContentChanged(); @@ -1107,10 +1177,11 @@ public void Dispose() _disposed = true; ReleaseViews(); - if (Texture != 0) + var texture = Texture; + Texture = 0; + if (texture != 0) { - MetalNative.SendVoid(Texture, SelRelease); - Texture = 0; + MetalNative.SendVoid(texture, SelRelease); } } } diff --git a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalCommandStream.cs b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalCommandStream.cs index f128773e0..0c6f5cb1e 100644 --- a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalCommandStream.cs +++ b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalCommandStream.cs @@ -716,7 +716,7 @@ private bool TryEncodeEndOfPipeLabel(in EndOfPipeWrite write) var copy = new DeferredLabelCopy( staging.Buffer, staging.Offset, destination.Buffer, destination.Offset, bytes, write.Destination); var serial = EncodeOrderedCopy(in copy, write.Destination, - GpuClobberDiagnostics.Enabled ? GpuClobberDiagnostics.LandedKind(write.Kind) : null, + CommandProcessorWriteHistory.Enabled ? GpuClobberDiagnostics.LandedKind(write.Kind) : null, bytes == sizeof(uint) ? (uint)write.Value : write.Value, MetalGpuWriteOrigin.EndOfPipeLabel, "eop label", null, (ulong)bytes); Interlocked.Increment(ref _gpuTimedLabels); @@ -879,9 +879,9 @@ private ulong EncodeOrderedCopies( }); } - if (landedKind is not null) + if (landedKind is not null && CommandProcessorWriteHistory.Enabled) { - // SHARPEMU_GPU_CLOBBER_DIAG: the bytes are in guest memory once this serial completes. + // SHARPEMU_GPU_CLOBBER_DIAG / SHARPEMU_CP_HISTORY: the bytes are in guest memory once this serial completes. var bytes = copies[0].Bytes; _completions.Enqueue(serial, () => CommandProcessorWriteHistory.Record( -1, guestAddress, bytes, landedValue, landedKind, deferred: true)); @@ -947,7 +947,7 @@ public void RecordCommandProcessorWrite(ulong address, ReadOnlySpan source Interlocked.Increment(ref _hostCommandProcessorWrites); var bytes = source.ToArray(); - var landedKind = GpuClobberDiagnostics.Enabled ? "landed:" + kind : null; + var landedKind = CommandProcessorWriteHistory.Enabled ? "landed:" + kind : null; var lazy = RidesNextCommit(clock: false); Interlocked.Increment(ref _owedHostLabels); QueueCompletion(() => @@ -961,7 +961,7 @@ public void RecordCommandProcessorWrite(ulong address, ReadOnlySpan source } _imageCache.InvalidateMemory(address, (ulong)bytes.Length); - if (landedKind is not null) + if (landedKind is not null && CommandProcessorWriteHistory.Enabled) { CommandProcessorWriteHistory.Record(-1, address, bytes.Length, FirstValue(bytes), landedKind, deferred: true); } @@ -1005,8 +1005,9 @@ private bool TryEncodeCommandProcessorWrite(ulong address, ReadOnlySpan so source.CopyTo(payload); var copy = new DeferredLabelCopy( staging.Buffer, staging.Offset, destination.Buffer, destination.Offset, source.Length, address); - var serial = EncodeOrderedCopy(in copy, address, GpuClobberDiagnostics.Enabled ? "landed:" + kind : null, - FirstValue(source), MetalGpuWriteOrigin.Copy, "cp write", kind, size); + var recordHistory = CommandProcessorWriteHistory.Enabled; + var serial = EncodeOrderedCopy(in copy, address, recordHistory ? "landed:" + kind : null, + recordHistory ? FirstValue(source) : 0, MetalGpuWriteOrigin.Copy, "cp write", kind, size); // Parse-time readers see the bytes as written until they land (MetalPendingWriteOverlay). using var bookkeepingScope = SharpEmu.Libs.VideoOut.RenderPhaseProfile.MeasureDetail( @@ -1699,7 +1700,7 @@ private void PublishEndOfPipeLabel(in EndOfPipeWrite write) $"address=0x{write.Destination:X16} bytes={bytes}."); } - if (GpuClobberDiagnostics.Enabled) + if (CommandProcessorWriteHistory.Enabled) { CommandProcessorWriteHistory.Record(-1, write.Destination, bytes, bytes == sizeof(uint) ? (uint)write.Value : write.Value, GpuClobberDiagnostics.LandedKind(write.Kind), deferred: true); @@ -2253,8 +2254,7 @@ private void CaptureFlip(int handle, int index, ulong requestId, int flipMode, l serial, VideoOutBackendServices.IsHdrPixelFormat(displayBuffer.PixelFormat)); VideoOutBackendServices.TraceGpuFlip( - _memory, handle, index, flipMode, flipArg, displayBuffer.Address, - VideoOutBackendServices.GetFlipEventCount(requestId)); + _memory, handle, index, flipMode, flipArg, displayBuffer.Address, requestId); // The shared phase profile measures the whole draw translation path, which v2 // runs on this thread exactly as the Vulkan render loop does on its own. A // flip is this host's frame boundary, so SHARPEMU_PROFILE_RENDER=1 reports the @@ -2353,10 +2353,17 @@ public void PrefetchDispatchPipeline( uint dispatchInitiator) { _prefetchingPipeline = true; + var abandoned = Interlocked.Read(ref _speculativeWaitsAbandoned); try { _translation?.PrefetchDispatchPipeline(banks, groupsX, groupsY, groupsZ, dispatchInitiator); } + catch (SharpEmu.Libs.Gpu.Scheduling.GpuFatal.AbandonedWorkException) when ( + Interlocked.Read(ref _speculativeWaitsAbandoned) != abandoned) + { + // This dispatch reads bytes the GPU is still writing (AbandonSpeculativeWait): its + // pipeline is left to the dispatch itself, and the look-ahead goes on to the next one. + } finally { _prefetchingPipeline = false; diff --git a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalCompileThread.cs b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalCompileThread.cs new file mode 100644 index 000000000..cf0b488cc --- /dev/null +++ b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalCompileThread.cs @@ -0,0 +1,117 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using SharpEmu.HLE.Host; + +namespace SharpEmu.Gpu.Metal.V2; + +/// +/// A shader or pipeline compile thread started at a QoS class of its own. Thread.Priority does +/// nothing on this host (CoreCLR's Unix PAL ignores it, HostThreadQualityOfService), so the +/// BelowNormal the background compiles asked for never applied, and a thread made with +/// new Thread from the GPU worker ran next to the worker and the guest's threads on the +/// performance cores. The class is also what the driver's compiler service runs a request at: +/// it travels with the XPC message. A thread made natively with the class in its attributes +/// is the one route that sets it (HostQosThread), and only such a thread may change its own +/// class later (). +/// What the frame or the guest waits for (the runtime's library and pipeline compiles, a +/// prewarm the guest is held for, the libraries a scene takes from the prewarm) runs at +/// UserInitiated, below the worker's UserInteractive; what nobody waits for (the static +/// precompile, the computes left once the guest runs, the archive and list writers) at Utility. +/// SHARPEMU_METAL_V2_COMPILE_QOS=0 makes them managed threads again. +/// +internal sealed class MetalCompileThread +{ + internal static readonly bool QosEnabled = + Environment.GetEnvironmentVariable("SHARPEMU_METAL_V2_COMPILE_QOS") != "0"; + + // A compile thread translates too (prewarm, precompile), which recurses as deep as on the worker. + private const int StackBytes = 16 * 1024 * 1024; + + private static readonly HashSet _reported = new(StringComparer.Ordinal); + + private readonly string _name; + private readonly HostThreadQos _qos; + private readonly Action _body; + private HostQosThread? _native; + private Thread? _managed; + + private MetalCompileThread(string name, HostThreadQos qos, Action body) + { + _name = name; + _qos = qos; + _body = body; + } + + /// A thread that runs at once started. + public static MetalCompileThread Create(string name, HostThreadQos qos, Action body) => new(name, qos, body); + + /// Creates and starts it. + public static MetalCompileThread Start(string name, HostThreadQos qos, Action body) + { + var thread = Create(name, qos, body); + thread.Start(); + return thread; + } + + public bool IsCurrentThread => + _native?.IsCurrentThread ?? ReferenceEquals(_managed, Thread.CurrentThread); + + public void Start() + { + if (QosEnabled) + { + _native = HostQosThread.TryStart(_name, _qos, StackBytes, Run, out _); + if (_native is not null) + { + return; + } + } + + _managed = new Thread(Run) { IsBackground = true, Name = _name }; + _managed.Start(); + } + + public bool Join(TimeSpan timeout) => + _native?.Join((int)timeout.TotalMilliseconds) ?? _managed?.Join(timeout) ?? true; + + public void Join() + { + if (_native is not null) + { + _native.Join(Timeout.Infinite); + } + else + { + _managed?.Join(); + } + } + + /// Moves the calling compile thread to (a prewarm thread whose + /// guest runs again): true when it applied; refused on a thread CoreCLR made, which keeps its class. + public static bool ApplyToCurrent(HostThreadQos qos) => + QosEnabled && HostThreadQualityOfService.TryApplyToCurrentThread(qos); + + private void Run() + { + ReportClass(); + _body(); + } + + // The class each kind of compile thread runs at, once per kind (the name without its index). + private void ReportClass() + { + var kind = _name.TrimEnd(' ', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9'); + lock (_reported) + { + if (!_reported.Add(kind)) + { + return; + } + } + + Console.Error.WriteLine( + $"[METAL-V2][QOS] thread=\"{kind}\" requested={(_native is null && _managed is not null ? "inherited" : _qos.ToString())} " + + $"class=0x{HostThreadQualityOfService.GetCurrentThreadClassForDiagnostics():X2}"); + } +} diff --git a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalGpuWorker.cs b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalGpuWorker.cs index 5b7a85f2c..661406c94 100644 --- a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalGpuWorker.cs +++ b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalGpuWorker.cs @@ -196,7 +196,7 @@ internal sealed class MetalCompletionQueue : IDisposable private readonly object _gate = new(); private readonly object _wakeGate = new(); private readonly List<(ulong Serial, Action Work)> _pending = new(); - private readonly Thread _thread; + private readonly SharpEmu.HLE.Host.HostHelperThread _thread; private readonly Action _wakeHandler; private readonly bool _handlerBound; private ulong _publishedSerial; @@ -217,12 +217,9 @@ public MetalCompletionQueue(nint sharedEvent) _handlerBound ? "[METAL-V2][INFO] Submission completions wake the host through a command-buffer handler." : "[METAL-V2][INFO] Submission completions are polled on the shared event."); - _thread = new Thread(Run) - { - IsBackground = true, - Name = "SharpEmu Metal completion", - }; - _thread.Start(); + // TryApplyToCurrentThread in Run cannot raise a thread CoreCLR created: the class is set + // when the thread is made. + _thread = SharpEmu.HLE.Host.HostHelperThread.Start("SharpEmu Metal completion", 4 * 1024 * 1024, Run); } /// Whether a committed command buffer should carry the wakeup handler. @@ -294,7 +291,7 @@ public void Enqueue(ulong serial, Action work) /// When the completion thread last saw the event reach a new value. public long LastSignalObservedTimestamp => Interlocked.Read(ref _lastSignalObserved); - /// How many completions are still waiting for their submission. + /// How many completions are queued or executing their host work. public int PendingCount { get @@ -371,7 +368,7 @@ public MetalCompletionWait WaitFor(ulong serial) var signalled = System.Diagnostics.Stopwatch.GetTimestamp(); lock (_gate) { - while (_drainedSerial < serial && !_stopping && HasWorkAtOrBefore(serial)) + while (!_stopping && HasWorkAtOrBefore(serial)) { Monitor.Wait(_gate, 2); } @@ -622,9 +619,9 @@ private void Drain(ulong signaled) { Action? work = null; ulong serial = 0; + var index = -1; lock (_gate) { - var index = -1; var best = ulong.MaxValue; for (var candidate = 0; candidate < _pending.Count; candidate++) { @@ -638,7 +635,9 @@ private void Drain(ulong signaled) if (index >= 0) { (serial, work) = _pending[index]; - _pending.RemoveAt(index); + // Keep it visible to PendingCount and WaitFor until its label writes, + // interrupts and other host work finish. Only this thread removes entries; + // concurrent enqueues append, so the index stays valid during the action. } else { @@ -664,6 +663,7 @@ private void Drain(ulong signaled) lock (_gate) { + _pending.RemoveAt(index); if (serial > _drainedSerial) { _drainedSerial = serial; diff --git a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalImageCache.AliasTransitions.cs b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalImageCache.AliasTransitions.cs index 831c3455a..e09aa884c 100644 --- a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalImageCache.AliasTransitions.cs +++ b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalImageCache.AliasTransitions.cs @@ -39,10 +39,13 @@ internal string TakeAliasTransitionText() _aliasTransitionRefusals.Clear(); var readRuns = Interlocked.Exchange(ref _aliasCpuReadRuns, 0); var readBytes = Interlocked.Exchange(ref _aliasCpuReadBytes, 0); + var consumers = TakeConsumerRuns(); return $"alias_transitions=[fused:{counts.Fused},discarded:{counts.Discarded},round-trip:{counts.RoundTrip}] " + $"alias_mib=[fused:{Mb(counts.FusedBytes)},discarded:{Mb(counts.DiscardedBytes)},round-trip:{Mb(counts.RoundTripBytes)}] " + $"alias_transition_refused=[{refusals}] alias_cpu_read_runs={readRuns} " + - $"alias_cpu_read_requested_mib={Mb(readBytes)}"; + $"alias_cpu_read_requested_mib={Mb(readBytes)} " + + $"consumer_runs=[published:{consumers.Runs},skipped:{consumers.AlreadyPublished},gpu_writes_kept:{consumers.GpuWritesKept}] " + + $"consumer_run_mib={Mb(consumers.RunBytes)}"; } // The pass replaces every visible texel of this image. Refuse when an alias owns any diff --git a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalImageCache.ConsumerRuns.cs b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalImageCache.ConsumerRuns.cs new file mode 100644 index 000000000..2240f0672 --- /dev/null +++ b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalImageCache.ConsumerRuns.cs @@ -0,0 +1,235 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +namespace SharpEmu.Gpu.Metal.V2; + +// A buffer binding, a storage descriptor or a host read that names a few KiB of a large +// GPU-written image publishes the run it names, not the image. A whole writeback retiles +// all of a 16-33 MiB render target for a 4 KiB V# inside it, on every draw that binds it. +// +// The image stays GPU-modified: its texture still owns every byte outside the published +// run, and the run is recorded (MetalCachedImage.MarkPublished) so the next consumer of the +// same bytes publishes nothing. A later reader of the rest, a guest store, a release or a +// drain still writes the whole image back. +// +// A consumer that also writes the bytes is followed by InvalidateMemoryFromGpu over them. +// That used to give the whole image to guest memory, which is only right once all of it is +// there. Here a GPU write into an image that stays GPU-modified marks just the written bytes +// stale, when the narrow reload that follows (TryRefreshDirtyRun) is certain to read only +// bytes guest memory holds for the image: ones published, or ones already stale. A written +// consumer publishes its run only when that will hold, and otherwise the whole image, as +// before. +// +// SHARPEMU_METAL_V2_SUBRANGE_CONSUMERS=0 writes every consumer's images back whole again. +internal sealed partial class MetalImageCache +{ + private static readonly bool SubRangeConsumersDefault = + Environment.GetEnvironmentVariable("SHARPEMU_METAL_V2_SUBRANGE_CONSUMERS") != "0"; + + /// Whether a consumer of part of a GPU-written image publishes only that part. + internal bool SubRangeConsumers { get; set; } = SubRangeConsumersDefault; + + private long _consumerRuns, _consumerRunBytes, _consumerRunsSkipped, _gpuWritesKeptOwned; + private long _writebackBytes; + + /// Image bytes every writeback, whole or run, has retiled into guest memory. + internal long WritebackBytes => Interlocked.Read(ref _writebackBytes); + + internal (long Runs, long RunBytes, long AlreadyPublished, long GpuWritesKept) TakeConsumerRuns() => + (Interlocked.Exchange(ref _consumerRuns, 0), Interlocked.Exchange(ref _consumerRunBytes, 0), + Interlocked.Exchange(ref _consumerRunsSkipped, 0), Interlocked.Exchange(ref _gpuWritesKeptOwned, 0)); + + // Caller holds the lock. True when guest memory now holds what the consumer reads of + // the image, without the image being written back whole. + private bool TryPublishConsumerRunLocked( + MetalCachedImage image, ulong address, ulong size, MetalWritebackCause cause, bool written) + { + if (!SubRangeConsumers || !SubRangeTransfersEnabled || + cause is not (MetalWritebackCause.HostRead or MetalWritebackCause.BufferBinding + or MetalWritebackCause.StorageDescriptor) || + !IsRunShaped(image)) + { + return false; + } + + var (start, end) = image.OverlapWithin(address, size); + if (end <= start || (start == 0 && end >= image.Description.Data.Size)) + { + return false; + } + + if (image.IsCpuDirty) + { + // Stale over these bytes already: guest memory, or the alias this same loop + // publishes, holds the newer copy, so the image has nothing of them to give. + // Anything else keeps the whole writeback, which reloads the stale run first. + if (start < image.DirtyStart || end > image.DirtyEnd || + (written && !CanKeepGpuOwnership(image, start, end))) + { + return false; + } + + _consumerRunsSkipped++; + return true; + } + + if (written) + { + // Published here, stale once the write lands (TryKeepGpuOwnershipOverGpuWriteLocked): + // the write keeps the texture the owner of the rest only if the reload of the run + // reads nothing guest memory lacks, so that is settled before anything is skipped. + if (!TryNarrowReloadRun(image, start, end, out var runStart, out var runEnd)) + { + return false; + } + + if (!image.IsPublished(runStart, runEnd)) + { + if (!TryWriteBackRun(image, runStart, runEnd, cause, out _)) + { + return false; + } + + NoteConsumerRun(runEnd - runStart); + } + else + { + _consumerRunsSkipped++; + } + + // A run recorded but not kept as published (a longer one is) leaves the write + // with no proof; the whole writeback then lands over it, in order. + return CanKeepGpuOwnership(image, start, end); + } + + if (image.IsPublished(start, end)) + { + _consumerRunsSkipped++; + return true; + } + + if (!TryWriteBackRun(image, start, end, cause, out var refusal)) + { + if (cause == MetalWritebackCause.HostRead) + { + RefuseAliasTransition($"cpu-read-{refusal}"); + } + + return false; + } + + if (cause == MetalWritebackCause.HostRead) + { + _aliasCpuReadRuns++; + _aliasCpuReadBytes += (long)(end - start); + } + + NoteConsumerRun(end - start); + return true; + } + + private void NoteConsumerRun(ulong bytes) + { + _consumerRuns++; + _consumerRunBytes += (long)bytes; + } + + // A run of the image may be published, and reloaded, on its own: one surface plane, + // the whole chain in an owned texture, fully resident and exact. + private static bool IsRunShaped(MetalCachedImage image) => + image.Texture != 0 && !image.Description.HasStencil && image.FirstLevel == 0 && + image.Sparse is null && !image.IsWindowLinearView && !image.Shape.Lossy; + + // Caller holds the lock. A GPU write into an image the GPU also wrote: instead of + // giving all of it to guest memory, mark the written bytes stale and keep the texture + // the owner of the rest, when the narrow reload of the stale bytes will read nothing + // but bytes guest memory holds for the image. + private bool TryKeepGpuOwnershipOverGpuWriteLocked(MetalCachedImage image, ulong address, ulong size) + { + if (!SubRangeConsumers || !SubRangeTransfersEnabled || !image.IsGpuModified) + { + return false; + } + + var (start, end) = image.OverlapWithin(address, size); + if (!CanKeepGpuOwnership(image, start, end)) + { + return false; + } + + image.MarkCpuDirty(MetalImageDirtyCause.GpuWrite, start, end); + _gpuWritesKeptOwned++; + return true; + } + + // Whether a GPU write of [start, end) may leave the image GPU-modified and stale over + // just those bytes. Changes nothing, so a written consumer asks it before the write + // and gets the answer the write's invalidation gets after. + private bool CanKeepGpuOwnership(MetalCachedImage image, ulong start, ulong end) + { + if (end <= start || !image.IsGpuModified || !IsRunShaped(image)) + { + return false; + } + + // One stale interval: a write apart from the one already stale would merge the + // bytes between them, which only the texture holds. + var (staleStart, staleEnd) = (start, end); + if (image.IsCpuDirty) + { + if (start > image.DirtyEnd || end < image.DirtyStart) + { + return false; + } + + staleStart = Math.Min(staleStart, image.DirtyStart); + staleEnd = Math.Max(staleEnd, image.DirtyEnd); + } + + return TryNarrowReloadRun(image, staleStart, staleEnd, out var runStart, out var runEnd) && + GuestHoldsImageBytes(image, runStart, runEnd, staleStart, staleEnd); + } + + // The run TryRefreshDirtyRun reloads for a stale interval, when it reloads just that run: + // the same conditions, in the same order, so the answer here is the answer there. + private bool TryNarrowReloadRun(MetalCachedImage image, ulong staleStart, ulong staleEnd, + out ulong runStart, out ulong runEnd) + { + runStart = runEnd = 0; + ref readonly var description = ref image.Description; + if (staleEnd <= staleStart || staleEnd - staleStart > description.Data.Size / 2 || + !TryCoveringRun(in description, staleStart, staleEnd, out runStart, out runEnd)) + { + return false; + } + + var source = _guestMemory.Resolve(description.Data.Address + runStart, runEnd - runStart); + return source.IsValid && source.Size >= runEnd - runStart; + } + + // Every byte of [runStart, runEnd) is one guest memory holds for the image: inside the + // published run, or inside the stale interval, where guest memory is the newer copy. + private static bool GuestHoldsImageBytes( + MetalCachedImage image, ulong runStart, ulong runEnd, ulong staleStart, ulong staleEnd) + { + if (runStart >= staleStart && runEnd <= staleEnd) + { + return true; + } + + var (publishedStart, publishedEnd) = (image.PublishedStart, image.PublishedEnd); + if (publishedEnd <= publishedStart) + { + return false; + } + + if (publishedStart <= staleEnd && staleStart <= publishedEnd) + { + // Touching or overlapping: together one interval. + publishedStart = Math.Min(publishedStart, staleStart); + publishedEnd = Math.Max(publishedEnd, staleEnd); + } + + return runStart >= publishedStart && runEnd <= publishedEnd; + } +} diff --git a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalImageCache.PendingWritebacks.cs b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalImageCache.PendingWritebacks.cs new file mode 100644 index 000000000..8eefcf399 --- /dev/null +++ b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalImageCache.PendingWritebacks.cs @@ -0,0 +1,123 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using SharpEmu.HLE.GpuMemory; + +namespace SharpEmu.Gpu.Metal.V2; + +// In the split GPU process the guest's managed writes reach this cache across processes, and the +// call that runs MarkPreciseCpuWrite before a write lands costs the guest thread a round trip. +// That call does something before the write only where this cache holds bytes guest memory lacks: +// an image the GPU wrote and has not written back (it is written back first), or a recorded +// writeback whose submission has not completed (the write waits for it). Everything else the call +// does, the write's ordered record (MarkCpuWriteLanded) does too. So the cache keeps those pages +// in the summary the guest process reads (IPendingWritebackPages, SplitSharedState), and the +// guest calls only where one is marked. +// +// A clear page must mean a call made at that moment would find nothing to publish or wait for. +// A call sees this cache's state as of a release of the lock, so the summary follows the same +// states: +// - a GPU-modified image claims its pages before it is marked GPU-modified, and gives them up +// when it is no longer (MetalCachedImage.PendingWritebacks), or when it leaves the cache; +// - a writeback claims its range as it is fenced, and gives it up once its submission completed +// (a call does not wait for a completed one); +// - a page given up is unmarked only as the lock is let go, and only if nothing claims it then. +// A writeback ends the image's claim before it starts its own, inside one hold of the lock. +// Marking is immediate and a full barrier, unmarking happens under the lock: so a page is marked +// whenever a call would act on it, and a guest write that saw it clear lands as one would after a +// call that found nothing (the GPU may still write the image after either; that race is the +// guest's, as on the hardware). +internal sealed partial class MetalImageCache +{ + private IPendingWritebackPages? _pendingWritebacks; + private readonly object _pendingFenceGate = new(); + private readonly Dictionary<(ulong Address, ulong Size), ulong> _pendingFences = new(); + private ulong _pendingFenceMinSerial = ulong.MaxValue; + private int _pendingFenceCount; + + /// + /// The split's pending-writeback summary, kept from here on (null in one process). Set it + /// before the first image is made. + /// + internal IPendingWritebackPages? PendingWritebacks + { + get => _pendingWritebacks; + set + { + _pendingWritebacks = value; + _lock.BeforeRelease = value is null ? null : PublishPendingWritebacks; + value?.Activate(); + } + } + + // A recorded writeback's range stays pending until its submission completes. + private void ClaimPendingFence(ulong address, ulong size, ulong serial) + { + if (_pendingWritebacks is not { } pending) + { + return; + } + + lock (_pendingFenceGate) + { + if (_pendingFences.TryGetValue((address, size), out var held)) + { + _pendingFences[(address, size)] = Math.Max(held, serial); + return; + } + + pending.Add(address, size); + _pendingFences.Add((address, size), serial); + _pendingFenceMinSerial = Math.Min(_pendingFenceMinSerial, serial); + Volatile.Write(ref _pendingFenceCount, _pendingFences.Count); + } + } + + // On every release of the lock (RegionLock.BeforeRelease), with the lock still held. + private void PublishPendingWritebacks() + { + var pending = _pendingWritebacks!; + if (Volatile.Read(ref _pendingFenceCount) != 0) + { + var completed = _owner.CompletedSerial; + lock (_pendingFenceGate) + { + if (completed >= _pendingFenceMinSerial) + { + ReleaseCompletedFencesLocked(pending, completed); + } + } + } + + pending.Publish(); + } + + private void ReleaseCompletedFencesLocked(IPendingWritebackPages pending, ulong completed) + { + List<(ulong Address, ulong Size)>? done = null; + var minimum = ulong.MaxValue; + foreach (var (range, serial) in _pendingFences) + { + if (serial <= completed) + { + (done ??= []).Add(range); + } + else + { + minimum = Math.Min(minimum, serial); + } + } + + if (done is not null) + { + foreach (var range in done) + { + _pendingFences.Remove(range); + pending.Remove(range.Address, range.Size); + } + } + + _pendingFenceMinSerial = minimum; + Volatile.Write(ref _pendingFenceCount, _pendingFences.Count); + } +} diff --git a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalImageCache.Tracking.cs b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalImageCache.Tracking.cs index a920a474f..8194a88e0 100644 --- a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalImageCache.Tracking.cs +++ b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalImageCache.Tracking.cs @@ -133,6 +133,8 @@ private void ReleaseImage(ResourceSlotIdentifier imageIdentifier, MetalImageRele _slots.Erase(imageIdentifier); RetireImage(image); image.Epochs = null; + // Out of the cache no call finds it: its bytes are written back (and fenced) or dropped. + image.PendingWritebacks = null; Epochs.NoteBindingChanged(); } diff --git a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalImageCache.WritebackFence.cs b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalImageCache.WritebackFence.cs index a0792b78b..1d5d39752 100644 --- a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalImageCache.WritebackFence.cs +++ b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalImageCache.WritebackFence.cs @@ -165,6 +165,7 @@ private void FenceWriteback(ulong address, ulong size, ulong serial, MetalWriteb return; } + ClaimPendingFence(address, size, serial); lock (_fenceGate) { using var fenceHold = MeasureFenceHold(); diff --git a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalImageCache.cs b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalImageCache.cs index da082c7e8..276f4700b 100644 --- a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalImageCache.cs +++ b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalImageCache.cs @@ -186,6 +186,11 @@ public MetalImageCache( MetalBlockCubeDump.Peek ??= guestMemory.TryPeek; _pages = pages; _backing = backing; + if (backing is IPendingWritebackSource { PendingWritebacks: { } pendingWritebacks }) + { + PendingWritebacks = pendingWritebacks; + } + InitializeLossy(); ConfigureMemoryPolicy( MetalImageBudgetSettings.FromEnvironment(MetalHostMemory.Physical), @@ -1622,9 +1627,10 @@ public void RefreshImage(ResourceSlotIdentifier imageIdentifier) /// /// Writes every GPU-modified image that overlaps the range back into guest /// memory. Called before a buffer binding, an incompatible image request or a - /// host read can observe the bytes. + /// host read can observe the bytes. says the consumer also + /// writes them, and is followed by over the range. /// - public void FlushGpuWrites(ulong address, ulong size, MetalWritebackCause cause) + public void FlushGpuWrites(ulong address, ulong size, MetalWritebackCause cause, bool written = false) { using var phaseScope = SharpEmu.Libs.VideoOut.RenderPhaseProfile.MeasureDetail(SharpEmu.Libs.VideoOut.RenderPhaseProfile.Phase.GpuWriteFlush); if (!IsValidRange(address, size) || !_pageOwners.MayHaveOwners(address, size)) @@ -1637,20 +1643,11 @@ public void FlushGpuWrites(ulong address, ulong size, MetalWritebackCause cause) { if (_slots.TryGet(imageIdentifier) is { } image && image.GpuOverlaps(address, size)) { - var (start, end) = image.OverlapWithin(address, size); - if (AliasTransitionsEnabled && cause == MetalWritebackCause.HostRead && - !image.Description.HasStencil && !image.IsCpuDirty && image.FirstLevel == 0 && - image.Sparse is null && (start != 0 || end != image.Description.Data.Size)) + if (TryPublishConsumerRunLocked(image, address, size, cause, written)) { - if (image.IsPublished(start, end)) continue; - if (TryWriteBackRun(image, start, end, cause, out var refusal)) - { - _aliasCpuReadRuns++; - _aliasCpuReadBytes += (long)(end - start); - continue; - } - RefuseAliasTransition($"cpu-read-{refusal}"); + continue; } + WriteBackToGuest(image, cause); } } @@ -1701,6 +1698,11 @@ public void InvalidateMemoryFromGpu(ulong address, ulong size) continue; } + if (TryKeepGpuOwnershipOverGpuWriteLocked(image, address, size)) + { + continue; + } + if (image.Shape.Lossy && image.Texture != 0) { // It reloads what the GPU wrote, which it may not approximate. The new @@ -1934,6 +1936,10 @@ private bool TryWriteBackRun( } image.MarkPublished(aligned, alignedEnd); NoteOwnGuestWrite(description.Data.Address + aligned, alignedEnd - aligned); + // Fenced as a whole writeback is: a store that replaces the image without asking it + // for its bytes (DiscardFullyOverwrittenByCpuLocked) waits for this run to land first. + FenceWriteback(description.Data.Address + aligned, alignedEnd - aligned, context.SubmissionSerial, cause); + _writebackBytes += (long)(alignedEnd - aligned); _writebackCauses[(int)cause]++; _partialWritebacks++; _partialWritebackBytesSaved += (long)(description.Data.Size - (alignedEnd - aligned)); @@ -2198,6 +2204,7 @@ private ResourceSlotIdentifier InsertImage( NoteRecreatedLocked(in admitted); image.Epochs = Epochs; + image.PendingWritebacks = _pendingWritebacks; var imageIdentifier = _slots.Insert(image); NoteImageSetChanged(image.Description.Data); Epochs.NoteBindingChanged(); @@ -2793,6 +2800,7 @@ private MetalWritebackOutcome WriteBackToGuest(MetalCachedImage image, MetalWrit context.ProfileEncoded(ProfileLabel("writeback", in description)); } image.ClearGpuModified(); + _writebackBytes += (long)written; // The retile is a GPU store into guest memory: until it completes the bytes // in guest memory are the old ones, so a read of them has to wait for it. NoteOwnGuestWrite(description.Data.Address, description.Data.Size); diff --git a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalImageTransfer.Aliases.cs b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalImageTransfer.Aliases.cs index 78cc1db9f..2a0e16662 100644 --- a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalImageTransfer.Aliases.cs +++ b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalImageTransfer.Aliases.cs @@ -159,9 +159,10 @@ plan.ElementBytes is not (4 or 8 or 16) || width == 0 || height == 0 || private static nint AliasView(nint texture, MtlPixelFormat format, uint bytes, bool writing, uint layers = 1) { var raw = TextureTileViewFormat(format); - var usage = (MtlTextureUsage)(uint)MetalV2Native.SendValue(texture, SelUsage); + var info = MetalTileViews.Read(texture); + var usage = info.Usage; var needed = writing ? MtlTextureUsage.ShaderWrite : MtlTextureUsage.ShaderRead; - var type = (MtlTextureType)(uint)MetalV2Native.SendValue(texture, SelTextureType); + var type = info.Type; if (raw == MtlPixelFormat.Invalid || ViewElementBytes(raw) != bytes || (usage & needed) == 0 || (raw != format && (usage & MtlTextureUsage.PixelFormatView) == 0) || type is not (MtlTextureType.Type2D or MtlTextureType.Type2DArray)) return 0; diff --git a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalImageTransfer.cs b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalImageTransfer.cs index 063f317f1..dc4e0e4b2 100644 --- a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalImageTransfer.cs +++ b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalImageTransfer.cs @@ -833,12 +833,13 @@ public bool EncodeTextureToGuest(IMetalSubmissionContext context, in ImageDescri return true; } - var compute = _tiler.BeginCompute(context); + var compute = _tiler.BeginCompute(context, concurrent: true); var source = linear; if (plan.Swap) { var swapped = _tiler.Acquire(context, plan.Tiled ? plan.LinearSize : staged); _tiler.EncodeSwapBgra16(compute, new MetalBufferSpan(linear.Buffer, linear.Offset, swapped.Size), swapped); + MetalTiler.Barrier(compute); source = swapped; } @@ -862,11 +863,16 @@ public bool EncodeTextureToGuest(IMetalSubmissionContext context, in ImageDescri private static readonly bool TextureTilerEnabled = Environment.GetEnvironmentVariable("SHARPEMU_METAL_V2_TEXTURE_TILER") != "0"; - private static readonly nint SelPixelFormat = MetalNative.Selector("pixelFormat"); - private static readonly nint SelUsage = MetalNative.Selector("usage"); - private static readonly nint SelTextureType = MetalNative.Selector("textureType"); - private static readonly nint SelMipmapLevelCount = MetalNative.Selector("mipmapLevelCount"); - private static readonly nint SelArrayLength = MetalNative.Selector("arrayLength"); + // SHARPEMU_METAL_V2_TEXTURE_TILER_TAIL=0 sends every mip chain that reaches its tail + // back through staging, as before the one-pass kernels placed the tail levels. + private static readonly bool TextureTilerTailEnabled = + Environment.GetEnvironmentVariable("SHARPEMU_METAL_V2_TEXTURE_TILER_TAIL") != "0"; + + // SHARPEMU_METAL_V2_TEXTURE_TILER_SUBDWORD=0 stages the uploads of one and two byte + // texels again instead of detiling them straight into the texture. + private static readonly bool TextureTilerSubDwordEnabled = + Environment.GetEnvironmentVariable("SHARPEMU_METAL_V2_TEXTURE_TILER_SUBDWORD") != "0"; + private static readonly nint SelNewTextureView = MetalNative.Selector("newTextureViewWithPixelFormat:textureType:levels:slices:"); private static readonly nint SelReleaseView = MetalNative.Selector("release"); @@ -875,10 +881,16 @@ public bool EncodeTextureToGuest(IMetalSubmissionContext context, in ImageDescri /// The unsigned-integer view format whose texel is exactly one element of this colour /// format, or Invalid when the format is not one a texture block copy may carry: an /// sRGB format (no shader writes, and a read would decode it), a shared-exponent or - /// compressed one, or one whose texel is not 4, 8 or 16 bytes. + /// compressed one (Metal refuses an integer view of a block format), a packed 16-bit + /// one, or one whose texel is not 1, 2, 4, 8 or 16 bytes. /// internal static MtlPixelFormat TextureTileViewFormat(MtlPixelFormat format) => format switch { + MtlPixelFormat.R8Unorm or MtlPixelFormat.R8Snorm or MtlPixelFormat.R8Uint or + MtlPixelFormat.R8Sint => MtlPixelFormat.R8Uint, + MtlPixelFormat.R16Unorm or MtlPixelFormat.R16Snorm or MtlPixelFormat.R16Uint or + MtlPixelFormat.R16Sint or MtlPixelFormat.R16Float or MtlPixelFormat.Rg8Unorm or + MtlPixelFormat.Rg8Snorm or MtlPixelFormat.Rg8Uint or MtlPixelFormat.Rg8Sint => MtlPixelFormat.R16Uint, MtlPixelFormat.R32Uint or MtlPixelFormat.R32Sint or MtlPixelFormat.R32Float or MtlPixelFormat.Rg16Unorm or MtlPixelFormat.Rg16Snorm or MtlPixelFormat.Rg16Uint or MtlPixelFormat.Rg16Sint or MtlPixelFormat.Rg16Float or @@ -895,6 +907,8 @@ MtlPixelFormat.Rgba16Unorm or MtlPixelFormat.Rgba16Snorm or MtlPixelFormat.Rgba1 private static uint ViewElementBytes(MtlPixelFormat view) => view switch { + MtlPixelFormat.R8Uint => 1, + MtlPixelFormat.R16Uint => 2, MtlPixelFormat.R32Uint => 4, MtlPixelFormat.Rg32Uint => 8, MtlPixelFormat.Rgba32Uint => 16, @@ -906,7 +920,8 @@ MtlPixelFormat.Rgba16Unorm or MtlPixelFormat.Rgba16Snorm or MtlPixelFormat.Rgba1 // a frame that ping-pongs aliased render targets through guest memory the transfers // are most of the GPU's time. Only what the one-pass kernels address exactly the way // the staged path does qualifies - whole-element texels of a plain 2D or 2D-array - // texture, no mip tail, no channel swap - and the staged path keeps the rest. + // texture, no channel swap, and for the tiling direction no element under a dword - + // and the staged path keeps the rest. private bool TryEncodeTextureTiles( IMetalSubmissionContext context, TransferPlan plan, nint texture, ulong guestAddress, MetalBufferSpan guest, MetalTileDirection direction) @@ -917,53 +932,47 @@ private bool TryEncodeTextureTiles( return false; } - var format = (MtlPixelFormat)(uint)MetalV2Native.SendValue(texture, SelPixelFormat); - var viewFormat = TextureTileViewFormat(format); - if (viewFormat == MtlPixelFormat.Invalid || ViewElementBytes(viewFormat) != plan.ElementBytes) - { - return false; - } - - var usage = (MtlTextureUsage)(uint)MetalV2Native.SendValue(texture, SelUsage); - var needed = direction == MetalTileDirection.Detile ? MtlTextureUsage.ShaderWrite : MtlTextureUsage.ShaderRead; - if ((usage & needed) == 0 || (format != viewFormat && (usage & MtlTextureUsage.PixelFormatView) == 0)) + // A texture a cached image owns keeps its properties and its view from the first + // transfer on (MetalTileViews); any other is asked again each time. + var owned = MetalTileViews.TryLookup(texture, out var info, out var known, out var view); + if (!known) { - return false; - } - - var type = (MtlTextureType)(uint)MetalV2Native.SendValue(texture, SelTextureType); - if (type is not (MtlTextureType.Type2D or MtlTextureType.Type2DArray)) - { - return false; + info = MetalTileViews.Read(texture); + if (owned) + { + MetalTileViews.Remember(texture, info); + } } - var levels = MetalV2Native.SendValue(texture, SelMipmapLevelCount); - var slices = MetalV2Native.SendValue(texture, SelArrayLength); - foreach (var transfer in _tiles) + try { - if (transfer.Tail || transfer.Depth != 1 || transfer.BytesPerElement != plan.ElementBytes || - transfer.Level >= levels || transfer.Slice >= slices) + if (!TextureTilesQualify(plan, info, direction, out var viewFormat)) { return false; } - } - var view = MetalV2Native.SendNewTextureView( - texture, SelNewTextureView, (nuint)viewFormat, (nuint)MtlTextureType.Type2DArray, - new NsRange { Location = 0, Length = levels }, new NsRange { Location = 0, Length = slices }); - if (view == 0) - { - throw GpuFatal.Fatal( - $"The texture tiler could not make its view: format={format} view={viewFormat} type={type} " + - $"levels={levels} slices={slices}."); - } + if (view == 0) + { + view = MetalV2Native.SendNewTextureView( + texture, SelNewTextureView, (nuint)viewFormat, (nuint)MtlTextureType.Type2DArray, + new NsRange { Location = 0, Length = info.Levels }, new NsRange { Location = 0, Length = info.Slices }); + if (view == 0) + { + throw GpuFatal.Fatal( + $"The texture tiler could not make its view: format={info.Format} view={viewFormat} " + + $"type={info.Type} levels={info.Levels} slices={info.Slices}."); + } + + if (owned) + { + MetalTileViews.RememberView(texture, view); + } + } - try - { // Nothing but dispatches: the open compute encoder takes them when the host // leaves it to this transfer, which saves the encoder boundary. var shared = context.TryShareComputeEncoder(guestAddress, guest.Size); - var compute = shared != 0 ? shared : _tiler.BeginCompute(context); + var compute = shared != 0 ? shared : _tiler.BeginCompute(context, concurrent: true); _tiler.EncodeTextureBlockCopies(compute, guest, view, _tiles, direction); if (shared != 0) { @@ -976,8 +985,46 @@ private bool TryEncodeTextureTiles( } finally { - // The command buffer holds the view for as long as the encoded work needs it. - MetalNative.SendVoid(view, SelReleaseView); + // The command buffer holds the view for as long as the encoded work needs it, + // and a kept one is still held by MetalTileViews. + if (view != 0) + { + MetalNative.SendVoid(view, SelReleaseView); + } + } + + return true; + } + + // Whether every selected transfer can move through the texture's integer view in one + // pass, and the view's format when they can. + private bool TextureTilesQualify( + TransferPlan plan, in MetalTileTextureInfo info, MetalTileDirection direction, out MtlPixelFormat viewFormat) + { + viewFormat = TextureTileViewFormat(info.Format); + if (viewFormat == MtlPixelFormat.Invalid || ViewElementBytes(viewFormat) != plan.ElementBytes || + !MetalTilerShaders.SupportsTextureCopy(plan.ElementBytes, direction) || + (plan.ElementBytes < 4 && !TextureTilerSubDwordEnabled)) + { + return false; + } + + var needed = direction == MetalTileDirection.Detile ? MtlTextureUsage.ShaderWrite : MtlTextureUsage.ShaderRead; + if ((info.Usage & needed) == 0 || + (info.Format != viewFormat && (info.Usage & MtlTextureUsage.PixelFormatView) == 0) || + info.Type is not (MtlTextureType.Type2D or MtlTextureType.Type2DArray)) + { + return false; + } + + foreach (var transfer in _tiles) + { + if ((transfer.Tail && !TextureTilerTailEnabled) || transfer.Depth != 1 || + transfer.BytesPerElement != plan.ElementBytes || + transfer.Level >= info.Levels || transfer.Slice >= info.Slices) + { + return false; + } } return true; @@ -1035,7 +1082,7 @@ private MetalBufferSpan StageUpload(IMetalSubmissionContext context, TransferPla } var guest = new MetalBufferSpan(source.Buffer, source.Offset, source.Size); - var compute = _tiler.BeginCompute(context); + var compute = _tiler.BeginCompute(context, concurrent: true); MetalBufferSpan linear; if (plan.Tiled) { @@ -1052,6 +1099,7 @@ private MetalBufferSpan StageUpload(IMetalSubmissionContext context, TransferPla if (plan.Swap) { var swapped = _tiler.Acquire(context, linear.Size); + MetalTiler.Barrier(compute); _tiler.EncodeSwapBgra16(compute, linear, swapped); linear = swapped; } @@ -1933,7 +1981,7 @@ private void EncodeDepthUpload(IMetalSubmissionContext context, in ImageDescript nint compute = 0; if (plan.Tiled) { - compute = _tiler.BeginCompute(context); + compute = _tiler.BeginCompute(context, concurrent: true); var detiled = _tiler.Acquire(context, depthPiece); _tiler.EncodeDetile(compute, depthPlane, detiled, _tiles); depthPlane = detiled; @@ -1950,7 +1998,15 @@ private void EncodeDepthUpload(IMetalSubmissionContext context, in ImageDescript var depthSliceBytes = depthRowBytes * description.Extent.Height; if (plan.TransferBytes != description.BytesPerBlock) { - compute = compute != 0 ? compute : _tiler.BeginCompute(context); + if (compute != 0) + { + MetalTiler.Barrier(compute); + } + else + { + compute = _tiler.BeginCompute(context, concurrent: true); + } + var widened = _tiler.Acquire(context, depthSliceBytes * layerCount); _tiler.EncodeDepthConversion(compute, depthPlane, widened, MetalDepthConversion.Widen, float32: true, new MetalDepthConversionLayout @@ -2049,7 +2105,7 @@ private void EncodeTiledDepthDownload(IMetalSubmissionContext context, in ImageD return; } - var compute = _tiler.BeginCompute(context); + var compute = _tiler.BeginCompute(context, concurrent: true); _tiler.EncodeDepthTextureWriteback(compute, texture, new MetalBufferSpan(destination.Buffer, destination.Offset, depthPiece), description.BytesPerBlock, plan.Layers > 1, _tiles); @@ -2088,7 +2144,7 @@ private void EncodeStagedTiledDepthDownload(IMetalSubmissionContext context, in } MetalTiler.EndEncoding(context, blit); - var compute = _tiler.BeginCompute(context); + var compute = _tiler.BeginCompute(context, concurrent: true); if (widening) { var narrowed = _tiler.Acquire(context, depthPiece); @@ -2104,6 +2160,7 @@ private void EncodeStagedTiledDepthDownload(IMetalSubmissionContext context, in TargetSliceStride = plan.SliceSize, }); staging = narrowed; + MetalTiler.Barrier(compute); } _tiler.EncodeTile(compute, staging, @@ -2173,7 +2230,7 @@ private void EncodeDepthDownload(IMetalSubmissionContext context, in ImageDescri return; } - var compute = _tiler.BeginCompute(context); + var compute = _tiler.BeginCompute(context, concurrent: true); if (widening) { var narrowed = _tiler.Acquire(context, depthPiece); @@ -2189,6 +2246,7 @@ private void EncodeDepthDownload(IMetalSubmissionContext context, in ImageDescri TargetSliceStride = plan.SliceSize, }); depthPlane = narrowed; + MetalTiler.Barrier(compute); } _tiler.EncodeRepackRows(compute, depthPlane, diff --git a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalLibraryCompilePool.cs b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalLibraryCompilePool.cs index cac5cf98f..5cccaf8e5 100644 --- a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalLibraryCompilePool.cs +++ b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalLibraryCompilePool.cs @@ -72,11 +72,8 @@ internal MetalLibraryCompilePool(int threads) { for (var index = 0; index < threads; index++) { - new Thread(Run) - { - IsBackground = true, - Name = $"SharpEmu Metal v2 library compile {index}", - }.Start(); + // The frame waits for these: below the worker, above the background prewarm. + MetalCompileThread.Start($"SharpEmu Metal v2 library compile {index}", SharpEmu.HLE.Host.HostThreadQos.UserInitiated, Run); } } diff --git a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalLibraryPrewarm.cs b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalLibraryPrewarm.cs index 8a9a50b52..ba2a0f942 100644 --- a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalLibraryPrewarm.cs +++ b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalLibraryPrewarm.cs @@ -93,8 +93,8 @@ private sealed class Slot(ulong key, bool usesPerVertexInputs, byte[] compressed private static ConcurrentDictionary? _slots; private static ConcurrentDictionary? _written; private static BlockingCollection<(byte Kind, ulong Key, bool UsesPerVertexInputs, string? Source)>? _writes; - private static Thread? _writer; - private static Thread[] _threads = []; + private static MetalCompileThread? _writer; + private static MetalCompileThread[] _threads = []; private static List _order = []; private static nint _device; private static volatile bool _stopping; @@ -204,12 +204,8 @@ public static void Open(string directory, nint device, string stamp) } var writes = new BlockingCollection<(byte, ulong, bool, string?)>(); - var writer = new Thread(() => WriteLoop(path, writes)) - { - IsBackground = true, - Priority = ThreadPriority.BelowNormal, - Name = "SharpEmu Metal v2 library list", - }; + var writer = MetalCompileThread.Create( + "SharpEmu Metal v2 library list", SharpEmu.HLE.Host.HostThreadQos.Utility, () => WriteLoop(path, writes)); lock (_sync) { _stopping = false; @@ -262,10 +258,13 @@ public static void Begin() _running = true; var started = Stopwatch.GetTimestamp(); Console.Error.WriteLine($"[LOADER][INFO] Library prewarm started: libraries={order.Count} workers={workerCount}"); - var threads = new Thread[workerCount]; + var threads = new MetalCompileThread[workerCount]; for (var index = 0; index < workerCount; index++) { - threads[index] = new Thread(() => + // UserInitiated, as the compute prewarm's while it holds the guest: a library the + // game takes from here waits for it, and the compiler service runs the request at + // this thread's class. + threads[index] = MetalCompileThread.Create($"SharpEmu library prewarm {index}", SharpEmu.HLE.Host.HostThreadQos.UserInitiated, () => { while (!_stopping && work.TryDequeue(out var slot)) { @@ -282,14 +281,7 @@ public static void Begin() $"claimed={Volatile.Read(ref _claimed)} ms={Stopwatch.GetElapsedTime(started).TotalMilliseconds:F0} " + $"library_avg_ms={(compiled == 0 ? 0 : TicksToMs(Interlocked.Read(ref _compileTicks)) / compiled):F1}")); } - }) - { - // Normal priority, as the compute prewarm's: a library the game takes from - // here waits for it, and the compiler service runs the request at this - // thread's priority. - IsBackground = true, - Name = $"SharpEmu library prewarm {index}", - }; + }); } lock (_sync) @@ -345,7 +337,8 @@ void drain() } var threads = Enumerable.Range(0, Math.Max(0, MetalShaderPrewarm.WorkerCount - 1)) - .Select(index => new Thread(drain) { IsBackground = true, Name = $"SharpEmu library precompile {index}" }) + .Select(index => MetalCompileThread.Create( + $"SharpEmu library precompile {index}", SharpEmu.HLE.Host.HostThreadQos.UserInitiated, drain)) .ToArray(); foreach (var thread in threads) { @@ -597,8 +590,8 @@ private static string Summary(string label) => /// Stops the workers, closes the list and releases the libraries nobody took. public static void Stop() { - Thread[] threads; - Thread? writer; + MetalCompileThread[] threads; + MetalCompileThread? writer; BlockingCollection<(byte, ulong, bool, string?)>? writes; ConcurrentDictionary? slots; lock (_sync) @@ -622,7 +615,7 @@ public static void Stop() Console.Error.WriteLine(Summary("Library prewarm session")); foreach (var thread in threads) { - if (thread != Thread.CurrentThread) + if (!thread.IsCurrentThread) { thread.Join(TimeSpan.FromSeconds(10)); } diff --git a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalPipelineCompiler.cs b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalPipelineCompiler.cs index 86db9d96e..05b9774ba 100644 --- a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalPipelineCompiler.cs +++ b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalPipelineCompiler.cs @@ -90,7 +90,7 @@ internal sealed class Pending(Func build) private readonly ConcurrentDictionary _pending = new(); private readonly BlockingCollection _queue = new(); - private readonly Thread[] _threads; + private readonly MetalCompileThread[] _threads; private readonly Action _onCompleted; private int _running; private long _started; @@ -103,15 +103,12 @@ internal sealed class Pending(Func build) public MetalPipelineCompiler(Action onCompleted, int parallelism) { _onCompleted = onCompleted; - _threads = new Thread[Math.Max(1, parallelism)]; + _threads = new MetalCompileThread[Math.Max(1, parallelism)]; for (var index = 0; index < _threads.Length; index++) { - _threads[index] = new Thread(Run) - { - IsBackground = true, - Name = $"SharpEmu Metal v2 pipeline compile {index}", - }; - _threads[index].Start(); + // A suspended packet or a waiting draw needs these: below the worker, above the background prewarm. + _threads[index] = MetalCompileThread.Start( + $"SharpEmu Metal v2 pipeline compile {index}", SharpEmu.HLE.Host.HostThreadQos.UserInitiated, Run); } } diff --git a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.Descriptors.cs b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.Descriptors.cs index d8084b952..689e2d694 100644 --- a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.Descriptors.cs +++ b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.Descriptors.cs @@ -828,7 +828,7 @@ public void BindResources(IPreparedBindings prepared) var size = mappedWhole ? requested : ClampMappedSize(descriptor.Address, requested); var written = info.Buffers[index].Written; - FlushGpuWrites(descriptor.Address, size, MetalWritebackCause.StorageDescriptor); + FlushGpuWrites(descriptor.Address, size, MetalWritebackCause.StorageDescriptor, written); var range = ResolveStorageBuffer(descriptor.Address, size, written); if (written) { @@ -923,7 +923,7 @@ private void BindDeviceAddressRanges( var size = ClampMappedSize(range.Base, range.Size); OrderAfterDeferredCopies(range.Base, size); OrderAfterStagedStores(range.Base, size); - FlushGpuWrites(range.Base, size, MetalWritebackCause.StorageDescriptor); + FlushGpuWrites(range.Base, size, MetalWritebackCause.StorageDescriptor, range.Written); if (!range.Written) { continue; @@ -1440,7 +1440,7 @@ private ulong PointerFieldValue(MetalStageBindings stage, DescriptorBindingKind ? ReachFeedbackAddress() : MetalNative.SendGpuAddress(_guestMemory.NullBuffer, SelGpuAddress); case DescriptorBindingKind.FlattenedResourceTable: - return UploadDwords(stage.StageResources.Resources.FlattenedResourceTable); + return UploadFlattenedTable(stage.StageResources.Resources.FlattenedResourceTable, stage.Program.FlattenedTableMinimumWords); case DescriptorBindingKind.ShaderData: return UploadDwords(stage.ShaderData.AsSpan(0, (int)stage.Program.Bindings!.ShaderDataDwordCount)); case DescriptorBindingKind.MipStatistics: @@ -1559,6 +1559,22 @@ internal static void AddDeviceAddressEntry( int spanCount) => MetalDeviceAddressTable.AddEntry(entries, entry, spanCount, MaxAddressRanges); + // The program reads the table's fixed slots below minimumWords without a bounds check + // (Gen5MslCodegenOptions.ConstantTableSlots), so a shorter table goes up zero-padded: + // those slots read the 0 the check gave. The padded words are part of the reuse key. + private ulong UploadFlattenedTable(uint[] table, uint minimumWords) + { + if ((uint)table.Length >= minimumWords) + { + return UploadDwords(table); + } + + Span padded = minimumWords <= 1024 ? stackalloc uint[(int)minimumWords] : new uint[minimumWords]; + padded.Clear(); + table.CopyTo(padded); + return UploadDwords(padded); + } + private ulong UploadDwords(ReadOnlySpan words) { var bytes = System.Runtime.InteropServices.MemoryMarshal.AsBytes(words); diff --git a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.Pipelines.cs b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.Pipelines.cs index 1148085d9..eb2e77ca8 100644 --- a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.Pipelines.cs +++ b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.Pipelines.cs @@ -661,17 +661,14 @@ private void StartQueuedCompile(MetalShaderModule? module) return; } - new Thread(() => + // The worker's own class: it waits for this one (MetalCompileThread). + MetalCompileThread.Start("SharpEmu Metal v2 library compile (needed now)", SharpEmu.HLE.Host.HostThreadQos.UserInteractive, () => { if (MetalLibraryCompilePool.Shared.TryRunHere(job)) { Interlocked.Increment(ref _libraryInlineCompiles); } - }) - { - IsBackground = true, - Name = "SharpEmu Metal v2 library compile (needed now)", - }.Start(); + }); } private void AwaitServingStores(MetalShaderModule? module) diff --git a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.Rendering.cs b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.Rendering.cs index 255a4b730..b462051b5 100644 --- a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.Rendering.cs +++ b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.Rendering.cs @@ -2353,9 +2353,9 @@ public void ShaderWriteBarrier(GpuPipelineStage sourceStages) private void RecordMemoryBarrier() => RecordMemoryBarrier(Mtl.RenderStageVertex | Mtl.RenderStageFragment); - private void RecordMemoryBarrier(nuint afterStages) + private void RecordMemoryBarrier(nuint afterStages, + nuint scope = Mtl.BarrierScopeBuffers | Mtl.BarrierScopeTextures | Mtl.BarrierScopeRenderTargets) { - const nuint scope = Mtl.BarrierScopeBuffers | Mtl.BarrierScopeTextures | Mtl.BarrierScopeRenderTargets; if (_renderEncoder != 0) { MetalV2Native.SendMemoryBarrierStages( @@ -2367,7 +2367,9 @@ private void RecordMemoryBarrier(nuint afterStages) return; } - if (_computeEncoder != 0) + // Every compute encoder this host opens is serial, where Metal ignores the barrier + // (MetalV2Diagnostics.SkipSerialComputeBarriers). + if (_computeEncoder != 0 && !MetalV2Diagnostics.SkipSerialComputeBarriers) { // A compute encoder has no render targets to order against, and Metal // rejects a scope that names them: validation fails the assertion rather @@ -2401,9 +2403,33 @@ public void EmitGlobalBarrier() } private Action? _emitOrderedGlobalBarrier; + private long _globalBarriersRecorded; + private long _globalBarriersElided; + + /// Guest barriers recorded and elided in an open render pass, for tests. + internal (long Recorded, long Elided) GlobalBarrierCounts => (_globalBarriersRecorded, _globalBarriersElided); private void EmitOrderedGlobalBarrier() { + if (_renderEncoder != 0 && MetalV2Diagnostics.ElideGlobalBarriers) + { + // What the guest's flush or invalidate must make visible to later work in this pass + // is shader stores; the executor has ordered every one but those it reports here. + // Attachments need no barrier: their loads and stores run in draw order per fragment. + if (_translation is not { HasUnorderedStoresInOpenPass: false }) + { + RecordMemoryBarrier(Mtl.RenderStageVertex | Mtl.RenderStageFragment, + Mtl.BarrierScopeBuffers | Mtl.BarrierScopeTextures); + _globalBarriersRecorded++; + } + else + { + _globalBarriersElided++; + } + + return; + } + if (_renderEncoder != 0 || _computeEncoder != 0) { RecordMemoryBarrier(); diff --git a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.StagedStorage.cs b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.StagedStorage.cs index 51b2d36b2..ec4836661 100644 --- a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.StagedStorage.cs +++ b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.StagedStorage.cs @@ -20,7 +20,7 @@ internal sealed partial class MetalRenderHost /// range, otherwise a staged copy that sees the GPU's earlier writes and whose stores /// reach guest memory (). /// - private MetalGuestRange ResolveStorageBuffer(ulong address, ulong size, bool written) + internal MetalGuestRange ResolveStorageBuffer(ulong address, ulong size, bool written) { using var phaseScope = SharpEmu.Libs.VideoOut.RenderPhaseProfile.MeasureDetail(SharpEmu.Libs.VideoOut.RenderPhaseProfile.Phase.StorageBufferResolve); OrderAfterDeferredCopies(address, size); @@ -87,9 +87,15 @@ private MetalGuestRange ResolveStorageBuffer(ulong address, ulong size, bool wri context.ProfileEncoded($"stage storage bytes=0x{staging.Size:X}"); } } + else if (SnapshotsOnGpu(_renderEncoder != 0, _computeEncoder != 0, staging.Size)) + { + pristine = _guestMemory.Allocate(staging.Size, 256, out _); + SnapshotOnGpu(staging, pristine, null); + } else { pristine = _guestMemory.Duplicate(staging); + Interlocked.Add(ref _cpuPristineBytes, (long)staging.Size); } if (!pristine.IsValid) @@ -149,6 +155,47 @@ private MetalGuestRange StageWrittenOnGpu(ulong address, ulong size) return staging; } + // SHARPEMU_METAL_V2_GPU_PRISTINE=0 copies the pristine twin of a written staged copy on the CPU again. + internal static readonly bool GpuPristine = + Environment.GetEnvironmentVariable("SHARPEMU_METAL_V2_GPU_PRISTINE") != "0"; + + // Below this a blit encoder costs the worker about what the memcpy does. + internal const ulong GpuPristineMinBytes = 64 * 1024; + + private long _cpuPristineBytes; + private long _gpuPristines; + private long _gpuPristineBytes; + + internal long GpuPristines => Interlocked.Read(ref _gpuPristines); + + /// + /// True when the pristine twin of a written copy the CPU staged is better copied from it by the GPU: a + /// blit ahead of the shader that stores into the copy, instead of a second memcpy of the whole range on the + /// worker, which costs as much as staging it did. Never with a pass open, which the blit would end; with a + /// compute encoder open only from the size at which the GPU stages a written range whole anyway + /// (), since the blit ends that encoder too. + /// + internal static bool SnapshotsOnGpu(bool renderOpen, bool computeOpen, ulong size) => + GpuPristine && !renderOpen && size >= GpuPristineMinBytes && (!computeOpen || size >= WholeGatherMinBytes); + + private void SnapshotOnGpu(in MetalGuestRange staging, in MetalGuestRange pristine, MetalStagedRangeCache.Entry? entry) + { + var context = ((IMetalImageCacheOwner)this).BeginTransfer(MetalTransferCause.GuestCopy); + if (entry is not null) + { + // The blit reads and writes the entry's buffers in this submission. + _stagedRanges!.Touch(entry, context.SubmissionSerial); + } + + StagedStorage.EncodeSnapshot(context.CommandBuffer, staging, pristine, context); + Interlocked.Increment(ref _gpuPristines); + Interlocked.Add(ref _gpuPristineBytes, (long)staging.Size); + if (context.WantsEncodedLabels) + { + context.ProfileEncoded($"snapshot staged storage bytes=0x{staging.Size:X}"); + } + } + private long _guestViewRefusedWrites; /// @@ -183,7 +230,9 @@ private string GuestViewCounters() $"guest_view_unmapped={v.Invalidated} guest_view_refused={v.CreateFailures} guest_view_unbacked={v.Unbacked} " + $"guest_view_refused_writes={Interlocked.Read(ref _guestViewRefusedWrites)} " + $"oversized_staged={_guestMemory.OversizedStaged} oversized_prefix={_guestMemory.OversizedInPlace} " + - $"gpu_staged={Interlocked.Read(ref _wholeGathers)} gpu_staged_mb={Interlocked.Read(ref _wholeGatherBytes) / (1024 * 1024)}"; + $"gpu_staged={Interlocked.Read(ref _wholeGathers)} gpu_staged_mb={Interlocked.Read(ref _wholeGatherBytes) / (1024 * 1024)} " + + $"pristine_cpu_mb={Interlocked.Read(ref _cpuPristineBytes) / (1024 * 1024)} " + + $"pristine_gpu={Interlocked.Read(ref _gpuPristines)} pristine_gpu_mb={Interlocked.Read(ref _gpuPristineBytes) / (1024 * 1024)}"; } /// @@ -262,7 +311,15 @@ private unsafe void FillStagedRange(MetalStagedRangeCache.Entry entry, MetalStag } else if (entry.Pristine.IsValid) { - bytes.CopyTo(new Span((void*)entry.PristineBuffer!.Contents, checked((int)size))); + if (SnapshotsOnGpu(_renderEncoder != 0, _computeEncoder != 0, size)) + { + SnapshotOnGpu(entry.Staging, entry.Pristine, entry); + } + else + { + bytes.CopyTo(new Span((void*)entry.PristineBuffer!.Contents, checked((int)size))); + Interlocked.Add(ref _cpuPristineBytes, (long)size); + } } } diff --git a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.WriteFences.cs b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.WriteFences.cs index 41df347b7..89eaa74ee 100644 --- a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.WriteFences.cs +++ b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.WriteFences.cs @@ -26,12 +26,23 @@ namespace SharpEmu.Gpu.Metal.V2; // CPU writes is unchanged: pending command-processor and label bytes are still applied by the // callers, and every other user of the write ranges still sees the serial pending. // +// Eager fences: a block is learned only after a read of it has waited for the whole buffer, so +// the first read of a word, and every read of a word a ring buffer moves to a new block, paid the +// full wait. By default any recorded write since the last fence asks for one, so the first read +// waits for its writer too. The fence is still planted only where a commit was legal and no render +// pass is open, so what it orders is exactly what the learned fence orders; only how often one is +// planted changes (at most once per coalesced submission, CoalesceMaxSubmissions per buffer). +// // SHARPEMU_METAL_V2_WRITE_FENCES=0 plants none and waits for whole command buffers again. +// SHARPEMU_METAL_V2_EAGER_WRITE_FENCES=0 plants them only for blocks reads have waited on. internal sealed partial class MetalRenderHost { private static readonly bool WriteFencesDefault = Environment.GetEnvironmentVariable("SHARPEMU_METAL_V2_WRITE_FENCES") != "0"; + private static readonly bool EagerWriteFencesDefault = + Environment.GetEnvironmentVariable("SHARPEMU_METAL_V2_EAGER_WRITE_FENCES") != "0"; + private static readonly nint SelNotifyListener = MetalNative.Selector("notifyListener:atValue:block:"); private const int ObservedBlockShift = 16; @@ -57,6 +68,9 @@ internal sealed partial class MetalRenderHost /// Whether reads may wait for a write fence instead of the whole command buffer. internal bool WriteFences { get; set; } = WriteFencesDefault; + /// Whether any recorded write asks for a fence, not only one into a block a read waited on. + internal bool EagerWriteFences { get; set; } = EagerWriteFencesDefault; + /// Tests only: runs with the command buffer just after a fence is planted. internal Action? WriteFencePlantedHook { get; set; } @@ -99,11 +113,22 @@ private void NoteObservedRead(ulong address, ulong size) } } - // A recorded write: one into a block the CPU has waited on asks for a fence at the next boundary. + // A recorded write: any one (eager), or one into a block the CPU has waited on, asks for a fence + // at the next boundary. private void NoteFencedWrite(ulong address, ulong size) { - if (_writeFenceWanted || _observedReadBlocks.Count == 0 || !WriteFences || size == 0 || - address > ulong.MaxValue - size) + if (_writeFenceWanted || !WriteFences || size == 0 || address > ulong.MaxValue - size) + { + return; + } + + if (EagerWriteFences && !_writeFenceUnavailable) + { + _writeFenceWanted = true; + return; + } + + if (_observedReadBlocks.Count == 0) { return; } diff --git a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.cs b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.cs index b95bf3886..a155ba749 100644 --- a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.cs +++ b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.cs @@ -1568,7 +1568,9 @@ public void CopyGuestMemory(ulong destination, ulong source, ulong size, bool al // A packet-sized copy the host can do itself costs nothing and records no GPU // write, so the poll that usually follows it does not wait for a submission. - if (allowHost && CanWriteGuestOnHost(destination, size)) + // Not when recorded work still writes the source: the host would have to read it, + // which waits for that work (a DMA of a count or a query result the GPU produced). + if (allowHost && CanWriteGuestOnHost(destination, size) && !HostCopyWouldWait(source, size)) { CopyGuestBytesOnHost(destination, source, size); _imageCache.InvalidateMemory(destination, size); @@ -1616,6 +1618,33 @@ public void CopyGuestMemory(ulong destination, ulong source, ulong size, bool al NoteGuestMemoryWrite(destination, size, MetalGpuWriteOrigin.Copy); } + // SHARPEMU_METAL_V2_HOST_COPY_PENDING_SOURCE=1 copies on the host even when recorded work + // still writes the source, which waits for that work, as before. + private static readonly bool HostCopyOfPendingSource = + Environment.GetEnvironmentVariable("SHARPEMU_METAL_V2_HOST_COPY_PENDING_SOURCE") == "1"; + + private long _hostCopiesDeclined; + + // Whether reading the source of a copy on the host would wait for the GPU. The blit that runs + // instead reads it where the copy sits in the stream, after the work that writes it, which is + // the order the guest asked for; the destination it writes is a recorded write like any other. + private bool HostCopyWouldWait(ulong source, ulong size) + { + if (HostCopyOfPendingSource || + (!_gpuWrites.MayCover(source, size) && !_imageCache.MayOwn(source, size))) + { + return false; + } + + var wait = _imageCache.HasGpuModifiedImageBytes(source, size) || PendingSerialHinted(source, size) != 0; + if (wait) + { + Interlocked.Increment(ref _hostCopiesDeclined); + } + + return wait; + } + private void CopyGuestBytesOnHost(ulong destination, ulong source, ulong size) { _cpOverlay.Forget(destination, size); @@ -1661,8 +1690,8 @@ private void CopyGuestBytesOnHostCore(ulong destination, ulong source, ulong siz } /// Publishes every GPU-owned image byte in the range into guest memory. - public void FlushGpuWrites(ulong address, ulong size, MetalWritebackCause cause) => - _imageCache.FlushGpuWrites(address, size, cause); + public void FlushGpuWrites(ulong address, ulong size, MetalWritebackCause cause, bool written = false) => + _imageCache.FlushGpuWrites(address, size, cause, written); public void RunPendingOperations() => _relay.RunPendingCommands(); @@ -1789,6 +1818,30 @@ public void SynchronizeShaderCode(ulong address, ulong size) } } + // SHARPEMU_METAL_V2_SPECULATIVE_WAITS=1 lets work done ahead of need (the pipeline look-ahead's + // program preparation, GpuFatal.IsSpeculative) wait for the GPU like the stream does. + private static readonly bool SpeculativeWaits = + Environment.GetEnvironmentVariable("SHARPEMU_METAL_V2_SPECULATIVE_WAITS") == "1"; + + private long _speculativeWaitsAbandoned; + + // Work done ahead of need reads registers and memory the stream has not reached: bytes the GPU + // is still writing there may not even be the ones the stream will read, and the stream reads + // them itself when it gets there. So that work is abandoned (as a fatal condition in it is) + // instead of the worker waiting for the GPU on its behalf. Nothing has been recorded or waited + // for when this throws, so the stream's own read later sees exactly what it saw before. + private void AbandonSpeculativeWait(ulong address, ulong size) + { + if (SpeculativeWaits || !SharpEmu.Libs.Gpu.Scheduling.GpuFatal.IsSpeculative) + { + return; + } + + Interlocked.Increment(ref _speculativeWaitsAbandoned); + throw new SharpEmu.Libs.Gpu.Scheduling.GpuFatal.AbandonedWorkException( + $"A read ahead of the stream would wait for the GPU: address=0x{address:X16} size=0x{size:X}."); + } + // Lands every recorded write into the range: an image's writeback, or a store or copy. private void WaitForGuestBytes(ulong address, ulong size) { @@ -1796,6 +1849,7 @@ private void WaitForGuestBytes(ulong address, ulong size) _lastReadSize = size; if (_imageCache.HasGpuModifiedImageBytes(address, size)) { + AbandonSpeculativeWait(address, size); // The writeback is recorded into the buffer being built, so this one has // to be committed before the bytes can be read. FlushGpuWrites(address, size, MetalWritebackCause.HostRead); @@ -1808,6 +1862,7 @@ private void WaitForGuestBytes(ulong address, ulong size) var pending = PendingSerialHinted(address, size); if (pending != 0) { + AbandonSpeculativeWait(address, size); if (size <= SmallReadWaitBytes) { ReportSmallReadWaitSite(address, size); @@ -2206,7 +2261,7 @@ public BufferBinding ObtainBuffer(ulong address, ulong size, bool isWritten) { // A buffer binding sees guest memory, so any image that owns these bytes // publishes them first; a written binding also invalidates those images. - FlushGpuWrites(address, size, MetalWritebackCause.BufferBinding); + FlushGpuWrites(address, size, MetalWritebackCause.BufferBinding, isWritten); // A read takes the storage path: its window, a guest view, or a staged copy that // holds the GPU's writes still in flight in this command buffer - whole, however // large, up to the largest window. @@ -2444,6 +2499,8 @@ public void ReportCounters() .Append(" gpu_gds=").Append(TakeGpuGdsTransfers()) .Append(" cleared_fills=").Append(Interlocked.Exchange(ref _absorbedFills, 0)) .Append(" host_copies=").Append(Interlocked.Exchange(ref _hostCopies, 0)) + .Append(" host_copies_declined=").Append(Interlocked.Exchange(ref _hostCopiesDeclined, 0)) + .Append(" speculative_waits_abandoned=").Append(Interlocked.Exchange(ref _speculativeWaitsAbandoned, 0)) .Append(TakeCopyReadCounters()) .Append(" inline_eop=").Append(TakeInlineCompletions()) .Append(" gpu_labels=").Append(labels.GpuTimed) diff --git a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalShaderPrewarm.cs b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalShaderPrewarm.cs index ee3c59806..e14a65436 100644 --- a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalShaderPrewarm.cs +++ b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalShaderPrewarm.cs @@ -184,16 +184,29 @@ internal static class MetalShaderPrewarm private static bool _graphicsRan; private static ConcurrentDictionary? _prewarmedShaderIdentities; private static ConcurrentDictionary? _runtimeComputeIdentities; - private static Thread[] _threads = []; + private static MetalCompileThread[] _threads = []; private static volatile bool _stopping; private static volatile bool _finished; private static int _compiled; private static int _failed; private static int _runtimeHits; private static int _deduplicated; + private static int _archived; + + // SHARPEMU_SHADER_PREWARM_SKIP_ARCHIVED=0 compiles the computes the binary archive holds too. + private static readonly bool SkipsArchived = + Environment.GetEnvironmentVariable("SHARPEMU_SHADER_PREWARM_SKIP_ARCHIVED") != "0"; private static readonly ConcurrentQueue _completed = new(); private static readonly ConcurrentDictionary _inFlight = new(StringComparer.Ordinal); + // Set on a prewarm thread once it compiles at Utility (the computes the guest is not held + // for): a runtime compile does not wait for one of its compiles, which a busy machine may + // leave behind every thread above it (WaitWhileCompiling). + [ThreadStatic] + private static bool _onUtilityThread; + + private static readonly ConcurrentDictionary _utilityCompiles = new(); + /// /// Compiles in flight at once. The MSL front end runs in the driver's compiler service /// and is CPU bound there. On a ten-core machine (four performance, six efficiency @@ -227,7 +240,7 @@ public static void Start(nint device) TranslateAsTheGameDoes(); Gate.Begin(); - new Thread(() => + MetalCompileThread.Start("SharpEmu Metal v2 shader prewarm", SharpEmu.HLE.Host.HostThreadQos.UserInitiated, () => { try { @@ -244,11 +257,7 @@ public static void Start(nint device) Gate.SetRunning(false); } } - }) - { - IsBackground = true, - Name = "SharpEmu Metal v2 shader prewarm", - }.Start(); + }); } /// @@ -333,11 +342,18 @@ private static bool StartCore(nint device) MetalLibraryPrewarm.Begin(); } - var threads = new Thread[workerCount]; + var threads = new MetalCompileThread[workerCount]; for (var index = 0; index < workerCount; index++) { - threads[index] = new Thread(() => + // UserInitiated while the guest is held for these compiles: the driver's compiler + // service runs a request at the requester's class, and from below-normal threads + // each compile took about a fifth longer even on an idle machine + // (ShaderCompileBench, 200 cold Demon's Souls computes) and fell far behind under + // load from other processes. The computes the guest is not held for compile at + // Utility (see the loop below). + threads[index] = MetalCompileThread.Create($"SharpEmu shader prewarm {index}", SharpEmu.HLE.Host.HostThreadQos.UserInitiated, () => { + var lowered = false; while (!_stopping && work.TryDequeue(out var item)) { if (item.Graphics is { } graphicsItem) @@ -351,9 +367,11 @@ private static bool StartCore(nint device) if (!item.Held) { // The guest runs again: what is left compiles behind its threads. - if (Thread.CurrentThread.Priority != ThreadPriority.BelowNormal) + if (!lowered) { + lowered = true; Thread.CurrentThread.Priority = ThreadPriority.BelowNormal; + _onUtilityThread = MetalCompileThread.ApplyToCurrent(SharpEmu.HLE.Host.HostThreadQos.Utility); } YieldToTheGame(); @@ -391,17 +409,7 @@ private static bool StartCore(nint device) { Finish(list, stamp, started); } - }) - { - // Normal priority while the guest is held for these compiles: the driver's - // compiler service runs a request at the requester's priority, and from - // below-normal threads each compile took about a fifth longer even on an idle - // machine (ShaderCompileBench, 200 cold Demon's Souls computes) and fell far - // behind under load from other processes. The computes the guest is not - // held for compile below normal (see the loop above). - IsBackground = true, - Name = $"SharpEmu shader prewarm {index}", - }; + }); } Volatile.Write(ref _threads, threads); @@ -674,7 +682,8 @@ public static void WaitWhileCompiling(MetalCompiledGuestShader shader) return; } - if (_inFlight.TryGetValue(Identity(shader), out var compiling) && !compiling.IsSet) + if (_inFlight.TryGetValue(Identity(shader), out var compiling) && !compiling.IsSet && + !_utilityCompiles.ContainsKey(compiling)) { var started = Stopwatch.GetTimestamp(); compiling.Wait(); @@ -752,6 +761,19 @@ private static void PrewarmComputePipeline( return; } + // The title's binary archive holds this compute's library and its pipeline's binary + // (MetalBinaryArchive): the dispatch opens both from there and asks the driver for + // neither, so compiling them again here would only fill the driver's own cache, at + // the cost of a front end and a back end per program (a second start of a title + // compiled every recorded compute once more, behind the guest's first frame for the + // held ones). The translation above still ran: its source names the file. + if (SkipsArchived && MetalBinaryArchive.For(device) is { } archive && + archive.IsStored(MetalBinaryArchive.LibraryKey(metal.Shader.Source, metal.Shader.UsesPerVertexInputs))) + { + Interlocked.Increment(ref _archived); + return; + } + // Waits while the game compiles (MetalCompileScheduler), and holds a prewarm slot // until this compile is done. using var prewarm = MetalCompileScheduler.Shared.EnterPrewarm(static () => _stopping); @@ -770,6 +792,11 @@ private static void PrewarmComputePipeline( // Not disposed: a dispatch may still be waking from it. var compiling = new ManualResetEventSlim(false); + if (_onUtilityThread) + { + _utilityCompiles[compiling] = 0; + } + _inFlight[identity] = compiling; try { @@ -778,6 +805,7 @@ private static void PrewarmComputePipeline( finally { _inFlight.TryRemove(identity, out _); + _utilityCompiles.TryRemove(compiling, out _); compiling.Set(); } } @@ -865,6 +893,7 @@ private static void Finish(ShaderPrewarmList list, string stamp, long started) var summary = $"[LOADER][INFO] Shader prewarm done: compiled={Volatile.Read(ref _compiled)} " + $"failed={Volatile.Read(ref _failed)} same_source={Volatile.Read(ref _deduplicated)} " + + $"archived={Volatile.Read(ref _archived)} " + $"runtime_first={Volatile.Read(ref _runtimeFirst)} " + $"graphics_libraries={Volatile.Read(ref _graphicsCompiled)} graphics_failed={Volatile.Read(ref _graphicsFailed)} " + $"ms={Stopwatch.GetElapsedTime(started).TotalMilliseconds:F0} " + @@ -921,7 +950,7 @@ public static void Stop() MetalDeviceDataLoadsStore.Stop(); foreach (var thread in Volatile.Read(ref _threads)) { - if (thread != Thread.CurrentThread) + if (!thread.IsCurrentThread) { thread.Join(TimeSpan.FromSeconds(10)); } diff --git a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalStagedStorage.cs b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalStagedStorage.cs index 6d1616c9c..675fc6a56 100644 --- a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalStagedStorage.cs +++ b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalStagedStorage.cs @@ -409,18 +409,27 @@ public void EncodeGather(nint commandBuffer, ulong address, in MetalGuestRange s MetalNative.SendVoid(gather, SelEndEncoding); } - if (!pristine.IsValid) + if (pristine.IsValid) { - return; + EncodeSnapshot(commandBuffer, staging, pristine, ordering); } + } + /// + /// Fills a pristine copy from its staged copy on the GPU, in an encoder of its own that + /// runs before whatever is recorded after it: before the + /// shader that stores into the staged copy, so the pristine bytes are what was staged. + /// + public void EncodeSnapshot(nint commandBuffer, in MetalGuestRange staging, in MetalGuestRange pristine, + IMetalSubmissionContext? ordering = null) + { var snapshot = BlitEncoder(commandBuffer, ordering); try { MetalNative.SendCopyBuffer( snapshot, SelCopyBuffer, staging.Buffer, (nuint)staging.Offset, - pristine.Buffer, (nuint)pristine.Offset, (nuint)size); + pristine.Buffer, (nuint)pristine.Offset, (nuint)staging.Size); } finally { diff --git a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalStaticPrecompile.cs b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalStaticPrecompile.cs index 28f0c63fd..08da1352f 100644 --- a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalStaticPrecompile.cs +++ b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalStaticPrecompile.cs @@ -92,7 +92,7 @@ internal static class MetalStaticPrecompile private static StaticShaderPlanner? _planner; private static FileStream? _doneFile; private static List _plan = []; - private static Thread[] _threads = []; + private static MetalCompileThread[] _threads = []; private static int _next = -1; private static bool _started; private static volatile bool _stopping; @@ -157,23 +157,22 @@ public static void Start(nint device, string metalDirectory, string stamp, IGues _started = true; } - new Thread(() => - { - try - { - Run(device, metalDirectory, stamp, compiler); - } - catch (Exception exception) + // Utility unless the guest is held until it completes (MetalCompileThread). + MetalCompileThread.Start( + "SharpEmu static shader precompile", + Precompile.HoldsUntilComplete ? SharpEmu.HLE.Host.HostThreadQos.UserInitiated : SharpEmu.HLE.Host.HostThreadQos.Utility, + () => { - Console.Error.WriteLine($"[LOADER][WARN] Shader precompile (static) stopped: {exception.Message}"); - _finished.Set(); - } - }) - { - IsBackground = true, - Priority = Precompile.HoldsUntilComplete ? ThreadPriority.Normal : ThreadPriority.BelowNormal, - Name = "SharpEmu static shader precompile", - }.Start(); + try + { + Run(device, metalDirectory, stamp, compiler); + } + catch (Exception exception) + { + Console.Error.WriteLine($"[LOADER][WARN] Shader precompile (static) stopped: {exception.Message}"); + _finished.Set(); + } + }); } /// @@ -224,7 +223,7 @@ public static void Stop() _scanCancel.Cancel(); foreach (var thread in Volatile.Read(ref _threads)) { - if (thread != Thread.CurrentThread) + if (!thread.IsCurrentThread) { thread.Join(TimeSpan.FromSeconds(10)); } @@ -297,28 +296,26 @@ private static void Run(nint device, string metalDirectory, string stamp, IGuest var jobs = hold ? MetalShaderPrewarm.WorkerCount : ReadInt(JobsVariable, DefaultJobs, 1, 16); // The workers take the created programs while the scan runs; the plan follows it. var remaining = jobs; - var threads = new Thread[jobs]; + var threads = new MetalCompileThread[jobs]; for (var index = 0; index < jobs; index++) { - threads[index] = new Thread(() => - { - try + threads[index] = MetalCompileThread.Create( + $"SharpEmu static shader precompile {index}", + hold ? SharpEmu.HLE.Host.HostThreadQos.UserInitiated : SharpEmu.HLE.Host.HostThreadQos.Utility, + () => { - Work(planner, archive, device, compiler, hold); - } - finally - { - if (Interlocked.Decrement(ref remaining) == 0) + try { - Finish(archive); + Work(planner, archive, device, compiler, hold); } - } - }) - { - IsBackground = true, - Priority = hold ? ThreadPriority.Normal : ThreadPriority.BelowNormal, - Name = $"SharpEmu static shader precompile {index}", - }; + finally + { + if (Interlocked.Decrement(ref remaining) == 0) + { + Finish(archive); + } + } + }); } Volatile.Write(ref _threads, threads); diff --git a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalTileViews.cs b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalTileViews.cs new file mode 100644 index 000000000..7da0197b6 --- /dev/null +++ b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalTileViews.cs @@ -0,0 +1,165 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +namespace SharpEmu.Gpu.Metal.V2; + +/// What a one-pass texture transfer asks a texture before it reads or writes it. +internal readonly record struct MetalTileTextureInfo( + MtlPixelFormat Format, MtlTextureUsage Usage, MtlTextureType Type, nuint Levels, nuint Slices); + +/// +/// The properties and the unsigned-integer view the tiler's one-pass kernels move a texture +/// through, made once for each texture a cached image owns instead of on every transfer: +/// creating the view alone was about 10 µs of the recording thread, the bulk of a render +/// target's transfer. A texture never changes its format, usage, type or extent, so both +/// stay valid for as long as the texture lives. +/// +/// The cached images count their references to a texture ( +/// moves an old texture to the image that retires it), and the last one to let go drops the +/// entry and releases the view, before the texture itself is released, so a new texture +/// that reuses the handle never finds an old entry. A texture no cached image owns has no +/// entry: its transfers read the properties and make a view each time, as before, since +/// nothing would ever release one held for it. +/// +internal static class MetalTileViews +{ + private static readonly nint SelPixelFormat = MetalNative.Selector("pixelFormat"); + private static readonly nint SelUsage = MetalNative.Selector("usage"); + private static readonly nint SelTextureType = MetalNative.Selector("textureType"); + private static readonly nint SelMipmapLevelCount = MetalNative.Selector("mipmapLevelCount"); + private static readonly nint SelArrayLength = MetalNative.Selector("arrayLength"); + private static readonly nint SelRetain = MetalNative.Selector("retain"); + private static readonly nint SelRelease = MetalNative.Selector("release"); + + private sealed class Entry + { + public int References; + public bool HasInfo; + public MetalTileTextureInfo Info; + public nint View; + } + + private static readonly Lock Gate = new(); + private static readonly Dictionary Entries = []; + + /// The view kept for a texture, or zero; for tests. + internal static nint KeptView(nint texture) + { + lock (Gate) + { + return Entries.TryGetValue(texture, out var entry) ? entry.View : 0; + } + } + + public static MetalTileTextureInfo Read(nint texture) => new( + (MtlPixelFormat)(uint)MetalV2Native.SendValue(texture, SelPixelFormat), + (MtlTextureUsage)(uint)MetalV2Native.SendValue(texture, SelUsage), + (MtlTextureType)(uint)MetalV2Native.SendValue(texture, SelTextureType), + MetalV2Native.SendValue(texture, SelMipmapLevelCount), + MetalV2Native.SendValue(texture, SelArrayLength)); + + /// A cached image now refers to this texture. + public static void Reference(nint texture) + { + if (texture == 0) + { + return; + } + + lock (Gate) + { + if (!Entries.TryGetValue(texture, out var entry)) + { + entry = new Entry(); + Entries[texture] = entry; + } + + entry.References++; + } + } + + /// A cached image no longer refers to this texture; the last one drops its view. + public static void Unreference(nint texture) + { + if (texture == 0) + { + return; + } + + nint view; + lock (Gate) + { + if (!Entries.TryGetValue(texture, out var entry) || --entry.References > 0) + { + return; + } + + Entries.Remove(texture); + view = entry.View; + } + + if (view != 0) + { + // Any command buffer that encoded the view holds it until it completes. + MetalNative.SendVoid(view, SelRelease); + } + } + + /// + /// The texture's properties and, when one was made, its integer view, retained for the + /// caller, who releases it. False when no cached image owns the texture; true with + /// false when one does but nothing was asked of it yet. + /// + public static bool TryLookup(nint texture, out MetalTileTextureInfo info, out bool hasInfo, out nint view) + { + lock (Gate) + { + if (!Entries.TryGetValue(texture, out var entry)) + { + info = default; + hasInfo = false; + view = 0; + return false; + } + + info = entry.Info; + hasInfo = entry.HasInfo; + view = entry.View; + if (view != 0) + { + MetalNative.SendVoid(view, SelRetain); + } + + return true; + } + } + + /// Keeps the properties of a texture a cached image owns. + public static void Remember(nint texture, in MetalTileTextureInfo info) + { + lock (Gate) + { + if (Entries.TryGetValue(texture, out var entry)) + { + entry.Info = info; + entry.HasInfo = true; + } + } + } + + /// + /// Keeps a view the caller made of a texture a cached image owns, taking a reference of + /// its own, or does nothing when the texture has no entry or already kept one. + /// + public static void RememberView(nint texture, nint view) + { + lock (Gate) + { + if (Entries.TryGetValue(texture, out var entry) && entry.View == 0) + { + MetalNative.SendVoid(view, SelRetain); + entry.View = view; + } + } + } +} diff --git a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalTiler.cs b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalTiler.cs index 9231c24bf..3de8faed1 100644 --- a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalTiler.cs +++ b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalTiler.cs @@ -93,6 +93,16 @@ internal sealed unsafe partial class MetalTiler : IDisposable private static readonly nint SelNewComputePipeline = MetalNative.Selector("newComputePipelineStateWithFunction:error:"); private static readonly nint SelSetConstantValue = MetalNative.Selector("setConstantValue:type:atIndex:"); private static readonly nint SelComputeEncoder = MetalNative.Selector("computeCommandEncoder"); + private static readonly nint SelComputeEncoderWithDispatchType = + MetalNative.Selector("computeCommandEncoderWithDispatchType:"); + private static readonly nint SelMemoryBarrierScope = MetalNative.Selector("memoryBarrierWithScope:"); + private const nint DispatchTypeConcurrent = 1; + + // Experimental: paired synthetic timings showed several slower writeback shapes, + // so serial dispatch remains the default. SHARPEMU_METAL_V2_CONCURRENT_TILER=1 + // enables overlap for further measurements; the dependency barriers stay in place. + internal static readonly bool ConcurrentDispatch = + Environment.GetEnvironmentVariable("SHARPEMU_METAL_V2_CONCURRENT_TILER") == "1"; private static readonly nint SelBlitEncoder = MetalNative.Selector("blitCommandEncoder"); private static readonly nint SelEndEncoding = MetalNative.Selector("endEncoding"); private static readonly nint SelSetPipeline = MetalNative.Selector("setComputePipelineState:"); @@ -198,10 +208,20 @@ public MetalBufferSpan Acquire(IMetalSubmissionContext context, ulong size) => /// Releases and makes purgeable the slabs idle since . public void TrimStaging(ulong completedSerial) => _staging.Trim(completedSerial); - public nint BeginCompute(IMetalSubmissionContext context) + /// + /// Opens a compute encoder for the tiler. A concurrent one lets its dispatches overlap: + /// one transfer is a dispatch per subresource (and per band of block rows), each writing + /// bytes no other one touches, and a serial encoder would drain the GPU between every + /// pair of them, which for the small levels of a mip chain is most of their time. A + /// caller that opens one concurrent puts a between a dispatch and + /// any later one that reads what it wrote. + /// + public nint BeginCompute(IMetalSubmissionContext context, bool concurrent = false) { context.ReleaseSharedEncoder(); - var encoder = MetalNative.Send(context.CommandBuffer, SelComputeEncoder); + var encoder = concurrent && ConcurrentDispatch + ? MetalNative.Send(context.CommandBuffer, SelComputeEncoderWithDispatchType, DispatchTypeConcurrent) + : MetalNative.Send(context.CommandBuffer, SelComputeEncoder); MetalMemoryTelemetry.ComputeEncoders.Add(); if (encoder == 0) { @@ -212,6 +232,19 @@ public nint BeginCompute(IMetalSubmissionContext context) return encoder; } + /// + /// Orders the dispatches encoded after this behind the buffer writes of the ones before + /// it, in an encoder opened concurrent. A serial encoder + /// already orders them, so there it records nothing. + /// + public static void Barrier(nint encoder) + { + if (ConcurrentDispatch) + { + MetalV2Native.SendMemoryBarrierScope(encoder, SelMemoryBarrierScope, Mtl.BarrierScopeBuffers); + } + } + public nint BeginBlit(IMetalSubmissionContext context) { context.ReleaseSharedEncoder(); @@ -410,8 +443,9 @@ private nint TextureCopyPipeline(TileBlockKind kind, MetalTileDirection directio /// Moves tiled guest bytes straight into a texture (detile) or straight out of one /// (tile), one dispatch per transfer, through an unsigned-integer 2D-array view whose /// texel is the transfer's element. The caller decides that every transfer qualifies - /// (, no mip tail, one slice deep); - /// one that does not is a caller bug, not a fallback. + /// (, one + /// slice deep); one that does not is a caller bug, not a fallback. A mip tail level is + /// placed inside its block exactly as the buffer kernels place it. /// public void EncodeTextureBlockCopies( nint encoder, MetalBufferSpan tiled, nint view, List transfers, MetalTileDirection direction) @@ -437,7 +471,7 @@ public void EncodeTextureBlockCopies( var subresource = stackalloc uint[2]; foreach (ref readonly var transfer in CollectionsMarshal.AsSpan(transfers)) { - if (!MetalTilerShaders.SupportsTextureCopy(transfer.BytesPerElement) || transfer.Tail || transfer.Depth != 1) + if (!MetalTilerShaders.SupportsTextureCopy(transfer.BytesPerElement, direction) || transfer.Depth != 1) { throw GpuFatal.Fatal( $"The texture tile transfer does not qualify: kind={(uint)transfer.Kind} bytes={transfer.BytesPerElement} " + diff --git a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalTilerShaders.cs b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalTilerShaders.cs index 4de6efb43..ff07828ce 100644 --- a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalTilerShaders.cs +++ b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalTilerShaders.cs @@ -56,9 +56,16 @@ internal static partial class MetalTilerShaders public const uint SubresourceIndex = 4; // The element sizes a texture block copy carries: one, two or four 32-bit channels - // of the view, which is every colour texel the tiled transfers move whole. + // of the view, which is every colour texel the tiled transfers move whole. A detile + // also carries the one and two byte elements, through an 8 or 16-bit channel: each + // thread reads its element out of the guest dword and writes one texel. The tiling + // direction would have to merge neighbouring elements into a guest dword, which the + // buffer path's dword-wide kernel does without atomics, so it keeps those. public static bool SupportsTextureCopy(uint elementBytes) => elementBytes is 4 or 8 or 16; + public static bool SupportsTextureCopy(uint elementBytes, MetalTileDirection direction) => + SupportsTextureCopy(elementBytes) || (direction == MetalTileDirection.Detile && elementBytes is 1 or 2); + public const string DepthWriteback = "writeback_depth"; public const string DepthWritebackArray = "writeback_depth_array"; @@ -626,7 +633,7 @@ public static string CreateTextureBlockCopyLibrary(TileBlockKind kind, MetalTile var any = false; foreach (var bytes in ElementSizes) { - if (SupportsTextureCopy(bytes) && TileGeometry.TryGetBlockLayout(kind, bytes, out _)) + if (SupportsTextureCopy(bytes, direction) && TileGeometry.TryGetBlockLayout(kind, bytes, out _)) { AppendOffsetFunction(source, kind, bytes); any = true; @@ -649,21 +656,32 @@ public static string CreateTextureBlockCopyLibrary(TileBlockKind kind, MetalTile source.Append($" constant TextureSubresource& subresource [[buffer({SubresourceIndex})]],\n"); source.Append(" uint3 gid [[thread_position_in_grid]])\n{\n"); source.Append(" if (gid.x >= args.width || gid.y >= args.height)\n {\n return;\n }\n\n"); - source.Append(" uint sy = gid.y + args.baseY;\n"); - source.Append(" uint2 texel = uint2(gid.x, sy);\n"); + // A mip tail level sits at (tailX, tailY) inside the one block the tail shares, + // exactly as the buffer kernel places it; its texels still start at the origin of + // its own level. A tail is never cut into block rows, so it has no baseY. + source.Append(" bool tail = args.tail != 0u;\n"); + source.Append(" uint2 texel = uint2(gid.x, tail ? gid.y : gid.y + args.baseY);\n"); + source.Append(" uint sx = tail ? gid.x + args.tailX : gid.x;\n"); + source.Append(" uint sy = tail ? gid.y + args.tailY : texel.y;\n"); foreach (var bytes in ElementSizes) { - if (!SupportsTextureCopy(bytes) || !TileGeometry.TryGetBlockLayout(kind, bytes, out var block)) + if (!SupportsTextureCopy(bytes, direction) || !TileGeometry.TryGetBlockLayout(kind, bytes, out var block)) { continue; } source.Append($" if (kElementBytes == {bytes}u)\n {{\n"); - source.Append($" uint block = (gid.y >> {Log2(block.BlockHeight)}u) * args.blocksPerRow + " + + source.Append($" uint block = tail ? 0u : (gid.y >> {Log2(block.BlockHeight)}u) * args.blocksPerRow + " + $"(gid.x >> {Log2(block.BlockWidth)}u);\n"); - source.Append($" uint tiled = block * {block.BlockSize}u + tiled_offset_{bytes}(gid.x, sy, args.surfaceZ);\n"); + source.Append($" uint tiled = block * {block.BlockSize}u + tiled_offset_{bytes}(sx, sy, args.surfaceZ);\n"); var words = bytes / 4; - if (detile) + if (detile && bytes < 4) + { + source.Append(" uint at = args.sourceBase + tiled;\n"); + source.Append($" uint value = (source[at >> 2] >> ((at & 3u) * 8u)) & {(bytes == 1 ? "0xffu" : "0xffffu")};\n"); + source.Append(" surface.write(uint4(value, 0u, 0u, 0u), texel, subresource.slice, subresource.level);\n"); + } + else if (detile) { source.Append(" uint at = (args.sourceBase + tiled) >> 2;\n"); source.Append(" uint4 value = uint4(source[at], "); diff --git a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalV2Diagnostics.cs b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalV2Diagnostics.cs index 484cb8fa6..d93db2a98 100644 --- a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalV2Diagnostics.cs +++ b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalV2Diagnostics.cs @@ -154,6 +154,25 @@ private static HashSet ParseHashes(string? text) public static readonly bool DeferFragmentWritePassEnd = !string.Equals(Environment.GetEnvironmentVariable("SHARPEMU_METAL_V2_DEFER_FRAGMENT_WRITE_PASS_END"), "0", StringComparison.Ordinal); + // SHARPEMU_METAL_V2_SKIP_SERIAL_COMPUTE_BARRIERS=0 records a memory barrier on the open + // compute encoder again for every barrier the executor asks for (two per dispatch). The + // encoder is serial, and Metal documents a barrier there as "allowed, but ignored" + // (MTLComputeCommandEncoder.h, memoryBarrierWithScope:): each dispatch completes, its + // stores visible, before the next starts, for untracked buffers as for tracked textures. + // Perf-audit bench 8b1/8b2: 2,700 dependent dispatches on untracked buffers read back + // right without the barriers and take the same GPU time with them. + public static readonly bool SkipSerialComputeBarriers = + !string.Equals(Environment.GetEnvironmentVariable("SHARPEMU_METAL_V2_SKIP_SERIAL_COMPUTE_BARRIERS"), "0", StringComparison.Ordinal); + + // SHARPEMU_METAL_V2_ELIDE_GLOBAL_BARRIERS=0 records the guest acquire or release barrier in + // an open render pass every time, as it did before. The executor already orders a draw's + // buffer stores from the vertex stage with a barrier right after the draw, and ends the pass + // over fragment stores before their consumer; attachments are ordered per fragment in draw + // order. So the barrier is recorded only while the pass holds stores nothing ordered yet + // (deferred fragment stores, or vertex stores through a device address or storage image). + public static readonly bool ElideGlobalBarriers = + !string.Equals(Environment.GetEnvironmentVariable("SHARPEMU_METAL_V2_ELIDE_GLOBAL_BARRIERS"), "0", StringComparison.Ordinal); + // SHARPEMU_METAL_V2_KERNEL_FILLS=0 runs a title's own compute memset or memcpy over the // exact bytes of a cached image as the dispatch it is, with the writeback before it // and the upload after it, instead of as a clear or texture copy of the image diff --git a/src/graphics/SharpEmu.Gpu.Vulkan/Presentation/VulkanVideoPresenter.CommandStream.cs b/src/graphics/SharpEmu.Gpu.Vulkan/Presentation/VulkanVideoPresenter.CommandStream.cs index 9871caf25..01f8dcf34 100644 --- a/src/graphics/SharpEmu.Gpu.Vulkan/Presentation/VulkanVideoPresenter.CommandStream.cs +++ b/src/graphics/SharpEmu.Gpu.Vulkan/Presentation/VulkanVideoPresenter.CommandStream.cs @@ -1071,7 +1071,7 @@ internal void CaptureFlip(int handle, int index, ulong requestId, int flipMode, IndexedDrawTrace.OnFlip(version); VisibilityResultTrace.OnFlip(version); RenderDocCapture.OnGuestFlipBoundary(version); - VideoOutBackendServices.TraceGpuFlip(_guestMemory, handle, index, flipMode, flipArg, displayBuffer.Address, VideoOutBackendServices.GetFlipEventCount(requestId)); + VideoOutBackendServices.TraceGpuFlip(_guestMemory, handle, index, flipMode, flipArg, displayBuffer.Address, requestId); } finally { diff --git a/tests/SharpEmu.Gpu.Metal.LeakTests/MetalCommandBufferPathLeakTests.cs b/tests/SharpEmu.Gpu.Metal.LeakTests/MetalCommandBufferPathLeakTests.cs index fcf990141..b473bb122 100644 --- a/tests/SharpEmu.Gpu.Metal.LeakTests/MetalCommandBufferPathLeakTests.cs +++ b/tests/SharpEmu.Gpu.Metal.LeakTests/MetalCommandBufferPathLeakTests.cs @@ -209,13 +209,10 @@ private static void RunFlushes(SceneRig rig, uint[] words, int count, Action objects) private void Report( string label, - (uint Blocks, ulong Bytes, ulong Footprint) baseline, - (uint Blocks, ulong Bytes, ulong Footprint) libraries, - (uint Blocks, ulong Bytes, ulong Footprint) functions, - (uint Blocks, ulong Bytes, ulong Footprint) pipelines, - (uint Blocks, ulong Bytes, ulong Footprint) pipelinesReleased, - (uint Blocks, ulong Bytes, ulong Footprint) released) => + (long Blocks, ulong Bytes, ulong Footprint) baseline, + (long Blocks, ulong Bytes, ulong Footprint) libraries, + (long Blocks, ulong Bytes, ulong Footprint) functions, + (long Blocks, ulong Bytes, ulong Footprint) pipelines, + (long Blocks, ulong Bytes, ulong Footprint) pipelinesReleased, + (long Blocks, ulong Bytes, ulong Footprint) released) => Output( label, Count, ("libraries", baseline, libraries), @@ -312,7 +312,7 @@ private void Report( ("pipelines released", pipelines, pipelinesReleased), ("all released", pipelines, released)); - private void Output(string label, int count, params (string Phase, (uint Blocks, ulong Bytes, ulong Footprint) From, (uint Blocks, ulong Bytes, ulong Footprint) To)[] phases) + private void Output(string label, int count, params (string Phase, (long Blocks, ulong Bytes, ulong Footprint) From, (long Blocks, ulong Bytes, ulong Footprint) To)[] phases) { var text = new StringBuilder(label); foreach (var (phase, from, to) in phases) @@ -339,27 +339,12 @@ private bool RequireDevice() return true; } - private static (uint Blocks, ulong Bytes, ulong Footprint) Sample() + private static (long Blocks, ulong Bytes, ulong Footprint) Sample() { - GC.Collect(); - GC.WaitForPendingFinalizers(); - GC.Collect(); - malloc_zone_statistics(0, out var statistics); - return (statistics.BlocksInUse, statistics.SizeInUse, PhysFootprint()); + var live = NativeHeap.Sample(); + return (live.Blocks, live.Bytes, PhysFootprint()); } - [StructLayout(LayoutKind.Sequential)] - private struct MallocStatistics - { - public uint BlocksInUse; - public nuint SizeInUse; - public nuint MaxSizeInUse; - public nuint SizeAllocated; - } - - [DllImport("/usr/lib/libSystem.dylib")] - private static extern void malloc_zone_statistics(nint zone, out MallocStatistics statistics); - [DllImport("/usr/lib/libSystem.dylib")] private static extern uint mach_task_self(); diff --git a/tests/SharpEmu.Gpu.Metal.LeakTests/NativeHeap.cs b/tests/SharpEmu.Gpu.Metal.LeakTests/NativeHeap.cs new file mode 100644 index 000000000..919b05983 --- /dev/null +++ b/tests/SharpEmu.Gpu.Metal.LeakTests/NativeHeap.cs @@ -0,0 +1,135 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using Xunit; + +namespace SharpEmu.Gpu.Metal.LeakTests; + +// Count live allocation ranges, as heap(1) does. On translated macOS 27.2 the +// malloc_zone_statistics block counter grows by 1-3 per flush even when heap's +// live nodes decrease; it can also wrap through uint.MaxValue. Those counters +// cannot distinguish a leaked command buffer from allocator bookkeeping drift. +internal static unsafe class NativeHeap +{ + internal struct Counts + { + public long Blocks; + public ulong Bytes; + } + + public static Counts Sample() + { + GC.Collect(); + GC.WaitForPendingFinalizers(); + GC.Collect(); + nint* zones; + uint count; + var task = mach_task_self(); + Check(malloc_get_all_zones(task, &Read, &zones, &count)); + Counts result = default; + for (var index = 0; index < count; index++) + Enumerate(task, zones[index], &result); + return result; + } + + internal static Counts CountZone(nint zone) + { + Counts result = default; + Enumerate(mach_task_self(), zone, &result); + return result; + } + + private static void Enumerate(uint task, nint zone, Counts* result) + { + // malloc/malloc.h's stable zone prefix: two reserved pointers, eight + // callbacks/name pointers, batch_malloc and batch_free, then introspect. + // The first introspection function is enumerator(task, context, mask, + // zone, reader, recorder). All pointers are in this process. + var introspect = *(nint**)(zone + 12 * IntPtr.Size); + if (introspect == null || introspect[0] == 0) + throw new InvalidOperationException("Malloc zone has no allocation enumerator."); + var enumerate = (delegate* unmanaged[Cdecl], + delegate* unmanaged[Cdecl], int>)introspect[0]; + Check(enumerate(task, result, 1, (nuint)zone, &Read, &Record)); // MALLOC_PTR_IN_USE_RANGE_TYPE + } + + [StructLayout(LayoutKind.Sequential)] + private struct Range + { + public nuint Address; + public nuint Size; + } + + [UnmanagedCallersOnly(CallConvs = [typeof(CallConvCdecl)])] + private static int Read(uint task, nuint address, nuint size, void** memory) + { + *memory = (void*)address; + return 0; + } + + [UnmanagedCallersOnly(CallConvs = [typeof(CallConvCdecl)])] + private static void Record(uint task, void* context, uint type, Range* ranges, uint count) + { + var result = (Counts*)context; + result->Blocks += count; + for (var index = 0; index < count; index++) result->Bytes += ranges[index].Size; + } + + private static void Check(int error) + { + if (error != 0) throw new InvalidOperationException($"Malloc range enumeration failed: {error}."); + } + + [DllImport("/usr/lib/libSystem.dylib")] + private static extern uint mach_task_self(); + + [DllImport("/usr/lib/libSystem.dylib")] + private static extern int malloc_get_all_zones(uint task, + delegate* unmanaged[Cdecl] reader, nint** zones, uint* count); +} + +public sealed unsafe class NativeHeapTests +{ + [Fact] + public void EnumerationCountsLiveBlocksAndTheirReleaseInAnIsolatedZone() + { + if (!OperatingSystem.IsMacOS()) return; + var zone = malloc_create_zone(0, 0); + Assert.NotEqual(0, zone); + nint* allocations = stackalloc nint[32]; + var allocated = 0; + try + { + var before = NativeHeap.CountZone(zone); + for (; allocated < 32; allocated++) + { + allocations[allocated] = malloc_zone_malloc(zone, 137); + Assert.NotEqual(0, allocations[allocated]); + } + var live = NativeHeap.CountZone(zone); + Assert.Equal(before.Blocks + 32, live.Blocks); + Assert.True(live.Bytes >= before.Bytes + 32 * 137); + while (allocated > 0) malloc_zone_free(zone, allocations[--allocated]); + var freed = NativeHeap.CountZone(zone); + Assert.Equal(before.Blocks, freed.Blocks); + Assert.Equal(before.Bytes, freed.Bytes); + } + finally + { + while (allocated > 0) malloc_zone_free(zone, allocations[--allocated]); + malloc_destroy_zone(zone); + } + } + + [DllImport("/usr/lib/libSystem.dylib")] + private static extern nint malloc_create_zone(nuint size, uint flags); + [DllImport("/usr/lib/libSystem.dylib")] + private static extern nint malloc_zone_malloc(nint zone, nuint size); + [DllImport("/usr/lib/libSystem.dylib")] + private static extern void malloc_zone_free(nint zone, nint pointer); + [DllImport("/usr/lib/libSystem.dylib")] + private static extern void malloc_destroy_zone(nint zone); +} diff --git a/tests/SharpEmu.Gpu.Metal.Tests/MetalBinaryArchiveTests.cs b/tests/SharpEmu.Gpu.Metal.Tests/MetalBinaryArchiveTests.cs index f8ca69709..cc14969c3 100644 --- a/tests/SharpEmu.Gpu.Metal.Tests/MetalBinaryArchiveTests.cs +++ b/tests/SharpEmu.Gpu.Metal.Tests/MetalBinaryArchiveTests.cs @@ -174,6 +174,28 @@ public void ARenderPipelineFromTheArchiveDrawsThePixelsTheCompiledOneDoes() Release(stored, archives, storedDescriptor, storedPixelLibrary, storedVertexLibrary, compiled, descriptor, pixelLibrary, vertexLibrary); } + // The shader prewarm skips a compute whose file is written (MetalShaderPrewarm): only a + // written file counts, in this session and the next. + [Fact] + public void AComputeLibraryIsStoredOnlyOnceItsFileIsWritten() + { + if (!RequireDevice()) return; + var computeKey = MetalBinaryArchive.LibraryKey(ComputeSource, usesPerVertexInputs: false); + var otherKey = MetalBinaryArchive.LibraryKey(ComputeSource + "\n// another\n", usesPerVertexInputs: false); + using (var writing = MetalBinaryArchive.Open(_root, Device.Value, "translator-a", Budget)) + { + Assert.False(writing.IsStored(computeKey)); + Assert.True(Store(writing, ComputeSource, "gen5_cs", Gen5MslStage.Compute, computeKey) > 0); + Assert.True(writing.IsStored(computeKey)); + Assert.False(writing.IsStored(otherKey)); + Assert.False(writing.IsStored(default)); + } + + using var reading = MetalBinaryArchive.Open(_root, Device.Value, "translator-a", Budget); + Assert.True(reading.IsStored(computeKey)); + Assert.False(reading.IsStored(otherKey)); + } + [Fact] public void LibrariesStoredAloneDrawAndComputeWhatTheCompiledOnesDo() { diff --git a/tests/SharpEmu.Gpu.Metal.Tests/MetalCompileThreadTests.cs b/tests/SharpEmu.Gpu.Metal.Tests/MetalCompileThreadTests.cs new file mode 100644 index 000000000..ac511e410 --- /dev/null +++ b/tests/SharpEmu.Gpu.Metal.Tests/MetalCompileThreadTests.cs @@ -0,0 +1,52 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using SharpEmu.Gpu.Metal.V2; +using SharpEmu.HLE.Host; +using Xunit; + +namespace SharpEmu.Gpu.Metal.Tests; + +// The compile threads run at the class they are started with, a thread of them may move +// itself to another later, and a join waits for the body. +public sealed class MetalCompileThreadTests +{ + private const uint QosClassUserInitiated = 0x19; + private const uint QosClassUtility = 0x11; + + [Fact] + public void AThreadRunsItsBodyAtTheRequestedClassAndCanLowerItself() + { + uint started = 0; + uint lowered = 0; + var applied = false; + var thread = MetalCompileThread.Start("SharpEmu test compile thread", HostThreadQos.UserInitiated, () => + { + started = HostThreadQualityOfService.GetCurrentThreadClassForDiagnostics(); + applied = MetalCompileThread.ApplyToCurrent(HostThreadQos.Utility); + lowered = HostThreadQualityOfService.GetCurrentThreadClassForDiagnostics(); + }); + + Assert.True(thread.Join(TimeSpan.FromSeconds(30))); + Assert.False(thread.IsCurrentThread); + if (!OperatingSystem.IsMacOS() || !MetalCompileThread.QosEnabled) + { + return; + } + + Assert.Equal(QosClassUserInitiated, started); + Assert.True(applied); + Assert.Equal(QosClassUtility, lowered); + } + + [Fact] + public void ACreatedThreadRunsOnlyOnceStarted() + { + using var ran = new ManualResetEventSlim(false); + var thread = MetalCompileThread.Create("SharpEmu test compile thread", HostThreadQos.Utility, ran.Set); + Assert.False(ran.Wait(TimeSpan.FromMilliseconds(100))); + thread.Start(); + Assert.True(ran.Wait(TimeSpan.FromSeconds(30))); + Assert.True(thread.Join(TimeSpan.FromSeconds(30))); + } +} diff --git a/tests/SharpEmu.Gpu.Metal.Tests/MetalEndOfPipeOrderTests.cs b/tests/SharpEmu.Gpu.Metal.Tests/MetalEndOfPipeOrderTests.cs index ad08b1933..cf56a5660 100644 --- a/tests/SharpEmu.Gpu.Metal.Tests/MetalEndOfPipeOrderTests.cs +++ b/tests/SharpEmu.Gpu.Metal.Tests/MetalEndOfPipeOrderTests.cs @@ -589,6 +589,71 @@ public void TheDeferredClockLandsInOrderWithItsParseTimeValue() /// A clock that rides the next commit is still committed by a wait packet polling it, so /// a queue waiting on the stamp never waits for the worker to park. /// + [Fact] + public void AWaitForASerialIncludesTheSecondCompletionWhileItsHostWorkRuns() + { + if (!RequireDevice()) return; + var sharedEvent = MetalNative.Send(Device.Value, MetalNative.Selector("newSharedEvent")); + Assert.NotEqual(0, sharedEvent); + try + { + using var completions = new MetalCompletionQueue(sharedEvent); + using var harness = new MetalTransferHarness(Device.Value); + using var running = new ManualResetEventSlim(); + using var release = new ManualResetEventSlim(); + using var waiting = new ManualResetEventSlim(); + using var finished = new ManualResetEventSlim(); + const ulong serial = 1; + var published = 0; + completions.Enqueue(serial, static () => { }); + completions.Enqueue(serial, () => + { + try + { + running.Set(); + if (release.Wait(TimeSpan.FromSeconds(10))) + Volatile.Write(ref published, 1); + } + finally + { + finished.Set(); + } + }); + Task? waiter = null; + try + { + // Signal exactly this producer on the real GPU before publishing its host work. + harness.Begin(); + MetalV2Native.SendEventValue(harness.CommandBuffer, MetalNative.Selector("encodeSignalEvent:value:"), sharedEvent, serial); + harness.Submit(); + completions.Publish(serial); + Assert.True(running.Wait(TimeSpan.FromSeconds(10))); + Assert.Equal(1, completions.PendingCount); + waiter = Task.Run(() => + { + waiting.Set(); + completions.WaitFor(serial); + }); + Assert.True(waiting.Wait(TimeSpan.FromSeconds(10))); + Assert.False(waiter.Wait(TimeSpan.FromMilliseconds(100)), "The serial completed while its second host action was held."); + release.Set(); + Assert.True(waiter.Wait(TimeSpan.FromSeconds(10))); + Assert.Equal(1, Volatile.Read(ref published)); + Assert.Equal(0, completions.PendingCount); + } + finally + { + release.Set(); + if (running.IsSet) Assert.True(finished.Wait(TimeSpan.FromSeconds(10))); + if (waiter is not null) Assert.True(waiter.Wait(TimeSpan.FromSeconds(10))); + } + } + finally + { + MetalNative.SendVoid(sharedEvent, MetalNative.Selector("release")); + } + } + [Fact] public void AWaitPacketPollingALazyClockCommitsItsBuffer() { @@ -604,14 +669,15 @@ public void AWaitPacketPollingALazyClockCommitsItsBuffer() Assert.Equal(open, host.SubmissionSerial); Span polled = stackalloc byte[sizeof(ulong)]; - Assert.True(host.TryReadGuestWaitOperand(clockLabel, polled, commitPendingWrite: false)); + Assert.True(host.TryReadGuestWaitOperand(clockLabel, polled, commitPendingWrite: false), "Reading the lazy clock without committing was refused."); Assert.Equal(open, host.SubmissionSerial); - Assert.True(host.TryReadGuestWaitOperand(clockLabel, polled, commitPendingWrite: true)); + Assert.True(host.TryReadGuestWaitOperand(clockLabel, polled, commitPendingWrite: true), "Reading the lazy clock while committing was refused."); Assert.NotEqual(open, host.SubmissionSerial); WaitForGpu(host, open); WaitForCompletions(host); - Assert.True(rig.Backing.Read64(clockLabel) > 1); + var actualClock = rig.Backing.Read64(clockLabel); + Assert.True(actualClock > 1, $"Lazy clock remained {actualClock}; producer={open}, completed={host.CompletedSerial}."); } private (ulong Parsed, ulong GpuDone, (ulong WhileHeld, ulong AfterDrain) Clock, diff --git a/tests/SharpEmu.Gpu.Metal.Tests/MetalGuestViewTests.cs b/tests/SharpEmu.Gpu.Metal.Tests/MetalGuestViewTests.cs index 764a85e28..f68037dd2 100644 --- a/tests/SharpEmu.Gpu.Metal.Tests/MetalGuestViewTests.cs +++ b/tests/SharpEmu.Gpu.Metal.Tests/MetalGuestViewTests.cs @@ -60,7 +60,7 @@ public void AScatteredRangePastTheRingCapIsReadWholeThroughAGuestView() memory.BeginSubmission(1, 0); Assert.False(memory.TryResolveInPlace(backing.Base, ViewBytes, out _)); Assert.True(memory.MayUseGuestView(ViewBytes)); - Assert.True(memory.TryResolveGuestView(backing.Base, ViewBytes, out var view)); + Assert.True(memory.TryResolveGuestView(backing.Base, ViewBytes, out var view), $"Guest-view counters: {memory.GuestViewCounts}"); Assert.Equal(ViewBytes, view.Size); Assert.Contains(view.Buffer, memory.ResidentBuffers.ToArray()); @@ -77,7 +77,7 @@ public void AScatteredRangePastTheRingCapIsReadWholeThroughAGuestView() // A smaller binding inside it reuses the view; the next submission names it only if it binds it. memory.BeginSubmission(2, 1); Assert.DoesNotContain(view.Buffer, memory.ResidentBuffers.ToArray()); - Assert.True(memory.TryResolveGuestView(backing.Base + 3 * ShmScatteredBacking.Piece, 8 * Mib, out var inner)); + Assert.True(memory.TryResolveGuestView(backing.Base + 3 * ShmScatteredBacking.Piece, 8 * Mib, out var inner), $"Guest-view counters: {memory.GuestViewCounts}"); Assert.Equal(view.Buffer, inner.Buffer); Assert.Equal(3 * ShmScatteredBacking.Piece, inner.Offset); Assert.Equal(1L, memory.GuestViewCounts.Created); @@ -231,6 +231,45 @@ public void AReadBehindAWriteInTheSameCommandBufferIsStagedAndSeesIt() AssertEqualBytes(expected, harness.Read(read, 0, size)); } + // A written scattered range bound with no encoder open has its pristine twin copied by the + // GPU from the staged copy, cached or in the ring, and the twin holds the staged bytes. + [Fact] + public void AWrittenScatteredRangeBoundBetweenPassesHasItsPristineTwinCopiedOnTheGpu() + { + if (!RequireDevice()) + { + return; + } + + using var backing = new ShmScatteredBacking(BackingBytes, ViewBytes); + backing.FillAliasPattern(0x2545F491u); + using var rig = new HostRig(Device.Value, backing); + var host = rig.Host; + var before = host.GpuPristines; + var expectedSnapshots = MetalRenderHost.GpuPristine ? 1 : 0; + + const ulong cachedSize = 1 * Mib; + var cachedAddress = backing.Base + (ShmScatteredBacking.Piece / 2); + var cached = host.ResolveStorageBuffer(cachedAddress, cachedSize, written: true); + Assert.True(cached.IsValid); + Assert.Equal(before + expectedSnapshots, host.GpuPristines); + + const ulong ringSize = 128 * 1024; + var ringAddress = backing.Base + (8 * Mib) + (ShmScatteredBacking.Piece / 2); + var ring = host.ResolveStorageBuffer(ringAddress, ringSize, written: true); + Assert.True(ring.IsValid); + Assert.Equal(before + (2 * expectedSnapshots), host.GpuPristines); + host.FlushAndWaitForGpu(); + + var expected = new byte[cachedSize]; + Assert.True(backing.TryRead(cachedAddress, expected)); + var cache = host.ImageCache.StagedRanges!; + Assert.True(cache.Acquire(cachedAddress, cachedSize, written: true, host.SubmissionSerial, + host.SubmissionSerial - 1, out var entry)); + AssertEqualBytes(expected, new ReadOnlySpan((void*)entry!.StagingBuffer!.Contents, (int)cachedSize).ToArray()); + AssertEqualBytes(expected, new ReadOnlySpan((void*)entry.PristineBuffer!.Contents, (int)cachedSize).ToArray()); + } + // A blit from a binding into a new shared buffer, recorded in the host's command buffer // the way its own transfers are. private static nint CopyOnHost(MetalRenderHost host, MetalTransferHarness harness, BufferBinding binding, ulong size) @@ -290,13 +329,13 @@ public void AnUnmapDropsTheViewAndTheNextBindReadsTheNewMapping() const ulong size = 2 * Mib; memory.BeginSubmission(1, 0); - Assert.True(memory.TryResolveGuestView(backing.Base, size, out var first)); + Assert.True(memory.TryResolveGuestView(backing.Base, size, out var first), $"Guest-view counters: {memory.GuestViewCounts}"); memory.OnGuestUnmap(backing.Base + ShmScatteredBacking.Piece, ShmScatteredBacking.Piece); backing.Remap(1, BackingBytes - ShmScatteredBacking.Piece); Assert.Equal(1L, memory.GuestViewCounts.Invalidated); - Assert.True(memory.TryResolveGuestView(backing.Base, size, out var second)); + Assert.True(memory.TryResolveGuestView(backing.Base, size, out var second), $"Guest-view counters: {memory.GuestViewCounts}"); Assert.NotEqual(first.Buffer, second.Buffer); Assert.Equal(2L, memory.GuestViewCounts.Created); var expected = new byte[size]; @@ -439,6 +478,7 @@ internal sealed unsafe class ShmScatteredBacking : IGuestBackedSpace, ICpuMemory private readonly int _descriptor; private readonly nint _alias; + private readonly nint _viewReservation; private readonly ulong _backingBytes; private readonly ulong _viewBytes; private readonly ulong[] _offsets; @@ -461,11 +501,15 @@ public ShmScatteredBacking(ulong backingBytes, ulong viewBytes) } _alias = mmap(0, (nuint)backingBytes, ProtRead | ProtWrite, MapShared, _descriptor, 0); - Base = (ulong)mmap(0, (nuint)viewBytes, ProtNone, MapPrivate | MapAnon, -1, 0); - if (_alias == -1 || Base == ulong.MaxValue) + // Rosetta mmap may return a 4 KiB boundary, while Metal's no-copy view + // requires 16 KiB. Keep an extra piece reserved and align inside our own + // reservation, so rounding a view down never reaches before its mapping. + _viewReservation = mmap(0, (nuint)(viewBytes + Piece), ProtNone, MapPrivate | MapAnon, -1, 0); + if (_alias == -1 || _viewReservation == -1) { throw new InvalidOperationException("mmap failed."); } + Base = ((ulong)_viewReservation + Piece - 1) & ~(Piece - 1); var pieces = viewBytes / Piece; var backingPieces = backingBytes / Piece; @@ -511,7 +555,7 @@ public ulong BackingOffsetOf(ulong address) => public void Dispose() { - munmap((nint)Base, (nuint)_viewBytes); + munmap(_viewReservation, (nuint)(_viewBytes + Piece)); munmap(_alias, (nuint)_backingBytes); close(_descriptor); } diff --git a/tests/SharpEmu.Gpu.Metal.Tests/MetalImageBudgetTests.PendingWritebacks.cs b/tests/SharpEmu.Gpu.Metal.Tests/MetalImageBudgetTests.PendingWritebacks.cs new file mode 100644 index 000000000..06bf60e54 --- /dev/null +++ b/tests/SharpEmu.Gpu.Metal.Tests/MetalImageBudgetTests.PendingWritebacks.cs @@ -0,0 +1,104 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using SharpEmu.HLE.GpuMemory; +using SharpEmu.Libs.Gpu.Images; +using Xunit; + +namespace SharpEmu.Gpu.Metal.Tests; + +public sealed partial class MetalImageBudgetTests +{ + [Fact] + public void PendingPagesFollowTheRealImageThroughRetirementAndWritebackCompletion() + { + if (!RequireDevice()) return; + using var rig = new Rig(Device.Value); + var pending = new PendingPageRecorder { RequirePending = true }; + rig.Cache.PendingWritebacks = pending; + Assert.True(pending.Active); + Assert.False(pending.HasImagePages); + + // This uses the real Metal image cache, texture upload, retirement and GPU writeback, + // rather than a second implementation of the image cache's state machine. + var (identifier, written) = ReleaseAGpuWrittenImage(rig); + Assert.False(rig.Cache.TryGetImage(identifier, out _)); + Assert.Equal(1, rig.Cache.FencedWritebackCount); + Assert.True(pending.HasImagePages); + Assert.False(pending.PublishedGap); + Assert.False(pending.NegativeClaim); + + rig.Cache.Collect(); + Assert.True(pending.HasImagePages); + Assert.False(pending.PublishedGap); + + pending.RequirePending = false; + rig.Harness.Submit(); + rig.Harness.Begin(); + rig.Cache.Collect(); + Assert.Equal(0, rig.Cache.FencedWritebackCount); + Assert.False(pending.HasImagePages); + Assert.False(pending.PublishedGap); + Assert.False(pending.NegativeClaim); + Assert.Equal(written, rig.ReadGuest(GuestBase, ImageSize)); + } + + [Fact] + public void AReadOnlyImageDoesNotClaimPendingWritebackPages() + { + if (!RequireDevice()) return; + using var rig = new Rig(Device.Value); + var pending = new PendingPageRecorder(); + rig.Cache.PendingWritebacks = pending; + var image = rig.Cache.GetImage(rig.Acquire(GuestBase, ImageRole.Texture)); + Assert.False(image.IsGpuModified); + rig.Cache.Collect(); + Assert.False(pending.HasImagePages); + Assert.False(pending.NegativeClaim); + } + + private sealed class PendingPageRecorder : IPendingWritebackPages + { + private readonly Dictionary _claims = new(); + private readonly HashSet _visible = new(); + private bool _everClaimed; + + public bool Active { get; private set; } + public bool RequirePending { get; set; } + public bool PublishedGap { get; private set; } + public bool NegativeClaim { get; private set; } + public bool HasImagePages => Enumerable.Range(0, (int)((ImageSize + 4095) / 4096)) + .All(page => _visible.Contains((GuestBase >> 12) + (ulong)page)); + + public void Activate() => Active = true; + + public void Add(ulong address, ulong size) + { + for (var page = address >> 12; page <= (address + size - 1) >> 12; page++) + { + _claims.TryGetValue(page, out var count); + _claims[page] = count + 1; + _visible.Add(page); + _everClaimed = true; + } + } + + public void Remove(ulong address, ulong size) + { + for (var page = address >> 12; page <= (address + size - 1) >> 12; page++) + { + _claims.TryGetValue(page, out var count); + _claims[page] = count - 1; + NegativeClaim |= count <= 0; + } + } + + public void Publish() + { + foreach (var (page, count) in _claims) + if (count <= 0) _visible.Remove(page); + // Record a violation without throwing inside RegionLock's release callback. + PublishedGap |= RequirePending && _everClaimed && !HasImagePages; + } + } +} diff --git a/tests/SharpEmu.Gpu.Metal.Tests/MetalImageTransferTests.Harness.cs b/tests/SharpEmu.Gpu.Metal.Tests/MetalImageTransferTests.Harness.cs index c74ad2f78..97b57ab86 100644 --- a/tests/SharpEmu.Gpu.Metal.Tests/MetalImageTransferTests.Harness.cs +++ b/tests/SharpEmu.Gpu.Metal.Tests/MetalImageTransferTests.Harness.cs @@ -45,6 +45,11 @@ public MetalTransferHarness(nint device) public MetalTiler Tiler => Transfer.Tiler; + /// The encoders the transfer and the tiler opened through this context. + public int EncodersStarted { get; private set; } + + void IMetalSubmissionContext.EncoderStarted(nint encoder, bool render) => EncodersStarted++; + public void Begin() => CommandBuffer = MetalNative.Send(MetalNative.Send(_queue, SelCommandBuffer), SelRetain); public void Submit() diff --git a/tests/SharpEmu.Gpu.Metal.Tests/MetalImageTransferTests.TextureTiles.cs b/tests/SharpEmu.Gpu.Metal.Tests/MetalImageTransferTests.TextureTiles.cs index 4fc65afa7..f63d3e014 100644 --- a/tests/SharpEmu.Gpu.Metal.Tests/MetalImageTransferTests.TextureTiles.cs +++ b/tests/SharpEmu.Gpu.Metal.Tests/MetalImageTransferTests.TextureTiles.cs @@ -23,6 +23,21 @@ public sealed partial class MetalImageTransferTests "rg32 render target", "rgba32f 64kb 2 mips", "rgba16f 4kb 3 layers", + "rgba8 64kb full chain", + "rgba16f 4kb full chain 2 layers", + "rg32 64kb full chain", + "rgba32f 4kb full chain", + }; + + // Elements under a dword go one pass on the way in only: the writeback merges + // neighbouring elements into a guest dword, which the staged path's kernel does. + public static TheoryData SubDwordTextureTileCases => new() + { + "r8 64kb full chain", + "rg8 4kb full chain 2 layers", + "r16f 64kb 3 mips into the tail", + "r8 render target", + "r16 render target", }; private static ImageDescription TextureTileCase(string name) => name switch @@ -39,18 +54,39 @@ public sealed partial class MetalImageTransferTests 256, 256, 2, 1, GuestTileMode.Standard64KB), "rgba16f 4kb 3 layers" => Color2D(GuestPixelFormat.Bits16_16_16_16Float, GpuFormat.R16G16B16A16Sfloat, 96, 48, 1, 3, GuestTileMode.Standard4KB), + "rgba8 64kb full chain" => Color2D(GuestPixelFormat.Bits8_8_8_8UNorm, GpuFormat.R8G8B8A8Unorm, + 256, 256, 9, 1, GuestTileMode.Standard64KB), + "rgba16f 4kb full chain 2 layers" => Color2D(GuestPixelFormat.Bits16_16_16_16Float, GpuFormat.R16G16B16A16Sfloat, + 128, 64, 8, 2, GuestTileMode.Standard4KB), + "rg32 64kb full chain" => Color2D(GuestPixelFormat.Bits32_32UInt, GpuFormat.R32G32Uint, + 200, 120, 8, 1, GuestTileMode.Standard64KB), + "rgba32f 4kb full chain" => Color2D(GuestPixelFormat.Bits32_32_32_32Float, GpuFormat.R32G32B32A32Sfloat, + 64, 64, 7, 1, GuestTileMode.Standard4KB), + "r8 64kb full chain" => Color2D(GuestPixelFormat.Bits8UNorm, GpuFormat.R8Unorm, + 512, 256, 10, 1, GuestTileMode.Standard64KB), + "rg8 4kb full chain 2 layers" => Color2D(GuestPixelFormat.Bits8_8UNorm, GpuFormat.R8G8Unorm, + 96, 80, 7, 2, GuestTileMode.Standard4KB), + "r16f 64kb 3 mips into the tail" => Color2D(GuestPixelFormat.Bits16Float, GpuFormat.R16Sfloat, + 300, 200, 3, 1, GuestTileMode.Standard64KB), + "r8 render target" => Color2D(GuestPixelFormat.Bits8UNorm, GpuFormat.R8Unorm, + 200, 120, 1, 1, GuestTileMode.RenderTarget), + "r16 render target" => Color2D(GuestPixelFormat.Bits16UNorm, GpuFormat.R16Unorm, + 160, 72, 1, 1, GuestTileMode.RenderTarget), _ => throw new ArgumentOutOfRangeException(nameof(name)), }; [Theory] [MemberData(nameof(TextureTileCases))] + [MemberData(nameof(SubDwordTextureTileCases))] public void ATiledColorUploadWritesTheTextureInOnePassExactlyAsTheOracleDetiles(string name) { if (!RequireDevice()) return; using var harness = new MetalTransferHarness(Device.Value); var description = TextureTileCase(name); var guest = BuildGuest(description, 91, out var linear); - var (_, copies, _, element) = BuildLayout(description); + var (_, copies, tiles, element) = BuildLayout(description); + // A full chain (or a small enough level) reaches the mip tail, which the one-pass kernel places itself. + Assert.Equal(name.Contains("full chain") || name.Contains("tail"), tiles.Exists(tile => tile.Tail)); var texture = harness.Texture(harness.Transfer.ShapeFor(description)); var range = new SubresourceRange(0, description.Resources.Levels, 0, description.Resources.Layers); @@ -88,7 +124,8 @@ public void ATiledColorWritebackReadsTheTextureInOnePassExactlyAsTheOracleTiles( using var harness = new MetalTransferHarness(Device.Value); var description = TextureTileCase(name); var expected = BuildGuest(description, 93, out var linear); - var (_, copies, _, element) = BuildLayout(description); + var (_, copies, tiles, element) = BuildLayout(description); + Assert.Equal(name.Contains("full chain") || name.Contains("tail"), tiles.Exists(tile => tile.Tail)); var texture = harness.Texture(harness.Transfer.ShapeFor(description)); var source = harness.Buffer(linear); harness.Begin(); @@ -153,3 +190,51 @@ private void AssertLiveTexelsEqual( output.WriteLine($"{name}: one-pass upload matched the oracle over 0x{compared:x} texel bytes"); } } + +// A texture a cached image owns is viewed once for every transfer of its lifetime, and the +// view goes with the last image that refers to the texture; any other texture is viewed +// per transfer and nothing is kept for it. +public sealed partial class MetalImageTransferTests +{ + [Fact] + public void ACachedImageKeepsOneTilerViewUntilItsTextureIsReleased() + { + if (!RequireDevice()) return; + using var harness = new MetalTransferHarness(Device.Value); + var description = TextureTileCase("rgba8 64kb full chain"); + var guest = BuildGuest(description, 97, out _); + var shape = harness.Transfer.ShapeFor(description); + var texture = harness.Texture(shape); + var loose = harness.Texture(shape); + var range = new SubresourceRange(0, description.Resources.Levels, 0, description.Resources.Layers); + var source = new MetalGuestRange(harness.Buffer(guest), 0, description.Data.Size, 0); + + // The image releases the texture it owns; the harness releases its own reference. + MetalNative.SendVoid(texture, MetalNative.Selector("retain")); + var image = new MetalCachedImage(in description, in shape, texture); + nint kept = 0; + for (var pass = 0; pass < 3; pass++) + { + harness.Begin(); + Assert.True(harness.Transfer.EncodeGuestToTexture(harness, description, range, source, default, texture, 0, + description.Data.Size)); + Assert.True(harness.Transfer.EncodeGuestToTexture(harness, description, range, source, default, loose, 0, + description.Data.Size)); + harness.Submit(); + var view = MetalTileViews.KeptView(texture); + Assert.NotEqual(0, view); + Assert.True(kept == 0 || kept == view, "the kept view changed between transfers"); + kept = view; + Assert.Equal(0, MetalTileViews.KeptView(loose)); + } + + // An image retiring the texture (PromoteToOwned and its kin) refers to it too, so + // the view survives until both have let go. + var retired = new MetalCachedImage(in description, in shape, texture); + MetalNative.SendVoid(texture, MetalNative.Selector("retain")); + image.Dispose(); + Assert.Equal(kept, MetalTileViews.KeptView(texture)); + retired.Dispose(); + Assert.Equal(0, MetalTileViews.KeptView(texture)); + } +} diff --git a/tests/SharpEmu.Gpu.Metal.Tests/MetalImageTransferTests.Timing.cs b/tests/SharpEmu.Gpu.Metal.Tests/MetalImageTransferTests.Timing.cs index eadf40cb8..933d07a57 100644 --- a/tests/SharpEmu.Gpu.Metal.Tests/MetalImageTransferTests.Timing.cs +++ b/tests/SharpEmu.Gpu.Metal.Tests/MetalImageTransferTests.Timing.cs @@ -1,6 +1,7 @@ // Copyright (C) 2026 SharpEmu Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later +using System.Diagnostics; using SharpEmu.Gpu.Metal.V2; using SharpEmu.Libs.Gpu.Abstractions; using SharpEmu.Libs.Gpu.Images; @@ -15,34 +16,48 @@ public sealed partial class MetalImageTransferTests private static readonly nint SelGpuStartTime = MetalNative.Selector("GPUStartTime"); private static readonly nint SelGpuEndTime = MetalNative.Selector("GPUEndTime"); + private static ImageDescription TimingCase(uint which) => which switch + { + 0 => Depth(GuestDepthFormat.Z32Float, false, 1024, 1024, 4), + 1 => Color2D(GuestPixelFormat.Bits16_16_16_16Float, GpuFormat.R16G16B16A16Sfloat, 1280, 720, 1, 1, GuestTileMode.RenderTarget), + 2 => Color2D(GuestPixelFormat.Bits16_16_16_16Float, GpuFormat.R16G16B16A16Sfloat, 3840, 2160, 1, 1, GuestTileMode.RenderTarget), + 3 => Color2D(GuestPixelFormat.Bits32Float, GpuFormat.R32Sfloat, 2560, 1440, 1, 1, GuestTileMode.Depth), + 4 => Color2D(GuestPixelFormat.Bc7UNorm, GpuFormat.BC7UnormBlock, 2048, 2048, 12, 1, GuestTileMode.Standard64KB), + 5 => Color2D(GuestPixelFormat.Bits8_8_8_8Srgb, GpuFormat.R8G8B8A8Srgb, 2048, 2048, 12, 1, GuestTileMode.Standard64KB), + 6 => Color2D(GuestPixelFormat.Bits8UNorm, GpuFormat.R8Unorm, 2048, 2048, 12, 1, GuestTileMode.Standard64KB), + 7 => Color2D(GuestPixelFormat.Bits8_8_8_8UNorm, GpuFormat.R8G8B8A8Unorm, 2048, 2048, 12, 1, GuestTileMode.Standard64KB), + _ => Color2D(GuestPixelFormat.Bits8_8_8_8UNorm, GpuFormat.R8G8B8A8Unorm, 256, 256, 9, 1, GuestTileMode.Standard64KB), + }; + [Theory] [InlineData("depth 1024x1024x4", 0u)] [InlineData("rgba16f 1280x720 RT", 1u)] [InlineData("rgba16f 3840x2160 RT", 2u)] [InlineData("r32f 2560x1440 depth-tiled", 3u)] + [InlineData("bc7 2048x2048 12 mips 64KB", 4u)] + [InlineData("rgba8 srgb 2048x2048 12 mips 64KB", 5u)] + [InlineData("r8 2048x2048 12 mips 64KB", 6u)] + [InlineData("rgba8 2048x2048 12 mips 64KB", 7u)] public void TransferGpuTime(string label, uint which) { if (!RequireDevice()) return; using var harness = new MetalTransferHarness(Device.Value); - var description = which switch - { - 0 => Depth(GuestDepthFormat.Z32Float, false, 1024, 1024, 4), - 1 => Color2D(GuestPixelFormat.Bits16_16_16_16Float, GpuFormat.R16G16B16A16Sfloat, 1280, 720, 1, 1, GuestTileMode.RenderTarget), - 2 => Color2D(GuestPixelFormat.Bits16_16_16_16Float, GpuFormat.R16G16B16A16Sfloat, 3840, 2160, 1, 1, GuestTileMode.RenderTarget), - _ => Color2D(GuestPixelFormat.Bits32Float, GpuFormat.R32Sfloat, 2560, 1440, 1, 1, GuestTileMode.Depth), - }; + var description = TimingCase(which); var layers = description.Resources.Layers; var size = description.Data.Size; var guest = harness.Buffer(size); var texture = harness.Texture(harness.Transfer.ShapeFor(description)); - var range = new SubresourceRange(0, 1, 0, layers); + var range = new SubresourceRange(0, description.Resources.Levels, 0, layers); double bestUp = double.MaxValue, bestDown = double.MaxValue; + int encodersUp = 0, encodersDown = 0; for (var rep = 0; rep < 5; rep++) { harness.Begin(); var buffer = harness.CommandBuffer; + var before = harness.EncodersStarted; harness.Transfer.EncodeGuestToTexture(harness, description, range, new MetalGuestRange(guest, 0, size, 0), default, texture, 0, size); + encodersUp = harness.EncodersStarted - before; MetalNative.SendVoid(buffer, MetalNative.Selector("retain")); harness.Submit(); bestUp = Math.Min(bestUp, MetalNative.SendDouble(buffer, SelGpuEndTime) - MetalNative.SendDouble(buffer, SelGpuStartTime)); @@ -50,15 +65,78 @@ public void TransferGpuTime(string label, uint which) harness.Begin(); buffer = harness.CommandBuffer; + before = harness.EncodersStarted; harness.Transfer.EncodeTextureToGuest(harness, description, range, texture, new MetalGuestRange(guest, 0, size, 0), default, 0, size); + encodersDown = harness.EncodersStarted - before; MetalNative.SendVoid(buffer, MetalNative.Selector("retain")); harness.Submit(); bestDown = Math.Min(bestDown, MetalNative.SendDouble(buffer, SelGpuEndTime) - MetalNative.SendDouble(buffer, SelGpuStartTime)); MetalNative.SendVoid(buffer, MetalNative.Selector("release")); } - output.WriteLine($"TIMING {label}: bytes=0x{size:x} upload={bestUp * 1000:F3} ms writeback={bestDown * 1000:F3} ms"); + output.WriteLine( + $"TIMING {label}: bytes=0x{size:x} upload={bestUp * 1000:F3} ms ({encodersUp} encoders) " + + $"writeback={bestDown * 1000:F3} ms ({encodersDown} encoders)"); + } + + // The CPU side of one transfer: what encoding it costs the thread that records it, + // as the median of batches of uploads and writebacks into one command buffer. + [Theory] + [InlineData("rgba16f 1280x720 RT", 1u, false)] + [InlineData("rgba16f 1280x720 RT", 1u, true)] + [InlineData("rgba8 256x256 9 mips 64KB", 8u, false)] + [InlineData("rgba8 256x256 9 mips 64KB", 8u, true)] + [InlineData("r8 2048x2048 12 mips 64KB", 6u, false)] + [InlineData("r8 2048x2048 12 mips 64KB", 6u, true)] + public void TransferCpuTime(string label, uint which, bool cached) + { + if (!RequireDevice()) return; + using var harness = new MetalTransferHarness(Device.Value); + var description = TimingCase(which); + var size = description.Data.Size; + var guest = harness.Buffer(size); + var texture = harness.Texture(harness.Transfer.ShapeFor(description)); + if (cached) MetalTileViews.Reference(texture); + try + { + var range = new SubresourceRange(0, description.Resources.Levels, 0, description.Resources.Layers); + const int Batch = 32; + var up = new List(); + var down = new List(); + for (var rep = 0; rep < 12; rep++) + { + harness.Begin(); + var watch = Stopwatch.StartNew(); + for (var index = 0; index < Batch; index++) + { + harness.Transfer.EncodeGuestToTexture(harness, description, range, + new MetalGuestRange(guest, 0, size, 0), default, texture, 0, size); + } + + up.Add(watch.Elapsed.TotalMicroseconds / Batch); + watch.Restart(); + for (var index = 0; index < Batch; index++) + { + harness.Transfer.EncodeTextureToGuest(harness, description, range, texture, + new MetalGuestRange(guest, 0, size, 0), default, 0, size); + } + + down.Add(watch.Elapsed.TotalMicroseconds / Batch); + harness.Submit(); + } + + // The first batches pay for pipeline builds and tier-up. + up = up.Skip(4).Order().ToList(); + down = down.Skip(4).Order().ToList(); + output.WriteLine( + $"CPUTIME {label} cached={cached}: upload={up[up.Count / 2]:F2} us writeback={down[down.Count / 2]:F2} us per transfer " + + $"(min {up[0]:F2} / {down[0]:F2})"); + } + finally + { + if (cached) MetalTileViews.Unreference(texture); + } } } diff --git a/tests/SharpEmu.Gpu.Metal.Tests/MetalLossyTextureTests.cs b/tests/SharpEmu.Gpu.Metal.Tests/MetalLossyTextureTests.cs index b479f932b..f36bee976 100644 --- a/tests/SharpEmu.Gpu.Metal.Tests/MetalLossyTextureTests.cs +++ b/tests/SharpEmu.Gpu.Metal.Tests/MetalLossyTextureTests.cs @@ -55,15 +55,18 @@ public void ASampledImageIsExactByDefaultAndLossyWithTheFlagOn() using var rig = new Rig(Device.Value); // Off by default (SHARPEMU_METAL_V2_LOSSY_TEXTURES=1 turns it on). Assert.False(rig.Cache.LossyTextures); + // A 64x64 texture rounds both modes to the same minimum allocation. Use a + // larger surface so the strict footprint assertion measures compression. + const uint CompressionSide = 256; rig.Cache.LossyTextures = true; - var on = rig.Find(ImageAddress + 0x40000, GpuFormat.R8G8B8A8Unorm, ImageRole.Texture); + var on = rig.Find(ImageAddress + 0x40000, GpuFormat.R8G8B8A8Unorm, ImageRole.Texture, height: CompressionSide, width: CompressionSide); var onImage = rig.Cache.GetImage(on); Assert.True(onImage.Shape.Lossy); Assert.Equal(Mtl.TextureUsageShaderRead, onImage.Shape.Usage); Assert.Equal(1, CompressionOf(onImage.Texture)); rig.Cache.LossyTextures = false; - var off = rig.Find(ImageAddress, GpuFormat.R8G8B8A8Unorm, ImageRole.Texture); + var off = rig.Find(ImageAddress, GpuFormat.R8G8B8A8Unorm, ImageRole.Texture, height: CompressionSide, width: CompressionSide); var offImage = rig.Cache.GetImage(off); Assert.False(offImage.Shape.Lossy); Assert.Equal(Breaking, offImage.Shape.Usage & Breaking); @@ -352,21 +355,21 @@ public Rig(nint device) public void Write(ulong address, ReadOnlySpan bytes) => Assert.True(_backing.TryWriteBacking(address, bytes)); // A linear 64x64 surface of four-byte texels. - public ImageRequest Request(ulong address, GpuFormat format, ImageRole role, uint height = Side) + public ImageRequest Request(ulong address, GpuFormat format, ImageRole role, uint height = Side, uint width = Side) { - var size = (ulong)Side * height * 4; + var size = (ulong)width * height * 4; var description = ImageDescription.Create(); description.Data = new GuestSpan(address, size); description.PixelFormat = format; description.GuestFormat = format == GpuFormat.R32Uint ? GuestPixelFormat.Bits32UInt : GuestPixelFormat.Bits8_8_8_8UNorm; description.Type = GuestImageType.Color2D; - description.Extent = new GpuExtent3D(Side, height, 1); + description.Extent = new GpuExtent3D(width, height, 1); description.Resources = new SubresourceCount(1, 1); - description.Pitch = Side; + description.Pitch = width; description.BytesPerBlock = 4; description.Samples = 1; description.TileMode = GuestTileMode.Linear; - description.MipLayout[0] = new MipLevelLayout { Offset = 0, Size = size, Pitch = Side, Height = height }; + description.MipLayout[0] = new MipLevelLayout { Offset = 0, Size = size, Pitch = width, Height = height }; var view = ImageViewDescription.Default with { Format = format, @@ -378,9 +381,9 @@ public ImageRequest Request(ulong address, GpuFormat format, ImageRole role, uin return new ImageRequest(description, view, role); } - public ResourceSlotIdentifier Find(ulong address, GpuFormat format, ImageRole role, uint height = Side) + public ResourceSlotIdentifier Find(ulong address, GpuFormat format, ImageRole role, uint height = Side, uint width = Side) { - var request = Request(address, format, role, height); + var request = Request(address, format, role, height, width); return Cache.FindImage(ref request); } diff --git a/tests/SharpEmu.Gpu.Metal.Tests/MetalStagedStorageTests.cs b/tests/SharpEmu.Gpu.Metal.Tests/MetalStagedStorageTests.cs index ec1790caf..2dd15f60e 100644 --- a/tests/SharpEmu.Gpu.Metal.Tests/MetalStagedStorageTests.cs +++ b/tests/SharpEmu.Gpu.Metal.Tests/MetalStagedStorageTests.cs @@ -225,6 +225,68 @@ public void AWritebackLeavesTheBytesTheShaderDidNotStore() } } + // A pristine twin the GPU copies from the staged copy, ahead of the shader's stores, is + // what the writeback compares against: the shader's bytes land, and a write from + // elsewhere after the copy was staged survives everywhere else. + [Fact] + public void AGpuSnapshotIsThePristineTheWritebackComparesAgainst() + { + if (!RequireDevice()) + { + return; + } + + using var backing = new ScatteredBacking(); + using var harness = new MetalTransferHarness(Device.Value); + using var memory = new MetalGuestMemory( + Device.Value, backing, new ScatteredMemory(backing), WindowBytes, MaxWindowBytes); + var staged = new MetalStagedStorage(memory); + backing.TryWriteBacking(Address, Enumerable.Repeat((byte)0x11, (int)Size).ToArray()); + + harness.Begin(); + var copy = memory.Stage(Address, Size); + var pristine = memory.Allocate(Size, 256, out var pristineBytes); + pristineBytes.Fill(0x77); + staged.EncodeSnapshot(harness.CommandBuffer, copy, pristine); + staged.Add(Address, Size, copy, pristine, written: true); + harness.Submit(); + Assert.All(harness.Read(pristine.Buffer, pristine.Offset, Size), value => Assert.Equal((byte)0x11, value)); + + backing.TryWriteBacking(Address, Enumerable.Repeat((byte)0x22, (int)Size).ToArray()); + harness.Begin(); + var stored = new MetalGuestRange(copy.Buffer, copy.Offset + (Size / 2) - 4096, 8192, copy.GpuAddress); + var store = MetalNative.Send(harness.CommandBuffer, SelBlit); + Fill(store, stored, 0xCD); + MetalNative.SendVoid(store, SelEnd); + harness.Submit(); + + harness.Begin(); + staged.Arm(); + staged.EncodeWritebacks(harness.CommandBuffer); + harness.Submit(); + + var bytes = new byte[Size]; + Assert.True(backing.TryReadBacking(Address, bytes)); + for (var index = 0; index < bytes.Length; index++) + { + var inside = index >= (int)(Size / 2) - 4096 && index < (int)(Size / 2) + 4096; + Assert.Equal(inside ? (byte)0xCD : (byte)0x22, bytes[index]); + } + } + + // The GPU copies the pristine twin only where the blit ends no pass, and ends a compute + // encoder only for a range it would stage on the GPU whole. + [Theory] + [InlineData(false, false, 64 * 1024, true)] + [InlineData(false, false, 64 * 1024 - 1, false)] + [InlineData(true, false, 64 * 1024 * 1024, false)] + [InlineData(false, true, 1024 * 1024, false)] + [InlineData(false, true, 4 * 1024 * 1024, true)] + public void TheGpuSnapshotsThePristineCopyOnlyWhereItEndsNoPass(bool renderOpen, bool computeOpen, ulong size, bool gpu) + { + Assert.Equal(gpu && MetalRenderHost.GpuPristine, MetalRenderHost.SnapshotsOnGpu(renderOpen, computeOpen, size)); + } + // A copy recorded after a GPU write to its bytes that has not run yet holds that write, // not what the CPU saw when it was recorded, and so does its pristine copy. [Fact] diff --git a/tests/SharpEmu.Gpu.Metal.Tests/Scenes/BarrierSceneTests.cs b/tests/SharpEmu.Gpu.Metal.Tests/Scenes/BarrierSceneTests.cs new file mode 100644 index 000000000..0860e3fd2 --- /dev/null +++ b/tests/SharpEmu.Gpu.Metal.Tests/Scenes/BarrierSceneTests.cs @@ -0,0 +1,99 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using Xunit; +using Xunit.Abstractions; + +namespace SharpEmu.Gpu.Metal.Tests.Scenes; + +/// +/// The orderings the host leaves to Metal: dispatches in one serial compute encoder with no +/// barrier between them, and guest cache flushes inside a render pass that holds no stores. +/// +public sealed class BarrierSceneTests(ITestOutputHelper output) : SceneTestBase(output) +{ + private const uint BufferFormat = 20; + private const uint Size = 32; + + /// + /// Eight dispatches with no guest flush between them, each loading through a guest pointer + /// what the one before stored and storing the value plus 1000 one KiB further on. + /// + [Fact] + public void ADispatchReadsWhatTheDispatchBeforeItStoredInTheSameEncoder() + { + using var rig = CreateRig(); + if (rig is null) return; + const uint threads = 128; + const int steps = 8; + var data = rig.Memory.Upload( + Enumerable.Range(0, 256 * (steps + 1)).Select(i => i < 256 ? (uint)i * 3 : 0xFFFFFFFFu).ToArray(), 256); + var returned = rig.Memory.Upload(Enumerable.Repeat(0xFFFFFFFFu, (int)threads).ToArray(), 256); + var pm4 = new ScenePm4(); + for (var step = 0; step < steps; step++) + { + var source = data + (ulong)step * 1024; + var state = new SceneState(1, 1); + state.Context.Clear(); + state.Uconfig.Clear(); + state.ComputeShader(rig.Program(ComputeScenePrograms.GlobalLoadStore), 64, 1, 1, + [Descriptors.Lo(source), Descriptors.Hi(source), 0, 0, .. Descriptors.Buffer(returned, 4, threads, BufferFormat)], true); + state.Emit(pm4).DispatchDirect(threads / 64, 1, 1); + } + + rig.Submit(pm4); + for (var step = 1; step <= steps; step++) + { + var values = rig.Read(data + (ulong)step * 1024, (int)threads); + for (var index = 0; index < threads; index++) + { + Assert.Equal((uint)(index * 3 + 1000 * step), values[index]); + } + } + + var last = rig.Read(returned, (int)threads); + for (var index = 0; index < threads; index++) + { + Assert.Equal((uint)(index * 3 + 1000 * steps), last[index]); + } + } + + /// + /// A release and an acquire between two draws into one target: the pass holds no shader + /// stores, so the host records no barrier for them, and both draws land. + /// + [Fact] + public void AGuestFlushBetweenDrawsWithoutStoresKeepsBothDraws() + { + using var rig = CreateRig(); + if (rig is null) return; + var target = rig.ColorTarget(Size, Size); + var state = new SceneState(Size, Size).ColorTarget(0, target, Size, Size); + var before = rig.Host.GlobalBarrierCounts; + // Lower-left half red, then the upper-right half green. + var pm4 = rig.DrawMesh(new ScenePm4(), state, + [ + SceneVertex.At(-1, -1, 0.5f, 1, 0, 0), + SceneVertex.At(1, -1, 0.5f, 1, 0, 0), + SceneVertex.At(-1, 1, 0.5f, 1, 0, 0), + ]); + pm4.Event(ScenePm4.EventPsPartialFlush).Event(ScenePm4.EventCacheFlushAndInvalidate); + rig.DrawMesh(pm4, state, + [ + SceneVertex.At(1, -1, 0.5f, 0, 1, 0), + SceneVertex.At(1, 1, 0.5f, 0, 1, 0), + SceneVertex.At(-1, 1, 0.5f, 0, 1, 0), + ]); + rig.Submit(pm4); + var after = rig.Host.GlobalBarrierCounts; + var pixels = rig.ReadImage(target, Size, Size); + + Assert.Equal(0xFF0000FFu, SceneDraws.Pixel(pixels, Size, 2, Size - 3)); + Assert.Equal(0xFF00FF00u, SceneDraws.Pixel(pixels, Size, Size - 3, 2)); + if (SharpEmu.Gpu.Metal.V2.MetalV2Diagnostics.ElideGlobalBarriers) + { + Assert.Equal(before.Recorded, after.Recorded); + Assert.True(after.Elided > before.Elided, $"elided {before.Elided} -> {after.Elided}"); + } + } +} diff --git a/tests/SharpEmu.Gpu.Metal.Tests/Scenes/ConsumerRunSceneTests.cs b/tests/SharpEmu.Gpu.Metal.Tests/Scenes/ConsumerRunSceneTests.cs new file mode 100644 index 000000000..855e6564e --- /dev/null +++ b/tests/SharpEmu.Gpu.Metal.Tests/Scenes/ConsumerRunSceneTests.cs @@ -0,0 +1,148 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using Xunit; +using Xunit.Abstractions; + +namespace SharpEmu.Gpu.Metal.Tests.Scenes; + +/// +/// A compute shader that binds 4 KiB inside a GPU-written 1080p RGBA16F render target as a +/// storage buffer publishes the run it binds, not the 16 MiB target (MetalImageCache.ConsumerRuns.cs). +/// Every scene runs with the run publication on and off: the bytes the shader sees, and the +/// whole target read back afterwards, must be the same both ways. +/// +public sealed class ConsumerRunSceneTests(ITestOutputHelper output) : SceneTestBase(output) +{ + private const uint Width = 1920; + private const uint Height = 1080; + private const uint BytesPerTexel = 8; + private const uint Dwords = 1024; + private const uint BufferFormat = 20; + + // The bound bytes start inside a row, a third of the way down the target. + private const ulong BoundOffset = (Width * BytesPerTexel * 360) + 0x1F00; + + /// buffer_load_dword from s[0:3], buffer_store_dword to s[4:7], one dword a thread. + private static readonly uint[] CopyDwords = + [ + 0x8F098608, // s_lshl_b32 s9, s8, 6 + 0x4A020009, // v_add_nc_u32_e32 v1, s9, v0 + 0xE0302000, 0x80000201, // buffer_load_dword v2, v1, s[0:3], 0 idxen + 0xBF8C3F70, // s_waitcnt vmcnt(0) + 0xE0702000, 0x80010201, // buffer_store_dword v2, v1, s[4:7], 0 idxen + 0xBF810000, // s_endpgm + ]; + + /// buffer_load_dword from s[0:3], plus 1000, buffer_store_dword back to s[0:3]. + private static readonly uint[] AddInPlace = + [ + 0x8F098608, // s_lshl_b32 s9, s8, 6 + 0x4A020009, // v_add_nc_u32_e32 v1, s9, v0 + 0xE0302000, 0x80000201, // buffer_load_dword v2, v1, s[0:3], 0 idxen + 0xBF8C3F70, // s_waitcnt vmcnt(0) + 0x4A0404FF, 0x000003E8, // v_add_nc_u32_e32 v2, 0x3e8, v2 + 0xE0702000, 0x80000201, // buffer_store_dword v2, v1, s[0:3], 0 idxen + 0xBF810000, // s_endpgm + ]; + + private readonly record struct Outcome(byte[] Read, byte[] Again, byte[] Whole, long BindRetiled, long WholeRetiled); + + private Outcome? Run(bool subRange, bool written, uint tileMode) + { + using var rig = CreateRig(); + if (rig is null) return null; + var cache = rig.Host.ImageCache; + cache.SubRangeConsumers = subRange; + var size = (ulong)Width * Height * BytesPerTexel; + // Room for a tiled target's padding rows too. + var target = rig.Memory.Allocate(size + (64 * 1024 * 16), 0x10000); + var fill = new byte[size + (64 * 1024 * 16)]; + Array.Fill(fill, (byte)0x11); + rig.Memory.Write(target, fill); + + // A gradient, so every row and column of the target holds different texels. + var state = new SceneState(Width, Height).ColorTarget(0, target, Width, Height, format: 12, numberType: 7, + tileMode: tileMode); + SceneVertex[] quad = + [ + SceneVertex.At(-1, -1, 0.5f, 0, 0, 1, 1), SceneVertex.At(1, -1, 0.5f, 1, 0, 0, 1), + SceneVertex.At(-1, 1, 0.5f, 0, 1, 0, 1), SceneVertex.At(-1, 1, 0.5f, 0, 1, 0, 1), + SceneVertex.At(1, -1, 0.5f, 1, 0, 0, 1), SceneVertex.At(1, 1, 0.5f, 1, 1, 1, 0.5f), + ]; + var draw = new ScenePm4(); + rig.DrawMesh(draw, state, quad); + rig.Submit(draw); + + var bound = target + BoundOffset; + var copy = rig.Memory.Allocate(Dwords * 4, 256); + var before = cache.WritebackBytes; + var pm4 = new ScenePm4(); + if (written) + { + rig.Dispatch(pm4, AddInPlace, + [.. Descriptors.Buffer(bound, 4, Dwords, BufferFormat), .. Descriptors.Buffer(copy, 4, Dwords, BufferFormat)], + Dwords / 64); + pm4.Event(ScenePm4.EventCsPartialFlush); + } + + rig.Dispatch(pm4, CopyDwords, + [.. Descriptors.Buffer(bound, 4, Dwords, BufferFormat), .. Descriptors.Buffer(copy, 4, Dwords, BufferFormat)], + Dwords / 64); + rig.Submit(pm4); + var bindRetiled = cache.WritebackBytes - before; + var read = rig.Read(copy, (int)(Dwords * 4)); + + // The same bytes bound again publish nothing more. + var again = rig.Memory.Allocate(Dwords * 4, 256); + var repeat = new ScenePm4(); + rig.Dispatch(repeat, CopyDwords, + [.. Descriptors.Buffer(bound, 4, Dwords, BufferFormat), .. Descriptors.Buffer(again, 4, Dwords, BufferFormat)], + Dwords / 64); + rig.Submit(repeat); + var repeatRetiled = cache.WritebackBytes - before - bindRetiled; + if (subRange && !written) + { + Assert.Equal(0, repeatRetiled); + } + + before = cache.WritebackBytes; + var whole = rig.Read(target, (int)size); + var wholeRetiled = cache.WritebackBytes - before; + Assert.Empty(rig.Fatals); + return new Outcome(read, rig.Read(again, (int)(Dwords * 4)), whole, bindRetiled, wholeRetiled); + } + + [Theory] + [InlineData(false, 0u)] + [InlineData(true, 0u)] + [InlineData(false, 27u)] + [InlineData(true, 27u)] + public void AComputeBindingOfFourKibInsideATargetRetilesItsRunAndReadsWhatAWholeWritebackDoes(bool written, uint tileMode) + { + if (Run(subRange: false, written, tileMode) is not { } whole) return; + var run = Run(subRange: true, written, tileMode)!.Value; + var size = (long)Width * Height * BytesPerTexel; + Output.WriteLine( + $"written={written} tile={tileMode}: bind retiled {whole.BindRetiled} bytes whole, {run.BindRetiled} as a run " + + $"({(double)whole.BindRetiled / Math.Max(1, run.BindRetiled):F0}x); the later full read retiled " + + $"{whole.WholeRetiled} and {run.WholeRetiled}"); + + // The bound bytes are the GPU's, not the fill the guest held before the draw. + Assert.Contains(whole.Read, value => value != 0x11); + var expected = whole.Whole.AsSpan((int)BoundOffset, (int)(Dwords * 4)).ToArray(); + Assert.Equal(expected, whole.Again); + Assert.Equal(whole.Read, run.Read); + Assert.Equal(whole.Again, run.Again); + Assert.Equal(whole.Whole, run.Whole); + if (!written) + { + Assert.Equal(expected, run.Read); + } + + Assert.True(whole.BindRetiled >= size, $"the whole writeback retiled {whole.BindRetiled} of {size} bytes"); + Assert.True(run.BindRetiled > 0 && run.BindRetiled <= size / 8, + $"the binding retiled {run.BindRetiled} of {size} bytes"); + Assert.True(run.WholeRetiled > 0, "the full read found nothing left to publish"); + } +} diff --git a/tests/SharpEmu.Gpu.Metal.Tests/Scenes/EarlyDepthSceneTests.cs b/tests/SharpEmu.Gpu.Metal.Tests/Scenes/EarlyDepthSceneTests.cs new file mode 100644 index 000000000..4f4f88b02 --- /dev/null +++ b/tests/SharpEmu.Gpu.Metal.Tests/Scenes/EarlyDepthSceneTests.cs @@ -0,0 +1,93 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using Xunit; +using Xunit.Abstractions; + +namespace SharpEmu.Gpu.Metal.Tests.Scenes; + +/// +/// DB_SHADER_CONTROL decides whether a pixel shader with memory writes runs for a fragment the +/// depth test rejects. A fully occluded quad draws with a pixel program that adds one to a +/// buffer counter per invocation; the counter shows how many invocations ran. The rule: the +/// shader is skipped (Metal [[early_fragment_tests]]) when the guest tests depth first without +/// EXEC_ON_HIER_FAIL / EXEC_ON_NOOP, or with DEPTH_BEFORE_SHADER; it runs for every fragment when +/// the order is LATE_Z, when either EXEC_ON_* bit is set, or when it can kill. +/// +public sealed class EarlyDepthSceneTests(ITestOutputHelper output) : SceneTestBase(output) +{ + private const uint Size = 16; + private const uint BufferFormat = 20; + private const uint ZOrderEarly = 1u << 4; + private const uint ZOrderRe = 2u << 4; + private const uint ZOrderEarlyThenRe = 3u << 4; + private const uint KillEnable = 1u << 6; + private const uint ExecOnHierFail = 1u << 9; + private const uint ExecOnNoop = 1u << 10; + private const uint DepthBeforeShader = 1u << 12; + + /// + /// s[0:3] = counter V#, s4 = primitive mask. Adds 1 to record 0, then MRT0 = interpolated PARAM0. + /// Assembled with llvm-mc -triple=amdgcn -mcpu=gfx1030 -show-encoding. + /// + private static readonly uint[] CountingPixel = + [ + 0xBEFC0304, // s_mov_b32 m0, s4 + 0x7E0C0281, // v_mov_b32_e32 v6, 1 + 0x7E0E0280, // v_mov_b32_e32 v7, 0 + 0xE0C82000, 0x80000607, // buffer_atomic_add v6, v7, s[0:3], 0 idxen + 0xC8080000, // v_interp_p1_f32_e32 v2, v0, attr0.x + 0xC8090001, // v_interp_p2_f32_e32 v2, v1, attr0.x + 0xC80C0100, // v_interp_p1_f32_e32 v3, v0, attr0.y + 0xC80D0101, // v_interp_p2_f32_e32 v3, v1, attr0.y + 0xC8100200, // v_interp_p1_f32_e32 v4, v0, attr0.z + 0xC8110201, // v_interp_p2_f32_e32 v4, v1, attr0.z + 0xC8140300, // v_interp_p1_f32_e32 v5, v0, attr0.w + 0xC8150301, // v_interp_p2_f32_e32 v5, v1, attr0.w + 0xF800180F, 0x05040302, // exp mrt0 v2, v3, v4, v5 done vm + 0xBF810000, // s_endpgm + ]; + + // Control value, whether the occluded fragments' shader runs. + public static TheoryData Cases => new() + { + { 0u, true }, // LATE_Z + { ZOrderEarly, false }, // EARLY_Z_THEN_LATE_Z + { ZOrderRe, false }, // RE_Z + { ZOrderEarlyThenRe, false }, // EARLY_Z_THEN_RE_Z + { ZOrderEarly | ExecOnHierFail, true }, + { ZOrderEarly | ExecOnNoop, true }, + { ZOrderRe | ExecOnHierFail | ExecOnNoop, true }, + { ZOrderEarlyThenRe | ExecOnNoop, true }, + { DepthBeforeShader, false }, // forced, whatever Z_ORDER says + { ZOrderEarly | ExecOnHierFail | ExecOnNoop | DepthBeforeShader, false }, + { ZOrderEarly | KillEnable, true }, // a kill is decided after the shader + }; + + [Theory] + [MemberData(nameof(Cases))] + public void AnOccludedFragmentStoresWhenTheGuestShaderWouldRun(uint control, bool occludedShaderRuns) + { + using var rig = CreateRig(); + if (rig is null) return; + var occludedCounter = rig.Memory.Allocate(64, 256); + var visibleCounter = rig.Memory.Allocate(64, 256); + rig.Memory.Write(occludedCounter, new uint[16]); + rig.Memory.Write(visibleCounter, new uint[16]); + var d = new DepthRig(rig, Size, Size, zFormat: 3); + + // The near quad owns the depth; the far quad is occluded; the last is nearer still and visible. + d.Draw(d.State().DepthControl(true, true, CompareFunc.Always), d.Full(0.25f, 1, 0, 0)); + foreach (var (z, counter) in new[] { (0.75f, occludedCounter), (0.1f, visibleCounter) }) + { + var state = d.State().DepthControl(true, true, CompareFunc.Less); + state.Context[Cx.DbShaderControl] = control; + rig.DrawMesh(d.Pm4, state, d.Full(z, 0, 1, 0), CountingPixel, Descriptors.Buffer(counter, 4, 16, BufferFormat)); + } + + rig.Submit(d.Pm4); + // Control: a fragment that passes the test always runs, so the program and the counter work. + Assert.Equal(Size * Size, rig.Read(visibleCounter, 1)[0]); + Assert.Equal(occludedShaderRuns ? Size * Size : 0u, rig.Read(occludedCounter, 1)[0]); + } +} diff --git a/tests/SharpEmu.Gpu.Metal.Tests/Scenes/GpuWordNoWaitSceneTests.cs b/tests/SharpEmu.Gpu.Metal.Tests/Scenes/GpuWordNoWaitSceneTests.cs new file mode 100644 index 000000000..938a99107 --- /dev/null +++ b/tests/SharpEmu.Gpu.Metal.Tests/Scenes/GpuWordNoWaitSceneTests.cs @@ -0,0 +1,283 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using SharpEmu.Gpu.Metal.V2; +using SharpEmu.Libs.Gpu.Scheduling; +using Xunit; +using Xunit.Abstractions; + +namespace SharpEmu.Gpu.Metal.Tests.Scenes; + +/// +/// Host reads of GPU-written guest bytes that no longer wait for more than they need: the first +/// read of a word waits for its writer's fence (eager write fences), a read ahead of the stream +/// gives up instead of waiting, and a packet-sized copy whose source the GPU still writes runs on +/// the GPU. Each holds the GPU on an event only the test signals, so a wait that included the held +/// work could not return before the test releases it. +/// +public sealed class GpuWordNoWaitSceneTests(ITestOutputHelper output) : SceneTestBase(output) +{ + private static readonly nint SelNewSharedEvent = MetalNative.Selector("newSharedEvent"); + private static readonly nint SelSetSignaledValue = MetalNative.Selector("setSignaledValue:"); + private static readonly nint SelEncodeWaitForEvent = MetalNative.Selector("encodeWaitForEvent:value:"); + private static readonly nint SelRelease = MetalNative.Selector("release"); + + private const int WatchdogMilliseconds = 30_000; + private const uint Words = 64; + private const uint LaterGroups = 16; + + [Fact] + public void TheFirstReadOfAWordAnEarlySubmissionWroteWaitsOnlyForItsWriter() + { + using var rig = CreateRig(); + if (rig is null) return; + var host = rig.Host; + Assert.True(host.WriteFences && host.EagerWriteFences, "eager write fences are on by default"); + var (word, later, writer, consumer) = WriterAndLater(rig); + var plantedBefore = host.WriteFenceCounters.Planted; + var waitsBefore = host.WriteFenceCounters; + + // No read has waited on these bytes before: the fence comes from the write alone. + var gate = MetalNative.Send(DeviceOf(host), SelNewSharedEvent); + Assert.NotEqual(0, gate); + var released = 0; + using var watchdog = Watchdog(gate, () => Interlocked.Exchange(ref released, 1)); + try + { + host.WriteFencePlantedHook = buffer => MetalV2Native.SendEventValue(buffer, SelEncodeWaitForEvent, gate, 1); + RunBoth(rig, writer, consumer); + host.WriteFencePlantedHook = null; + Assert.True(host.WriteFenceCounters.Planted > plantedBefore, "a fence was planted after the writer's submission"); + + Assert.Equal(5u ^ 0x5A5A, ReadWord(host, word + 5 * 4)); + Assert.Equal(0, Volatile.Read(ref released)); + // The later submission is still held: its bytes are the zeros the rig allocated. + Assert.Equal(0u, BitConverter.ToUInt32(rig.Memory.ReadRaw(later + 4 * 9, 4))); + var counters = host.WriteFenceCounters; + Assert.True(counters.Waits + counters.Reads > waitsBefore.Waits + waitsBefore.Reads, $"fence counters {counters}"); + } + finally + { + host.WriteFencePlantedHook = null; + MetalNative.SendVoid(gate, SelSetSignaledValue, (nint)1); + } + + rig.Drain(); + AssertStored(rig, later, Words * LaterGroups); + watchdog.Dispose(); + MetalNative.SendVoid(gate, SelRelease); + Assert.Empty(rig.Fatals); + } + + [Fact] + public void WithEagerFencesOffTheFirstReadWaitsForTheWholeBuffer() + { + using var rig = CreateRig(); + if (rig is null) return; + var host = rig.Host; + host.EagerWriteFences = false; + var (word, later, writer, consumer) = WriterAndLater(rig); + var plantedBefore = host.WriteFenceCounters.Planted; + + // The whole buffer is held from its start and released shortly after the read begins: + // a read that waits for the whole buffer returns with the later submission's bytes written. + var gate = MetalNative.Send(DeviceOf(host), SelNewSharedEvent); + var released = 0; + try + { + MetalV2Native.SendEventValue(host.CommandBuffer, SelEncodeWaitForEvent, gate, 1); + RunBoth(rig, writer, consumer); + Assert.Equal(plantedBefore, host.WriteFenceCounters.Planted); + var signaller = new Thread(() => + { + Thread.Sleep(100); + Interlocked.Exchange(ref released, 1); + MetalNative.SendVoid(gate, SelSetSignaledValue, (nint)1); + }) { IsBackground = true }; + signaller.Start(); + Assert.Equal(5u ^ 0x5A5A, ReadWord(host, word + 5 * 4)); + Assert.Equal(1, Volatile.Read(ref released)); + Assert.Equal(9u ^ 0x5A5A, BitConverter.ToUInt32(rig.Memory.ReadRaw(later + 4 * 9, 4))); + signaller.Join(); + } + finally + { + MetalNative.SendVoid(gate, SelSetSignaledValue, (nint)1); + } + + rig.Drain(); + MetalNative.SendVoid(gate, SelRelease); + Assert.Empty(rig.Fatals); + } + + [Fact] + public void AReadAheadOfTheStreamGivesUpInsteadOfWaitingForTheGpu() + { + using var rig = CreateRig(); + if (rig is null) return; + var host = rig.Host; + var word = rig.Memory.Allocate(Words * 4, 256); + var writer = rig.Dispatch(new ScenePm4(), ScenePrograms.StoreGlobalId, Descriptors.Buffer(word, 4, Words, format: 20), 1).ToArray(); + rig.Submit(writer); + rig.Memory.Write(word, new byte[Words * 4]); + + var gate = MetalNative.Send(DeviceOf(host), SelNewSharedEvent); + var released = 0; + using var watchdog = Watchdog(gate, () => Interlocked.Exchange(ref released, 1)); + try + { + MetalV2Native.SendEventValue(host.CommandBuffer, SelEncodeWaitForEvent, gate, 1); + rig.Enqueue(writer); + host.RunPendingCommands(); + host.RunCommandStreamSlices(64); + var waitsBefore = host.HostGpuWaits; + + Assert.Throws(() => + GpuFatal.Speculatively(() => ReadWord(host, word + 3 * 4))); + Assert.Equal(waitsBefore, host.HostGpuWaits); + Assert.Equal(0, Volatile.Read(ref released)); + } + finally + { + MetalNative.SendVoid(gate, SelSetSignaledValue, (nint)1); + } + + // The stream's own read waits as before and sees the written word. + Assert.Equal(3u ^ 0x5A5A, ReadWord(host, word + 3 * 4)); + rig.Drain(); + watchdog.Dispose(); + MetalNative.SendVoid(gate, SelRelease); + Assert.Empty(rig.Fatals); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void APacketSizedCopyOfAWordTheGpuStillWritesIsCopiedOnTheGpu(bool queuedBehindHeldWork) + { + using var rig = CreateRig(); + if (rig is null) return; + var host = rig.Host; + // Command-processor writes land as parsed, so a small DMA copy may take the host path. + host.CommandProcessorWritesAtParse = true; + var produced = rig.Memory.Allocate(Words * 4, 256); + var copied = rig.Memory.Allocate(16, 256); + var dispatch = rig.Dispatch(new ScenePm4(), ScenePrograms.StoreGlobalId, Descriptors.Buffer(produced, 4, Words, format: 20), 1); + rig.Submit(dispatch.ToArray()); + rig.Memory.Write(produced, new byte[Words * 4]); + + // DMA_DATA, memory to memory, 8 bytes from produced + 6 * 4 to copied. + var source = produced + 6 * 4; + var stream = rig.Dispatch(new ScenePm4(), ScenePrograms.StoreGlobalId, Descriptors.Buffer(produced, 4, Words, format: 20), 1) + .Packet(0x50, 0u, (uint)source, (uint)(source >> 32), (uint)copied, (uint)(copied >> 32), 8u) + .ToArray(); + var gate = MetalNative.Send(DeviceOf(host), SelNewSharedEvent); + var released = 0; + using var watchdog = Watchdog(gate, () => Interlocked.Exchange(ref released, 1)); + try + { + if (queuedBehindHeldWork) + { + MetalV2Native.SendEventValue(host.CommandBuffer, SelEncodeWaitForEvent, gate, 1); + } + + var waitsBefore = host.HostGpuWaits; + rig.Enqueue(stream); + host.RunPendingCommands(); + host.RunCommandStreamSlices(64); + Assert.False(host.CommandStream.HasUnblockedPending); + // Parsing the copy waited for nothing. + Assert.Equal(waitsBefore, host.HostGpuWaits); + Assert.Equal(0, Volatile.Read(ref released)); + } + finally + { + MetalNative.SendVoid(gate, SelSetSignaledValue, (nint)1); + } + + rig.Drain(); + var words = rig.Read(copied, 2); + Assert.Equal(6u ^ 0x5A5A, words[0]); + Assert.Equal(7u ^ 0x5A5A, words[1]); + watchdog.Dispose(); + MetalNative.SendVoid(gate, SelRelease); + Assert.Empty(rig.Fatals); + } + + // A word buffer one group writes and, apart from it, a buffer a later, larger dispatch writes; + // both programs are compiled first so neither submission parks on a pipeline. + private static (ulong Word, ulong Later, uint[] Writer, uint[] Consumer) WriterAndLater(SceneRig rig) + { + var word = rig.Memory.Allocate(Words * 4, 256); + var later = rig.Memory.Allocate(Words * 4 * LaterGroups, 0x10000); + var writer = rig.Dispatch(new ScenePm4(), ScenePrograms.StoreGlobalId, Descriptors.Buffer(word, 4, Words, format: 20), 1).ToArray(); + var consumer = rig.Dispatch(new ScenePm4(), ScenePrograms.StoreGlobalId, + Descriptors.Buffer(later, 4, Words * LaterGroups, format: 20), LaterGroups).ToArray(); + rig.Submit(writer); + rig.Submit(consumer); + rig.Memory.Write(word, new byte[Words * 4]); + rig.Memory.Write(later, new byte[Words * 4 * LaterGroups]); + return (word, later, writer, consumer); + } + + private static void AssertStored(SceneRig rig, ulong address, uint count) + { + var words = rig.Read(address, (int)count); + for (var index = 0u; index < words.Length; index++) + { + Assert.Equal(index ^ 0x5A5A, words[index]); + } + } + + // Both submissions run in one worker pass, so the coalescer keeps one command buffer open + // across them and commits it when the stream runs dry. + private static void RunBoth(SceneRig rig, uint[] first, uint[] second) + { + rig.Enqueue(first); + rig.Enqueue(second); + rig.Host.RunPendingCommands(); + rig.Host.RunCommandStreamSlices(64); + Assert.False(rig.Host.CommandStream.HasPending); + } + + private static IDisposable Watchdog(nint gate, Action fired) + { + var cancel = new CancellationTokenSource(); + var thread = new Thread(() => + { + if (!cancel.Token.WaitHandle.WaitOne(WatchdogMilliseconds)) + { + fired(); + MetalNative.SendVoid(gate, SelSetSignaledValue, (nint)1); + } + }) { IsBackground = true }; + thread.Start(); + return new WatchdogStop(cancel, thread); + } + + private sealed class WatchdogStop(CancellationTokenSource cancel, Thread thread) : IDisposable + { + private int _stopped; + + public void Dispose() + { + if (Interlocked.Exchange(ref _stopped, 1) != 0) + { + return; + } + + cancel.Cancel(); + thread.Join(); + cancel.Dispose(); + } + } + + private static uint ReadWord(MetalRenderHost host, ulong address) + { + Span bytes = stackalloc byte[4]; + Assert.True(host.TryReadGuest(address, bytes)); + return BitConverter.ToUInt32(bytes); + } + + private static nint DeviceOf(MetalRenderHost host) => ((IMetalImageCacheOwner)host).Device; +} diff --git a/tests/SharpEmu.Gpu.Metal.Tests/Scenes/GpuWordWaitSceneTests.cs b/tests/SharpEmu.Gpu.Metal.Tests/Scenes/GpuWordWaitSceneTests.cs index fe2a615db..1f5bddc9d 100644 --- a/tests/SharpEmu.Gpu.Metal.Tests/Scenes/GpuWordWaitSceneTests.cs +++ b/tests/SharpEmu.Gpu.Metal.Tests/Scenes/GpuWordWaitSceneTests.cs @@ -34,6 +34,8 @@ public void AWordAnEarlySubmissionWroteIsReadWhileALaterSubmissionInTheSameBuffe if (rig is null) return; var host = rig.Host; Assert.True(host.WriteFences, "write fences are on by default"); + // The learned fences: an eager one would answer the first read too (GpuWordNoWaitSceneTests). + host.EagerWriteFences = false; var word = rig.Memory.Allocate(Words * 4, 256); // Apart, so the two submissions' write ranges stay two records rather than merging into one. var later = rig.Memory.Allocate(Words * 4 * LaterGroups, 0x10000); diff --git a/tests/SharpEmu.GpuRemote.Tests/GpuRemoteChannelTests.cs b/tests/SharpEmu.GpuRemote.Tests/GpuRemoteChannelTests.cs index d5bad2bc8..e6ff1a193 100644 --- a/tests/SharpEmu.GpuRemote.Tests/GpuRemoteChannelTests.cs +++ b/tests/SharpEmu.GpuRemote.Tests/GpuRemoteChannelTests.cs @@ -83,6 +83,20 @@ public void AttachRefusesAnotherProtocolVersion() Assert.Contains("protocol version", exception.Message); } + [Theory] + [InlineData(4u)] + [InlineData(5u)] + [InlineData(6u)] + public void CombinedLayoutRefusesEveryIndependentRecoveryProtocol(uint version) + { + if (!OperatingSystem.IsMacOS()) return; + using var host = GpuRemoteChannel.Create(SharedMemoryRegion.CreateUniqueName("setest")); + using (var raw = SharedMemoryRegion.Open(host.Name)) + ((RegionHeader*)raw.Pointer)->Version = version; + Assert.Equal(7u, GpuRemoteProtocol.Version); + Assert.Throws(() => GpuRemoteChannel.Attach(host.Name)); + } + [MacOSFact] public void AttachRefusesSomethingThatIsNotARegion() { diff --git a/tests/SharpEmu.GpuRemote.Tests/SplitTransportTests.cs b/tests/SharpEmu.GpuRemote.Tests/SplitTransportTests.cs index a2670249e..8b2cecf24 100644 --- a/tests/SharpEmu.GpuRemote.Tests/SplitTransportTests.cs +++ b/tests/SharpEmu.GpuRemote.Tests/SplitTransportTests.cs @@ -218,6 +218,7 @@ public void SplitPayloadsKeepTheirSizesOnEveryArchitecture() Assert.Equal(40, sizeof(ProtectRequestMessage)); Assert.Equal(40, sizeof(GamepadOutputMessage)); Assert.Equal(48, sizeof(DisplayBufferResponse)); + Assert.Equal(64, sizeof(ReserveFlipResponse)); Assert.Equal(32, sizeof(SpanRequest)); Assert.True(sizeof(InputMessage) < 512); } diff --git a/tests/SharpEmu.GpuSplit.Tests/InputDeliveryBenchmarks.cs b/tests/SharpEmu.GpuSplit.Tests/InputDeliveryBenchmarks.cs new file mode 100644 index 000000000..a7c4a9d3d --- /dev/null +++ b/tests/SharpEmu.GpuSplit.Tests/InputDeliveryBenchmarks.cs @@ -0,0 +1,55 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using System.Diagnostics; +using SharpEmu.Libs.Pad; +using Xunit; +using Xunit.Abstractions; +namespace SharpEmu.GpuSplit.Tests; +[Collection(SplitCollection.Name)] +public sealed class InputDeliveryBenchmarks(ITestOutputHelper output) +{ + [MacOSFact] + public void Measure() + { + if (Environment.GetEnvironmentVariable("SHARPEMU_INPUT_BENCH") != "1") return; + var lines = new List(); + var previousMode = System.Environment.GetEnvironmentVariable("SHARPEMU_SPLIT_INPUT_POLL"); + try + { + foreach (var mode in new[] { "1", "0", "1", "0" }) + { + System.Environment.SetEnvironmentVariable("SHARPEMU_SPLIT_INPUT_POLL", mode); + using var split = new InProcessSplit(inputPoll: TimeSpan.FromMilliseconds(4)); + HostWindowInput.Connect(); + Thread.Sleep(1500); + var proc = Process.GetCurrentProcess(); + proc.Refresh(); var c0 = proc.TotalProcessorTime; var w0 = Stopwatch.GetTimestamp(); + Thread.Sleep(8000); + proc.Refresh(); var c1 = proc.TotalProcessorTime; + var cpu = (c1 - c0).TotalMilliseconds / Stopwatch.GetElapsedTime(w0).TotalSeconds; + var lat = new List(); + var rnd = new Random(1); + for (var i = 0; i < 60; i++) + { + Thread.Sleep(10 + rnd.Next(27)); + var n = split.GuestSide.Inputs.Count; + var t = Stopwatch.GetTimestamp(); + HostWindowInput.SetKey(33, down: i % 2 == 0); + while (split.GuestSide.Inputs.Count == n && Stopwatch.GetElapsedTime(t).TotalMilliseconds < 500) { } + lat.Add(Stopwatch.GetElapsedTime(t).TotalMilliseconds); + } + HostWindowInput.Disconnect(); + lat.Sort(); + lines.Add($"poll_mode={mode} process_cpu_ms_per_s_idle={cpu:F2} latency_ms p50={lat[30]:F2} p90={lat[54]:F2} max={lat[^1]:F2}"); + } + } + finally + { + System.Environment.SetEnvironmentVariable("SHARPEMU_SPLIT_INPUT_POLL", previousMode); + HostWindowInput.Disconnect(); + File.WriteAllLines("/tmp/r2-k4-input.txt", lines); + foreach (var line in lines) output.WriteLine(line); + } + } +} diff --git a/tests/SharpEmu.GpuSplit.Tests/RemoteGuestAddressSpaceTests.cs b/tests/SharpEmu.GpuSplit.Tests/RemoteGuestAddressSpaceTests.cs index 5ef092e28..f9ccc2e0e 100644 --- a/tests/SharpEmu.GpuSplit.Tests/RemoteGuestAddressSpaceTests.cs +++ b/tests/SharpEmu.GpuSplit.Tests/RemoteGuestAddressSpaceTests.cs @@ -68,6 +68,64 @@ public void AWriteToAnUnwatchedPageNextToAWatchedOneIsNotOfInterest() Assert.False(state.MayInterest(Group, 16 * 0x1000)); } + [MacOSFact] + public void EveryPageIsPendingUntilTheGpuSideKeepsTheSummary() + { + using var channel = GpuRemoteChannel.Create(SharedMemoryRegion.CreateUniqueName("sesplit"), SplitSharedState.Layout); + var state = new SplitSharedState(channel); + Assert.True(state.MayPendWriteback(0x10_0000, 8)); + + var pending = new PendingWritebackPages(state); + Assert.True(state.MayPendWriteback(0x10_0000, 8)); + pending.Activate(); + Assert.True(state.PendingLive); + Assert.False(state.MayPendWriteback(0x10_0000, 8)); + Assert.False(state.MayPendWriteback(0, TrackerLayout.SpaceBytes)); + } + + [MacOSFact] + public void APendingPageIsMarkedAtOnceAndUnmarkedOnlyWhenPublishedUnclaimed() + { + using var channel = GpuRemoteChannel.Create(SharedMemoryRegion.CreateUniqueName("sesplit"), SplitSharedState.Layout); + var state = new SplitSharedState(channel); + var pending = new PendingWritebackPages(state); + pending.Activate(); + const ulong Image = 0x4_0000_0000 + 0x3F000; // straddles a summary word: pages 63 and 64 of the block + const ulong Page = 0x1000; + + // An image of two pages and a writeback over its second page and the next. + pending.Add(Image + 0x10, 2 * Page - 0x20); + Assert.True(state.MayPendWriteback(Image, 1)); + Assert.True(state.MayPendWriteback(Image + Page + 0xFF8, 8)); + Assert.False(state.MayPendWriteback(Image - 8, 8)); + Assert.False(state.MayPendWriteback(Image + 2 * Page, 8)); + pending.Add(Image + Page, 2 * Page); + Assert.Equal(2, pending.ClaimsAt(Image + Page)); + + // The image's claim ends: nothing changes until published, and then only its own page goes. + pending.Remove(Image + 0x10, 2 * Page - 0x20); + Assert.True(state.MayPendWriteback(Image, 8)); + pending.Publish(); + Assert.False(state.MayPendWriteback(Image, Page)); + Assert.True(state.MayPendWriteback(Image + Page, 8)); + Assert.True(state.MayPendWriteback(Image + 2 * Page, 8)); + + // Claimed again before the publish: the page stays marked. + pending.Remove(Image + Page, 2 * Page); + pending.Add(Image + 2 * Page, 8); + pending.Publish(); + Assert.False(state.MayPendWriteback(Image, 2 * Page)); + Assert.True(state.MayPendWriteback(Image + 2 * Page, 1)); + pending.Remove(Image + 2 * Page, 8); + pending.Publish(); + Assert.False(state.MayPendWriteback(Image - Page, 8 * Page)); + + // A release of what was never claimed changes nothing. + pending.Remove(Image, Page); + pending.Publish(); + Assert.Equal(0, pending.ClaimsAt(Image)); + } + [MacOSFact] public void AProcessWithoutTheBlockIsRefused() { diff --git a/tests/SharpEmu.GpuSplit.Tests/SplitInProcessTests.cs b/tests/SharpEmu.GpuSplit.Tests/SplitInProcessTests.cs index 9119163e3..b377132fa 100644 --- a/tests/SharpEmu.GpuSplit.Tests/SplitInProcessTests.cs +++ b/tests/SharpEmu.GpuSplit.Tests/SplitInProcessTests.cs @@ -30,7 +30,7 @@ internal sealed class InProcessSplit : IDisposable private readonly GpuProcessClient _client; private readonly Thread _loop; - public InProcessSplit(Action? beforeStart = null, TimeSpan? inputPoll = null) + public InProcessSplit(Action? beforeStart = null, TimeSpan? inputPoll = null, IGuestImageStore? gpuStore = null) { Guest = new SharedGuestMemory(); _server = MacOSHost.ServeGuestMemory(Guest.Export, Guest.Table.Region, out var service); @@ -43,7 +43,7 @@ public InProcessSplit(Action? beforeStart = null, TimeSpan? inpu ComposeBackend = environment => { Environment = environment; - environment.Memory.AttachStores(null, GpuStore); + environment.Memory.AttachStores(null, gpuStore ?? GpuStore); return Backend; }, ConfigureVideo = options => ConfiguredVideo = options, @@ -293,6 +293,44 @@ public void AMappingChangeParksTheWorkerAndUnregistersOnIt() } } + [MacOSFact] + public void AReservedDisplayBufferCannotCrossBridgeInstances() => CheckReservationInvalidation(crossBridge: true); + + [MacOSFact] + public void AReservedDisplayBufferCannotSurviveAnInterveningSlotLookup() => CheckReservationInvalidation(crossBridge: false); + + private static void CheckReservationInvalidation(bool crossBridge) + { + using var split = new InProcessSplit(); + var bridge = split.Host.VideoOut!; + var other = new RemoteVideoOutBridge(split.Host, "other-session", false); + Exception? failure = null; + var caller = new Thread(() => + { + try + { + { + split.GuestSide.Calls.Clear(); + Assert.Equal(0, bridge.TryReserveFlipRequest(1, 2, 1, 42, gpuQueued: true, out _)); + if (crossBridge) + Assert.True(other.TryGetDisplayBufferInfo(1, 2, out _)); + else + Assert.True(bridge.TryGetDisplayBufferInfo(1, 3, out _)); + Assert.True(bridge.TryGetDisplayBufferInfo(1, 2, out _)); + // Reservation plus both lookups must ask the guest for their own buffer. + Assert.Equal(3, split.GuestSide.Calls.Count(call => call.StartsWith("buffer ", StringComparison.Ordinal))); + } + } + catch (Exception exception) + { + failure = exception; + } + }) { IsBackground = true }; + caller.Start(); + Assert.True(caller.Join(20_000)); + Assert.Null(failure); + } + [MacOSFact] public void TheGpuSideReachesTheGuestsVideoOutState() { @@ -312,6 +350,11 @@ public void TheGpuSideReachesTheGuestsVideoOutState() Assert.Equal(1100, bridge.NextFlipPresentTimestamp(77, 1000)); Assert.True(bridge.TryGetDisplayBufferInfo(1, 2, out var buffer)); Assert.Equal(new VideoOutExports.DisplayBufferInfo(0x5000, 0x8000_0000, 0, 1920, 1080, 1920, 3), buffer); + // The reservation carried that buffer, so no second call was made for it; the + // next lookup of the slot asks the guest again, and a flip that names a buffer + // is never released without a call to the guest. + Assert.True(bridge.TryGetDisplayBufferInfo(1, 2, out _)); + Assert.False(bridge.ReleaseNoBufferFlip(2, 77)); Assert.Equal(2, bridge.GetFlipEventCount(77)); Assert.True(bridge.ReleaseNoBufferFlip(-1, 77)); Assert.True(bridge.IsFlipPresentationPending(77)); @@ -336,7 +379,7 @@ public void TheGpuSideReachesTheGuestsVideoOutState() Assert.True(caller.Join(20_000)); Assert.Null(failure); Assert.Equal( - new[] { "reserve 1 2 1 42", "done 1 2", "can 77 1000", "next 77 1000", "buffer 1 2", "events 77", "release -1 77", "pending 77" }, + new[] { "reserve 1 2 1 42", "buffer 1 2", "done 1 2", "can 77 1000", "next 77 1000", "buffer 1 2", "events 77", "release -1 77", "pending 77" }, split.GuestSide.Calls); bridge.SetSelectedGpuName("Apple M5 Pro"); diff --git a/tests/SharpEmu.GpuSplit.Tests/SplitPendingWritebackTests.cs b/tests/SharpEmu.GpuSplit.Tests/SplitPendingWritebackTests.cs new file mode 100644 index 000000000..070602dba --- /dev/null +++ b/tests/SharpEmu.GpuSplit.Tests/SplitPendingWritebackTests.cs @@ -0,0 +1,276 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using SharpEmu.HLE; +using SharpEmu.HLE.GpuMemory; +using SharpEmu.Libs.Gpu.GpuCommands; +using Xunit; + +namespace SharpEmu.GpuSplit.Tests; + +/// +/// The GPU process's store for one image at the start of a page, kept as the Metal image cache +/// keeps its images (MetalImageCache.PendingWritebacks.cs): a render makes it GPU-modified, a +/// writeback is recorded under the lock (the image's claim ends, the writeback's begins) and lands +/// later, a managed write's call writes the image back or waits for the writeback before the write +/// lands, and the write's record marks the image stale (dropping the GPU's bytes if it is still +/// GPU-modified then). The pending summary is published as the lock is let go. +/// +internal sealed unsafe class WritebackImageStore : IGuestImageStore +{ + public const ulong ImageBytes = 2048; + // What a render changes; the rest of the image keeps what the last reload read. + public const int GpuBytes = 1024; + + private readonly object _gate = new(); + private readonly byte[] _texture = new byte[ImageBytes]; + private readonly ManualResetEventSlim _landed = new(true); + private IPendingWritebackPages? _pending; + private ulong _address; + private bool _modified; + private bool _stale = true; + private bool _inFlight; + private int _precise; + private int _landedWrites; + private int _lostGpuBytes; + + public int PreciseCalls => Volatile.Read(ref _precise); + + public int LandedWrites => Volatile.Read(ref _landedWrites); + + public int LostGpuBytes => Volatile.Read(ref _lostGpuBytes); + + /// Spins between the image's claim ending and its writeback's beginning, to widen that gap. + public int GapSpins { get; set; } + + public void Attach(ulong address, IPendingWritebackPages pending) + { + _address = address; + _pending = pending; + pending.Activate(); + } + + public void Render(byte value) + { + lock (_gate) + { + if (_stale) + { + new ReadOnlySpan((void*)_address, (int)ImageBytes).CopyTo(_texture); + _stale = false; + } + + if (!_modified) + { + _pending!.Add(_address, ImageBytes); + _modified = true; + } + + _texture.AsSpan(0, GpuBytes).Fill(value); + _pending!.Publish(); + } + } + + /// The bytes the writeback will store, or null when the image holds nothing to write back. + public byte[]? RecordWriteback() + { + lock (_gate) + { + byte[]? snapshot = null; + if (_modified) + { + snapshot = (byte[])_texture.Clone(); + _modified = false; + _pending!.Remove(_address, ImageBytes); + Thread.SpinWait(GapSpins); + _pending.Add(_address, ImageBytes); + _inFlight = true; + _landed.Reset(); + } + + _pending!.Publish(); + return snapshot; + } + } + + public void LandWriteback(byte[] snapshot) + { + snapshot.CopyTo(new Span((void*)_address, (int)ImageBytes)); + lock (_gate) + { + _inFlight = false; + _pending!.Remove(_address, ImageBytes); + _pending.Publish(); + _landed.Set(); + } + } + + public bool MarkPreciseCpuWrite(ulong address, ulong size) + { + Interlocked.Increment(ref _precise); + while (true) + { + _landed.Wait(); + lock (_gate) + { + if (_inFlight) + { + continue; + } + + if (_modified) + { + _texture.CopyTo(new Span((void*)_address, (int)ImageBytes)); + _modified = false; + _pending!.Remove(_address, ImageBytes); + } + + _stale = true; + _pending!.Publish(); + return true; + } + } + } + + public bool MarkCpuWriteLanded(ulong address, ulong size) + { + Interlocked.Increment(ref _landedWrites); + lock (_gate) + { + if (_modified) + { + // The write is in guest memory already: nothing may go under it, so the GPU's bytes are lost. + Interlocked.Increment(ref _lostGpuBytes); + _modified = false; + _pending!.Remove(_address, ImageBytes); + } + + _stale = true; + _pending!.Publish(); + return true; + } + } + + public bool MarkCpuWrite(ulong address, ulong size) => false; + + public void Unregister(ulong address, ulong size) + { + } +} + +[Collection(SplitCollection.Name)] +public sealed unsafe class SplitPendingWritebackTests +{ + private const GuestPageProtection ReadWrite = InProcessSplit.ReadWrite; + + private static InProcessSplit Start(WritebackImageStore store, out ulong image) + { + var split = new InProcessSplit(gpuStore: store); + image = split.Guest.Address; + split.GuestMemory.Register(image, SharedGuestMemory.ViewBytes, ReadWrite); + var page = image; + var watcher = new Thread(() => split.GpuMemory.Pages.AddWatch(page, 0x1000, blockReads: false)); + watcher.Start(); + Assert.True(watcher.Join(10_000)); + var pending = ((IPendingWritebackSource)split.Environment!.AddressSpace).PendingWritebacks; + Assert.NotNull(pending); + store.Attach(image, pending); + return split; + } + + // A managed write the way PhysicalVirtualMemory makes one. + private static void ManagedWrite(InProcessSplit split, ulong address, ulong value) + { + Assert.True(split.GuestMemory.MarkCpuWrite(address, 8)); + *(ulong*)address = value; + split.GuestMemory.MarkCpuWriteLanded(address, 8); + } + + [MacOSFact] + public void AWatchedWriteCallsOnlyWhereTheGpuHoldsBytesGuestMemoryLacks() + { + var store = new WritebackImageStore(); + using var split = Start(store, out var image); + var slot = image + (ulong)WritebackImageStore.GpuBytes; + + // Watched, nothing GPU-written: no call, and the records invalidate the image. + ManagedWrite(split, slot, 1); + Wait.Until(() => store.LandedWrites >= 1, "the write's record"); + Assert.Equal(0, store.PreciseCalls); + // Direct fixture renders bypass the worker's ordered command stream. Drain + // both CPU-write records before creating the next generation of GPU bytes. + Assert.Equal(IdleOutcome.Completed, split.Session.Backend.SubmitDone(null!)); + + // GPU-written: the call writes the image back before the write lands over its page. + store.Render(0x5A); + ManagedWrite(split, slot, 2); + Assert.Equal(1, store.PreciseCalls); + Assert.True(new ReadOnlySpan((void*)image, WritebackImageStore.GpuBytes).IndexOfAnyExcept((byte)0x5A) < 0); + Assert.Equal(2UL, *(ulong*)slot); + Assert.Equal(IdleOutcome.Completed, split.Session.Backend.SubmitDone(null!)); + + // A recorded writeback not landed yet: the call waits for it. + store.Render(0x6B); + var snapshot = store.RecordWriteback()!; + var writer = Task.Run(() => ManagedWrite(split, slot, 3)); + Assert.False(writer.Wait(100)); + store.LandWriteback(snapshot); + Assert.True(writer.Wait(10_000)); + Assert.Equal(2, store.PreciseCalls); + Assert.Equal(3UL, *(ulong*)slot); + + // Landed and published: no call again. + ManagedWrite(split, slot, 4); + _ = split.Session.Backend.SubmitDone(null!); + Assert.Equal(2, store.PreciseCalls); + Assert.Equal(0, store.LostGpuBytes); + } + + // CPU writes racing a GPU writeback into the same image: neither side's bytes may be lost. + [MacOSFact] + public void ManagedWritesRacingAWritebackLoseNeitherTheGpusBytesNorTheirOwn() + { + const int Rounds = 1500; + var store = new WritebackImageStore { GapSpins = 2_000 }; + using var split = Start(store, out var image); + var random = new Random(18); + for (var round = 1; round <= Rounds; round++) + { + var value = (byte)(round % 251 + 1); + var slot = image + (ulong)WritebackImageStore.GpuBytes + (ulong)(round % 128) * 8; + var cpuValue = 0xC0DE_0000_0000UL | (uint)round; + store.Render(value); + var gpuDelay = random.Next(0, 4_000); + var landDelay = random.Next(0, 4_000); + var cpuDelay = random.Next(0, 6_000); + using var go = new ManualResetEventSlim(); + var gpu = Task.Run(() => + { + go.Wait(); + Thread.SpinWait(gpuDelay); + if (store.RecordWriteback() is { } snapshot) + { + Thread.SpinWait(landDelay); + store.LandWriteback(snapshot); + } + }); + var cpu = Task.Run(() => + { + go.Wait(); + Thread.SpinWait(cpuDelay); + ManagedWrite(split, slot, cpuValue); + }); + go.Set(); + Assert.True(Task.WaitAll([gpu, cpu], 10_000)); + _ = split.Session.Backend.SubmitDone(null!); + + Assert.Equal(cpuValue, *(ulong*)slot); + var gpuBytes = new ReadOnlySpan((void*)image, WritebackImageStore.GpuBytes); + Assert.True(gpuBytes.IndexOfAnyExcept(value) < 0, $"round {round}: the GPU's bytes were lost"); + Assert.Equal(0, store.LostGpuBytes); + } + + // Both paths ran: writes that met a pending writeback called, the others went as records. + Assert.InRange(store.PreciseCalls, 1, Rounds - 1); + } +} diff --git a/tests/SharpEmu.GpuSplit.Tests/WorkerBracketTests.cs b/tests/SharpEmu.GpuSplit.Tests/WorkerBracketTests.cs new file mode 100644 index 000000000..72179ab46 --- /dev/null +++ b/tests/SharpEmu.GpuSplit.Tests/WorkerBracketTests.cs @@ -0,0 +1,76 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using Xunit; + +namespace SharpEmu.GpuSplit.Tests; + +/// +/// The threads that park the GPU worker for a bracket: one after another, brackets share a parked +/// thread instead of making one each; brackets that overlap still each have their own. +/// +[Collection(SplitCollection.Name)] +public sealed class WorkerBracketTests +{ + private static readonly TimeSpan Wait = TimeSpan.FromSeconds(10); + + // The fake worker runs the park on the thread that hands it over. + private static WorkerBracket Begin(ulong id, Action? seen = null) + { + var bracket = new WorkerBracket(id, static () => true); + bracket.Start(park => + { + seen?.Invoke(Thread.CurrentThread); + park(); + }); + Assert.True(bracket.WaitParked(Wait)); + return bracket; + } + + [Fact] + public void BracketsOneAfterAnotherShareOneThread() + { + var threads = new HashSet(); + // Settle the pool: whatever an earlier test left idle is reused from here on. + Assert.True(Begin(1, thread => threads.Add(thread)).End(static () => { }, Wait)); + var created = WorkerBracket.BracketRunner.Created; + + for (ulong id = 2; id < 50; id++) + { + var ran = 0; + var bracket = Begin(id, thread => threads.Add(thread)); + Assert.True(bracket.TryRun(() => ran++, Wait)); + Assert.True(bracket.End(() => ran += 10, Wait)); + Assert.Equal(11, ran); + // Ended: the work goes elsewhere. + Assert.False(bracket.TryRun(static () => { }, Wait)); + } + + Assert.Equal(created, WorkerBracket.BracketRunner.Created); + Assert.Single(threads); + } + + [Fact] + public void OverlappingBracketsEachHaveAThread() + { + var threads = new HashSet(); + var first = Begin(100, thread => { lock (threads) threads.Add(thread); }); + var second = Begin(101, thread => { lock (threads) threads.Add(thread); }); + Assert.Equal(2, threads.Count); + Assert.True(second.End(static () => { }, Wait)); + Assert.True(first.End(static () => { }, Wait)); + } + + [Fact] + public void ABracketWhoseHandOverThrowsStillEndsAndFreesItsThread() + { + var bracket = new WorkerBracket(200, static () => true); + bracket.Start(static _ => throw new InvalidOperationException("no worker")); + Assert.True(bracket.WaitParked(Wait)); + Assert.True(bracket.End(static () => { }, Wait)); + + var created = WorkerBracket.BracketRunner.Created; + Assert.True(Begin(201).End(static () => { }, Wait)); + Assert.Equal(created, WorkerBracket.BracketRunner.Created); + } +} diff --git a/tests/SharpEmu.Libs.Tests/Audio/AjmTraceHandlerTests.cs b/tests/SharpEmu.Libs.Tests/Audio/AjmTraceHandlerTests.cs new file mode 100644 index 000000000..9bc5a969d --- /dev/null +++ b/tests/SharpEmu.Libs.Tests/Audio/AjmTraceHandlerTests.cs @@ -0,0 +1,47 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using SharpEmu.Libs.Audio; +using Xunit; + +namespace SharpEmu.Libs.Tests.Audio; + +public sealed class AjmTraceHandlerTests +{ + [Fact] + public void DisabledTraceDoesNotFormatOrAllocate() + { + if (AjmExports.TraceEnabled) + { + return; // SHARPEMU_LOG_AJM=1 in this process: the disabled path is not reachable. + } + + var evaluated = 0; + Func expensive = () => + { + evaluated++; + return new string('x', 64); + }; + + static AjmTraceInterpolatedStringHandler Build(ulong a, uint b, string c, Func expensive) + { + AjmTraceInterpolatedStringHandler handler = $"batch a=0x{a:X16} b=0x{b:X8} c={c} d={expensive()}"; + return handler; + } + + var handler = Build(1, 2, "abc", expensive); + Assert.False(handler.Enabled); + Assert.Equal(0, evaluated); + Assert.Equal(string.Empty, handler.ToStringAndClear()); + + var before = GC.GetAllocatedBytesForCurrentThread(); + for (var i = 0; i < 1000; i++) + { + var h = Build((ulong)i, (uint)i, "abc", expensive); + _ = h.Enabled; + } + + Assert.Equal(0, GC.GetAllocatedBytesForCurrentThread() - before); + Assert.Equal(0, evaluated); + } +} diff --git a/tests/SharpEmu.Libs.Tests/Cpu/ImportResultLoggingTests.cs b/tests/SharpEmu.Libs.Tests/Cpu/ImportResultLoggingTests.cs index a723fcbe0..c9b399fd7 100644 --- a/tests/SharpEmu.Libs.Tests/Cpu/ImportResultLoggingTests.cs +++ b/tests/SharpEmu.Libs.Tests/Cpu/ImportResultLoggingTests.cs @@ -20,7 +20,19 @@ public void RepeatedImportFailuresAreSampledPerImportAndResult() using var backend = new DirectExecutionBackend(modules); var method = typeof(DirectExecutionBackend).GetMethod( "ShouldLogImportResult", BindingFlags.Instance | BindingFlags.NonPublic)!; - var filter = method.CreateDelegate(backend); + var entryType = method.GetParameters()[0].ParameterType.GetElementType()!; + var entries = new Dictionary(); + bool Filter(string nid, OrbisGen2Result result, out int occurrence) + { + if (!entries.TryGetValue(nid, out var entry)) + entries[nid] = entry = Activator.CreateInstance(entryType, + [0UL, nid, null, false, false, false, false, 0UL])!; + object?[] arguments = [entry, result, 0]; + var logged = (bool)method.Invoke(backend, arguments)!; + occurrence = (int)arguments[2]!; + return logged; + } + ImportResultFilter filter = Filter; bool ShouldLog(string nid, OrbisGen2Result result) => filter(nid, result, out _); foreach (var nid in new[] { "9UK1vLZQft4", "7H0iTOciTLo" }) diff --git a/tests/SharpEmu.Libs.Tests/Cpu/Sse4aPosixSignalRecoveryTests.cs b/tests/SharpEmu.Libs.Tests/Cpu/Sse4aPosixSignalRecoveryTests.cs index edf8ebe3b..2ba3273c5 100644 --- a/tests/SharpEmu.Libs.Tests/Cpu/Sse4aPosixSignalRecoveryTests.cs +++ b/tests/SharpEmu.Libs.Tests/Cpu/Sse4aPosixSignalRecoveryTests.cs @@ -2,7 +2,6 @@ // SPDX-License-Identifier: GPL-2.0-or-later using System.Reflection; -using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using SharpEmu.Core.Cpu.Emulation; using SharpEmu.Core.Cpu.Native; @@ -19,7 +18,24 @@ public sealed unsafe class Sse4aPosixSignalRecoveryTests : IDisposable { private readonly object? _previousBackend = FaultRecoveryBackend.GetValue(null); - public void Dispose() => FaultRecoveryBackend.SetValue(null, _previousBackend); + private DirectExecutionBackend? _backend; + + public void Dispose() + { + _backend?.Dispose(); + FaultRecoveryBackend.SetValue(null, _previousBackend); + } + + private DirectExecutionBackend Backend() + { + if (_backend is null) + { + var modules = new ModuleManager(); + modules.Freeze(); + _backend = new DirectExecutionBackend(modules, installFaultHandling: false); + } + return _backend; + } private const int PosixSigIll = 4; private const int LinuxUcontextGregsOffset = 40; @@ -119,7 +135,7 @@ public void ExtrqSigillRoundTripsXmmThroughTheBridge() try { const ulong value = 0x1234_5678_9ABC_DEF0UL; - var frame = new FakeSignalFrame((ulong)code); + var frame = new FakeSignalFrame((ulong)code, Backend()); frame.SetXmmLow(FxsaveXmm0Offset, value); var emulatedBefore = (long)EmulatedCounter.GetValue(null)!; @@ -153,7 +169,7 @@ public void InsertqSigillReadsSourceXmmThroughTheBridge() { const ulong destination = 0x1111_2222_3333_4444UL; const ulong source = 0xAAAA_BBBB_CCCC_DDDDUL; - var frame = new FakeSignalFrame((ulong)code); + var frame = new FakeSignalFrame((ulong)code, Backend()); frame.SetXmmLow(FxsaveXmm0Offset, destination); frame.SetXmmLow(FxsaveXmm1Offset, source); @@ -180,7 +196,7 @@ public void ExtractSignalPreservesLivePointerAndOtherVectorRegisters() var code = AllocateProbeVisibleCode([0x66, 0x0F, 0x78, 0xC1, 0x28, 0x00]); try { - var frame = new FakeSignalFrame((ulong)code) { Accumulator = 0x1B86FD090 }; + var frame = new FakeSignalFrame((ulong)code, Backend()) { Accumulator = 0x1B86FD090 }; for (int registerIndex = 0; registerIndex < 16; registerIndex++) { frame.SetXmmLow(FxsaveXmm0Offset + registerIndex * 16, (ulong)registerIndex); @@ -221,7 +237,7 @@ public void InsertInstructionPreservesOtherVectorRegisters() var code = AllocateProbeVisibleCode([0xF2, 0x0F, 0x78, 0xC0, 0x08, 0x08]); try { - var frame = new FakeSignalFrame((ulong)code); + var frame = new FakeSignalFrame((ulong)code, Backend()); frame.SetXmmLow(FxsaveXmm0Offset, 0x10); frame.SetXmmLow(FxsaveXmm0Offset + 8, ulong.MaxValue); for (int registerIndex = 1; registerIndex < 16; registerIndex++) @@ -264,7 +280,7 @@ public void RecoveryDeclinesWhenNoXmmStateWasBridged() try { DirectExecutionFaultContext.VectorRegistersBridged = false; - var backend = RuntimeHelpers.GetUninitializedObject(typeof(DirectExecutionBackend)); + var backend = Backend(); var contextRecord = stackalloc byte[0x4D0]; var recovered = (bool)TryRecoverAmdCompat.Invoke( @@ -288,8 +304,11 @@ private sealed class FakeSignalFrame private readonly byte[] _machineContext = new byte[1032]; private readonly bool _includeFloatingPointState; - public FakeSignalFrame(ulong instructionPointer, bool includeFloatingPointState = true) + private readonly DirectExecutionBackend _backend; + + public FakeSignalFrame(ulong instructionPointer, DirectExecutionBackend backend, bool includeFloatingPointState = true) { + _backend = backend; _includeFloatingPointState = includeFloatingPointState; fixed (byte* userContext = _userContext) fixed (byte* machineContext = _machineContext) @@ -362,7 +381,7 @@ public void AssertMacOsExtendedVectorStateEquals(byte value) public bool Dispatch() { - EnsureBridgeBackend(); + EnsureBridgeBackend(_backend); fixed (byte* userContext = _userContext) fixed (byte* floatingPointState = _floatingPointState) fixed (byte* machineContext = _machineContext) @@ -384,24 +403,12 @@ public bool Dispatch() } } - /// - /// The bridge only runs the recovery chain when a backend instance is - /// registered. The tests do not need any of the constructor's state (and must not run - /// it: it installs process-wide signal handlers), so register an uninitialized - /// instance - the SIGILL recovery path only touches static state. - /// - private static void EnsureBridgeBackend() + private static void EnsureBridgeBackend(DirectExecutionBackend backend) { - // Point the bridge at the shared recovery entries without installing - // the signal handlers, which is process-wide and cannot be undone. + // Use the shared entry points, but do not install irreversible process signal handlers. UseSharedRecoveryEntries!.Invoke(null, [(nint)(delegate* managed)&DirectExecutionBackend.SharedRawFaultRecovery, (nint)(delegate* managed)&DirectExecutionBackend.SharedFaultRecovery]); - if (FaultRecoveryBackend.GetValue(null) == null) - { - FaultRecoveryBackend.SetValue( - null, - RuntimeHelpers.GetUninitializedObject(typeof(DirectExecutionBackend))); - } + FaultRecoveryBackend.SetValue(null, backend); } // Use the host allocator so Linux memory probes can read the instruction bytes. diff --git a/tests/SharpEmu.Libs.Tests/Gpu/Pipelines/PixelDepthExportResolverTests.cs b/tests/SharpEmu.Libs.Tests/Gpu/Pipelines/PixelDepthExportResolverTests.cs index 31a30812f..ee25da973 100644 --- a/tests/SharpEmu.Libs.Tests/Gpu/Pipelines/PixelDepthExportResolverTests.cs +++ b/tests/SharpEmu.Libs.Tests/Gpu/Pipelines/PixelDepthExportResolverTests.cs @@ -73,5 +73,52 @@ public void StencilExportFollowsTheBoundStencilAspect( Assert.False(info.EarlyDepth && control != 0); } + // DB_SHADER_CONTROL: Z_ORDER bits 4-5, KILL 6, MASK_EXPORT 8, EXEC_ON_HIER_FAIL 9, EXEC_ON_NOOP 10, + // DEPTH_BEFORE_SHADER 12. Metal tests depth early only where the guest does and skips the shader for + // the fragments it rejects: EXEC_ON_* make that shader run for them, so those stay late. + [Theory] + [InlineData(0x000u, false)] // LATE_Z + [InlineData(0x010u, true)] // EARLY_Z_THEN_LATE_Z + [InlineData(0x020u, true)] // RE_Z + [InlineData(0x030u, true)] // EARLY_Z_THEN_RE_Z + [InlineData(0x210u, false)] // + EXEC_ON_HIER_FAIL + [InlineData(0x410u, false)] // + EXEC_ON_NOOP + [InlineData(0x620u, false)] // RE_Z + both + [InlineData(0x1000u, true)] // DEPTH_BEFORE_SHADER alone + [InlineData(0x1630u, true)] // DEPTH_BEFORE_SHADER wins over EXEC_ON_* + [InlineData(0x050u, false)] // KILL_ENABLE + [InlineData(0x1050u, false)] // KILL_ENABLE, even with DEPTH_BEFORE_SHADER + [InlineData(0x011u, false)] // Z_EXPORT_ENABLE + [InlineData(0x012u, false)] // STENCIL_TEST_VAL_EXPORT_ENABLE + [InlineData(0x110u, false)] // MASK_EXPORT_ENABLE + [InlineData(0x8000u, false)] // DUAL_QUAD_DISABLE and friends do not matter + [InlineData(0x8010u, true)] + public void EarlyDepthFollowsTheGuestsDepthOrderAndExecuteBits(uint control, bool early) + { + var shaderInterface = new ShaderInterfaceRegisters { DepthShaderControl = DepthShaderControlRegisters.Decode(control) }; + var info = PixelStageInputResolver.Resolve( + new CpuContext(_memory, Generation.Gen5), + new RegisteredShader(MemoryBase, MemoryBase, 16, 0, 0, 0, 0, 0, 0), + shaderInterface, + new byte[PixelInputInfo.TargetCount], + new ColorComponentMap[PixelInputInfo.TargetCount], + inputCount: 0, + depthAspectBound: true, + stencilAspectBound: true); + + Assert.Equal(early, info.EarlyDepth); + } + + [Fact] + public void TheDecodedBitsMatchTheHardwareRegister() + { + var control = DepthShaderControlRegisters.Decode((1u << 9) | (1u << 12)); + Assert.True(control.ExecuteOnHierFail); + Assert.True(control.DepthBeforeShader); + Assert.False(control.ExecuteOnNoop); + Assert.Equal(0u, control.RemainingBits); + Assert.True(DepthShaderControlRegisters.Decode(1u << 10).ExecuteOnNoop); + } + public void Dispose() => _fatal.Dispose(); } diff --git a/tests/SharpEmu.Libs.Tests/Gpu/Rendering/RenderExecutorDrawTests.cs b/tests/SharpEmu.Libs.Tests/Gpu/Rendering/RenderExecutorDrawTests.cs index 7ef9b21e3..6596ccce9 100644 --- a/tests/SharpEmu.Libs.Tests/Gpu/Rendering/RenderExecutorDrawTests.cs +++ b/tests/SharpEmu.Libs.Tests/Gpu/Rendering/RenderExecutorDrawTests.cs @@ -1395,6 +1395,54 @@ public void DeferredPixelBufferStores_EndThePassBeforeAnIndirectDrawOfTheBytesTh AssertOrder("end_rendering", FragmentBarrier, "obtain 100600000 14 written=False", "begin_rendering", "draw_indexed_indirect"); } + [Fact] + public void DeferredPixelStores_AreUnorderedInTheOpenPassUntilTheGuestReleasesThem() + { + Assert.False(_executor.HasUnorderedStoresInOpenPass); + DrawWithPixelStores(); + Assert.True(_executor.HasUnorderedStoresInOpenPass); + + _executor.ReleaseShaderWrites(); + Assert.False(_executor.HasUnorderedStoresInOpenPass); + } + + [Fact] + public void DeferredPixelStores_AreNotUnorderedOnceTheHostEndedTheirPass() + { + DrawWithPixelStores(); + _host.RenderPassSerial++; + + Assert.False(_executor.HasUnorderedStoresInOpenPass); + } + + [Fact] + public void VertexBufferStores_TheDrawBarriersLeaveNothingUnordered() + { + _host.OrdersVertexWritesInsidePass = true; + var written = new BufferResourceInfo(false, true, false, false, false, 4, 0); + _pipelines.Graphics = Programs( + vertexStage: Stage(Program(ShaderStageKind.Vertex, buffers: [written]), buffers: [BufferDescriptor(VertexBase, 4, 16)]), + pixelStage: Stage(Program(ShaderStageKind.Pixel))); + _executor.DrawIndexed(1, Banks(), Indexed(3)); + + Assert.Contains($"write_barrier {GpuPipelineStage.VertexShaderBit}", _host.Calls); + Assert.False(_executor.HasUnorderedStoresInOpenPass); + } + + [Fact] + public void VertexDeviceAddressStores_AreUnorderedInTheirPass() + { + _host.OrdersVertexWritesInsidePass = true; + _pipelines.Graphics = Programs( + vertexStage: Stage(Program(ShaderStageKind.Vertex, usesDeviceAddresses: true, writesDeviceAddresses: true)), + pixelStage: Stage(Program(ShaderStageKind.Pixel))); + _executor.DrawIndexed(1, Banks(), Indexed(3)); + + Assert.True(_executor.HasUnorderedStoresInOpenPass); + _host.RenderPassSerial++; + Assert.False(_executor.HasUnorderedStoresInOpenPass); + } + [Fact] public void DeferredPixelStores_NeedNothingOnceTheHostEndedTheirPass() { diff --git a/tests/SharpEmu.Libs.Tests/Gpu/Split/SplitHookTests.cs b/tests/SharpEmu.Libs.Tests/Gpu/Split/SplitHookTests.cs index b713c532e..e1d52c3f9 100644 --- a/tests/SharpEmu.Libs.Tests/Gpu/Split/SplitHookTests.cs +++ b/tests/SharpEmu.Libs.Tests/Gpu/Split/SplitHookTests.cs @@ -141,7 +141,7 @@ public void TheVideoOutBridgeTakesEveryStateCall() VideoOutBackendServices.CancelOutstandingFlips(); VideoOutBackendServices.NotifyPresentationWindowClosed(); VideoOutBackendServices.SetSelectedGpuName("Apple M5"); - VideoOutBackendServices.TraceGpuFlip(null!, 1, 2, 3, 4, 0x5000, 1); + VideoOutBackendServices.TraceGpuFlip(null!, 1, 2, 3, 4, 0x5000, 1UL); Assert.Equal( new[] @@ -340,7 +340,7 @@ public bool IsFlipDone(int handle, int bufferIndex) public void SetSelectedGpuName(string gpuName) => Calls.Add($"gpu-name {gpuName}"); - public void TraceGpuFlip(int handle, int bufferIndex, int flipMode, long flipArg, ulong address, int flipEventCount) => + public void TraceGpuFlip(int handle, int bufferIndex, int flipMode, long flipArg, ulong address, ulong requestId) => Calls.Add($"trace {handle} {bufferIndex} 0x{address:X}"); public bool TryGetDisplayBufferInfo(int handle, int bufferIndex, out VideoOutExports.DisplayBufferInfo info) diff --git a/tests/SharpEmu.Libs.Tests/GuestThreadQosPolicyTests.cs b/tests/SharpEmu.Libs.Tests/GuestThreadQosPolicyTests.cs index 9bdb67165..56c31d257 100644 --- a/tests/SharpEmu.Libs.Tests/GuestThreadQosPolicyTests.cs +++ b/tests/SharpEmu.Libs.Tests/GuestThreadQosPolicyTests.cs @@ -85,4 +85,29 @@ public void NativeQosThreadRunsBodyAtRequestedClassAndKeepsName() Assert.Equal("qos-test", name); Assert.Equal(0, errno); } + + [Fact] + public void HelperThreadRunsAtItsClassJoinsAndKnowsItself() + { + uint actual = 0; + var self = false; + HostHelperThread? thread = null; + var started = new ManualResetEventSlim(false); + thread = HostHelperThread.Start("helper-test", 1024 * 1024, () => + { + started.Wait(); + actual = HostThreadQualityOfService.GetCurrentThreadClassForDiagnostics(); + self = thread!.IsCurrentThread; + }, HostThreadQos.UserInitiated); + started.Set(); + + Assert.True(thread.Join(TimeSpan.FromSeconds(10))); + Assert.True(self); + Assert.False(thread.IsCurrentThread); + if (OperatingSystem.IsMacOS() && HostHelperThread.QosEnabled) + { + Assert.True(thread.HasQos); + Assert.Equal(0x19u, actual); + } + } } diff --git a/tests/SharpEmu.Libs.Tests/Kernel/KernelEventQueueSignalSkipTests.cs b/tests/SharpEmu.Libs.Tests/Kernel/KernelEventQueueSignalSkipTests.cs new file mode 100644 index 000000000..52e8f79b9 --- /dev/null +++ b/tests/SharpEmu.Libs.Tests/Kernel/KernelEventQueueSignalSkipTests.cs @@ -0,0 +1,338 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using System.Reflection; +using SharpEmu.HLE; +using SharpEmu.Libs.Kernel; +using Xunit; + +namespace SharpEmu.Libs.Tests.Kernel; + +[CollectionDefinition(Name, DisableParallelization = true)] +public sealed class KernelEventQueueSignalSkipCollection +{ + public const string Name = "KernelEventQueueSignalSkip"; +} + +// Signal skips the scheduler scan while no guest waiter is registered on the queue. These tests pin the +// waiter count and that skipping cannot lose a wake against a waiter that blocks concurrently. +[Collection(KernelEventQueueSignalSkipCollection.Name)] +public sealed class KernelEventQueueSignalSkipTests +{ + private const ulong BaseAddress = 0x1_0000_0000; + private const ulong HandleAddress = BaseAddress + 0x100; + private const ulong EventsAddress = BaseAddress + 0x200; + private const ulong OutCountAddress = BaseAddress + 0x300; + + private static KernelEventQueueCompatExports.KernelQueuedEvent UserEvent(ulong ident) => + new( + Ident: ident, + Filter: KernelEventQueueCompatExports.KernelEventFilterUser, + Flags: KernelEventQueueCompatExports.KernelEventFlagClear, + Fflags: 1, + Data: 0x1234, + UserData: 0x5678); + + [Fact] + public void SignalWithoutWaiter_SkipsTheSchedulerScan() + { + var (scheduler, parking) = CreateScheduler(); + var previous = GuestThreadExecution.Scheduler; + GuestThreadExecution.Scheduler = scheduler; + try + { + var (_, ctx, handle) = CreateEqueue(); + Assert.Equal(0, KernelEventQueueCompatExports.BlockedWaiterCountForTest(handle)); + + Assert.True(KernelEventQueueCompatExports.EnqueueEvent(handle, UserEvent(1))); + Assert.Equal(0, parking.WakeCalls); + + DeleteEqueue(ctx, handle); + } + finally + { + GuestThreadExecution.Scheduler = previous; + } + } + + [Fact] + public void SignalWithWaiter_ScansAndTheCountReturnsToZeroOnResume() + { + var (scheduler, parking) = CreateScheduler(); + var previous = GuestThreadExecution.Scheduler; + GuestThreadExecution.Scheduler = scheduler; + try + { + var (_, ctx, handle) = CreateEqueue(); + var waiter = StageGuestWait(handle, threadHandle: 0x801, out _); + Assert.NotNull(waiter); + Assert.Equal(1, KernelEventQueueCompatExports.BlockedWaiterCountForTest(handle)); + + Assert.True(KernelEventQueueCompatExports.EnqueueEvent(handle, UserEvent(2))); + Assert.True(parking.WakeCalls >= 1); + + Assert.True(waiter!.TryWake()); + Assert.Equal((int)OrbisGen2Result.ORBIS_GEN2_OK, waiter.Resume()); + Assert.Equal(0, KernelEventQueueCompatExports.BlockedWaiterCountForTest(handle)); + + DeleteEqueue(ctx, handle); + } + finally + { + GuestThreadExecution.Scheduler = previous; + } + } + + [Fact] + public void DeleteWithWaiter_StillWakesIt() + { + var (scheduler, parking) = CreateScheduler(); + var previous = GuestThreadExecution.Scheduler; + GuestThreadExecution.Scheduler = scheduler; + try + { + var (_, ctx, handle) = CreateEqueue(); + var waiter = StageGuestWait(handle, threadHandle: 0x802, out _); + Assert.NotNull(waiter); + + DeleteEqueue(ctx, handle); + Assert.True(parking.WakeCalls >= 1); + Assert.True(waiter!.TryWake()); + Assert.Equal((int)OrbisGen2Result.ORBIS_GEN2_ERROR_DELETED, waiter.Resume()); + } + finally + { + GuestThreadExecution.Scheduler = previous; + } + } + + // Models the scheduler's contract: a blocking waiter is parked under one gate and re-checked with + // TryWake after parking; a wake scans the parked waiters under the same gate. The signaller enqueues + // while the waiter is blocking, with varying offsets. A lost wake leaves the waiter parked forever. + [Fact] + public void ConcurrentSignalAndWait_NeverLosesAWake() + { + const int Iterations = 3000; + var (scheduler, parking) = CreateScheduler(); + var previous = GuestThreadExecution.Scheduler; + GuestThreadExecution.Scheduler = scheduler; + try + { + var (_, ctx, firstHandle) = CreateEqueue(); + var handles = new ulong[Iterations]; + handles[0] = firstHandle; + for (var i = 1; i < Iterations; i++) + { + handles[i] = CreateEqueue().Handle; + } + + using var barrier = new Barrier(2); + using var stop = new CancellationTokenSource(); + Exception? signalerFailure = null; + var signaler = new Thread(() => + { + try + { + for (var i = 0; i < Iterations; i++) + { + barrier.SignalAndWait(stop.Token); + Spin(i * 5 % 97); + Assert.True(KernelEventQueueCompatExports.EnqueueEvent(handles[i], UserEvent((ulong)i))); + } + } + catch (OperationCanceledException) when (stop.IsCancellationRequested) + { + } + catch (Exception ex) + { + signalerFailure = ex; + stop.Cancel(); + } + }) + { IsBackground = true }; + signaler.Start(); + + var lost = 0; + var delivered = 0; + try + { + for (var i = 0; i < Iterations; i++) + { + barrier.SignalAndWait(stop.Token); + Spin(i * 3 % 89); + var waiter = StageGuestWait(handles[i], threadHandle: 0x900, out var immediate); + if (waiter is null) + { + Assert.Equal((int)OrbisGen2Result.ORBIS_GEN2_OK, immediate); + delivered++; + continue; + } + + if (!parking.Park(waiter, handles[i], TimeSpan.FromSeconds(3))) + { + lost++; + break; + } + + delivered++; + Assert.Equal(0, KernelEventQueueCompatExports.BlockedWaiterCountForTest(handles[i])); + } + } + finally + { + // A lost wake must fail the test, not leave the signaller at the next barrier forever. + stop.Cancel(); + Assert.True(signaler.Join(TimeSpan.FromSeconds(5)), "The signal worker did not stop."); + foreach (var handle in handles) + DeleteEqueue(ctx, handle); + } + + Assert.Null(signalerFailure); + Assert.Equal(0, lost); + Assert.Equal(Iterations, delivered); + } + finally + { + GuestThreadExecution.Scheduler = previous; + } + } + + private static void Spin(int iterations) + { + for (var i = 0; i < iterations; i++) + { + Thread.SpinWait(20); + } + } + + private static (IGuestThreadScheduler Scheduler, ParkingScheduler Parking) CreateScheduler() + { + var proxy = DispatchProxy.Create(); + return (proxy, (ParkingScheduler)(object)proxy); + } + + private static (FakeCpuMemory Memory, CpuContext Context, ulong Handle) CreateEqueue() + { + var memory = new FakeCpuMemory(BaseAddress, 0x1000); + var ctx = new CpuContext(memory, Generation.Gen5); + ctx[CpuRegister.Rdi] = HandleAddress; + Assert.Equal( + (int)OrbisGen2Result.ORBIS_GEN2_OK, + KernelEventQueueCompatExports.KernelCreateEqueue(ctx)); + Span bytes = stackalloc byte[sizeof(ulong)]; + Assert.True(memory.TryRead(HandleAddress, bytes)); + return (memory, ctx, System.Buffers.Binary.BinaryPrimitives.ReadUInt64LittleEndian(bytes)); + } + + private static void DeleteEqueue(CpuContext ctx, ulong handle) + { + ctx[CpuRegister.Rdi] = handle; + Assert.Equal( + (int)OrbisGen2Result.ORBIS_GEN2_OK, + KernelEventQueueCompatExports.KernelDeleteEqueue(ctx)); + } + + // Returns the blocking waiter, or null when the wait completed without blocking (result in immediate). + private static IGuestThreadBlockWaiter? StageGuestWait(ulong handle, ulong threadHandle, out int immediate) + { + var memory = new FakeCpuMemory(BaseAddress, 0x1000); + var ctx = new CpuContext(memory, Generation.Gen5); + var previousThread = GuestThreadExecution.EnterGuestThread(threadHandle); + var previousFrame = GuestThreadExecution.EnterImportCallFrame( + returnRip: 0x1_0000 + threadHandle, + resumeRsp: 0x2_0000 + threadHandle, + returnSlotAddress: 0x3_0000 + threadHandle); + try + { + ctx[CpuRegister.Rdi] = handle; + ctx[CpuRegister.Rsi] = EventsAddress; + ctx[CpuRegister.Rdx] = 1; + ctx[CpuRegister.Rcx] = OutCountAddress; + ctx[CpuRegister.R8] = 0; + immediate = KernelEventQueueCompatExports.KernelWaitEqueue(ctx); + if (!GuestThreadExecution.TryConsumeCurrentThreadBlock( + out _, + out _, + out var hasContinuation, + out _, + out var waiter, + out _)) + { + return null; + } + + Assert.True(hasContinuation); + return Assert.IsAssignableFrom(waiter); + } + finally + { + GuestThreadExecution.RestoreImportCallFrame(previousFrame); + GuestThreadExecution.RestoreGuestThread(previousThread); + } + } + + public class ParkingScheduler : DispatchProxy + { + private readonly object _gate = new(); + private readonly Dictionary _parked = new(); + private int _wakeCalls; + + public int WakeCalls => Volatile.Read(ref _wakeCalls); + + // Parks the waiter the way the scheduler does: register, then re-check TryWake under the gate. + public bool Park(IGuestThreadBlockWaiter waiter, ulong handle, TimeSpan timeout) + { + using var woken = new ManualResetEventSlim(false); + lock (_gate) + { + if (waiter.TryWake()) + { + woken.Set(); + } + else + { + _parked[handle] = (waiter, woken); + } + } + + if (!woken.Wait(timeout)) + { + // The signaller must not retain an event that this method is about to dispose. + lock (_gate) + _parked.Remove(handle); + return false; + } + + Assert.Equal((int)OrbisGen2Result.ORBIS_GEN2_OK, waiter.Resume()); + return true; + } + + protected override object? Invoke(MethodInfo? targetMethod, object?[]? args) + { + if (targetMethod?.Name == nameof(IGuestThreadScheduler.WakeBlockedThreads)) + { + Interlocked.Increment(ref _wakeCalls); + var woke = 0; + lock (_gate) + { + foreach (var (handle, entry) in _parked.ToArray()) + { + if (entry.Waiter.TryWake()) + { + _parked.Remove(handle); + entry.Woken.Set(); + woke++; + } + } + } + + return woke; + } + + var returnType = targetMethod?.ReturnType; + return returnType is null || returnType == typeof(void) || !returnType.IsValueType + ? null + : Activator.CreateInstance(returnType); + } + } +} diff --git a/tests/SharpEmu.Libs.Tests/Loader/RosettaVectorStorePatchTests.cs b/tests/SharpEmu.Libs.Tests/Loader/RosettaVectorStorePatchTests.cs index b175ed0b1..196725e4b 100644 --- a/tests/SharpEmu.Libs.Tests/Loader/RosettaVectorStorePatchTests.cs +++ b/tests/SharpEmu.Libs.Tests/Loader/RosettaVectorStorePatchTests.cs @@ -53,8 +53,10 @@ public void SplitStoresPreserveAddressAndSourceAfterRelocation(byte[] instructio Assert.Equal(unchecked(originalInstruction.MemoryDisplacement64 + 16), upperStore.MemoryDisplacement64); } - [Fact] - public unsafe void RosettaSplitsStoresWithoutRedZoneUseAndPreservesAllBytes() + [Theory] + [InlineData(false, 1)] + [InlineData(true, 2)] + public unsafe void RosettaSplitsStoresWithoutRedZoneUseAndPreservesAllBytes(bool separateSpans, int expectedSites) { if (!RosettaVectorStorePatch.IsRequired) return; @@ -71,6 +73,10 @@ public unsafe void RosettaSplitsStoresWithoutRedZoneUseAndPreservesAllBytes() 0xC5, 0xFE, 0x7F, 0x02, 0xC5, 0xF8, 0x77, 0xC3, ]; + // A branch target closes the first span; straight-line stores share one trampoline. + // Both forms must expand both vector stores and preserve all 32 bytes. + if (separateSpans) + function = [.. function[..12], 0xEB, 0x00, .. function[12..]]; Assert.True(memory.TryWrite(imageBase, function)); var exceptionFrameHeader = new byte[32]; exceptionFrameHeader[0] = 1; @@ -87,7 +93,7 @@ public unsafe void RosettaSplitsStoresWithoutRedZoneUseAndPreservesAllBytes() var result = GuestRedZonePatcher.Patch(memory, memory, programHeaders, imageBase, imageSize); Assert.Equal(0, result.RedZoneFunctions); Assert.Equal(2, result.VectorStoreCount); - Assert.Equal(2, result.PatchedSites); + Assert.Equal(expectedSites, result.PatchedSites); Assert.Equal(0, result.FailedSites); byte* source = stackalloc byte[32]; diff --git a/tests/SharpEmu.Libs.Tests/Memory/GuestMemory/GuestViewTableTests.cs b/tests/SharpEmu.Libs.Tests/Memory/GuestMemory/GuestViewTableTests.cs index 61145a2ce..dd634104c 100644 --- a/tests/SharpEmu.Libs.Tests/Memory/GuestMemory/GuestViewTableTests.cs +++ b/tests/SharpEmu.Libs.Tests/Memory/GuestMemory/GuestViewTableTests.cs @@ -215,6 +215,79 @@ public void TranslatorCacheSurvivesProtectionChangesButNotUnmaps() Assert.False(translator.TryTranslate(Base + 5 * Block, 1, out _)); } + [Fact] + public void TranslatorCachesViewsAtAPowerOfTwoStrideAndMissesWithoutAllocating() + { + // Every view starts a 64 KiB block whose index is a multiple of 64: one slot each, not one between them. + const int Views = 48; + const ulong Stride = 4UL << 20; + using var table = NewTable(capacity: 128); + for (var view = 0; view < Views; view++) + table.Map(Base + (ulong)view * Stride, Block, (ulong)view * 2 * Block, HostPageProtection.ReadWrite); + var translator = new GuestMirrorTranslator(table.Region.Address, table.Region.Size, MirrorBase, MirrorBytes) + { + CountCacheAccesses = true, + }; + + // The thread's first translation sets up its thread-static storage: not a miss's cost. + _ = new GuestMirrorTranslator(table.Region.Address, table.Region.Size, MirrorBase, MirrorBytes).TryTranslate(Base, 8, out _); + // Measure the translator only. xUnit's successful value assertions allocate too; + // keep every result check, but run it after the allocation sample. + Span translated = stackalloc bool[Views]; + Span mirrors = stackalloc ulong[Views]; + var allocated = GC.GetAllocatedBytesForCurrentThread(); + for (var view = 0; view < Views; view++) + translated[view] = translator.TryTranslate(Base + (ulong)view * Stride + 0x40, 8, out mirrors[view]); + var translationBytes = GC.GetAllocatedBytesForCurrentThread() - allocated; + + Assert.Equal(0, translationBytes); + for (var view = 0; view < Views; view++) + { + Assert.True(translated[view]); + Assert.Equal(MirrorBase + (ulong)view * 2 * Block + 0x40, mirrors[view]); + } + Assert.Equal(Views, translator.CacheMisses); + for (var view = 0; view < Views; view++) + Assert.True(translator.TryTranslate(Base + (ulong)view * Stride + 0x80, 8, out _)); + Assert.Equal(Views, translator.CacheMisses); + Assert.Equal(Views, translator.CacheHits); + } + + [Fact] + public void TranslatorServesTheOtherBlocksOfARangeFromTheThreadsLastRead() + { + using var table = NewTable(); + table.Map(Base, 256 * Block, 5 * Block, HostPageProtection.ReadWrite); + var translator = new GuestMirrorTranslator(table.Region.Address, table.Region.Size, MirrorBase, MirrorBytes) + { + CountCacheAccesses = true, + }; + + for (ulong block = 0; block < 256; block++) + { + Assert.True(translator.TryTranslate(Base + block * Block + 0x10, 8, out var mirror)); + Assert.Equal(MirrorBase + (5 + block) * Block + 0x10, mirror); + } + + Assert.Equal(1, translator.CacheMisses); + + // Another thread has its own last read, and an unmap retires both it and the slots. + long otherMisses = 0; + var other = new Thread(() => + { + var before = translator.CacheMisses; + for (ulong block = 0; block < 256; block++) translator.TryTranslate(Base + block * Block, 8, out _); + otherMisses = translator.CacheMisses - before; + }); + other.Start(); + other.Join(); + Assert.Equal(0, otherMisses); + table.Unmap(Base, Block); + Assert.False(translator.TryTranslate(Base, 8, out _)); + Assert.True(translator.TryTranslate(Base + 7 * Block, 8, out var moved)); + Assert.Equal(MirrorBase + 12 * Block, moved); + } + [Fact] public void TranslatorStopsAtTheMirrorAndRefusesARegionWithoutATable() { diff --git a/tests/SharpEmu.Libs.Tests/Pad/HostWindowInputTests.cs b/tests/SharpEmu.Libs.Tests/Pad/HostWindowInputTests.cs index 32a4819f1..938404c3d 100644 --- a/tests/SharpEmu.Libs.Tests/Pad/HostWindowInputTests.cs +++ b/tests/SharpEmu.Libs.Tests/Pad/HostWindowInputTests.cs @@ -9,6 +9,50 @@ namespace SharpEmu.Libs.Tests.Pad; public sealed class HostWindowInputTests { + [Fact] + public void WaitForChangeReturnsAtOnceWhenTheVersionAlreadyMoved() + { + var seen = HostWindowInput.Version; + HostWindowInput.SetKey(0x41, true); + try + { + Assert.True(HostWindowInput.WaitForChange(seen, TimeSpan.FromSeconds(30))); + } + finally + { + HostWindowInput.SetKey(0x41, false); + } + } + + [Fact] + public void WaitForChangeTimesOutWithoutAChangeAndWakesOnOne() + { + var seen = HostWindowInput.Version; + Assert.False(HostWindowInput.WaitForChange(seen, TimeSpan.FromMilliseconds(20))); + + using var waiting = new ManualResetEventSlim(); + var woke = false; + var thread = new Thread(() => + { + waiting.Set(); + woke = HostWindowInput.WaitForChange(seen, TimeSpan.FromSeconds(30)); + }); + thread.Start(); + Assert.True(waiting.Wait(TimeSpan.FromSeconds(10))); + var started = System.Diagnostics.Stopwatch.StartNew(); + HostWindowInput.SetKey(0x42, true); + try + { + Assert.True(thread.Join(TimeSpan.FromSeconds(10))); + Assert.True(woke); + Assert.True(started.Elapsed < TimeSpan.FromSeconds(10)); + } + finally + { + HostWindowInput.SetKey(0x42, false); + } + } + [Fact] public void ConnectPublishesWindowKeyboardState() { diff --git a/tests/SharpEmu.ShaderCompiler.Metal.Tests/Gen5MslMrtzExportTests.cs b/tests/SharpEmu.ShaderCompiler.Metal.Tests/Gen5MslMrtzExportTests.cs index 20c8cb3fd..e3d23fe3a 100644 --- a/tests/SharpEmu.ShaderCompiler.Metal.Tests/Gen5MslMrtzExportTests.cs +++ b/tests/SharpEmu.ShaderCompiler.Metal.Tests/Gen5MslMrtzExportTests.cs @@ -107,11 +107,35 @@ public void SampleMaskExportNeedsTheControlBit() Assert.DoesNotContain("[[sample_mask]]", shader.Source, StringComparison.Ordinal); } - private static Gen5MslShader Compile(uint[] words, Gen5ColorExportFormat? format, bool stencil, bool sampleMask) + // The guest asked for the depth test before the shader, but a depth, stencil or coverage export decides + // the fragment after the shader ran, and Metal rejects depth or stencil outputs with + // [[early_fragment_tests]]: no export may get the attribute; without one the request's flag does. + [Theory] + [InlineData(true, false, false, false)] + [InlineData(false, true, false, false)] + [InlineData(false, false, true, false)] + [InlineData(true, true, true, false)] + [InlineData(false, false, false, true)] + public void AnEarlyDepthRequestGetsTheAttributeOnlyWithoutDepthStencilOrCoverageExports( + bool depth, bool stencil, bool sampleMask, bool attribute) + { + var shader = Compile(Program([Depth, 0x1A5, 0xFFFF, 0], 0xF, 0), Gen5ColorExportFormat.Abgr32, stencil, sampleMask, + depth, earlyDepth: true); + Assert.Equal(attribute, shader.Source.Contains("[[early_fragment_tests]]", StringComparison.Ordinal)); + if (Environment.GetEnvironmentVariable("SHARPEMU_TEST_REQUIRE_DEVICE") == "1") + { + Assert.True(MetalNative.TryCompileLibrary(shader.Source, out var library, out var error), $"{error}\n{shader.Source}"); + MetalNative.ReleaseCollisionLibrary(library); + } + } + + private static Gen5MslShader Compile(uint[] words, Gen5ColorExportFormat? format, bool stencil, bool sampleMask, + bool depth = false, bool earlyDepth = false) { var program = Gen5ComputeFixtures.DecodeOrThrow(new Gen5ComputeFixture("pixel-mrtz-export", words, 0, 0)); var request = Gen5ComputeFixtures.RequestOrThrow(program, ShaderStage.Pixel, pixelDepthExportFormat: format, - pixelStencilExportEnable: stencil, pixelSampleMaskExportEnable: sampleMask); + pixelDepthExportEnable: depth, pixelStencilExportEnable: stencil, pixelSampleMaskExportEnable: sampleMask, + pixelEarlyDepth: earlyDepth); Assert.True(Gen5MslTranslator.TryCompileProgram(request, out var shader, out var error), error); return shader; } diff --git a/tests/SharpEmu.ShaderCompiler.Metal.Tests/Goldens/exec-store-layout.msl b/tests/SharpEmu.ShaderCompiler.Metal.Tests/Goldens/exec-store-layout.msl index 06b33903d..2bb658f3f 100644 --- a/tests/SharpEmu.ShaderCompiler.Metal.Tests/Goldens/exec-store-layout.msl +++ b/tests/SharpEmu.ShaderCompiler.Metal.Tests/Goldens/exec-store-layout.msl @@ -279,6 +279,32 @@ static inline uint sharpemu_load_dword(device uint* b, uint bytes, uint addr) return index < (bytes >> 2) ? b[index] : 0u; } +// Up to four consecutive raw dwords of an S_BUFFER_LOAD/BUFFER_LOAD_DWORDXn, each what +// sharpemu_load_dword reads at addr + 4 * i. A run that lies inside the binding is one +// bounds check and adjacent loads; one that crosses the end reads dword by dword. +template +static inline uint4 sharpemu_load_dwords(device uint* b, uint bytes, uint addr) +{ + uint4 result = uint4(0u); + uint index = addr >> 2; + uint words = bytes >> 2; + if (index < words && (uint)N <= words - index) + { + for (int i = 0; i < N; i++) + { + result[i] = b[index + (uint)i]; + } + } + else + { + for (int i = 0; i < N; i++) + { + result[i] = sharpemu_load_dword(b, bytes, addr + (uint)(4 * i)); + } + } + return result; +} + // A dword a format access may straddle, assembled from the two it spans. Only the // format path needs this: a descriptor's stride is a byte count, so a component can // start at any byte. @@ -561,7 +587,7 @@ static inline void sharpemu_note_step_cap(device atomic_uint* report) } } -kernel void gen5_cs( +[[max_total_threads_per_threadgroup(32)]] kernel void gen5_cs( constant Gen5StageResources& sharpemu_resources [[buffer(0)]], constant uint* sharpemu_push_data [[buffer(1)]], uint3 sharpemu_local_id [[thread_position_in_threadgroup]], diff --git a/tests/SharpEmu.ShaderCompiler.Metal.Tests/Goldens/fmac-layout.msl b/tests/SharpEmu.ShaderCompiler.Metal.Tests/Goldens/fmac-layout.msl index cc2105fc0..cf5dde091 100644 --- a/tests/SharpEmu.ShaderCompiler.Metal.Tests/Goldens/fmac-layout.msl +++ b/tests/SharpEmu.ShaderCompiler.Metal.Tests/Goldens/fmac-layout.msl @@ -277,6 +277,32 @@ static inline uint sharpemu_load_dword(device uint* b, uint bytes, uint addr) return index < (bytes >> 2) ? b[index] : 0u; } +// Up to four consecutive raw dwords of an S_BUFFER_LOAD/BUFFER_LOAD_DWORDXn, each what +// sharpemu_load_dword reads at addr + 4 * i. A run that lies inside the binding is one +// bounds check and adjacent loads; one that crosses the end reads dword by dword. +template +static inline uint4 sharpemu_load_dwords(device uint* b, uint bytes, uint addr) +{ + uint4 result = uint4(0u); + uint index = addr >> 2; + uint words = bytes >> 2; + if (index < words && (uint)N <= words - index) + { + for (int i = 0; i < N; i++) + { + result[i] = b[index + (uint)i]; + } + } + else + { + for (int i = 0; i < N; i++) + { + result[i] = sharpemu_load_dword(b, bytes, addr + (uint)(4 * i)); + } + } + return result; +} + // A dword a format access may straddle, assembled from the two it spans. Only the // format path needs this: a descriptor's stride is a byte count, so a component can // start at any byte. @@ -559,7 +585,7 @@ static inline void sharpemu_note_step_cap(device atomic_uint* report) } } -kernel void gen5_cs( +[[max_total_threads_per_threadgroup(32)]] kernel void gen5_cs( constant Gen5StageResources& sharpemu_resources [[buffer(0)]], constant uint* sharpemu_push_data [[buffer(1)]], uint3 sharpemu_local_id [[thread_position_in_threadgroup]], diff --git a/tests/SharpEmu.ShaderCompiler.Metal.Tests/Goldens/lds-layout.msl b/tests/SharpEmu.ShaderCompiler.Metal.Tests/Goldens/lds-layout.msl index 91fd93d02..b55c4b5b9 100644 --- a/tests/SharpEmu.ShaderCompiler.Metal.Tests/Goldens/lds-layout.msl +++ b/tests/SharpEmu.ShaderCompiler.Metal.Tests/Goldens/lds-layout.msl @@ -279,6 +279,32 @@ static inline uint sharpemu_load_dword(device uint* b, uint bytes, uint addr) return index < (bytes >> 2) ? b[index] : 0u; } +// Up to four consecutive raw dwords of an S_BUFFER_LOAD/BUFFER_LOAD_DWORDXn, each what +// sharpemu_load_dword reads at addr + 4 * i. A run that lies inside the binding is one +// bounds check and adjacent loads; one that crosses the end reads dword by dword. +template +static inline uint4 sharpemu_load_dwords(device uint* b, uint bytes, uint addr) +{ + uint4 result = uint4(0u); + uint index = addr >> 2; + uint words = bytes >> 2; + if (index < words && (uint)N <= words - index) + { + for (int i = 0; i < N; i++) + { + result[i] = b[index + (uint)i]; + } + } + else + { + for (int i = 0; i < N; i++) + { + result[i] = sharpemu_load_dword(b, bytes, addr + (uint)(4 * i)); + } + } + return result; +} + // A dword a format access may straddle, assembled from the two it spans. Only the // format path needs this: a descriptor's stride is a byte count, so a component can // start at any byte. @@ -561,7 +587,7 @@ static inline void sharpemu_note_step_cap(device atomic_uint* report) } } -kernel void gen5_cs( +[[max_total_threads_per_threadgroup(32)]] kernel void gen5_cs( constant Gen5StageResources& sharpemu_resources [[buffer(0)]], constant uint* sharpemu_push_data [[buffer(1)]], uint3 sharpemu_local_id [[thread_position_in_threadgroup]], diff --git a/tests/SharpEmu.ShaderCompiler.Metal.Tests/Goldens/loop-layout.msl b/tests/SharpEmu.ShaderCompiler.Metal.Tests/Goldens/loop-layout.msl index 166c51f1e..9f108ba94 100644 --- a/tests/SharpEmu.ShaderCompiler.Metal.Tests/Goldens/loop-layout.msl +++ b/tests/SharpEmu.ShaderCompiler.Metal.Tests/Goldens/loop-layout.msl @@ -279,6 +279,32 @@ static inline uint sharpemu_load_dword(device uint* b, uint bytes, uint addr) return index < (bytes >> 2) ? b[index] : 0u; } +// Up to four consecutive raw dwords of an S_BUFFER_LOAD/BUFFER_LOAD_DWORDXn, each what +// sharpemu_load_dword reads at addr + 4 * i. A run that lies inside the binding is one +// bounds check and adjacent loads; one that crosses the end reads dword by dword. +template +static inline uint4 sharpemu_load_dwords(device uint* b, uint bytes, uint addr) +{ + uint4 result = uint4(0u); + uint index = addr >> 2; + uint words = bytes >> 2; + if (index < words && (uint)N <= words - index) + { + for (int i = 0; i < N; i++) + { + result[i] = b[index + (uint)i]; + } + } + else + { + for (int i = 0; i < N; i++) + { + result[i] = sharpemu_load_dword(b, bytes, addr + (uint)(4 * i)); + } + } + return result; +} + // A dword a format access may straddle, assembled from the two it spans. Only the // format path needs this: a descriptor's stride is a byte count, so a component can // start at any byte. @@ -561,7 +587,7 @@ static inline void sharpemu_note_step_cap(device atomic_uint* report) } } -kernel void gen5_cs( +[[max_total_threads_per_threadgroup(32)]] kernel void gen5_cs( constant Gen5StageResources& sharpemu_resources [[buffer(0)]], constant uint* sharpemu_push_data [[buffer(1)]], uint3 sharpemu_local_id [[thread_position_in_threadgroup]], diff --git a/tests/SharpEmu.ShaderCompiler.Metal.Tests/Goldens/muls-layout.msl b/tests/SharpEmu.ShaderCompiler.Metal.Tests/Goldens/muls-layout.msl index 51bc14335..1a83885c1 100644 --- a/tests/SharpEmu.ShaderCompiler.Metal.Tests/Goldens/muls-layout.msl +++ b/tests/SharpEmu.ShaderCompiler.Metal.Tests/Goldens/muls-layout.msl @@ -277,6 +277,32 @@ static inline uint sharpemu_load_dword(device uint* b, uint bytes, uint addr) return index < (bytes >> 2) ? b[index] : 0u; } +// Up to four consecutive raw dwords of an S_BUFFER_LOAD/BUFFER_LOAD_DWORDXn, each what +// sharpemu_load_dword reads at addr + 4 * i. A run that lies inside the binding is one +// bounds check and adjacent loads; one that crosses the end reads dword by dword. +template +static inline uint4 sharpemu_load_dwords(device uint* b, uint bytes, uint addr) +{ + uint4 result = uint4(0u); + uint index = addr >> 2; + uint words = bytes >> 2; + if (index < words && (uint)N <= words - index) + { + for (int i = 0; i < N; i++) + { + result[i] = b[index + (uint)i]; + } + } + else + { + for (int i = 0; i < N; i++) + { + result[i] = sharpemu_load_dword(b, bytes, addr + (uint)(4 * i)); + } + } + return result; +} + // A dword a format access may straddle, assembled from the two it spans. Only the // format path needs this: a descriptor's stride is a byte count, so a component can // start at any byte. @@ -559,7 +585,7 @@ static inline void sharpemu_note_step_cap(device atomic_uint* report) } } -kernel void gen5_cs( +[[max_total_threads_per_threadgroup(32)]] kernel void gen5_cs( constant Gen5StageResources& sharpemu_resources [[buffer(0)]], constant uint* sharpemu_push_data [[buffer(1)]], uint3 sharpemu_local_id [[thread_position_in_threadgroup]], diff --git a/tests/SharpEmu.ShaderCompiler.Metal.Tests/Goldens/pixel-layout.msl b/tests/SharpEmu.ShaderCompiler.Metal.Tests/Goldens/pixel-layout.msl index 02c5e10d5..0001548a8 100644 --- a/tests/SharpEmu.ShaderCompiler.Metal.Tests/Goldens/pixel-layout.msl +++ b/tests/SharpEmu.ShaderCompiler.Metal.Tests/Goldens/pixel-layout.msl @@ -277,6 +277,32 @@ static inline uint sharpemu_load_dword(device uint* b, uint bytes, uint addr) return index < (bytes >> 2) ? b[index] : 0u; } +// Up to four consecutive raw dwords of an S_BUFFER_LOAD/BUFFER_LOAD_DWORDXn, each what +// sharpemu_load_dword reads at addr + 4 * i. A run that lies inside the binding is one +// bounds check and adjacent loads; one that crosses the end reads dword by dword. +template +static inline uint4 sharpemu_load_dwords(device uint* b, uint bytes, uint addr) +{ + uint4 result = uint4(0u); + uint index = addr >> 2; + uint words = bytes >> 2; + if (index < words && (uint)N <= words - index) + { + for (int i = 0; i < N; i++) + { + result[i] = b[index + (uint)i]; + } + } + else + { + for (int i = 0; i < N; i++) + { + result[i] = sharpemu_load_dword(b, bytes, addr + (uint)(4 * i)); + } + } + return result; +} + // A dword a format access may straddle, assembled from the two it spans. Only the // format path needs this: a descriptor's stride is a byte count, so a component can // start at any byte. diff --git a/tests/SharpEmu.ShaderCompiler.Metal.Tests/Goldens/typed-load-layout.msl b/tests/SharpEmu.ShaderCompiler.Metal.Tests/Goldens/typed-load-layout.msl index 716bd9594..9d1385081 100644 --- a/tests/SharpEmu.ShaderCompiler.Metal.Tests/Goldens/typed-load-layout.msl +++ b/tests/SharpEmu.ShaderCompiler.Metal.Tests/Goldens/typed-load-layout.msl @@ -451,6 +451,32 @@ static inline uint sharpemu_load_dword(device uint* b, uint bytes, uint addr) return index < (bytes >> 2) ? b[index] : 0u; } +// Up to four consecutive raw dwords of an S_BUFFER_LOAD/BUFFER_LOAD_DWORDXn, each what +// sharpemu_load_dword reads at addr + 4 * i. A run that lies inside the binding is one +// bounds check and adjacent loads; one that crosses the end reads dword by dword. +template +static inline uint4 sharpemu_load_dwords(device uint* b, uint bytes, uint addr) +{ + uint4 result = uint4(0u); + uint index = addr >> 2; + uint words = bytes >> 2; + if (index < words && (uint)N <= words - index) + { + for (int i = 0; i < N; i++) + { + result[i] = b[index + (uint)i]; + } + } + else + { + for (int i = 0; i < N; i++) + { + result[i] = sharpemu_load_dword(b, bytes, addr + (uint)(4 * i)); + } + } + return result; +} + // A dword a format access may straddle, assembled from the two it spans. Only the // format path needs this: a descriptor's stride is a byte count, so a component can // start at any byte. @@ -733,7 +759,7 @@ static inline void sharpemu_note_step_cap(device atomic_uint* report) } } -kernel void gen5_cs( +[[max_total_threads_per_threadgroup(32)]] kernel void gen5_cs( constant Gen5StageResources& sharpemu_resources [[buffer(0)]], constant uint* sharpemu_push_data [[buffer(1)]], uint3 sharpemu_local_id [[thread_position_in_threadgroup]], diff --git a/tests/SharpEmu.ShaderCompiler.Metal.Tests/Goldens/typed-store-layout.msl b/tests/SharpEmu.ShaderCompiler.Metal.Tests/Goldens/typed-store-layout.msl index d036b4a60..404d02c71 100644 --- a/tests/SharpEmu.ShaderCompiler.Metal.Tests/Goldens/typed-store-layout.msl +++ b/tests/SharpEmu.ShaderCompiler.Metal.Tests/Goldens/typed-store-layout.msl @@ -451,6 +451,32 @@ static inline uint sharpemu_load_dword(device uint* b, uint bytes, uint addr) return index < (bytes >> 2) ? b[index] : 0u; } +// Up to four consecutive raw dwords of an S_BUFFER_LOAD/BUFFER_LOAD_DWORDXn, each what +// sharpemu_load_dword reads at addr + 4 * i. A run that lies inside the binding is one +// bounds check and adjacent loads; one that crosses the end reads dword by dword. +template +static inline uint4 sharpemu_load_dwords(device uint* b, uint bytes, uint addr) +{ + uint4 result = uint4(0u); + uint index = addr >> 2; + uint words = bytes >> 2; + if (index < words && (uint)N <= words - index) + { + for (int i = 0; i < N; i++) + { + result[i] = b[index + (uint)i]; + } + } + else + { + for (int i = 0; i < N; i++) + { + result[i] = sharpemu_load_dword(b, bytes, addr + (uint)(4 * i)); + } + } + return result; +} + // A dword a format access may straddle, assembled from the two it spans. Only the // format path needs this: a descriptor's stride is a byte count, so a component can // start at any byte. @@ -733,7 +759,7 @@ static inline void sharpemu_note_step_cap(device atomic_uint* report) } } -kernel void gen5_cs( +[[max_total_threads_per_threadgroup(32)]] kernel void gen5_cs( constant Gen5StageResources& sharpemu_resources [[buffer(0)]], constant uint* sharpemu_push_data [[buffer(1)]], uint3 sharpemu_local_id [[thread_position_in_threadgroup]], diff --git a/tests/SharpEmu.ShaderCompiler.Metal.Tests/Goldens/vertex-layout.msl b/tests/SharpEmu.ShaderCompiler.Metal.Tests/Goldens/vertex-layout.msl index d6670ccd7..2d6178478 100644 --- a/tests/SharpEmu.ShaderCompiler.Metal.Tests/Goldens/vertex-layout.msl +++ b/tests/SharpEmu.ShaderCompiler.Metal.Tests/Goldens/vertex-layout.msl @@ -277,6 +277,32 @@ static inline uint sharpemu_load_dword(device uint* b, uint bytes, uint addr) return index < (bytes >> 2) ? b[index] : 0u; } +// Up to four consecutive raw dwords of an S_BUFFER_LOAD/BUFFER_LOAD_DWORDXn, each what +// sharpemu_load_dword reads at addr + 4 * i. A run that lies inside the binding is one +// bounds check and adjacent loads; one that crosses the end reads dword by dword. +template +static inline uint4 sharpemu_load_dwords(device uint* b, uint bytes, uint addr) +{ + uint4 result = uint4(0u); + uint index = addr >> 2; + uint words = bytes >> 2; + if (index < words && (uint)N <= words - index) + { + for (int i = 0; i < N; i++) + { + result[i] = b[index + (uint)i]; + } + } + else + { + for (int i = 0; i < N; i++) + { + result[i] = sharpemu_load_dword(b, bytes, addr + (uint)(4 * i)); + } + } + return result; +} + // A dword a format access may straddle, assembled from the two it spans. Only the // format path needs this: a descriptor's stride is a byte count, so a component can // start at any byte. diff --git a/tests/SharpEmu.ShaderCompiler.Metal.Tests/MetalNative.Benchmark.cs b/tests/SharpEmu.ShaderCompiler.Metal.Tests/MetalNative.Benchmark.cs index bda5383e5..1017b98ef 100644 --- a/tests/SharpEmu.ShaderCompiler.Metal.Tests/MetalNative.Benchmark.cs +++ b/tests/SharpEmu.ShaderCompiler.Metal.Tests/MetalNative.Benchmark.cs @@ -40,7 +40,8 @@ public static bool TryBenchmark( out BenchmarkResult result, out string error, IReadOnlyList? tables = null, - nuint? mathMode = null) + nuint? mathMode = null, + bool groupSizeIsMultipleOfWidth = false) { result = default; if (!TryCompileLibrary(shader.Source, out var library, out error, mathMode)) @@ -79,15 +80,9 @@ public static bool TryBenchmark( return false; } - nint nsError = 0; - var pipeline = Send( - Device.Value, - Selector("newComputePipelineStateWithFunction:error:"), - function, - ref nsError); + var pipeline = NewBenchmarkPipeline(function, groupSizeIsMultipleOfWidth, out error); if (pipeline == 0) { - error = $"pipeline creation failed: {DescribeError(nsError)}"; return false; } @@ -166,6 +161,114 @@ public static bool TryBenchmark( return true; } + [LibraryImport(ObjCLibrary, EntryPoint = "objc_msgSend")] + private static partial nint SendComputeDescriptor( + nint receiver, nint selector, nint descriptor, nuint options, nint reflection, ref nint error); + + // A compute pipeline the way the emulator makes one, or through a descriptor that + // declares the threadgroup size a multiple of the SIMD width. + private static nint NewBenchmarkPipeline(nint function, bool groupSizeIsMultipleOfWidth, out string error) + { + error = string.Empty; + nint nsError = 0; + nint pipeline; + if (groupSizeIsMultipleOfWidth) + { + var descriptor = Send(Send(objc_getClass("MTLComputePipelineDescriptor"), Selector("alloc")), Selector("init")); + SendVoid(descriptor, Selector("setComputeFunction:"), function); + SendVoidBool(descriptor, Selector("setThreadGroupSizeIsMultipleOfThreadExecutionWidth:"), true); + pipeline = SendComputeDescriptor( + Device.Value, Selector("newComputePipelineStateWithDescriptor:options:reflection:error:"), descriptor, 0, 0, ref nsError); + } + else + { + pipeline = Send(Device.Value, Selector("newComputePipelineStateWithFunction:error:"), function, ref nsError); + } + + if (pipeline == 0) + { + error = $"pipeline creation failed: {DescribeError(nsError)}"; + } + + return pipeline; + } + + /// Compiles and times a hand-written kernel with one buffer at indices 0 and 1, + /// the same way as (warm-up, then the best of five). + public static bool TryBenchmarkKernel( + string source, + string entryPoint, + byte[] bufferContents, + uint threadgroups, + uint threadsPerThreadgroup, + uint iterations, + out double gpuMilliseconds, + out uint maxThreads, + out string error, + bool groupSizeIsMultipleOfWidth = false) + { + gpuMilliseconds = 0; + maxThreads = 0; + if (!TryCompileLibrary(source, out var library, out error)) + { + return false; + } + + var function = Send(library, Selector("newFunctionWithName:"), NsString(entryPoint)); + if (function == 0) + { + error = $"entry point '{entryPoint}' not found"; + return false; + } + + var pipeline = NewBenchmarkPipeline(function, groupSizeIsMultipleOfWidth, out error); + var buffer = SharedBuffer(bufferContents); + var queue = Send(Device.Value, Selector("newCommandQueue")); + if (pipeline == 0 || buffer == 0 || queue == 0) + { + error = error.Length != 0 ? error : "failed to create the buffer or queue"; + return false; + } + + maxThreads = (uint)SendULong(pipeline, Selector("maxTotalThreadsPerThreadgroup")); + var best = double.MaxValue; + for (var pass = 0; pass < 6; pass++) + { + var commandBuffer = Send(queue, Selector("commandBuffer")); + var encoder = Send(commandBuffer, Selector("computeCommandEncoder")); + SendVoid(encoder, Selector("setComputePipelineState:"), pipeline); + SendSetBuffer(encoder, Selector("setBuffer:offset:atIndex:"), buffer, 0, 0); + SendSetBuffer(encoder, Selector("setBuffer:offset:atIndex:"), buffer, 0, 1); + for (var iteration = 0u; iteration < iterations; iteration++) + { + SendDispatch( + encoder, + Selector("dispatchThreadgroups:threadsPerThreadgroup:"), + new MtlSize { Width = threadgroups, Height = 1, Depth = 1 }, + new MtlSize { Width = threadsPerThreadgroup, Height = 1, Depth = 1 }); + } + + SendVoid(encoder, Selector("endEncoding")); + SendVoid(commandBuffer, Selector("commit")); + SendVoid(commandBuffer, Selector("waitUntilCompleted")); + var commandError = Send(commandBuffer, Selector("error")); + if (commandError != 0) + { + error = "GPU execution failed: " + DescribeError(commandError); + return false; + } + + if (pass != 0) + { + best = Math.Min(best, (SendDouble(commandBuffer, Selector("GPUEndTime")) - + SendDouble(commandBuffer, Selector("GPUStartTime"))) * 1000.0); + } + } + + gpuMilliseconds = best; + return true; + } + private static byte[] BuildArgumentBuffer( Gen5MslArgumentLayout layout, ulong bufferAddress, diff --git a/tests/SharpEmu.ShaderCompiler.Metal.Tests/MslLoadWordsTests.cs b/tests/SharpEmu.ShaderCompiler.Metal.Tests/MslLoadWordsTests.cs index e1733d657..c639fb024 100644 --- a/tests/SharpEmu.ShaderCompiler.Metal.Tests/MslLoadWordsTests.cs +++ b/tests/SharpEmu.ShaderCompiler.Metal.Tests/MslLoadWordsTests.cs @@ -59,7 +59,18 @@ kernel void load_words_test(device uint* b [[buffer(0)]], uint t [[thread_positi [Theory] [MemberData(nameof(Cases))] - public void TheRunMatchesPerDwordLoads(uint address, uint bytes) + public void TheRunMatchesPerDwordLoads(uint address, uint bytes) => CheckRun(address, bytes, raw: false); + + [Theory] + [MemberData(nameof(Cases))] + [InlineData(0xFFFF_FFFCu, 64u)] + [InlineData(0xFFFF_FFFFu, 64u)] + [InlineData(1u, 3u)] + [InlineData(3u, 7u)] + public void RawDwordRunMatchesScalarLoadsIncludingAddressWrap(uint address, uint bytes) => + CheckRun(address, bytes, raw: true); + + private static void CheckRun(uint address, uint bytes, bool raw) { if (!MetalNative.IsAvailable) { @@ -70,7 +81,11 @@ public void TheRunMatchesPerDwordLoads(uint address, uint bytes) } var prelude = Gen5NggFixtures.CompileOrThrow(Gen5NggFixtures.CullingTriangle()).Source; - Assert.True(MetalNative.TryCompileLibrary(prelude + Kernel, out var library, out var compileError), compileError); + var kernel = raw + ? Kernel.Replace("sharpemu_load_words", "sharpemu_load_dwords", StringComparison.Ordinal) + .Replace("sharpemu_load_word(", "sharpemu_load_dword(", StringComparison.Ordinal) + : Kernel; + Assert.True(MetalNative.TryCompileLibrary(prelude + kernel, out var library, out var compileError), compileError); var contents = new byte[(OutputBase + OutputStride) * sizeof(uint)]; var words = new uint[contents.Length / sizeof(uint)]; @@ -100,7 +115,13 @@ public void TheRunMatchesPerDwordLoads(uint address, uint bytes) // A dword wholly inside both the binding and the data reads as the bytes that sit there. for (var i = 0; i < 4; i++) { - var at = address + (uint)(4 * i); + var at = unchecked(address + (uint)(4 * i)); + if (raw) + { + Assert.Equal((at >> 2) < (bytes >> 2) ? words[at >> 2] : 0u, output[i]); + continue; + } + if ((address & 3) == 0 && at + 4 <= bytes && at + 4 <= DataWords * 4) { Assert.Equal(words[at / 4], output[i]); diff --git a/tests/SharpEmu.ShaderCompiler.Metal.Tests/MslNativeMinMaxSweepTests.cs b/tests/SharpEmu.ShaderCompiler.Metal.Tests/MslNativeMinMaxSweepTests.cs new file mode 100644 index 000000000..f07dddb42 --- /dev/null +++ b/tests/SharpEmu.ShaderCompiler.Metal.Tests/MslNativeMinMaxSweepTests.cs @@ -0,0 +1,249 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using System.Buffers.Binary; +using SharpEmu.ShaderCompiler.Resources; +using Xunit; +using Xunit.Abstractions; + +namespace SharpEmu.ShaderCompiler.Metal.Tests; + +/// +/// V_MIN/MAX/MIN3/MAX3/MED3_F32 lowered without the bit-level helpers +/// () give the helpers' bits for every +/// pair of a sweep of special values: zeros of both signs, denormals, infinities, quiet and +/// signalling NaNs with payloads, and ordinary numbers. The operands come from a buffer, so +/// nothing is folded at compile time; each variant runs with the switch on and off, in the +/// flushing FLOAT_MODE and in the one that keeps f32 denormals, and the outputs must match. +/// +public sealed class MslNativeMinMaxSweepTests(ITestOutputHelper output) +{ + private const int Lanes = 1024; + + private static readonly uint[] Values = + [ + 0x00000000u, 0x80000000u, 0x3F800000u, 0xBF800000u, 0x3F000000u, 0xBF000000u, 0x7F800000u, 0xFF800000u, + 0x7F7FFFFFu, 0xFF7FFFFFu, 0x00800000u, 0x80800000u, 0x00000001u, 0x80000001u, 0x007FFFFFu, 0x807FFFFFu, + 0x7FC00000u, 0xFFC00000u, 0x7F800001u, 0xFF800002u, 0x7FC12345u, 0x7FFFFFFFu, 0x40000000u, 0xC0000000u, + 0x0DA24260u, 0x8DA24260u, 0x3F800001u, 0x3F7FFFFFu, 0x40490FDBu, 0xC0490FDBu, 0x4B000000u, 0x00400000u, + ]; + + // v1..v3 are read from the buffer, the inserted instruction writes v4 (see Fixture). + public static readonly TheoryData Variants = new() + { + { "v_max_f32_e32 v4, v1, v2", [0x20080501u] }, + { "v_min_f32_e32 v4, v1, v2", [0x1E080501u] }, + { "v_max_f32_e64 v4, v1, v2 clamp", [0xD5108004u, 0x02020501u] }, + { "v_max_f32_e64 v4, -v1, |v2|", [0xD5100204u, 0x22020501u] }, + { "v_max3_f32 v4, v1, v2, v3", [0xD5540004u, 0x040E0501u] }, + { "v_min3_f32 v4, v1, v2, v3", [0xD5510004u, 0x040E0501u] }, + { "v_med3_f32 v4, v1, v2, v3", [0xD5570004u, 0x040E0501u] }, + { "v_max_f32_e32 v4, 0, v1", [0x20080280u] }, + { "v_min_f32_e32 v4, 0, v1", [0x1E080280u] }, + { "v_max_f32_e32 v4, 0x80000000, v1", [0x200802FFu, 0x80000000u] }, + { "v_min_f32_e32 v4, 0x80000000, v1", [0x1E0802FFu, 0x80000000u] }, + { "v_max_f32_e64 v4, neg(0), v1", [0xD5100004u, 0x22020280u] }, + { "v_min_f32_e64 v4, neg(0), v1", [0xD50F0004u, 0x22020280u] }, + { "v_max_f32_e32 v4, 1.0, v1", [0x200802F2u] }, + { "v_min_f32_e32 v4, 1.0, v1", [0x1E0802F2u] }, + { "v_max_f32_e32 v4, -1.0, v1", [0x200802F3u] }, + { "v_min_f32_e32 v4, -1.0, v1", [0x1E0802F3u] }, + { "v_max_f32_e32 v4, 0x7f800000, v1", [0x200802FFu, 0x7F800000u] }, + { "v_min_f32_e32 v4, 0x7f800000, v1", [0x1E0802FFu, 0x7F800000u] }, + { "v_max_f32_e32 v4, 0xff800000, v1", [0x200802FFu, 0xFF800000u] }, + { "v_min_f32_e32 v4, 0xff800000, v1", [0x1E0802FFu, 0xFF800000u] }, + { "v_max_f32_e32 v4, 1, v1", [0x20080281u] }, + { "v_min_f32_e32 v4, 0x80000001, v1", [0x1E0802FFu, 0x80000001u] }, + { "v_max_f32_e32 v4, 0x7f7fffff, v1", [0x200802FFu, 0x7F7FFFFFu] }, + { "v_max_f32_e64 v4, v1, 0", [0xD5100004u, 0x02010101u] }, + { "v_max_f32_e64 v4, |v1|, 0", [0xD5100104u, 0x02010101u] }, + { "v_min_f32_e64 v4, -v1, 0", [0xD50F0004u, 0x22010101u] }, + { "v_max_f32_e64 v4, 0, v1 clamp", [0xD5108004u, 0x02020280u] }, + { "v_max3_f32 v4, v1, v2, 0", [0xD5540004u, 0x02020501u] }, + { "v_min3_f32 v4, v1, 1.0, v2", [0xD5510004u, 0x0409E501u] }, + { "v_max3_f32 v4, 0, v1, -1.0", [0xD5540004u, 0x03CE0280u] }, + { "v_med3_f32 v4, v1, 0, 1.0", [0xD5570004u, 0x03C90101u] }, + { "v_med3_f32 v4, v1, -1.0, 1.0", [0xD5570004u, 0x03C9E701u] }, + { "v_med3_f32 v4, 0, v1, 1.0", [0xD5570004u, 0x03CA0280u] }, + { "v_med3_f32 v4, 1.0, 0, v1", [0xD5570004u, 0x040500F2u] }, + { "v_med3_f32 v4, v1, 0x80000000, 1.0", [0xD5570004u, 0x03C9FF01u, 0x80000000u] }, + { "v_med3_f32 v4, v1, -1.0, 0x80000000", [0xD5570004u, 0x03FDE701u, 0x80000000u] }, + { "v_med3_f32 v4, v1, 1.0, 1.0", [0xD5570004u, 0x03C9E501u] }, + { "v_med3_f32 v4, -v1, 0, 1.0", [0xD5570004u, 0x23C90101u] }, + { "v_med3_f32 v4, v1, 0.5, 2.0", [0xD5570004u, 0x03D1E101u] }, + }; + + private static Gen5ComputeFixture Fixture(uint[] operation) => new( + "minmax-sweep", + [ + 0x34140082u, // v_lshlrev_b32 v10, 2, v0 + 0xE0301000u, 0x8002010Au, // buffer_load_dword v1, v10, s[8:11], 0 offen + 0x4A1614FFu, 0x00001000u, // v_add_nc_u32 v11, 0x1000, v10 + 0xE0301000u, 0x8002020Bu, // buffer_load_dword v2, v11, s[8:11], 0 offen + 0x4A1614FFu, 0x00002000u, // v_add_nc_u32 v11, 0x2000, v10 + 0xE0301000u, 0x8002030Bu, // buffer_load_dword v3, v11, s[8:11], 0 offen + 0xBF8C3F70u, // s_waitcnt vmcnt(0) + .. operation, + 0x4A1614FFu, 0x00003000u, // v_add_nc_u32 v11, 0x3000, v10 + 0xE0701000u, 0x8002040Bu, // buffer_store_dword v4, v11, s[8:11], 0 offen + 0xBF810000u, // s_endpgm + ], + StoreScalarResourceBase: 8, + StoreBackingBytes: 4 * Lanes * sizeof(uint)); + + private static Gen5MslShader Compile(uint[] operation, uint floatMode, bool native) + { + var request = Gen5ComputeFixtures.CreateComputeRequest(Fixture(operation), localSizeX: Lanes, floatMode: floatMode); + Assert.True( + Gen5MslTranslator.TryCompileProgram(request, out var shader, out var error, new Gen5MslCodegenOptions(NativeFloatMinMax: native)), + error); + return shader; + } + + [Theory] + [MemberData(nameof(Variants))] + public void ConstantOperandsAndFlushedPairsSkipTheHelpers(string assembly, uint[] operation) + { + var native = Compile(operation, ShaderCompileRequest.DefaultFloatMode, native: true); + var helper = Compile(operation, ShaderCompileRequest.DefaultFloatMode, native: false); + // The prelude's own definitions and calls are in both sources alike. + static int Calls(string source) => + source.Split("sharpemu_max_f32(").Length + source.Split("sharpemu_min_f32(").Length + + source.Split("sharpemu_med3_f32(").Length; + var keepsHelper = assembly is "v_med3_f32 v4, v1, v2, v3" or "v_med3_f32 v4, v1, 1.0, 1.0" or "v_med3_f32 v4, v1, -1.0, 0x80000000"; + if (keepsHelper) + { + Assert.Equal(Calls(helper.Source), Calls(native.Source)); + } + else + { + Assert.True(Calls(native.Source) < Calls(helper.Source), assembly); + } + } + + [Theory] + [MemberData(nameof(Variants))] + public void NativeLoweringMatchesTheHelpersBitForBit(string assembly, uint[] operation) + { + if (Environment.GetEnvironmentVariable("SHARPEMU_TEST_REQUIRE_DEVICE") != "1") + { + output.WriteLine("Numerical execution requires the exclusive Metal device gate."); + return; + } + + Assert.True(MetalNative.IsAvailable, "The required execution gate needs a Metal device."); + var threeOperands = assembly.Contains("3_f32", StringComparison.Ordinal); + foreach (var floatMode in new[] { ShaderCompileRequest.DefaultFloatMode, 0xF0u }) + { + var native = Compile(operation, floatMode, native: true); + var helper = Compile(operation, floatMode, native: false); + for (var run = 0; run < (threeOperands ? Values.Length : 1); run++) + { + var buffer = new byte[4 * Lanes * sizeof(uint)]; + for (var lane = 0; lane < Lanes; lane++) + { + BinaryPrimitives.WriteUInt32LittleEndian(buffer.AsSpan(lane * 4), Values[lane % Values.Length]); + BinaryPrimitives.WriteUInt32LittleEndian(buffer.AsSpan((Lanes + lane) * 4), Values[lane / Values.Length]); + BinaryPrimitives.WriteUInt32LittleEndian(buffer.AsSpan((2 * Lanes + lane) * 4), Values[(run + lane) % Values.Length]); + } + + var expected = Execute(helper, buffer); + var actual = Execute(native, buffer); + for (var lane = 0; lane < Lanes; lane++) + { + var want = BinaryPrimitives.ReadUInt32LittleEndian(expected.AsSpan((3 * Lanes + lane) * 4)); + var got = BinaryPrimitives.ReadUInt32LittleEndian(actual.AsSpan((3 * Lanes + lane) * 4)); + Assert.True( + want == got, + $"{assembly} float_mode=0x{floatMode:X2} a=0x{Values[lane % Values.Length]:X8} b=0x{Values[lane / Values.Length]:X8} " + + $"c=0x{Values[(run + lane) % Values.Length]:X8}: helper 0x{want:X8}, native 0x{got:X8}"); + } + } + } + } + + public static readonly TheoryData Reciprocals = new() + { + { "v_rcp_f32 v4, v1", [0x7E085501u] }, + { "v_rsq_f32 v4, v1", [0x7E085D01u] }, + { "v_sqrt_f32 v4, v1", [0x7E086701u] }, + }; + + // The fast forms (Gen5MslCodegenOptions.FastReciprocals) against the precise ones over + // the special values and a spread of numbers of every exponent and sign: a special input + // (zero, denormal, infinity, NaN, negative under a root) or a result that is not a normal + // number must come out with the same bits, and a normal result within 1 ULP. + [Theory] + [MemberData(nameof(Reciprocals))] + public void FastReciprocalsKeepTheSpecialResults(string assembly, uint[] operation) + { + if (Environment.GetEnvironmentVariable("SHARPEMU_TEST_REQUIRE_DEVICE") != "1") + { + output.WriteLine("Numerical execution requires the exclusive Metal device gate."); + return; + } + + Assert.True(MetalNative.IsAvailable, "The required execution gate needs a Metal device."); + foreach (var floatMode in new[] { ShaderCompileRequest.DefaultFloatMode, 0xF0u }) + { + var request = Gen5ComputeFixtures.CreateComputeRequest(Fixture(operation), localSizeX: Lanes, floatMode: floatMode); + Assert.True(Gen5MslTranslator.TryCompileProgram(request, out var fast, out var error, new Gen5MslCodegenOptions(FastReciprocals: true)), error); + Assert.True(Gen5MslTranslator.TryCompileProgram(request, out var precise, out error, new Gen5MslCodegenOptions(FastReciprocals: false)), error); + var squareRoot = assembly.StartsWith("v_sqrt", StringComparison.Ordinal); + if (squareRoot) + Assert.DoesNotContain("fast::sqrt(", fast.Source.Split("kernel void")[1], StringComparison.Ordinal); + else + Assert.Contains("fast::", fast.Source.Split("kernel void")[1], StringComparison.Ordinal); + var worst = 0L; + var state = 0x12345678u; + for (var run = 0; run < 8; run++) + { + var inputs = new uint[Lanes]; + for (var lane = 0; lane < Lanes; lane++) + { + state = state * 1664525u + 1013904223u; + inputs[lane] = run == 0 && lane < Values.Length ? Values[lane] : state; + // fast::sqrt gave 0x4EEFAD10 instead of 0x4EEFAD0E for this input. + if (run == 0 && lane == Values.Length) + inputs[lane] = 0x5E606496u; + } + + var buffer = new byte[4 * Lanes * sizeof(uint)]; + for (var lane = 0; lane < Lanes; lane++) + { + BinaryPrimitives.WriteUInt32LittleEndian(buffer.AsSpan(lane * 4), inputs[lane]); + } + + var expected = Execute(precise, buffer); + var actual = Execute(fast, buffer); + for (var lane = 0; lane < Lanes; lane++) + { + var want = BinaryPrimitives.ReadUInt32LittleEndian(expected.AsSpan((3 * Lanes + lane) * 4)); + var got = BinaryPrimitives.ReadUInt32LittleEndian(actual.AsSpan((3 * Lanes + lane) * 4)); + var input = inputs[lane]; + var exponent = want & 0x7F800000u; + var special = (input & 0x7F800000u) is 0 or 0x7F800000u || exponent is 0 or 0x7F800000u || + (input & 0x80000000u) != 0 && !assembly.StartsWith("v_rcp", StringComparison.Ordinal); + var ulps = Math.Abs((long)(int)(want & 0x7FFFFFFFu) - (int)(got & 0x7FFFFFFFu)); + Assert.True( + special || squareRoot ? want == got : (want & 0x80000000u) == (got & 0x80000000u) && ulps <= 1, + $"{assembly} float_mode=0x{floatMode:X2} input=0x{input:X8}: precise 0x{want:X8}, fast 0x{got:X8}"); + if (!special) + { + worst = Math.Max(worst, ulps); + } + } + } + + output.WriteLine($"{assembly} float_mode=0x{floatMode:X2}: worst {worst} ulp from the precise form"); + } + } + + private static byte[] Execute(Gen5MslShader shader, byte[] buffer) + { + Assert.True(MetalNative.TryCompileLibrary(shader.Source, out var library, out var error), error); + Assert.True(MetalNative.TryExecuteWithArgumentBuffer(library, shader.EntryPoint, buffer, + new byte[PushData.ByteSize], shader.ArgumentLayout!, Lanes, out var result, out error), error); + return result; + } +} diff --git a/tests/SharpEmu.ShaderCompiler.Metal.Tests/MslTranslatorOverheadBenchTests.cs b/tests/SharpEmu.ShaderCompiler.Metal.Tests/MslTranslatorOverheadBenchTests.cs new file mode 100644 index 000000000..a59af67da --- /dev/null +++ b/tests/SharpEmu.ShaderCompiler.Metal.Tests/MslTranslatorOverheadBenchTests.cs @@ -0,0 +1,316 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using System.Text; +using Xunit; +using Xunit.Abstractions; + +namespace SharpEmu.ShaderCompiler.Metal.Tests; + +/// +/// GPU time of the per-instruction overheads the emitted MSL carried, before and after each +/// codegen switch, on the device (round-2 perf audit, workstream k21). Translated programs are +/// compiled both ways in one process and timed with the command buffer's clock, best of three +/// after a warm-up; where the overhead is a pattern rather than an opcode the kernel is the +/// pattern written out in MSL. Opt-in, because a timing is not a pass/fail property: +/// SHARPEMU_GPU_BENCH=1 with SHARPEMU_TEST_REQUIRE_DEVICE=1, and read the output +/// (dotnet test --filter MslTranslatorOverheadBench --logger "console;verbosity=detailed"). +/// +public sealed class MslTranslatorOverheadBenchTests(ITestOutputHelper output) +{ + private const uint Groups = 4096; + private const uint Threads = 64; + private const uint Iterations = 4; + private const int Rounds = 3; + + private static bool Enabled => + Environment.GetEnvironmentVariable("SHARPEMU_GPU_BENCH") == "1" && + Environment.GetEnvironmentVariable("SHARPEMU_TEST_REQUIRE_DEVICE") == "1" && + MetalNative.IsAvailable; + + // 256 trips of S_BUFFER_LOAD_DWORDX4 and BUFFER_LOAD_DWORDX4, all eight dwords summed. + private static readonly Gen5ComputeFixture Loads = new( + "bench-loads", + [ + 0x34120084, // v_lshlrev_b32 v9, 4, v0 + 0x361212FF, 0x000003F0, // v_and_b32 v9, 0x3f0, v9 + 0x7E020280, // v_mov_b32 v1, 0 + 0xBE8003FF, 0x00000100, // s_mov_b32 s0, 256 + 0xBE810380, // s_mov_b32 s1, 0 + // loop: + 0xF4280304, 0x02000000, // s_buffer_load_dwordx4 s[12:15], s[8:11], s1 + 0xE0381000, 0x80020409, // buffer_load_dwordx4 v[4:7], v9, s[8:11], 0 offen + 0xBF8C0070, // s_waitcnt vmcnt(0) lgkmcnt(0) + 0x4A020304, // v_add_nc_u32 v1, v4, v1 + 0x4A020305, // v_add_nc_u32 v1, v5, v1 + 0x4A020306, // v_add_nc_u32 v1, v6, v1 + 0x4A020307, // v_add_nc_u32 v1, v7, v1 + 0x4A02020C, // v_add_nc_u32 v1, s12, v1 + 0x4A02020D, // v_add_nc_u32 v1, s13, v1 + 0x4A02020E, // v_add_nc_u32 v1, s14, v1 + 0x4A02020F, // v_add_nc_u32 v1, s15, v1 + 0x4A121290, // v_add_nc_u32 v9, 16, v9 + 0x361212FF, 0x000003F0, // v_and_b32 v9, 0x3f0, v9 + 0x80019001, // s_add_u32 s1, s1, 16 + 0x8701FF01, 0x000003F0, // s_and_b32 s1, s1, 0x3f0 + 0x81808100, // s_sub_i32 s0, s0, 1 + 0xBF078000, // s_cmp_lg_u32 s0, 0 + 0xBF85FFEA, // s_cbranch_scc1 loop + 0xE0700000, 0x80020100, // buffer_store_dword v1, off, s[8:11], 0 + 0xBF810000, // s_endpgm + ], + StoreScalarResourceBase: 8, + StoreBackingBytes: 4096); + + // 256 trips of twelve f32 min/max/med3 between registers and against 0, 1.0 and -1.0. + private static readonly Gen5ComputeFixture MinMax = new( + "bench-minmax", + [ + 0x7E020D00, // v_cvt_f32_u32 v1, v0 + 0x100202FF, 0x3C23D70A, // v_mul_f32 v1, 0x3c23d70a, v1 + 0x7E0402F0, // v_mov_b32 v2, 0.5 + 0x7E100280, // v_mov_b32 v8, 0 + 0xBE8003FF, 0x00000100, // s_mov_b32 s0, 256 + // loop: + 0x20060501, // v_max_f32 v3, v1, v2 + 0x1E081103, // v_min_f32 v4, v3, v8 + 0x200A0880, // v_max_f32 v5, 0, v4 + 0x1E0C0AF2, // v_min_f32 v6, 1.0, v5 + 0xD5570007, 0x03C90106, // v_med3_f32 v7, v6, 0, 1.0 + 0x20060307, // v_max_f32 v3, v7, v1 + 0x1E080503, // v_min_f32 v4, v3, v2 + 0x200A0880, // v_max_f32 v5, 0, v4 + 0x1E0C0AF2, // v_min_f32 v6, 1.0, v5 + 0xD5570007, 0x03C9E706, // v_med3_f32 v7, v6, -1.0, 1.0 + 0xD5540003, 0x02020307, // v_max3_f32 v3, v7, v1, 0 + 0xD5510004, 0x03CA0503, // v_min3_f32 v4, v3, v2, 1.0 + 0x06020304, // v_add_f32 v1, v4, v1 + 0x08100F01, // v_sub_f32 v8, v1, v7 + 0x81808100, // s_sub_i32 s0, s0, 1 + 0xBF078000, // s_cmp_lg_u32 s0, 0 + 0xBF85FFEB, // s_cbranch_scc1 loop + 0xE0700000, 0x80020100, // buffer_store_dword v1, off, s[8:11], 0 + 0xBF810000, // s_endpgm + ], + StoreScalarResourceBase: 8, + StoreBackingBytes: 16); + + // 256 trips of two each of V_RCP_F32, V_RSQ_F32 and V_SQRT_F32. + private static readonly Gen5ComputeFixture Reciprocals = new( + "bench-rcp", + [ + 0x7E020D00, // v_cvt_f32_u32 v1, v0 + 0x060202F2, // v_add_f32 v1, 1.0, v1 + 0xBE8003FF, 0x00000100, // s_mov_b32 s0, 256 + // loop: + 0x7E045501, // v_rcp_f32 v2, v1 + 0x7E065D01, // v_rsq_f32 v3, v1 + 0x7E086701, // v_sqrt_f32 v4, v1 + 0x060A0702, // v_add_f32 v5, v2, v3 + 0x060A0905, // v_add_f32 v5, v5, v4 + 0x7E045505, // v_rcp_f32 v2, v5 + 0x7E065D05, // v_rsq_f32 v3, v5 + 0x7E086705, // v_sqrt_f32 v4, v5 + 0x060A0702, // v_add_f32 v5, v2, v3 + 0x06020905, // v_add_f32 v1, v5, v4 + 0x81808100, // s_sub_i32 s0, s0, 1 + 0xBF078000, // s_cmp_lg_u32 s0, 0 + 0xBF85FFF3, // s_cbranch_scc1 loop + 0xE0700000, 0x80020100, // buffer_store_dword v1, off, s[8:11], 0 + 0xBF810000, // s_endpgm + ], + StoreScalarResourceBase: 8, + StoreBackingBytes: 16); + + // 1024 trips of V_MAD_F32 and V_MUL_F32 + V_ADD_F32 pairs a contraction could fuse. + private static readonly Gen5ComputeFixture MultiplyAdds = new( + "bench-mad", + [ + 0x7E020D00, // v_cvt_f32_u32 v1, v0 + 0x7E0402FF, 0x3F7FF972, // v_mov_b32 v2, 0x3f7ff972 + 0x7E0602FF, 0x3A83126F, // v_mov_b32 v3, 0x3a83126f + 0xBE8003FF, 0x00000400, // s_mov_b32 s0, 1024 + // loop: + 0xD5410001, 0x040E0501, // v_mad_f32 v1, v1, v2, v3 + 0x10080501, // v_mul_f32 v4, v1, v2 + 0x06080704, // v_add_f32 v4, v4, v3 + 0xD5410001, 0x04060504, // v_mad_f32 v1, v4, v2, v1 + 0x100A0301, // v_mul_f32 v5, v1, v1 + 0x06020705, // v_add_f32 v1, v5, v3 + 0x81808100, // s_sub_i32 s0, s0, 1 + 0xBF078000, // s_cmp_lg_u32 s0, 0 + 0xBF85FFF5, // s_cbranch_scc1 loop + 0xE0700000, 0x80020100, // buffer_store_dword v1, off, s[8:11], 0 + 0xBF810000, // s_endpgm + ], + StoreScalarResourceBase: 8, + StoreBackingBytes: 16); + + private static Gen5MslShader Compile(Gen5ComputeFixture fixture, Gen5MslCodegenOptions options) + { + var request = Gen5ComputeFixtures.CreateComputeRequest(fixture, localSizeX: Threads); + Assert.True(Gen5MslTranslator.TryCompileProgram(request, out var shader, out var error, options), error); + return shader; + } + + private double Time(Gen5MslShader shader, string label, int bufferBytes, bool multipleOfWidth = false) + { + Assert.True( + MetalNative.TryBenchmark( + shader, new byte[bufferBytes], new byte[Resources.PushData.ByteSize], Groups, Threads, Iterations, + out var result, out var error, groupSizeIsMultipleOfWidth: multipleOfWidth), + error); + output.WriteLine($"{label,-44} {result.GpuMilliseconds,9:F3} ms GPU maxThreads {result.MaxThreadsPerThreadgroup}"); + return result.GpuMilliseconds; + } + + private void Compare(Gen5ComputeFixture fixture, string name, Gen5MslCodegenOptions before, Gen5MslCodegenOptions after, int bufferBytes) + { + var old = Compile(fixture, before); + var current = Compile(fixture, after); + Assert.NotEqual(old.Source, current.Source); + var (bestOld, bestNew) = (double.MaxValue, double.MaxValue); + for (var round = 0; round < Rounds; round++) + { + bestOld = Math.Min(bestOld, Time(old, $"{name}, off", bufferBytes)); + bestNew = Math.Min(bestNew, Time(current, $"{name}, on", bufferBytes)); + } + + output.WriteLine($"{name}: {bestOld:F3} -> {bestNew:F3} ms ({(bestNew - bestOld) / bestOld * 100:+0.0;-0.0}%)"); + } + + private static readonly Gen5MslCodegenOptions Baseline = new( + VectorDwordLoads: false, ThreadgroupSizeHint: false, NativeFloatMinMax: false, ConstantTableSlots: false, + FastReciprocals: false, ContractPixelCompute: false); + + [Fact] + public void VectorDwordLoads() + { + if (!Enabled) return; + Compare(Loads, "s_buffer/buffer_load_dwordx4", Baseline, Baseline with { VectorDwordLoads = true }, 4096); + } + + [Fact] + public void NativeFloatMinMax() + { + if (!Enabled) return; + Compare(MinMax, "f32 min/max/med3", Baseline, Baseline with { NativeFloatMinMax = true }, 16); + } + + [Fact] + public void FastReciprocals() + { + if (!Enabled) return; + Compare(Reciprocals, "rcp/rsq/sqrt f32", Baseline, Baseline with { FastReciprocals = true }, 16); + } + + [Fact] + public void ContractPixelCompute() + { + if (!Enabled) return; + Compare(MultiplyAdds, "mad/mul+add f32", Baseline, Baseline with { ContractPixelCompute = true }, 16); + } + + [Fact] + public void ThreadgroupSizeHintOnTranslatedPrograms() + { + if (!Enabled) return; + foreach (var (fixture, name, bytes) in new[] { (Loads, "loads", 4096), (MinMax, "minmax", 16), (Reciprocals, "rcp", 16) }) + { + var old = Compile(fixture, Baseline); + var hinted = Compile(fixture, Baseline with { ThreadgroupSizeHint = true }); + Assert.Contains("[[max_total_threads_per_threadgroup(64)]]", hinted.Source, StringComparison.Ordinal); + for (var round = 0; round < Rounds; round++) + { + Time(old, $"{name}, no hint", bytes); + Time(hinted, $"{name}, max_total_threads 64", bytes); + Time(hinted, $"{name}, + multiple of SIMD width", bytes, multipleOfWidth: true); + } + } + } + + // A register-heavy kernel (96 floats live across a loop), the shape where the register + // budget the hint buys matters. + private static string RegisterHeavySource(bool hint) + { + const int live = 96; + var source = new StringBuilder(); + source.AppendLine("#include \nusing namespace metal;"); + source.AppendLine($"{(hint ? "[[max_total_threads_per_threadgroup(64)]] " : string.Empty)}kernel void bench_regs(device uint* b [[buffer(0)]], uint tid [[thread_position_in_grid]])"); + source.AppendLine("{"); + source.AppendLine($" float r[{live}];"); + source.AppendLine($" for (int i = 0; i < {live}; i++) r[i] = float(tid + (uint)i) * 0.001f;"); + source.AppendLine(" for (int k = 0; k < 32; k++)"); + source.AppendLine(" {"); + for (var i = 0; i < live; i++) + { + source.AppendLine($" r[{i}] = fma(r[{i}], r[{(i + 1) % live}], r[{(i + 37) % live}]);"); + } + + source.AppendLine(" }"); + source.AppendLine(" float sum = 0.0f;"); + source.AppendLine($" for (int i = 0; i < {live}; i++) sum += r[i];"); + source.AppendLine(" b[tid & 255u] = as_type(sum);"); + source.AppendLine("}"); + return source.ToString(); + } + + [Fact] + public void ThreadgroupSizeHintOnARegisterHeavyKernel() + { + if (!Enabled) return; + for (var round = 0; round < Rounds; round++) + { + foreach (var (hint, width) in new[] { (false, false), (true, false), (true, true) }) + { + Assert.True(MetalNative.TryBenchmarkKernel(RegisterHeavySource(hint), "bench_regs", new byte[1024], Groups, Threads, Iterations, + out var ms, out var maxThreads, out var error, width), error); + output.WriteLine($"96 live floats, hint {hint}, multiple of width {width}: {ms,9:F3} ms GPU maxThreads {maxThreads}"); + } + } + } + + // The flattened-table pattern: 128 fixed-slot reads per trip, with and without the bounds + // select the translator put on every one (the word count is read at run time, as the + // argument buffer's is), through the constant address space the table lives in. + private static string TableSource(bool select) + { + var source = new StringBuilder(); + source.AppendLine("#include \nusing namespace metal;"); + source.AppendLine("kernel void bench_table(device uint* b [[buffer(0)]], constant uint* t [[buffer(1)]], uint tid [[thread_position_in_grid]])"); + source.AppendLine("{"); + source.AppendLine(" constant uint* table = t + 16;"); + source.AppendLine(" uint words = t[0];"); + source.AppendLine(" uint acc = tid;"); + source.AppendLine(" for (uint k = 0; k < 64u; k++)"); + source.AppendLine(" {"); + for (var slot = 0; slot < 128; slot++) + { + source.AppendLine(select + ? $" acc = (acc ^ ({slot}u < words ? table[{slot}u] : 0u)) * 0x9E3779B1u;" + : $" acc = (acc ^ table[{slot}u]) * 0x9E3779B1u;"); + } + + source.AppendLine(" }"); + source.AppendLine(" b[4096u + (tid & 255u)] = acc;"); + source.AppendLine("}"); + return source.ToString(); + } + + [Fact] + public void FlattenedTableSelect() + { + if (!Enabled) return; + var buffer = new byte[5120 * sizeof(uint)]; + BitConverter.TryWriteBytes(buffer, 200u); + for (var round = 0; round < Rounds; round++) + { + foreach (var select in new[] { true, false }) + { + Assert.True(MetalNative.TryBenchmarkKernel(TableSource(select), "bench_table", buffer, Groups, Threads, Iterations, + out var ms, out _, out var error), error); + output.WriteLine($"128 table words x64, select {select}: {ms,9:F3} ms GPU"); + } + } + } +} diff --git a/tests/SharpEmu.ShaderCompiler.Tests/Resources/ResourceMaterializationCacheTests.cs b/tests/SharpEmu.ShaderCompiler.Tests/Resources/ResourceMaterializationCacheTests.cs index 53c8f1349..c257498d9 100644 --- a/tests/SharpEmu.ShaderCompiler.Tests/Resources/ResourceMaterializationCacheTests.cs +++ b/tests/SharpEmu.ShaderCompiler.Tests/Resources/ResourceMaterializationCacheTests.cs @@ -9,6 +9,58 @@ namespace SharpEmu.ShaderCompiler.Tests.Resources; public sealed class ResourceMaterializationCacheTests { + [Theory] + [InlineData(0, 16, 32)] + [InlineData(32, 16, 0)] + [InlineData(16, 0, 32)] + public void RecordedReadOrderingPreservesDescriptorsAndFreshInvalidation(int first, int second, int third) + { + var plan = Extract(Program( + ScalarLoad(0, 0, 8, 4, first), ScalarLoad(8, 0, 12, 4, second), ScalarLoad(16, 0, 16, 4, third), + BufferLoad(24, 8), BufferLoad(32, 12), BufferLoad(40, 16), EndProgram(48)), userDataCount: 2); + var words = new uint[] { 0x2000, 0, 256, 0x20000, 0x4000, 0, 128, 0x20000, 0x6000, 0, 64, 0x20000 }; + bool Read(ulong address, out uint word) + { + var index = (address - 0x1000) / 4; + word = index < (ulong)words.Length ? words[index] : 0; + return address >= 0x1000 && index < (ulong)words.Length; + } + var readable = true; + bool Resident(ulong address, Span bytes, bool clean) + { + if (!readable) return false; + for (var offset = 0; offset < bytes.Length; offset += 4) + { + if (!Read(address + (ulong)offset, out var word)) return false; + System.Buffers.Binary.BinaryPrimitives.WriteUInt32LittleEndian(bytes[offset..], word); + } + return true; + } + var inputs = new ResourceRuntimeInputs + { + UserData = new uint[] { 0x1000, 0 }, ReadMemory = Read, ReadCleanMemory = Read, ReadResidentMemory = Resident, + }; + var cache = new ResourceMaterializationCache(); + var snapshot = new ResourceSnapshot(); + var specialization = new ResourceSpecialization(); + for (var draw = 0; draw < 4; draw++) + { + // Reuse, descriptor/bounds change, and unreadable resident memory must all agree with a fresh walk. + if (draw == 2) { words[0] = 0x8000; words[2] = 8; } + if (draw == 3) readable = false; + var expected = new ResourceSnapshot(); + var expectedSpecialization = new ResourceSpecialization(); + Assert.True(ResourceMaterializer.Materialize(plan, inputs, ref expected, ref expectedSpecialization, out _)); + var hits = cache.Hits; + Assert.True(cache.Materialize(plan, inputs, Resident, ref snapshot, ref specialization, out _)); + Assert.Equal(expected.Buffers.Length, snapshot.Buffers.Length); + for (var buffer = 0; buffer < expected.Buffers.Length; buffer++) + Assert.Equal(expected.Buffers[buffer], snapshot.Buffers[buffer]); + Assert.Equal(expected.FlattenedResourceTable, snapshot.FlattenedResourceTable); + Assert.Equal(hits + (draw == 1 ? 1 : 0), cache.Hits); + } + } + private const ulong HeapBase = 0x1000; // A mask word, a two-entry index table and six 32-byte image records. diff --git a/tests/SharpEmu.ShaderCompiler.Tests/Resources/ResourceMaterializationContentKeyTests.cs b/tests/SharpEmu.ShaderCompiler.Tests/Resources/ResourceMaterializationContentKeyTests.cs index c29476b34..3004ccd98 100644 --- a/tests/SharpEmu.ShaderCompiler.Tests/Resources/ResourceMaterializationContentKeyTests.cs +++ b/tests/SharpEmu.ShaderCompiler.Tests/Resources/ResourceMaterializationContentKeyTests.cs @@ -100,6 +100,121 @@ private static void AssertFullWalk(ShaderResourcePlan plan, TestWordMemory memor Assert.Equal(expectedSpecialization, specialization); } + [Theory] + [InlineData(false)] + [InlineData(true)] + public void ColdScanAdmissionPreservesFreshResultsAndReadmitsRecurringContents(bool sameAddress) + { + var plan = TablePlan(2, 1); + var memory = NewMemory(); + var cache = new ResourceMaterializationCache(generationCapacity: 2); + ResourceSnapshot? first = null; + for (var draw = 0; draw < 96; draw++) + { + var table = MemoryBase + (sameAddress ? 0UL : (ulong)draw * 256); + WriteTable(memory, table, draw, 2, 1); + Assert.True(Run(cache, plan, memory, Pointer(table), out var snapshot, out var specialization)); + AssertFullWalk(plan, memory, Pointer(table), snapshot, specialization); + first ??= snapshot; + } + if (Environment.GetEnvironmentVariable("SHARPEMU_RESOURCE_CACHE_ADMISSION") == "0") + Assert.Equal(0, cache.ColdBypasses); + else + Assert.True(cache.ColdBypasses > 0); + Assert.Equal(Texture(0x0100_0000u), first!.Images[0]); + var last = MemoryBase + (sameAddress ? 0UL : 95UL * 256); + Assert.True(Run(cache, plan, memory, Pointer(last), out var repeated, out var repeatedSpecialization)); + AssertFullWalk(plan, memory, Pointer(last), repeated, repeatedSpecialization); + var hits = cache.Hits; + Assert.True(Run(cache, plan, memory, Pointer(last), out repeated, out repeatedSpecialization)); + Assert.Equal(hits + 1, cache.Hits); + AssertFullWalk(plan, memory, Pointer(last), repeated, repeatedSpecialization); + } + + [Fact] + public void ColdBypassOwnsUserDataAfterTheCallerReusesItsArray() + { + if (Environment.GetEnvironmentVariable("SHARPEMU_RESOURCE_CACHE_ADMISSION") == "0") return; + var plan = TablePlan(2, 1); + var memory = NewMemory(); + var cache = new ResourceMaterializationCache(generationCapacity: 2); + var userData = Pointer(MemoryBase); + ResourceSnapshot snapshot = new(); + for (var draw = 0; draw < 96; draw++) + { + WriteTable(memory, MemoryBase, draw, 2, 1); + Assert.True(Run(cache, plan, memory, userData, out snapshot, out _)); + if (cache.ColdBypasses > 0) break; + } + Assert.True(cache.ColdBypasses > 0); + userData[0] += 256; + Assert.Equal(Pointer(MemoryBase), snapshot.UserData); + } + + [Fact] + public void FailedRecordedAdmissionRewalkPreservesCallerOutputs() + { + if (Environment.GetEnvironmentVariable("SHARPEMU_RESOURCE_CACHE_ADMISSION") == "0") return; + var plan = TablePlan(2, 1); + var memory = NewMemory(); + var cache = new ResourceMaterializationCache(generationCapacity: 2); + for (var draw = 0; draw < 96; draw++) + { + WriteTable(memory, MemoryBase, draw, 2, 1); + Assert.True(Run(cache, plan, memory, Pointer(MemoryBase), out _, out _)); + } + Assert.True(cache.ColdBypasses > 0); + + var readsPerWalk = 0; + bool CountRead(ulong address, out uint word) + { + readsPerWalk++; + return memory.Read(address, out word); + } + var snapshot = new ResourceSnapshot(); + var specialization = new ResourceSpecialization(); + Assert.True(ResourceMaterializer.Materialize(plan, Inputs(Pointer(MemoryBase), CountRead, CountRead), + ref snapshot, ref specialization)); + var originalSnapshot = snapshot; + var originalSpecialization = specialization; + var reads = 0; + bool FailRewalk(ulong address, out uint word) + { + word = 0; + return ++reads <= readsPerWalk && memory.Read(address, out word); + } + Assert.False(cache.Materialize(plan, Inputs(Pointer(MemoryBase), FailRewalk, FailRewalk), + (address, destination, clean) => false, ref snapshot, ref specialization, out _)); + Assert.True(reads > readsPerWalk); + Assert.Same(originalSnapshot, snapshot); + Assert.Same(originalSpecialization, specialization); + } + + [Fact] + public void AWorkingSetWithinTheBudgetIsAdmittedEvenAfterManyDistinctMisses() + { + var plan = TablePlan(1, 1); + var memory = NewMemory(); + var cache = new ResourceMaterializationCache(generationCapacity: 128); + for (var draw = 0; draw < 96; draw++) + { + var table = MemoryBase + (ulong)draw * 256; + WriteTable(memory, table, draw, 1, 1); + Assert.True(Run(cache, plan, memory, Pointer(table), out var snapshot, out var specialization)); + AssertFullWalk(plan, memory, Pointer(table), snapshot, specialization); + } + Assert.Equal(0, cache.ColdBypasses); + var hits = cache.Hits; + for (var draw = 0; draw < 96; draw++) + { + var table = MemoryBase + (ulong)draw * 256; + Assert.True(Run(cache, plan, memory, Pointer(table), out var snapshot, out var specialization)); + AssertFullWalk(plan, memory, Pointer(table), snapshot, specialization); + } + Assert.Equal(hits + 96, cache.Hits); + Assert.Equal(0, cache.ColdBypasses); + } + [Fact] public void IdenticalDescriptorsAtDifferentTableAddressesHit() { @@ -178,6 +293,84 @@ public void AChangedDescriptorWordIsNeverReused() Assert.Equal(hits + 1, cache.Hits); } + // A hit through a pointer seen before guesses its key from the print the pointer gave last time. + // Descriptors rewritten behind the pointer since then must never hit the entry of the old ones, + // and when they now equal another object's, the draw finds that object's entry instead. + [Fact] + public void ARecalledPrintNeverHitsDescriptorsRewrittenBehindThePointer() + { + var plan = TablePlan(textures: 4, buffers: 2); + var memory = NewMemory(); + var tables = Enumerable.Range(0, 8).Select(index => MemoryBase + 0x1000 + (ulong)index * 0x300).ToArray(); + foreach (var table in tables) + WriteTable(memory, table, material: 3, textures: 4, buffers: 2); + var cache = new ResourceMaterializationCache(); + for (var draw = 0; draw < 32; draw++) + Assert.True(Run(cache, plan, memory, Pointer(tables[draw % tables.Length]), out _, out _)); + var hits = cache.Hits; + var misses = cache.Misses; + + void Draw(ulong table) + { + Assert.True(Run(cache, plan, memory, Pointer(table), out var snapshot, out var specialization)); + AssertFullWalk(plan, memory, Pointer(table), snapshot, specialization); + } + + // Every descriptor behind tables[5] rewritten: the recalled print points at material 3's entry. + WriteTable(memory, tables[5], material: 7, textures: 4, buffers: 2); + Draw(tables[5]); + Assert.Equal((hits, misses + 1), (cache.Hits, cache.Misses)); + + // tables[6] now holds what tables[5] does: its recalled print is stale, its real one finds + // the entry tables[5] just built. + WriteTable(memory, tables[6], material: 7, textures: 4, buffers: 2); + Draw(tables[6]); + Assert.Equal((hits + 1, misses + 1), (cache.Hits, cache.Misses)); + + // One word rewritten behind a pointer whose print was just read. + Write(memory, tables[6] + 32 + 4, [21u << 20]); + Draw(tables[6]); + Assert.Equal((hits + 1, misses + 2), (cache.Hits, cache.Misses)); + + // Pointers left alone keep hitting their own descriptors. + Draw(tables[5]); + Draw(tables[2]); + Assert.Equal((hits + 3, misses + 2), (cache.Hits, cache.Misses)); + } + + [Fact] + public void ARecalledPrintChecksReadabilityAndReplacementMemoryAtTheSameAddress() + { + var plan = TablePlan(textures: 4, buffers: 2); + var memory = NewMemory(); + var tables = Enumerable.Range(0, 8).Select(index => MemoryBase + 0x1000 + (ulong)index * 0x300).ToArray(); + foreach (var table in tables) + WriteTable(memory, table, material: 3, textures: 4, buffers: 2); + var cache = new ResourceMaterializationCache(); + for (var draw = 0; draw < 32; draw++) + Assert.True(Run(cache, plan, memory, Pointer(tables[draw % tables.Length]), out _, out _)); + + var hits = cache.Hits; + // The table pointer and its last print survive, but a descriptor word is no longer readable. + memory.FailAddress = tables[5] + 4; + var expected = new ResourceSnapshot(); + var expectedSpecialization = new ResourceSpecialization(); + var expectedResult = ResourceMaterializer.Materialize(plan, + Inputs(Pointer(tables[5]), memory.Read, memory.Read), ref expected, ref expectedSpecialization); + Assert.Equal(expectedResult, Run(cache, plan, memory, Pointer(tables[5]), out _, out _)); + Assert.Equal(hits, cache.Hits); + + // A new mapping at the same guest address must use its own contents, including buffer bounds. + var replacement = NewMemory(); + foreach (var table in tables) + WriteTable(replacement, table, material: 9, textures: 4, buffers: 2); + Write(replacement, tables[5] + 4 * 32 + 16 + 8, [1024u]); + Assert.True(Run(cache, plan, replacement, Pointer(tables[5]), out var snapshot, out var specialization)); + AssertFullWalk(plan, replacement, Pointer(tables[5]), snapshot, specialization); + Assert.Equal(1024u, snapshot.Buffers[0][2]); + Assert.Equal(hits, cache.Hits); + } + // A hit through a moved pointer gives each pointer its own snapshot; draws that come back to a // pointer share the one made for it, and nothing a later draw does changes an earlier one's. [Fact] diff --git a/tools/SharpEmu.Tools.MaterializationBench/Benches.Materialization.cs b/tools/SharpEmu.Tools.MaterializationBench/Benches.Materialization.cs new file mode 100644 index 000000000..782e96ab8 --- /dev/null +++ b/tools/SharpEmu.Tools.MaterializationBench/Benches.Materialization.cs @@ -0,0 +1,205 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using SharpEmu.PerfAudit.Materialization; +using SharpEmu.ShaderCompiler; +using SharpEmu.ShaderCompiler.Resources; +using static SharpEmu.PerfAudit.Materialization.ResourceTestProgram; + +namespace SharpEmu.PerfAudit; + +/// +/// ResourceMaterializer / ResourceMaterializationCache over a synthetic shader that reads 8 T# + 2 S# + 4 V# +/// through a descriptor table in guest memory (user data = table pointer), with entirely fabricated descriptor words. +/// +internal static unsafe class MaterializationBenches +{ + private const int Textures = 8, Samplers = 2, Buffers = 4; + private const ulong TableStride = 512; + private const ulong TextureRegion = NativeGuestMemory.Base + 0x100_0000; + private const ulong BufferRegion = NativeGuestMemory.Base + 0x200_0000; + private const ulong TableRegion = NativeGuestMemory.Base + 0x300_0000; + + public static ShaderResourcePlan BuildPlan() + { + var program = new List(); + uint pc = 0; + // s[8+8i : 15+8i] = T#i, s[72+4j : 75+4j] = S#j, s[80+4k : 83+4k] = V#k, all loaded through the pointer in s[0:1]. + for (uint i = 0; i < Textures; i++) { program.Add(ScalarLoad(pc, 0, 8 + 8 * i, 8, (int)(i * 32))); pc += 8; } + for (uint j = 0; j < Samplers; j++) { program.Add(ScalarLoad(pc, 0, 72 + 4 * j, 4, 256 + (int)(j * 16))); pc += 8; } + for (uint k = 0; k < Buffers; k++) { program.Add(ScalarLoad(pc, 0, 80 + 4 * k, 4, 288 + (int)(k * 16))); pc += 8; } + for (uint i = 0; i < Textures; i++) { program.Add(Image(pc, "ImageSample", 8 + 8 * i, 72 + 4 * (i % Samplers))); pc += 8; } + for (uint k = 0; k < Buffers; k++) { program.Add(BufferLoad(pc, 80 + 4 * k, 0, 4)); pc += 8; } + program.Add(EndProgram(pc)); + return Extract(Program([.. program]), userDataBase: 0, userDataCount: 2, stage: ShaderStage.Pixel); + } + + /// Writes one descriptor table; makes its texture/buffer addresses distinct. + private static void WriteTable(NativeGuestMemory memory, ulong table, int salt) + { + for (var i = 0; i < Textures; i++) + { + var words = Descriptors.Texture(TextureRegion + ((ulong)salt * 16 + (ulong)i) * 0x1_0000, 56, 256, 256); + memory.WriteWords(table + (ulong)i * 32, words); + } + + for (var j = 0; j < Samplers; j++) memory.WriteWords(table + 256 + (ulong)j * 16, Descriptors.Sampler()); + for (var k = 0; k < Buffers; k++) + memory.WriteWords(table + 288 + (ulong)k * 16, Descriptors.Buffer(BufferRegion + ((ulong)salt * 8 + (ulong)k) * 0x1000, 16, 256, format: 77)); + } + + // Historical control assemblies predate the admission counter. + private static long ColdBypasses(ResourceMaterializationCache cache) => + (long?)typeof(ResourceMaterializationCache).GetProperty("ColdBypasses")?.GetValue(cache) ?? 0; + + public static void Run() + { + Console.WriteLine("--- 3. Resource materialization (ResourceMaterializer / ResourceMaterializationCache) ---"); + var plan = BuildPlan(); + Console.WriteLine($"plan: images={plan.Info.Images.Count} buffers={plan.Info.Buffers.Count} samplers={plan.Info.Samplers.Count} descriptorSources={plan.DescriptorSources.Count} memoryOps={plan.Memory.Count}"); + using var memory = new NativeGuestMemory(64 << 20); + const int Pool = 4096; + for (var t = 0; t < Pool; t++) WriteTable(memory, TableRegion + (ulong)t * TableStride, t); + + bool ReadWord(ulong address, out uint word) + { + Span bytes = stackalloc byte[4]; + if (!memory.TryRead(address, bytes)) { word = 0; return false; } + word = System.Buffers.Binary.BinaryPrimitives.ReadUInt32LittleEndian(bytes); + return true; + } + + bool ReadResident(ulong address, Span destination, bool clean) => memory.TryRead(address, destination); + + var inputs = new ResourceRuntimeInputs { ReadMemory = ReadWord, ReadCleanMemory = ReadWord, ReadResidentMemory = ReadResident }; + var snapshot = new ResourceSnapshot(); + var specialization = new ResourceSpecialization(); + var table0 = TableRegion; + inputs.UserData = new uint[] { (uint)table0, (uint)(table0 >> 32) }; + const int Batch = 200; + + // 3a: the full walk, no cache, same descriptors. + Harness.Measure("3a", "FULL WALK ResourceMaterializer.Materialize (no cache), 8T+2S+4V via table", Batch, 1404, _ => + { + for (var i = 0; i < Batch; i++) + { + snapshot = new ResourceSnapshot(); specialization = new ResourceSpecialization(); + if (!ResourceMaterializer.Materialize(plan, inputs, ref snapshot, ref specialization, out var failure)) throw new InvalidOperationException(failure.ToString()); + } + }, unit: "full walk", note: "uncached control"); + + // 3b: warm cache hit, same descriptors every draw. + var cache = new ResourceMaterializationCache(); + if (!cache.Materialize(plan, inputs, ReadResident, ref snapshot, ref specialization, out var warmFailure)) throw new InvalidOperationException(warmFailure.ToString()); + var hitsBefore = cache.Hits; + var warm = Harness.Measure("3b", "WARM cache hit, same descriptors every call (validation re-read)", Batch * 5, 13825, _ => + { + for (var i = 0; i < Batch * 5; i++) + { + if (!cache.Materialize(plan, inputs, ReadResident, ref snapshot, ref specialization, out var failure)) throw new InvalidOperationException(failure.ToString()); + } + }, unit: "cache-hit call", note: "validation re-read"); + Console.WriteLine($" -> cache hits {cache.Hits - hitsBefore}, misses {cache.Misses}, refreshes {cache.TableRefreshes}"); + + // 3c: cold, new table each call (new key), cache too small to retain: every call is a miss + entry build. + var cold = new ResourceMaterializationCache(generationCapacity: 8); + var cursor = 0; + Harness.Measure("3c", "COLD miss: new table address + new descriptors each call (key not found)", Batch, 1404, _ => + { + for (var i = 0; i < Batch; i++) + { + var table = TableRegion + (ulong)(cursor++ % Pool) * TableStride; + inputs.UserData = new uint[] { (uint)table, (uint)(table >> 32) }; + if (!cold.Materialize(plan, inputs, ReadResident, ref snapshot, ref specialization, out var failure)) throw new InvalidOperationException(failure.ToString()); + } + }, unit: "miss", note: "cold: includes allocating the user-data array and the entry build"); + Console.WriteLine($" -> hits {cold.Hits} misses {cold.Misses} refreshes {cold.TableRefreshes} uncacheable {cold.Uncacheable} cold_bypasses {ColdBypasses(cold)}"); + + // 3d: same key, the table's descriptors change between calls (stale same-key): tries refresh, falls back to the walk. + var stale = new ResourceMaterializationCache(); + inputs.UserData = new uint[] { (uint)table0, (uint)(table0 >> 32) }; + stale.Materialize(plan, inputs, ReadResident, ref snapshot, ref specialization, out _); + var salt = 0; + var refreshesBefore = stale.TableRefreshes; + Harness.Measure("3d", "STALE same key: descriptor words change every call (refresh or re-walk)", Batch, 1404, _ => + { + for (var i = 0; i < Batch; i++) + { + WriteTable(memory, table0, 1 + (salt++ % (Pool - 1))); + if (!stale.Materialize(plan, inputs, ReadResident, ref snapshot, ref specialization, out var failure)) throw new InvalidOperationException(failure.ToString()); + } + }, unit: "stale", note: "includes WriteTable (14 small writes) in the timed loop"); + Console.WriteLine($" -> hits {stale.Hits} misses {stale.Misses} refreshes {stale.TableRefreshes - refreshesBefore} (slot refreshes {stale.SlotRefreshes}, cold_bypasses {ColdBypasses(stale)})"); + + // 3f: 50 draws per batch alternate between 8 descriptor tables at different addresses whose contents are identical: + // only the table pointer in user data differs, so every draw could reuse one materialization. + { + const int Tables = 8, Draws = 50; + var identical = new ulong[Tables]; + for (var t = 0; t < Tables; t++) + { + identical[t] = TableRegion + (ulong)(Pool + 8 + t * 3) * TableStride; + WriteTable(memory, identical[t], 7); + } + + var userData = new uint[Tables][]; + for (var t = 0; t < Tables; t++) userData[t] = new uint[] { (uint)identical[t], (uint)(identical[t] >> 32) }; + var alternating = new ResourceMaterializationCache(); + for (var i = 0; i < Draws; i++) + { + inputs.UserData = userData[i % Tables]; + if (!alternating.Materialize(plan, inputs, ReadResident, ref snapshot, ref specialization, out var failure)) throw new InvalidOperationException(failure.ToString()); + } + Console.WriteLine($" 3f first {Draws} draws (cold cache): hits {alternating.Hits} misses {alternating.Misses} refreshes {alternating.TableRefreshes} -> hit rate {100.0 * alternating.Hits / Draws:F1}%"); + var hits0 = alternating.Hits; var misses0 = alternating.Misses; var refreshes0 = alternating.TableRefreshes; + var draw = 0; + Harness.Measure("3f", "8 tables at different addresses, identical contents, 50 draws alternating (pointer-only change)", Draws, 1404, _ => + { + for (var i = 0; i < Draws; i++) + { + inputs.UserData = userData[draw++ % Tables]; + if (!alternating.Materialize(plan, inputs, ReadResident, ref snapshot, ref specialization, out var failure)) throw new InvalidOperationException(failure.ToString()); + } + }, unit: "draw"); + var calls = (alternating.Hits - hits0) + (alternating.Misses - misses0) + (alternating.TableRefreshes - refreshes0); + Console.WriteLine($" 3f steady: hits {alternating.Hits - hits0} misses {alternating.Misses - misses0} refreshes {alternating.TableRefreshes - refreshes0} -> hit rate {100.0 * (alternating.Hits - hits0) / Math.Max(1, calls):F1}%"); + inputs.UserData = new uint[] { (uint)table0, (uint)(table0 >> 32) }; + } + + // 3g: a per-frame descriptor ring. One shader draws 32 materials (distinct tables) per frame; every frame the engine + // writes each material's table again at a new ring position with the same contents, so a material's pointer differs + // from its previous frame's while its descriptors do not. Includes WriteTable per draw (subtract 3e). + { + const int Materials = 32; + const int RingFirst = 8192, RingSlots = 24576 - Materials; + var ring = new ResourceMaterializationCache(); + var frame = 0; + var ringUserData = new uint[Materials][]; + void Frame() + { + var start = (frame++ * 977) % RingSlots; + for (var m = 0; m < Materials; m++) + { + var table = TableRegion + (ulong)(RingFirst + start + m) * TableStride; + WriteTable(memory, table, 100 + m); + inputs.UserData = new uint[] { (uint)table, (uint)(table >> 32) }; + if (!ring.Materialize(plan, inputs, ReadResident, ref snapshot, ref specialization, out var failure)) throw new InvalidOperationException(failure.ToString()); + } + } + + for (var f = 0; f < 8; f++) Frame(); + Console.WriteLine($" 3g first 8 frames x {Materials} draws: hits {ring.Hits} misses {ring.Misses} refreshes {ring.TableRefreshes} -> hit rate {100.0 * ring.Hits / (8 * Materials):F1}%"); + var hits0 = ring.Hits; var misses0 = ring.Misses; var refreshes0 = ring.TableRefreshes; + Harness.Measure("3g", "descriptor ring: 32 materials/frame, same contents at a new address each frame (+WriteTable)", Materials, 1404, _ => Frame(), unit: "draw"); + var calls = (ring.Hits - hits0) + (ring.Misses - misses0) + (ring.TableRefreshes - refreshes0); + Console.WriteLine($" 3g steady: hits {ring.Hits - hits0} misses {ring.Misses - misses0} refreshes {ring.TableRefreshes - refreshes0} -> hit rate {100.0 * (ring.Hits - hits0) / Math.Max(1, calls):F1}%"); + inputs.UserData = new uint[] { (uint)table0, (uint)(table0 >> 32) }; + } + + // 3e: write-only cost of 3d for subtraction. + Harness.Measure("3e", "baseline: WriteTable only (subtract from 3d)", Batch, 1404, _ => + { + for (var i = 0; i < Batch; i++) WriteTable(memory, table0, 1 + (salt++ % (Pool - 1))); + }, unit: "write"); + } +} diff --git a/tools/SharpEmu.Tools.MaterializationBench/Descriptors.cs b/tools/SharpEmu.Tools.MaterializationBench/Descriptors.cs new file mode 100644 index 000000000..0ae35aae3 --- /dev/null +++ b/tools/SharpEmu.Tools.MaterializationBench/Descriptors.cs @@ -0,0 +1,65 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +namespace SharpEmu.PerfAudit; + +internal static class Descriptors +{ + public const uint IdentitySelect = 0xFAC; + + /// A V# for structured (idxen) loads: bytes per record. + public static uint[] Buffer(ulong address, uint stride, uint records, uint format = 77, uint oobSelect = 0) => + [ + (uint)address, + ((uint)(address >> 32) & 0xFFFF) | (stride << 16), + records, + IdentitySelect | (format << 12) | (1u << 24) | (oobSelect << 28), + ]; + + /// + /// A T# over a linear or tiled image. 9 = 2D, 13 = 2D array, 11 = cube, 10 = 3D. Word 5 MAX_MIP is + /// the resource's mip count minus one, independent of the view's LAST_LEVEL; defaults to LAST_LEVEL. + /// + public static uint[] Texture(ulong address, uint format, uint width, uint height, uint type = 9, uint tileMode = 0, + uint baseLevel = 0, uint lastLevel = 0, uint depthOrLastLayer = 0, uint baseArray = 0, uint dstSelect = IdentitySelect, + uint minLodFixed = 0, uint? maxMip = null) + { + var shifted = address >> 8; + var lastX = width - 1; + var lastY = height - 1; + return + [ + (uint)shifted, + ((uint)(shifted >> 32) & 0xFF) | (minLodFixed << 8) | (format << 20) | ((lastX & 0x3) << 30), + ((lastX >> 2) & 0xFFF) | (lastY << 14), + dstSelect | (baseLevel << 12) | (lastLevel << 16) | (tileMode << 20) | (type << 28), + depthOrLastLayer | (baseArray << 16), + (maxMip ?? lastLevel) << 4, + 0, + 0, + ]; + } + + /// + /// An S#. Clamp 0 = wrap, 1 = mirror, 2 = clamp-to-last, 4 = clamp-half-border, 6 = clamp-to-border; filters 0 = point, 1 = bilinear; + /// mip filter 0 = none, 1 = point, 2 = linear. LOD fields are unsigned 4.8 fixed point; bias is signed 6.8. + /// + public static uint[] Sampler(uint clampX = 2, uint clampY = 2, uint clampZ = 2, uint depthCompare = 0, + uint magFilter = 0, uint minFilter = 0, uint mipFilter = 0, float minLod = 0, float maxLod = 15, float lodBias = 0, + uint borderColorType = 0, uint maxAniso = 0, bool forceUnnormalized = false, uint zFilter = 0) + { + static uint Fixed(float value, int bits) => (uint)Math.Clamp((int)MathF.Round(value * 256f), 0, (1 << bits) - 1); + var bias = (uint)((int)MathF.Round(lodBias * 256f) & 0x3FFF); + return + [ + clampX | (clampY << 3) | (clampZ << 6) | (maxAniso << 9) | ((uint)depthCompare << 12) | (forceUnnormalized ? 1u << 15 : 0u), + Fixed(minLod, 12) | (Fixed(maxLod, 12) << 12), + bias | (magFilter << 20) | (minFilter << 22) | (zFilter << 24) | (mipFilter << 26), + borderColorType << 30, + ]; + } + + public static uint Lo(ulong address) => (uint)address; + + public static uint Hi(ulong address) => (uint)(address >> 32); +} diff --git a/tools/SharpEmu.Tools.MaterializationBench/Harness.cs b/tools/SharpEmu.Tools.MaterializationBench/Harness.cs new file mode 100644 index 000000000..e13bb9bad --- /dev/null +++ b/tools/SharpEmu.Tools.MaterializationBench/Harness.cs @@ -0,0 +1,107 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using System.Diagnostics; +using System.Globalization; +using System.Text; + +namespace SharpEmu.PerfAudit; + +/// One synthetic benchmark: per-operation latency and allocation. +internal sealed record BenchResult( + string Id, string Name, double MedianNs, double P95Ns, double MinNs, double BytesPerOp, double OpsPerFrame, string Unit, string Note); + +/// +/// Hand-rolled Stopwatch harness: untimed warmup, timed runs, median and p95 of +/// the per-run ns/op, and managed allocation measured on the benchmarking thread (or process-wide +/// for the cross-thread cases). +/// +internal static class Harness +{ + public static int Runs = int.TryParse(Environment.GetEnvironmentVariable("PA_RUNS"), out var runs) ? runs : 5; + public static double TargetRunMs = double.TryParse(Environment.GetEnvironmentVariable("PA_RUN_MS"), out var runMs) ? runMs : 150; + public static double WarmupMs = 1500; + public static readonly string[]? OnlyScenarios = Environment.GetEnvironmentVariable("PA_SCENARIOS")?.Split(','); + public static readonly List Results = []; + + /// operations the body performs per call (ns/op = elapsed / calls / ops). + /// Historical workload weighting, unused in timing results. + /// the measured work; receives a running iteration index. + /// count every thread's allocations (for benchmarks that cross threads). + public static BenchResult Measure(string id, string name, double opsPerInvoke, double opsPerFrame, Action body, + string unit = "draw", string note = "", int warmupInvokes = 3, int maxInvokesPerRun = int.MaxValue, + bool processWideAllocations = false, Action? beforeEachRun = null) + { + if (OnlyScenarios is not null && !OnlyScenarios.Contains(id)) + return new BenchResult(id, name, 0, 0, 0, 0, opsPerFrame, unit, note); + // Warm until both the minimum invoke count and WarmupMs have passed, so tiered JIT / PGO promotion has settled + // (the shipped app runs R2R then tier-1 after a long session; a cold tier-0 loop would overstate every number). + var warm = Stopwatch.StartNew(); + for (var i = 0; i < warmupInvokes || warm.Elapsed.TotalMilliseconds < WarmupMs; i++) body(i); + // Calibrate how many invokes make a ~TargetRunMs run. + var sw = Stopwatch.StartNew(); + var calibrationInvokes = 0; + while (sw.Elapsed.TotalMilliseconds < 30 && calibrationInvokes < maxInvokesPerRun) + { + body(calibrationInvokes++); + } + + var perInvokeMs = sw.Elapsed.TotalMilliseconds / Math.Max(calibrationInvokes, 1); + var invokes = (int)Math.Clamp(TargetRunMs / Math.Max(perInvokeMs, 1e-6), 1, maxInvokesPerRun); + var times = new double[Runs]; + var bytes = new double[Runs]; + var index = 0; + for (var run = 0; run < Runs; run++) + { + beforeEachRun?.Invoke(); + GC.Collect(); + GC.WaitForPendingFinalizers(); + var allocatedBefore = processWideAllocations ? GC.GetTotalAllocatedBytes(true) : GC.GetAllocatedBytesForCurrentThread(); + var start = Stopwatch.GetTimestamp(); + for (var i = 0; i < invokes; i++) body(index++); + var elapsed = Stopwatch.GetElapsedTime(start); + var allocatedAfter = processWideAllocations ? GC.GetTotalAllocatedBytes(true) : GC.GetAllocatedBytesForCurrentThread(); + var ops = (double)invokes * opsPerInvoke; + times[run] = elapsed.TotalNanoseconds / ops; + bytes[run] = (allocatedAfter - allocatedBefore) / ops; + } + + Array.Sort(times); + Array.Sort(bytes); + var median = times[Runs / 2]; + var p95 = times[(int)Math.Ceiling(0.95 * Runs) - 1]; + var result = new BenchResult(id, name, median, p95, times[0], bytes[Runs / 2], opsPerFrame, unit, note); + Results.Add(result); + Console.WriteLine(Format(result)); + Console.WriteLine(string.Create(CultureInfo.InvariantCulture, $"MIN {id} {times[0]:F1} {times[Runs / 4]:F1}")); + return result; + } + + /// Records a result computed elsewhere (a long-running scenario measured once per run). + public static BenchResult Record(string id, string name, IReadOnlyList nsPerOp, IReadOnlyList bytesPerOp, + double opsPerFrame, string unit = "draw", string note = "") + { + var times = nsPerOp.OrderBy(v => v).ToArray(); + var bytes = bytesPerOp.OrderBy(v => v).ToArray(); + var result = new BenchResult(id, name, times[times.Length / 2], times[(int)Math.Ceiling(0.95 * times.Length) - 1], times[0], + bytes[bytes.Length / 2], opsPerFrame, unit, note); + Results.Add(result); + Console.WriteLine(Format(result)); + return result; + } + + public static string Format(BenchResult r) => + string.Create(CultureInfo.InvariantCulture, + $"{r.Id,-6} {r.Name,-62} {r.MedianNs,12:N1} ns/op p95 {r.P95Ns,12:N1} {r.BytesPerOp,10:N1} B/op"); + + public static string Table() + { + var sb = new StringBuilder(); + sb.AppendLine("| ID | Benchmark | median ns/op | p95 ns/op | B/op |"); + sb.AppendLine("|---|---|---:|---:|---:|"); + foreach (var r in Results) + sb.AppendLine(string.Create(CultureInfo.InvariantCulture, + $"| {r.Id} | {r.Name} | {r.MedianNs:N1} | {r.P95Ns:N1} | {r.BytesPerOp:N1} |")); + return sb.ToString(); + } +} diff --git a/tools/SharpEmu.Tools.MaterializationBench/NativeGuestMemory.cs b/tools/SharpEmu.Tools.MaterializationBench/NativeGuestMemory.cs new file mode 100644 index 000000000..667932818 --- /dev/null +++ b/tools/SharpEmu.Tools.MaterializationBench/NativeGuestMemory.cs @@ -0,0 +1,43 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using System.Runtime.InteropServices; + +namespace SharpEmu.PerfAudit; + +/// One native block seen as guest memory at ; no tracking, no wrappers. +internal sealed unsafe class NativeGuestMemory : IDisposable +{ + public const ulong Base = 0x2_0000_0000; + private readonly byte* _block; + private readonly ulong _size; + + public NativeGuestMemory(ulong size) + { + _size = size; + _block = (byte*)NativeMemory.AlignedAlloc((nuint)size, 16384); + NativeMemory.Clear(_block, (nuint)size); + } + + public byte* Raw(ulong address) => _block + (address - Base); + public ulong Size => _size; + public void Dispose() => NativeMemory.AlignedFree(_block); + + public bool TryRead(ulong address, Span destination) + { + if (address < Base || address - Base + (ulong)destination.Length > _size) return false; + new ReadOnlySpan(_block + (address - Base), destination.Length).CopyTo(destination); + return true; + } + + public bool TryWrite(ulong address, ReadOnlySpan source) + { + if (address < Base || address - Base + (ulong)source.Length > _size) return false; + source.CopyTo(new Span(_block + (address - Base), source.Length)); + return true; + } + + public bool CanRead(ulong address, ulong size) => address >= Base && address - Base + size <= _size; + + public void WriteWords(ulong address, ReadOnlySpan words) => TryWrite(address, MemoryMarshal.AsBytes(words)); +} diff --git a/tools/SharpEmu.Tools.MaterializationBench/Program.cs b/tools/SharpEmu.Tools.MaterializationBench/Program.cs new file mode 100644 index 000000000..a99e5ea9d --- /dev/null +++ b/tools/SharpEmu.Tools.MaterializationBench/Program.cs @@ -0,0 +1,12 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using System.Runtime.InteropServices; +using SharpEmu.PerfAudit; +using SharpEmu.ShaderCompiler.Resources; + +Console.WriteLine($"arch={RuntimeInformation.ProcessArchitecture} runtime={RuntimeInformation.FrameworkDescription}"); +Harness.WarmupMs = double.TryParse(Environment.GetEnvironmentVariable("PA_WARMUP_MS"), out var warmup) ? warmup : 1500; +MaterializationBenches.Run(); +Console.WriteLine(Harness.Table()); +ResourceMaterializationProfile.WriteReport(); diff --git a/tools/SharpEmu.Tools.MaterializationBench/README.md b/tools/SharpEmu.Tools.MaterializationBench/README.md new file mode 100644 index 000000000..6db122bf7 --- /dev/null +++ b/tools/SharpEmu.Tools.MaterializationBench/README.md @@ -0,0 +1,38 @@ + + +# Synthetic materialization benchmark + +Build with `dotnet build tools/SharpEmu.Tools.MaterializationBench -c Release` and run +`dotnet artifacts/bin/Release/net10.0/SharpEmu.Tools.MaterializationBench.dll`. +Use the same SDK, process architecture and executable/dependencies for every arm. +On Apple Silicon the audit used an x64 .NET host under Rosetta. No GPU or captured +shader/game data is needed. + +The harness warms each workload for 1,500 ms, calibrates batches and reports five +150 ms run samples as median/p95 ns/op and managed bytes/op. `PA_RUNS` and +`PA_RUN_MS` override those sampling settings. `PA_WARMUP_MS=5000` extends warmup; +`PA_SCENARIOS=3c` selects an individual workload for closer alternating arms. `DOTNET_TieredCompilation=0` gives a +separate diagnostic arm; normal tiered/PGO results must be measured separately. +`SHARPEMU_PROFILE_MATERIALIZATION=1` adds phase/read counters; do not compare those +timings against an uninstrumented arm. + +| ID | Workload | +|---|---| +| 3a | Uncached full walk: 8 images, 2 samplers, 4 buffers | +| 3b | Repeated hit with fresh resident-byte validation | +| 3c | Distinct descriptors/addresses cycling 4,096 tables through an 8-entry generation | +| 3d | Changed descriptors at the same address; includes table writes | +| 3f | Eight repeated table pointers with equal descriptor contents | +| 3g | 32 materials moved through a large ring with unchanged descriptor contents; includes writes | +| 3e | Table-write control | + +For source comparisons, build each compiler revision into a separate build profile. +Run the same benchmark executable against each compiler assembly, with the common +HLE/logging dependencies fixed. Record the source revisions and assembly hashes. +Alternate arms in forward and reverse order, retain every run and report ranges +as well as central values. Monitor the full-walk and table-write controls for host +load drift. Cache counters must distinguish actual hits from rebuilds; correctness +and invalidation remain the responsibility of the regression suites. + +These are synthetic per-operation timings, not game frame-time projections. diff --git a/tools/SharpEmu.Tools.MaterializationBench/ResourceTestProgram.cs b/tools/SharpEmu.Tools.MaterializationBench/ResourceTestProgram.cs new file mode 100644 index 000000000..6810b05ef --- /dev/null +++ b/tools/SharpEmu.Tools.MaterializationBench/ResourceTestProgram.cs @@ -0,0 +1,231 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using SharpEmu.ShaderCompiler; +using SharpEmu.ShaderCompiler.Resources; + +namespace SharpEmu.PerfAudit.Materialization; + +// Hand-built instruction records for the resource-plan tests, one helper per shape. +internal static class ResourceTestProgram +{ + public const uint NullOperand = 125; + public const ulong Hash = 0x1234_5678_9ABC_DEF0; + + private static Gen5Operand Literal(uint value) => new(Gen5OperandKind.LiteralConstant, value); + + private static Gen5Operand Immediate(uint value) => Gen5Operand.Source(128 + value); + + public static Gen5Operand Operand(uint value) => + value <= 64 ? Immediate(value) : Literal(value); + + public static Gen5ShaderInstruction EndProgram(uint pc) => + new(pc, Gen5ShaderEncoding.Sopp, "SEndpgm", [0xBF810000], [], [], null); + + public static Gen5ShaderInstruction Nop(uint pc) => + new(pc, Gen5ShaderEncoding.Sopp, "SNop", [0xBF800000], [], [], null); + + public static Gen5ShaderInstruction Branch(uint pc, string opcode, short wordOffset) => + new(pc, Gen5ShaderEncoding.Sopp, opcode, [unchecked((uint)(ushort)wordOffset)], [], [], null); + + public static Gen5ShaderInstruction MoveScalar(uint pc, uint destination, uint value) => + new(pc, Gen5ShaderEncoding.Sop1, "SMovB32", [0u], [Operand(value)], [Gen5Operand.Scalar(destination)], null); + + public static Gen5ShaderInstruction MoveScalarRegister(uint pc, uint destination, uint source) => + new(pc, Gen5ShaderEncoding.Sop1, "SMovB32", [0u], [Gen5Operand.Scalar(source)], [Gen5Operand.Scalar(destination)], null); + + public static Gen5ShaderInstruction Sop1(uint pc, string opcode, uint destination, Gen5Operand source) => + new(pc, Gen5ShaderEncoding.Sop1, opcode, [0u], [source], [Gen5Operand.Scalar(destination)], null); + + public static Gen5ShaderInstruction Sop2(uint pc, string opcode, uint destination, Gen5Operand source0, Gen5Operand source1) => + new(pc, Gen5ShaderEncoding.Sop2, opcode, [0u], [source0, source1], [Gen5Operand.Scalar(destination)], null); + + public static Gen5ShaderInstruction Sopc(uint pc, string opcode, Gen5Operand source0, Gen5Operand source1) => + new(pc, Gen5ShaderEncoding.Sopc, opcode, [0u], [source0, source1], [], null); + + public static Gen5ShaderInstruction Vop1(uint pc, string opcode, uint destination, Gen5Operand source) => + new(pc, Gen5ShaderEncoding.Vop1, opcode, [0u], [source], [Gen5Operand.Vector(destination)], null); + + public static Gen5ShaderInstruction Vop2(uint pc, string opcode, uint destination, Gen5Operand source0, Gen5Operand source1) => + new(pc, Gen5ShaderEncoding.Vop2, opcode, [0u], [source0, source1], [Gen5Operand.Vector(destination)], null); + + public static Gen5ShaderInstruction Vopc(uint pc, string opcode, Gen5Operand source0, uint vectorSource1) => + new(pc, Gen5ShaderEncoding.Vopc, opcode, [0u], [source0, Gen5Operand.Vector(vectorSource1)], [], null); + + public static Gen5ShaderInstruction Vop3(uint pc, string opcode, uint destination, params Gen5Operand[] sources) => + new(pc, Gen5ShaderEncoding.Vop3, opcode, [0u, 0u], sources, [Gen5Operand.Vector(destination)], + new Gen5Vop3Control(0, 0, 0, false, 0, null)); + + public static Gen5ShaderInstruction ReadFirstLane(uint pc, uint scalarDestination, uint vectorSource) => + new(pc, Gen5ShaderEncoding.Vop1, "VReadfirstlaneB32", [0u], [Gen5Operand.Vector(vectorSource)], [Gen5Operand.Scalar(scalarDestination)], null); + + public static Gen5ShaderInstruction WriteLane(uint pc, uint vectorRegister, uint scalarRegister, uint lane) => + new(pc, Gen5ShaderEncoding.Vop3, "VWritelaneB32", [0u, 0u], + [Gen5Operand.Scalar(scalarRegister), Immediate(lane), Gen5Operand.Scalar(0)], [Gen5Operand.Vector(vectorRegister)], null); + + public static Gen5ShaderInstruction ReadLane(uint pc, uint scalarRegister, uint vectorRegister, uint lane) => + new(pc, Gen5ShaderEncoding.Vop3, "VReadlaneB32", [0u, 0u], + [Gen5Operand.Vector(vectorRegister), Immediate(lane), Gen5Operand.Scalar(0)], [Gen5Operand.Scalar(scalarRegister)], null); + + // s_load_dwordxN through the address pair at baseRegister. + public static Gen5ShaderInstruction ScalarLoad(uint pc, uint baseRegister, uint destination, uint count = 1, int immediateOffset = 0, uint? dynamicOffsetRegister = null) => + new(pc, Gen5ShaderEncoding.Smem, ScalarLoadName("SLoadDword", count), [0u, 0u], + [Gen5Operand.Scalar(baseRegister), dynamicOffsetRegister is { } register ? Gen5Operand.Scalar(register) : Gen5Operand.Source(NullOperand)], + Enumerable.Range((int)destination, (int)count).Select(index => Gen5Operand.Scalar((uint)index)).ToArray(), + new Gen5ScalarMemoryControl(count, immediateOffset, dynamicOffsetRegister)); + + // s_buffer_load_dwordxN through the descriptor at resourceRegister. + public static Gen5ShaderInstruction ScalarBufferLoad(uint pc, uint resourceRegister, uint destination, uint count = 1, int immediateOffset = 0, uint? dynamicOffsetRegister = null) => + new(pc, Gen5ShaderEncoding.Smem, ScalarLoadName("SBufferLoadDword", count), [0u, 0u], + [Gen5Operand.Scalar(resourceRegister), dynamicOffsetRegister is { } register ? Gen5Operand.Scalar(register) : Gen5Operand.Source(NullOperand)], + Enumerable.Range((int)destination, (int)count).Select(index => Gen5Operand.Scalar((uint)index)).ToArray(), + new Gen5ScalarMemoryControl(count, immediateOffset, dynamicOffsetRegister)); + + private static string ScalarLoadName(string prefix, uint count) => count == 1 ? prefix : $"{prefix}x{count}"; + + public static Gen5ShaderInstruction BufferAccess(uint pc, string opcode, uint resourceRegister, int offset = 0, uint dwords = 1, uint vectorData = 4, bool indexEnabled = false, bool offsetEnabled = false, uint vectorAddress = 0) + { + var isStore = opcode.Contains("Store", StringComparison.Ordinal) || opcode.Contains("Atomic", StringComparison.Ordinal); + var data = Enumerable.Range((int)vectorData, (int)dwords).Select(index => Gen5Operand.Vector((uint)index)).ToArray(); + return new Gen5ShaderInstruction(pc, Gen5ShaderEncoding.Mubuf, opcode, [0u, 0u], + isStore ? [Gen5Operand.Vector(vectorAddress), Gen5Operand.Scalar(resourceRegister), Gen5Operand.Source(NullOperand), .. data] + : [Gen5Operand.Vector(vectorAddress), Gen5Operand.Scalar(resourceRegister), Gen5Operand.Source(NullOperand)], + isStore ? [] : data, + new Gen5BufferMemoryControl(dwords, vectorAddress, vectorData, resourceRegister, offset, indexEnabled, offsetEnabled, Glc: false, Slc: false)); + } + + public static Gen5ShaderInstruction BufferLoad(uint pc, uint resourceRegister, int offset = 0, uint dwords = 1, bool formatted = false) => + BufferAccess(pc, formatted ? FormatName("BufferLoadFormat", dwords) : DwordName("BufferLoadDword", dwords), resourceRegister, offset, dwords); + + public static Gen5ShaderInstruction BufferStore(uint pc, uint resourceRegister, int offset = 0, uint dwords = 1, bool formatted = false) => + BufferAccess(pc, formatted ? FormatName("BufferStoreFormat", dwords) : DwordName("BufferStoreDword", dwords), resourceRegister, offset, dwords); + + public static Gen5ShaderInstruction BufferAtomicAdd(uint pc, uint resourceRegister, int offset = 0) => + BufferAccess(pc, "BufferAtomicAdd", resourceRegister, offset); + + private static string FormatName(string prefix, uint dwords) => prefix + dwords switch { 1 => "X", 2 => "Xy", 3 => "Xyz", _ => "Xyzw" }; + + private static string DwordName(string prefix, uint dwords) => dwords == 1 ? prefix : $"{prefix}x{dwords}"; + + // global_load/store through the scalar address pair, or a flat access through v[address:address+1]. + public static Gen5ShaderInstruction GlobalAccess(uint pc, string opcode, uint scalarAddress, int offset = 0, uint vectorAddress = 0, uint dwords = 1) + { + var flat = opcode.StartsWith("Flat", StringComparison.Ordinal); + var isStore = opcode.Contains("Store", StringComparison.Ordinal); + Gen5Operand[] addresses = flat + ? [Gen5Operand.Vector(vectorAddress), Gen5Operand.Vector(vectorAddress + 1)] + : [Gen5Operand.Vector(vectorAddress), Gen5Operand.Scalar(scalarAddress)]; + var data = Enumerable.Range(8, (int)dwords).Select(index => Gen5Operand.Vector((uint)index)).ToArray(); + return new Gen5ShaderInstruction(pc, Gen5ShaderEncoding.Flat, opcode, [0u, 0u], + isStore ? [.. addresses, .. data] : addresses, + isStore ? [] : data, + new Gen5GlobalMemoryControl(dwords, vectorAddress, 8, 8, flat ? uint.MaxValue : scalarAddress, offset, Glc: false, Slc: false, flat)); + } + + public static Gen5ShaderInstruction Image(uint pc, string opcode, uint resourceRegister, uint samplerRegister = 0, uint dimension = 1, uint dmask = 0xF, uint vectorAddress = 0, bool r128 = false) + { + var addressRegisters = new uint[] { vectorAddress, vectorAddress + 1, vectorAddress + 2 }; + var control = new Gen5ImageControl(dmask, vectorAddress, addressRegisters, 4, resourceRegister, samplerRegister, dimension, + dimension is 4 or 5 or 7, Glc: false, Slc: false, A16: false, D16: false); + var sources = new List { Gen5Operand.Vector(vectorAddress), Gen5Operand.Scalar(resourceRegister), Gen5Operand.Scalar(samplerRegister) }; + var destinations = opcode.StartsWith("ImageStore", StringComparison.Ordinal) ? Array.Empty() : [Gen5Operand.Vector(4)]; + return new Gen5ShaderInstruction(pc, Gen5ShaderEncoding.Mimg, opcode, [r128 ? 1u << 15 : 0u, 0u], sources, destinations, control); + } + + public static Gen5ShaderInstruction DataShareWrite(uint pc, bool gds) => + new(pc, Gen5ShaderEncoding.Ds, "DsWriteB32", [gds ? 1u << 17 : 0u, 0u], [Gen5Operand.Vector(0), Gen5Operand.Vector(1)], [], + new Gen5DataShareControl(0, 0, gds)); + + // Any DS instruction: sources first (address, data...), then the vector destinations. + public static Gen5ShaderInstruction DataShare(uint pc, string opcode, bool gds, Gen5Operand[] sources, uint[] destinations, uint offset0 = 0, uint offset1 = 0) => + new(pc, Gen5ShaderEncoding.Ds, opcode, [gds ? 1u << 17 : 0u, 0u], sources, destinations.Select(Gen5Operand.Vector).ToArray(), + new Gen5DataShareControl(offset0, offset1, gds)); + + // A global or flat access with explicit data registers: a load writes destination, + // a store or atomic reads source (an atomic with glc also writes destination). + public static Gen5ShaderInstruction GlobalMemory(uint pc, string opcode, uint scalarAddress, uint vectorAddress, uint destination, uint source, int offset = 0, uint dwords = 1, bool glc = false) + { + var flat = opcode.StartsWith("Flat", StringComparison.Ordinal); + var noScalarAddress = flat || scalarAddress >= NullOperand; + var isStore = opcode.Contains("Store", StringComparison.Ordinal) || opcode.Contains("Atomic", StringComparison.Ordinal); + Gen5Operand[] addresses = noScalarAddress + ? [Gen5Operand.Vector(vectorAddress), Gen5Operand.Vector(vectorAddress + 1)] + : [Gen5Operand.Vector(vectorAddress), Gen5Operand.Scalar(scalarAddress)]; + var data = Enumerable.Range((int)source, (int)dwords).Select(index => Gen5Operand.Vector((uint)index)).ToArray(); + var results = Enumerable.Range((int)destination, (int)dwords).Select(index => Gen5Operand.Vector((uint)index)).ToArray(); + return new Gen5ShaderInstruction(pc, Gen5ShaderEncoding.Flat, opcode, [0u, 0u], + isStore ? [.. addresses, .. data] : addresses, + isStore && !glc ? [] : results, + new Gen5GlobalMemoryControl(dwords, vectorAddress, source, destination, noScalarAddress ? NullOperand : scalarAddress, offset, Glc: glc, Slc: false, flat)); + } + + public static Gen5ShaderInstruction MoveVector(uint pc, uint destination, uint value) => + Vop1(pc, "VMovB32", destination, Operand(value)); + + public static Gen5ShaderInstruction MoveVectorFromScalar(uint pc, uint destination, uint scalarRegister) => + Vop1(pc, "VMovB32", destination, Gen5Operand.Scalar(scalarRegister)); + + public static Gen5ShaderProgram Program(params Gen5ShaderInstruction[] instructions) => new(0, instructions); + + public static ShaderResourcePlan Extract(Gen5ShaderProgram program, uint userDataBase = 0, uint userDataCount = 64, ShaderStage stage = ShaderStage.Compute, + uint waveSize = 64) => + ShaderResourcePlan.Extract(program, stage, Hash, userDataBase, userDataCount, waveSize: waveSize); + + // The plan, its default specialization applied, and the layout of one program. + public static (ShaderResourcePlan Plan, SpecializedResourceInfo Resources, BindingLayout Layout) Prepare( + Gen5ShaderProgram program, ShaderStage stage = ShaderStage.Compute, uint userDataBase = 0, uint userDataCount = 64, uint pushDataStartDword = 0, + bool usesDispatchThreadLimits = false) + { + var plan = ShaderResourcePlan.Extract(program, stage, Hash, userDataBase, userDataCount); + var resources = ResourceMaterializer.ApplyTo(plan, ResourceSpecialization.Default(plan.Info)); + var layout = BindingLayout.Allocate( + resources.Info, + BindingLayout.CollectUserDataRegisters(program, userDataBase, userDataCount, plan.Graph.ExcludedUserDataRegisters), + BindingLayout.UsesGlobalDataShare(program), + ShaderCompileRequest.RequiresFlattenedTable(plan, resources), + BindingLayout.ReadsShaderBase(program), + pushDataStartDword, + usesDispatchThreadLimits, + usesMeshDrawParameters: stage == ShaderStage.Mesh); + return (plan, resources, layout); + } + + public static ShaderCompileRequest Request(Gen5ShaderProgram program, ShaderStage stage = ShaderStage.Compute, uint userDataBase = 0, uint userDataCount = 64, uint pushDataStartDword = 0) + { + var (plan, resources, layout) = Prepare(program, stage, userDataBase, userDataCount, pushDataStartDword); + return new ShaderCompileRequest(plan, resources, layout) + { + PixelOutputs = stage == ShaderStage.Pixel ? [new Gen5PixelOutputBinding(0, 0, Gen5PixelOutputKind.Float)] : [], + }; + } + + public static ResourceRuntimeInputs Inputs(uint[] userData, GuestWordReader? readMemory = null, GuestWordReader? readCleanMemory = null, ulong shaderBase = 0) => + new() { UserData = userData, ReadMemory = readMemory, ReadCleanMemory = readCleanMemory, ShaderBase = shaderBase }; +} + +// A small dword-addressed memory image whose reads can be counted and refused. +internal sealed class TestWordMemory +{ + public ulong Base { get; init; } = 0x1000; + public uint[] Words { get; init; } = new uint[8]; + public uint Reads { get; set; } + public uint FailAfter { get; set; } = uint.MaxValue; + public ulong FailAddress { get; set; } = ulong.MaxValue; + public bool RequireAlignment { get; init; } + + public bool Read(ulong address, out uint value) + { + value = 0; + if (address < Base || address - Base >= (ulong)Words.Length * sizeof(uint) || Reads >= FailAfter || + address == FailAddress || (RequireAlignment && (address & 3) != 0)) + { + return false; + } + + value = Words[(address - Base) / sizeof(uint)]; + Reads++; + return true; + } + + public ref uint At(ulong address) => ref Words[(address - Base) / sizeof(uint)]; +} diff --git a/tools/SharpEmu.Tools.MaterializationBench/SharpEmu.Tools.MaterializationBench.csproj b/tools/SharpEmu.Tools.MaterializationBench/SharpEmu.Tools.MaterializationBench.csproj new file mode 100644 index 000000000..92a39ba97 --- /dev/null +++ b/tools/SharpEmu.Tools.MaterializationBench/SharpEmu.Tools.MaterializationBench.csproj @@ -0,0 +1,14 @@ + + + + Exe + true + false + false + false + + + + + From 379fa0273a2a18dfe3b3ce6ee5c7e6829d9ab8d0 Mon Sep 17 00:00:00 2001 From: kuba Date: Mon, 26 Oct 2026 16:54:00 +0100 Subject: [PATCH 29/82] [Bench] Publish synthetic audit harness and change coverage --- docs/performance/benchmark-index.md | 104 +++ docs/performance/combined-audit.md | 3 + .../SharpEmu.Gpu.Metal.csproj | 1 + tools/SharpEmu.Tools.PerfAudit/Benches.E2E.cs | 324 +++++++ .../Benches.Lookup.cs | 121 +++ .../Benches.Materialization.cs | 202 +++++ .../Benches.Memory.cs | 185 ++++ .../SharpEmu.Tools.PerfAudit/Benches.Metal.cs | 285 ++++++ .../SharpEmu.Tools.PerfAudit/Benches.Misc.cs | 27 + .../Benches.Planning.cs | 121 +++ tools/SharpEmu.Tools.PerfAudit/Benches.Pm4.cs | 100 +++ .../SharpEmu.Tools.PerfAudit/Benches.Queue.cs | 115 +++ tools/SharpEmu.Tools.PerfAudit/Benches.Rig.cs | 36 + .../Benches.Translator.cs | 91 ++ tools/SharpEmu.Tools.PerfAudit/Benches.W8.cs | 128 +++ .../Benches.WriteRanges.cs | 27 + .../SharpEmu.Tools.PerfAudit/FakeCpuMemory.cs | 101 +++ tools/SharpEmu.Tools.PerfAudit/Harness.cs | 111 +++ tools/SharpEmu.Tools.PerfAudit/LeanHosts.cs | 86 ++ .../PlanningSupport.cs | 812 ++++++++++++++++++ tools/SharpEmu.Tools.PerfAudit/Program.cs | 34 + tools/SharpEmu.Tools.PerfAudit/README.md | 57 ++ .../SharpEmu.Tools.PerfAudit/RegisterWords.cs | 85 ++ .../ResourceTestProgram.cs | 231 +++++ .../Rig/SceneDraws.cs | 164 ++++ .../Rig/SceneMemory.cs | 135 +++ .../SharpEmu.Tools.PerfAudit/Rig/ScenePm4.cs | 102 +++ .../Rig/ScenePrograms.cs | 74 ++ .../SharpEmu.Tools.PerfAudit/Rig/SceneRig.cs | 247 ++++++ .../Rig/SceneState.cs | 397 +++++++++ .../SharpEmu.Tools.PerfAudit.csproj | 25 + 31 files changed, 4531 insertions(+) create mode 100644 docs/performance/benchmark-index.md create mode 100644 tools/SharpEmu.Tools.PerfAudit/Benches.E2E.cs create mode 100644 tools/SharpEmu.Tools.PerfAudit/Benches.Lookup.cs create mode 100644 tools/SharpEmu.Tools.PerfAudit/Benches.Materialization.cs create mode 100644 tools/SharpEmu.Tools.PerfAudit/Benches.Memory.cs create mode 100644 tools/SharpEmu.Tools.PerfAudit/Benches.Metal.cs create mode 100644 tools/SharpEmu.Tools.PerfAudit/Benches.Misc.cs create mode 100644 tools/SharpEmu.Tools.PerfAudit/Benches.Planning.cs create mode 100644 tools/SharpEmu.Tools.PerfAudit/Benches.Pm4.cs create mode 100644 tools/SharpEmu.Tools.PerfAudit/Benches.Queue.cs create mode 100644 tools/SharpEmu.Tools.PerfAudit/Benches.Rig.cs create mode 100644 tools/SharpEmu.Tools.PerfAudit/Benches.Translator.cs create mode 100644 tools/SharpEmu.Tools.PerfAudit/Benches.W8.cs create mode 100644 tools/SharpEmu.Tools.PerfAudit/Benches.WriteRanges.cs create mode 100644 tools/SharpEmu.Tools.PerfAudit/FakeCpuMemory.cs create mode 100644 tools/SharpEmu.Tools.PerfAudit/Harness.cs create mode 100644 tools/SharpEmu.Tools.PerfAudit/LeanHosts.cs create mode 100644 tools/SharpEmu.Tools.PerfAudit/PlanningSupport.cs create mode 100644 tools/SharpEmu.Tools.PerfAudit/Program.cs create mode 100644 tools/SharpEmu.Tools.PerfAudit/README.md create mode 100644 tools/SharpEmu.Tools.PerfAudit/RegisterWords.cs create mode 100644 tools/SharpEmu.Tools.PerfAudit/ResourceTestProgram.cs create mode 100644 tools/SharpEmu.Tools.PerfAudit/Rig/SceneDraws.cs create mode 100644 tools/SharpEmu.Tools.PerfAudit/Rig/SceneMemory.cs create mode 100644 tools/SharpEmu.Tools.PerfAudit/Rig/ScenePm4.cs create mode 100644 tools/SharpEmu.Tools.PerfAudit/Rig/ScenePrograms.cs create mode 100644 tools/SharpEmu.Tools.PerfAudit/Rig/SceneRig.cs create mode 100644 tools/SharpEmu.Tools.PerfAudit/Rig/SceneState.cs create mode 100644 tools/SharpEmu.Tools.PerfAudit/SharpEmu.Tools.PerfAudit.csproj diff --git a/docs/performance/benchmark-index.md b/docs/performance/benchmark-index.md new file mode 100644 index 000000000..2b887fda6 --- /dev/null +++ b/docs/performance/benchmark-index.md @@ -0,0 +1,104 @@ + + +# Audit changes and benchmark index + +The combined PR contains the complete round-one integration ending at +`cde04b713e60de7ab02dfa092e8f91d086df0c83` (37 commits above the audit's main +baseline), the two round-two committed changes and the accepted round-two WIP. +The original combined implementation is `18323b71870897ea657939d621a1bacfbde83a2b`. +The later benchmark packaging adds reusable source and documentation; it does not +turn historical component measurements into measurements of the combined build. + +## Implementation coverage + +| Area | Changes included | +|---|---| +| CPU and HLE crossings | AGC builder leaf path, native clock stubs, host-thread pthread identity, shared thread state, in-place large memset, expected-result logging and disabled-trace formatting guards | +| Guest scheduling | Direct wake handoff/shared resume stub, owner-aware bounded mutex spinning, per-fiber lock stripes, targeted ULT wakes, TLS caching, sleep/yield backoff and thread/exit behavior | +| Draw preparation | Pipeline/shader identity memoization with fresh-memory fallback, table/color-target lookup reuse, scratch index expansion, unchanged register snapshot reuse and diagnostic-only write history | +| Resource materialization | Content-keyed reuse across moving descriptors, pointer-key costs, linear merge of recorded reads and bounded cold-scan retention; descriptor, mapping, bounds and snapshot invalidation tests | +| Metal storage/binding | Page-granular staged refresh, reusable retile staging, binding/residency reuse, GPU-pristine snapshots, owned texture-view cache and alias/narrow-consumer transitions | +| Synchronization | Exact GPU-written word fences, host completion publication through callbacks at the same serial, tracked store/barrier handling and waiter registration/lost-wake coverage | +| Split transport | One protocol-7 layout, frame-boundary waits outside the submission gate, event-driven input, one-use display reservations, pending-writeback page summary and cross-architecture smoke coverage | +| Shader translation | Relaxed-math/early-depth handling, global-record reuse, grouped raw loads, native min/max and reciprocal lowering; precise sqrt retained; contraction remains opt-in | +| Audio and host overhead | AJM disabled-formatting path, environment lookup and audio context work, helper/compiler thread policy and archive/prewarm handling | +| Regression fixes | Six Libs fixture/API failures, scattered-view alignment, lossy allocation footprint, enumerated native leak counters and ordered fake-store writes | + +The [K1–K21 disposition table](combined-audit.md#workstream-dispositions) records +what was accepted, rejected or left opt-in. Concurrent tiling is not the default; +GC/tiering changes, unsupported lock-free/parallel planning, extra barrier/store +elision, decoder pooling and ABI changes were not accepted. + +A source inventory check found 124 distinct recovered changed paths: 122 are +tracked in the combined PR. The two other paths are the temporary input harness +(replaced by `InputDeliveryBenchmarks.cs`) and `SpinNapElfSurvey.cs` (an ELF survey, +not an accepted emulator fix). K1's survey-only loader field was also omitted. +This is an accounting of accepted implementation, not a claim that every discarded +experiment or local artifact belongs in the PR. Original synthetic eboot binaries, +commercial captures, raw logs, build outputs and machine diagnostics stay outside it. + +## Runnable benchmark sources + +| Benchmark | Source / instructions | +|---|---| +| CPU, queue, memory, translator, pipeline and synthetic Metal paths | [PerfAudit harness](../../tools/SharpEmu.Tools.PerfAudit/README.md) | +| Final cold/stale/hit/ring comparison | [Materialization harness](../../tools/SharpEmu.Tools.MaterializationBench/README.md) | +| Split input latency | [InputDeliveryBenchmarks](../../tests/SharpEmu.GpuSplit.Tests/InputDeliveryBenchmarks.cs), `SHARPEMU_INPUT_BENCH=1` | +| Grouped loads, min/max, reciprocals, contraction and hints | [MslTranslatorOverheadBenchTests](../../tests/SharpEmu.ShaderCompiler.Metal.Tests/MslTranslatorOverheadBenchTests.cs) | +| Relaxed math, global records and early depth | [MslGpuShaderTimeBenchTests](../../tests/SharpEmu.ShaderCompiler.Metal.Tests/MslGpuShaderTimeBenchTests.cs) | +| Texture transfer and alias CPU/GPU costs | [MetalImageTransferTests.Timing](../../tests/SharpEmu.Gpu.Metal.Tests/MetalImageTransferTests.Timing.cs) | +| Narrow consumer transfer work and exact bytes | [ConsumerRunSceneTests](../../tests/SharpEmu.Gpu.Metal.Tests/Scenes/ConsumerRunSceneTests.cs) | + +Run the shader benchmarks with `SHARPEMU_GPU_BENCH=1 SHARPEMU_TEST_REQUIRE_DEVICE=1`, +Release configuration, a class filter and `shader-serial.runsettings`. For example: + +```sh +SHARPEMU_GPU_BENCH=1 SHARPEMU_TEST_REQUIRE_DEVICE=1 ~/.dotnet-x64/dotnet test tests/SharpEmu.ShaderCompiler.Metal.Tests -c Release -m:2 --settings docs/performance/shader-serial.runsettings --filter FullyQualifiedName~MslTranslatorOverheadBenchTests --logger 'console;verbosity=detailed' +SHARPEMU_INPUT_BENCH=1 ~/.dotnet-x64/dotnet test tests/SharpEmu.GpuSplit.Tests -c Release -m:2 --filter FullyQualifiedName~InputDeliveryBenchmarks --logger 'console;verbosity=detailed' +``` + +## Measurements and limits + +These are different experiments with different baselines; they must not be added +together or converted into a game-FPS estimate. Final admission figures are from +alternating runs of the combined build. Other figures are historical isolated +component comparisons retained from the recovery audit. + +| Experiment | Before | After | Interpretation | +|---|---:|---:|---| +| Final cold allocation, tiering disabled | 8,264 B/op | 6,379.9–6,382.6 B/op | About 23% less allocation; timing remains inconclusive | +| Final cold allocation, tiered/PGO | 8,160 B/op | 6,378.9–6,384.2 B/op | About 22% less allocation; host controls drifted | +| Final stale descriptor allocation, tiering disabled | 8,392.2–8,392.3 B/op | 7,032.6–7,035.4 B/op | About 16% less allocation | +| K17 strided translator, x64 tiering-disabled ABBA | 29.9 / 25.9 ns; 47 B/op | 4.9 / 5.3 ns; 0 B/op | Historical component result | +| K17 fragmented translator, same experiment | 153.1 / 155.8 ns; 45.2 B/op | 20.8 / 22.2 ns; 0 B/op | Single-view case instead slowed slightly; ~40 KiB inline storage per translator | +| K4 input delivery p50 | 3.48 / 3.11 ms | 0.10 / 0.10 ms | Two transport halves in one process; not guest draw latency | +| K12 narrow linear consumer retile | 16,588,800 bytes | 15,360 bytes | Deferred work; later full consumption still does the remaining work | +| K12 narrow tiled consumer retile | 16,711,680 bytes | 983,040 bytes | Exact-byte correctness checked; not a timing result | +| K21 grouped-load GPU kernel | 9.173 ms | 4.972 ms | Best of three warmed synthetic GPU measurements | +| K21 min/max GPU kernel | 13.700 ms | 8.215 ms | Same narrow methodology | +| K21 reciprocal/rsqrt GPU kernel | 8.628 ms | 6.303 ms | Precise sqrt in both arms; not a whole-shader result | + +[The full report](combined-audit.md#measurements-and-validation) retains both +medians and p95 ranges for every final A/B arm, historical main/cde cold controls, +allocation results, source identities, test counts and reproducible commands. +The linear merge alone did not prove a timing improvement. Concurrent tiling +regressed representative writebacks and remains opt-in. Compiler QoS and shader +hints have no established general speedup. Two admission-disabled parallel test-host +stalls remain unresolved; the full serial run passes unchanged assertions. + +The original combined validation totals are **16,741 passed, 0 failed, 13 existing +skips**, plus REUSE, script checks and successful macOS CI at `18323b718`. +No commercial game was run. Benchmark smoke checks for this packaging change are +reported separately from those full-suite results. + +## Benchmark packaging validation + +On x64 .NET 10.0.3/macOS, the standalone project builds with zero errors. +The documented short runtime invocation completes **75 benchmark rows** across +10 selectors. These short-run numbers are smoke evidence, not new A/B claims. +The generated Metal triangle completes and asserts the exact `0xFF0000FF` pixel; +a mismatch exits with an error. REUSE passes for all 2,816 files and whitespace +checks pass. The emulator algorithms are unchanged by this packaging follow-up; +the Metal project only grants the uniquely named tool access to its internals. +Full-suite/CI counts above belong to the original combined implementation. diff --git a/docs/performance/combined-audit.md b/docs/performance/combined-audit.md index 7278b8f4e..d3c425cac 100644 --- a/docs/performance/combined-audit.md +++ b/docs/performance/combined-audit.md @@ -8,6 +8,9 @@ and the valid round-two changes. Validation uses fabricated guest programs, synthetic descriptors and real Metal device tests. No commercial game was run. Microkernel, byte-work and microbenchmark results do not establish game performance. +See the [change and benchmark index](benchmark-index.md) for round-one coverage, +source-inventory exclusions, runnable benchmark locations and a measurement summary. + ## Integration and correctness The combined wire protocol is revision **7**. Independent revisions 4 (frame boundary), diff --git a/src/graphics/SharpEmu.Gpu.Metal/SharpEmu.Gpu.Metal.csproj b/src/graphics/SharpEmu.Gpu.Metal/SharpEmu.Gpu.Metal.csproj index e3a8046eb..6244e26e5 100644 --- a/src/graphics/SharpEmu.Gpu.Metal/SharpEmu.Gpu.Metal.csproj +++ b/src/graphics/SharpEmu.Gpu.Metal/SharpEmu.Gpu.Metal.csproj @@ -35,6 +35,7 @@ SPDX-License-Identifier: GPL-2.0-or-later + diff --git a/tools/SharpEmu.Tools.PerfAudit/Benches.E2E.cs b/tools/SharpEmu.Tools.PerfAudit/Benches.E2E.cs new file mode 100644 index 000000000..ae1a6465d --- /dev/null +++ b/tools/SharpEmu.Tools.PerfAudit/Benches.E2E.cs @@ -0,0 +1,324 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using System.Diagnostics; +using SharpEmu.Gpu.Metal; +using SharpEmu.Gpu.Metal.Tests.Scenes; + +namespace SharpEmu.PerfAudit; + +/// +/// Whole-path scenario: a PM4 stream of textured draws through the real interpreter, RenderExecutor, +/// ShaderPipelineCache, resource materialization, Metal host binding/encoding, commit and GPU wait, +/// driven the way the Metal worker drives it, over the scene rig (real device, real shader compiler). +/// +internal static class E2EBenches +{ + private const uint Target = 256; + private const int Pairs = 50; + private const int StateVariants = 4; + private const int Textures = 8; + private const int Meshes = 16; + + private static uint[] WithNops(uint[] code, int nops) + { + // Insert nops before the final s_endpgm so each variant has a distinct hash and code size. + var result = new uint[code.Length + nops]; + Array.Copy(code, result, code.Length - 1); + for (var i = 0; i < nops; i++) result[code.Length - 1 + i] = 0xBF800000; + result[^1] = code[^1]; + return result; + } + + /// + /// A pixel program that s_loads 8 T#, 2 S# and 4 V# through the pointer in s[0:1] (the user data), samples all + /// 8 textures through the two samplers, loads a record from each of the 4 buffers and sums everything into MRT0. + /// + private static uint[] HeavyPixel() + { + var code = new List + { + 0xBEFC0302, // s_mov_b32 m0, s2 (two user SGPRs, so the primitive mask is s2) + 0xC8080000, 0xC8090001, // v_interp p1/p2 attr0.x -> v2 + 0xC80C0100, 0xC80D0101, // attr0.y -> v3 + }; + for (var i = 0u; i < 8; i++) { code.Add(0xF40C0000 | ((16 + 8 * i) << 6)); code.Add(0xFA000000 | (i * 0x20)); } // s_load_dwordx8 T#i + for (var j = 0u; j < 2; j++) { code.Add(0xF4080000 | ((80 + 4 * j) << 6)); code.Add(0xFA000100 | (j * 0x10)); } // s_load_dwordx4 S#j + for (var k = 0u; k < 4; k++) { code.Add(0xF4080000 | ((88 + 4 * k) << 6)); code.Add(0xFA000000 | (0x120 + k * 0x10)); } // s_load_dwordx4 V#k + code.Add(0xBF8CC07F); // s_waitcnt lgkmcnt(0) + for (var i = 0u; i < 8; i++) // image_sample v[4+4i:7+4i], v[2:3], T#i, S#(i%2) + { + code.Add(0xF0800F08); + code.Add(((20 + (i % 2)) << 21) | ((4 + 2 * i) << 16) | ((4 + 4 * i) << 8) | 2); + } + for (var k = 0u; k < 4; k++) // buffer_load_dwordx4 v[36+4k:39+4k], v0, V#k idxen + { + code.Add(0xE0382000); + code.Add(0x80000000 | ((22 + k) << 16) | ((36 + 4 * k) << 8) | 0); + } + code.Add(0xBF8C3F70); // s_waitcnt vmcnt(0) + for (var i = 1u; i < 8; i++) code.Add(0x06000000 | (4u << 17) | (4u << 9) | (256 + 4 + 4 * i)); // v_add_f32 v4, v(4+4i), v4 + for (var k = 0u; k < 4; k++) code.Add(0x06000000 | (4u << 17) | (4u << 9) | (256 + 36 + 4 * k)); + code.Add(0xF800180F); code.Add(0x07060504); // exp mrt0 v4, v5, v6, v7 done vm + code.Add(0xBF810000); // s_endpgm + return [.. code]; + } + + private sealed class Scenario + { + public required SceneRig Rig; + public required uint[] Stream; + public required int Draws; + public required int PacketWords; + public ulong Address; + } + + private static ulong _submissionId = 1_000_000; + + /// 0 = 8 rotating texture descriptors (warm); 1 = every draw gets fresh T#/S# words (cold materialization). + private static Scenario Build(SceneRig rig, int draws, int pairs, int variants, int descriptorChurn, bool delta = false, bool heavy = false) + { + var vs = new ulong[pairs]; + var ps = new ulong[pairs]; + for (var p = 0; p < pairs; p++) + { + vs[p] = rig.Program(WithNops(ScenePrograms.FetchVertex, p)); + ps[p] = rig.Program(WithNops(heavy ? HeavyPixel() : ScenePrograms.SampledTexture, p)); + } + + // Eight small linear textures, sixteen meshes (VS user data = position V# + param V#). + var images = new ulong[Textures]; + for (var t = 0; t < Textures; t++) + { + var texels = new byte[16 * 16 * 4]; + for (var i = 0; i < texels.Length; i += 4) BitConverter.TryWriteBytes(texels.AsSpan(i), 0xFF000000u | (uint)(i * 2654435761u >> 8 & 0xFFFFFF) | (uint)t); + images[t] = rig.LinearImage(16, 16, 4, texels); + } + + var meshes = new uint[Meshes][]; + for (var m = 0; m < Meshes; m++) + { + var x = -1f + (m % 4) * 0.5f; + var y = -1f + (m / 4) * 0.5f; + meshes[m] = rig.UploadMesh( + [ + SceneVertex.At(x, y, 0.5f, 1, 0, 0), + SceneVertex.At(x + 0.4f, y, 0.5f, 1, 1, 0), + SceneVertex.At(x, y + 0.4f, 0.5f, 0, 0, 1), + ]); + } + + // Heavy: descriptor tables in guest memory. Warm = 8 tables reused; churn = one table (distinct T#/S# words) per draw. + var tableCount = descriptorChurn == 0 ? 8 : Math.Min(draws, 6000); + var tables = new ulong[heavy ? tableCount : 0]; + if (heavy) + { + var bufferBase = rig.Memory.Allocate(4 * 0x1000, 0x1000); + for (var t = 0; t < tableCount; t++) + { + var table = new uint[0x160 / 4]; + for (var i = 0; i < 8; i++) + Descriptors.Texture(images[(t + i) % Textures], 56, 16, 16, minLodFixed: descriptorChurn == 0 ? 0u : (uint)((t * 8 + i) & 0xFFF)).CopyTo(table, i * 8); + for (var j = 0; j < 2; j++) + Descriptors.Sampler(lodBias: descriptorChurn == 0 ? 0 : ((t + j) % 64) / 64f).CopyTo(table, 0x100 / 4 + j * 4); + for (var k = 0; k < 4; k++) + Descriptors.Buffer(bufferBase + (ulong)k * 0x1000, 16, 256).CopyTo(table, 0x120 / 4 + k * 4); + tables[t] = rig.Memory.Upload(table, 256); + } + } + + var target = rig.ColorTarget(Target, Target); + var pm4 = new ScenePm4(); + var sampler = Descriptors.Sampler(); + for (var d = 0; d < draws; d++) + { + var pair = d % pairs; + var variant = (d / pairs) % variants; + var state = new SceneState(Target, Target).ColorTarget(0, target, Target, Target); + // Four pipeline-state variants: write mask and blending change the pipeline. + state.Context[Cx.CbTargetMask] = variant < 2 ? 0xFu : 0x7u; + state.Context[Cx.CbBlend0Control] = (variant & 1) == 0 ? 0u : (1u << 30) | 4u | (5u << 8) | (4u << 16) | (5u << 24); + var texture = Descriptors.Texture(images[d % Textures], 56, 16, 16, + minLodFixed: descriptorChurn == 0 ? 0u : (uint)(d & 0xFFF)); + var smp = descriptorChurn == 0 ? sampler : Descriptors.Sampler(lodBias: descriptorChurn == 0 ? 0 : (d % 64) / 64f); + state.VertexShader(vs[pair], meshes[d % Meshes]); + state.PixelShader(ps[pair], heavy ? [(uint)tables[d % tableCount], (uint)(tables[d % tableCount] >> 32)] : [.. texture, .. smp]); + if (!delta || d == 0) + { + state.Emit(pm4); + } + else + { + // What a lean engine re-sends per draw: batched program/user-data blocks plus the two pipeline-state registers. + var psWords = state.Shader.Where(r => r.Key is >= Sh.PgmLoPs and <= Sh.PgmRsrc2Ps).Select(r => r.Value).ToArray(); + pm4.Shader(Sh.PgmLoPs, psWords); + pm4.Shader(Sh.UserDataPs0, state.Shader.Where(r => r.Key >= Sh.UserDataPs0 && r.Key < Sh.UserDataPs0 + 16).Select(r => r.Value).ToArray()); + pm4.Shader(Sh.PgmLoEs, state.Shader[Sh.PgmLoEs], state.Shader[Sh.PgmHiEs]); + pm4.Shader(Sh.PgmRsrc1Gs, state.Shader[Sh.PgmRsrc1Gs], state.Shader[Sh.PgmRsrc2Gs]); + pm4.Shader(Sh.UserDataGs0, state.Shader.Where(r => r.Key >= Sh.UserDataGs0 && r.Key < Sh.UserDataGs0 + 16).Select(r => r.Value).ToArray()); + pm4.Context(Cx.CbTargetMask, state.Context[Cx.CbTargetMask]); + pm4.Context(Cx.CbBlend0Control, state.Context[Cx.CbBlend0Control]); + } + + pm4.NumInstances(1); + pm4.DrawAuto(3); + } + + var words = pm4.ToArray(); + return new Scenario { Rig = rig, Stream = words, Draws = draws, PacketWords = words.Length, Address = rig.Memory.Upload(words, 256) }; + } + + private sealed record RunTimes(double CpuMs, double CommitWaitMs, double AllocBytes, double GcCount); + + /// Enqueue the stream and drain it like SceneRig.Drain, timing the CPU part and the commit/GPU wait separately. + private static RunTimes RunOnce(Scenario scenario) + { + var rig = scenario.Rig; + rig.Host.CommandStream.EnqueueGraphics(scenario.Address, (uint)scenario.Stream.Length, _submissionId++, null); + var gcBefore = GC.CollectionCount(0); + var allocBefore = GC.GetAllocatedBytesForCurrentThread(); + var cpu = new Stopwatch(); + var wait = new Stopwatch(); + var host = rig.Host; + var deadline = DateTime.UtcNow.AddSeconds(120); + while (true) + { + cpu.Start(); + host.RunPendingCommands(); + host.RunCommandStreamSlices(64); + cpu.Stop(); + if (host.CommandStream.HasUnblockedPending || rig.Relay.HasPendingCommands) continue; + wait.Start(); + host.CommitCoalescedFlush(); + host.FlushAndWaitForGpu(); + host.PollGpuCompletions(); + wait.Stop(); + if (!host.CommandStream.HasPending) break; + if (DateTime.UtcNow > deadline) throw new TimeoutException("stream stayed blocked"); + host.CommandStream.RetryBlocked(); + } + + return new RunTimes(cpu.Elapsed.TotalMilliseconds, wait.Elapsed.TotalMilliseconds, + GC.GetAllocatedBytesForCurrentThread() - allocBefore, GC.CollectionCount(0) - gcBefore); + } + + public static string[]? OnlyScenarios; + public static double SoakSeconds; + public static bool TrackAllocations; + + /// GCAllocationTick sampler (one event per ~100 KB allocated): bytes by type, all threads. + private sealed class AllocationTypes : System.Diagnostics.Tracing.EventListener + { + private readonly Dictionary _types = []; + private readonly object _gate = new(); + + protected override void OnEventSourceCreated(System.Diagnostics.Tracing.EventSource source) + { + if (source.Name == "Microsoft-Windows-DotNETRuntime") + EnableEvents(source, System.Diagnostics.Tracing.EventLevel.Verbose, (System.Diagnostics.Tracing.EventKeywords)0x1); + } + + protected override void OnEventWritten(System.Diagnostics.Tracing.EventWrittenEventArgs e) + { + if (e.EventName is null || !e.EventName.StartsWith("GCAllocationTick")) return; + var type = e.Payload![e.PayloadNames!.IndexOf("TypeName")] as string ?? "?"; + var amount = Convert.ToInt64(e.Payload[e.PayloadNames.IndexOf("AllocationAmount64")]); + lock (_gate) + { + _types.TryGetValue(type, out var current); + _types[type] = (current.Bytes + amount, current.Ticks + 1); + } + } + + public void Report(string id, double draws) + { + lock (_gate) + { + var total = _types.Values.Sum(v => v.Bytes); + Console.WriteLine($"[{id}] allocation by type (GCAllocationTick sample, {total / draws:F0} B/draw sampled over {draws:F0} draws):"); + foreach (var (type, v) in _types.OrderByDescending(t => t.Value.Bytes).Take(25)) + Console.WriteLine($"[{id}] {v.Bytes / draws,8:F1} B/draw {100.0 * v.Bytes / total,5:F1}% ticks={v.Ticks,5} {type}"); + } + } + } + + private static void Scenario5(string id, string name, SceneRig rig, int draws, int pairs, int variants, int churn, int runs, bool delta = false, bool heavy = false) + { + if (OnlyScenarios is { } only && !only.Contains(id)) return; + var scenario = Build(rig, draws, pairs, variants, churn, delta, heavy); + Console.WriteLine($"[{id}] stream {scenario.PacketWords * 4 / 1024} KiB, {scenario.PacketWords / (double)draws:F0} dwords/draw, pairs={pairs} pipelineStates<={pairs * variants}"); + var sw = Stopwatch.StartNew(); + var first = RunOnce(scenario); + Console.WriteLine($"[{id}] first (cold) run: cpu {first.CpuMs:F0} ms, wait {first.CommitWaitMs:F0} ms ({sw.Elapsed.TotalSeconds:F1} s total incl. shader+pipeline compile)"); + // Pipelines compile asynchronously and draws may be left out meanwhile: warm until two runs agree within 8%. + var previous = double.MaxValue; + for (var warm = 0; warm < 12; warm++) + { + var t = RunOnce(scenario); + if (Math.Abs(t.CpuMs - previous) / previous < 0.08) break; + previous = t.CpuMs; + } + + if (SoakSeconds > 0 || TrackAllocations) + { + AllocationTypes? listener = TrackAllocations ? new AllocationTypes() : null; + Console.WriteLine($"[{id}] SOAK START {DateTime.Now:HH:mm:ss} pid={Environment.ProcessId}"); + var soak = Stopwatch.StartNew(); + var iterations = 0; + do { RunOnce(scenario); iterations++; } while (soak.Elapsed.TotalSeconds < SoakSeconds); + Console.WriteLine($"[{id}] SOAK END {iterations} iterations in {soak.Elapsed.TotalSeconds:F1} s"); + listener?.Report(id, iterations * (double)draws); + listener?.Dispose(); + } + + var cpuNs = new List(); + var waitNs = new List(); + var bytes = new List(); + for (var run = 0; run < runs; run++) + { + var t = RunOnce(scenario); + cpuNs.Add(t.CpuMs * 1e6 / draws); + waitNs.Add(t.CommitWaitMs * 1e6 / draws); + bytes.Add(t.AllocBytes / draws); + Console.WriteLine($"[{id}] run {run}: cpu {t.CpuMs:F1} ms = {t.CpuMs * 1e6 / draws:F0} ns/draw, commit+wait {t.CommitWaitMs:F1} ms, alloc {t.AllocBytes / draws:F0} B/draw, gen0 GCs {t.GcCount}"); + } + + Harness.Record(id, name, cpuNs, bytes, Pm4Benches.FrameDraws, note: "CPU worker time per draw, GPU wait excluded"); + Harness.Record(id + "w", name + " [commit + GPU wait only]", waitNs, bytes, Pm4Benches.FrameDraws); + if (SharpEmu.Libs.VideoOut.RenderPhaseProfile.Enabled) PhaseReport(id, scenario); + } + + private static void PhaseReport(string id, Scenario scenario) + { + var type = typeof(SharpEmu.Libs.VideoOut.RenderPhaseProfile); + var flags = System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Static; + var ticks = (long[])type.GetField("_ticks", flags)!.GetValue(null)!; + var entries = (long[])type.GetField("_entries", flags)!.GetValue(null)!; + Array.Clear(ticks); + Array.Clear(entries); + const int Runs = 3; + for (var run = 0; run < Runs; run++) RunOnce(scenario); + var total = Runs * (double)scenario.Draws; + var rows = Enum.GetValues() + .Where(phase => (int)phase < ticks.Length).Select(phase => (Phase: phase, Us: ticks[(int)phase] * 1e6 / Stopwatch.Frequency / total, Entries: entries[(int)phase] / total)) + .Where(row => row.Us > 0.005) + .OrderByDescending(row => row.Us) + .ToList(); + Console.WriteLine($"[{id}] exclusive phase time per draw over {Runs} profiled runs (profile on: scope overhead inflates small phases), sum {rows.Sum(r => r.Us):F2} us/draw"); + foreach (var row in rows.Take(32)) + Console.WriteLine($"[{id}] {row.Phase,-34} {row.Us,8:F3} us/draw {row.Entries,7:F2} entries/draw x6331 = {row.Us * 6331 / 1000.0,7:F2} ms/scaled workload"); + } + + public static void Run(int draws = 6000) + { + Console.WriteLine("--- 8. End-to-end per draw over the real Metal host (SceneRig) ---"); + var device = MetalNative.MTLCreateSystemDefaultDevice(); + using var rig = new SceneRig(device); + Scenario5("8a", $"E2E {draws} textured draws, 50 VS/PS pairs x 4 pipeline states, 8 rotating T#/S# (warm)", rig, draws, Pairs, StateVariants, 0, 5); + Scenario5("8b", $"E2E {draws} draws, same but fresh T#/S# words every draw (cold materialization)", rig, draws, Pairs, StateVariants, 1, 5); + Scenario5("8d", $"E2E {draws} draws, lean engine (delta state: ~11 packets/draw), 50 pairs x 4 states, 8 rotating T#/S#", rig, draws, Pairs, StateVariants, 0, 5, delta: true); + Scenario5("8e", $"E2E {draws} draws, HEAVY bind: 8 T# + 2 S# + 4 V# via table pointer (warm, 8 tables), 50 pairs x 4 states", rig, draws, Pairs, StateVariants, 0, 5, delta: true, heavy: true); + Scenario5("8f", $"E2E {draws} draws, HEAVY bind, fresh table + T#/S# words every draw (cold descriptors)", rig, draws, Pairs, StateVariants, 1, 5, delta: true, heavy: true); + Scenario5("8c", $"E2E {draws} draws, ONE shader pair, ONE pipeline state, 8 rotating T#/S#", rig, draws, 1, 1, 0, 5); + } +} diff --git a/tools/SharpEmu.Tools.PerfAudit/Benches.Lookup.cs b/tools/SharpEmu.Tools.PerfAudit/Benches.Lookup.cs new file mode 100644 index 000000000..01cb1e1e0 --- /dev/null +++ b/tools/SharpEmu.Tools.PerfAudit/Benches.Lookup.cs @@ -0,0 +1,121 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using SharpEmu.Gpu.Metal.V2; +using SharpEmu.Libs.Gpu.Abstractions; +using SharpEmu.Libs.Gpu.Pipelines; +using SharpEmu.Libs.Gpu.Rendering; +using SharpEmu.ShaderCompiler; + +namespace SharpEmu.PerfAudit; + +/// +/// 5. Pipeline-state lookup with 50 VS/PS pairs x 4 states = 200 cached pipelines. The program-cache probe +/// (ShaderProgramCache.ProgramKeyProbe) is internal to SharpEmu.Libs and is measured through the e2e phase +/// profile (ProgramCacheLookup / ProgramPermutationLookup) instead. +/// +internal static class LookupBenches +{ + private const int Pairs = 50; + private const int States = 4; + + private static void Fill(GraphicsPipelineKey key, int pair, int state) + { + key.Rendering.Reset(); + key.VertexInput.Reset(); + key.StaticParameters.Reset(); + key.Rendering.ColorCount = 1; + key.Rendering.ColorFormats[0] = GpuFormat.R8G8B8A8Unorm; + key.StaticParameters.ColorCount = 1; + key.StaticParameters.Topology = GpuPrimitiveTopology.TriangleList; + key.StaticParameters.Samples = 1; + key.StaticParameters.SetColorMask(0, 0xF); + key.StaticParameters.CullBack = (state & 1) != 0; + key.StaticParameters.SetBlendEnable(0, (state & 2) != 0); + if ((state & 2) != 0) + { + key.StaticParameters.SetColorSourceBlend(0, 4); + key.StaticParameters.SetColorDestinationBlend(0, 5); + } + + key.VertexInput.Bindings[0] = new PipelineVertexBinding(32, false); + key.VertexInput.Attributes[0] = new PipelineVertexAttribute(0, 0); + key.VertexInput.Attributes[1] = new PipelineVertexAttribute(0, 16); + key.VertexInput.BindingCount = 1; + key.VertexInput.AttributeCount = 2; + key.VertexProgramId = 0x1000UL + (ulong)pair * 2; + key.PixelProgramId = 0x1001UL + (ulong)pair * 2; + } + + private static GraphicsPipelineKey Make(int pair, int state) + { + var key = new GraphicsPipelineKey + { + Rendering = new PipelineRenderingState(), + VertexInput = new PipelineVertexInputState(), + StaticParameters = new PipelineStaticParameters(), + }; + Fill(key, pair, state); + return key; + } + + public static void Run() + { + Console.WriteLine("--- 5. Shader / pipeline-state lookup (50 pairs x 4 states = 200) ---"); + var cache = new Dictionary(); + for (var pair = 0; pair < Pairs; pair++) + for (var state = 0; state < States; state++) + cache[Make(pair, state)] = (ulong)(pair * States + state); + var lookup = Make(0, 0); + long sink = 0; + // Pre-hashed variant: lookup key fully filled once, only ids change (best case). + Harness.Measure("5a", "GraphicsPipelineKey dictionary lookup, 200 entries, ids swapped (hash + equals only)", 1, 6331, i => + { + lookup.VertexProgramId = 0x1000UL + (ulong)(i % Pairs) * 2; + lookup.PixelProgramId = lookup.VertexProgramId + 1; + if (cache.TryGetValue(lookup, out var h)) sink += (long)h; + }, note: "Libs/Gpu/Pipelines/PipelineKeys.cs:234; ShaderPipelineCache.cs:1127"); + // The per-draw path: Reset x3 + refill the lookup key from the draw's state, then probe. + Harness.Measure("5b", "GraphicsPipelineKey refill (Reset x3 + ~20 setters) + lookup, 200 entries", 1, 6331, i => + { + Fill(lookup, i % Pairs, (i >> 3) % States); + if (cache.TryGetValue(lookup, out var h)) sink += (long)h; + }, note: "ShaderPipelineCache.cs:1109-1127 (FillGraphicsState itself is internal; this is a faithful subset)"); + + // Metal side: the descriptor key built when a pipeline state is created, and the shared-handle table probe. + var metalInput = new PipelineVertexInputState(); + metalInput.Bindings[0] = new PipelineVertexBinding(8, false); + metalInput.Attributes[0] = new PipelineVertexAttribute(0, 0); + metalInput.BindingCount = 1; + metalInput.AttributeCount = 1; + var stage = new ShaderProgramInfo { Stage = ShaderStageKind.Vertex, Hash = 1 }; + stage.VertexFetchComponents[0] = 2; + var description = new GraphicsPipelineDescription + { + Rendering = lookup.Rendering, + VertexInput = metalInput, + VertexInfo = new VertexInputInfo + { + Attributes = [new VertexAttributeResource(new BufferDescriptorWords(0, 8u << 16, 6, 64u << 12), 0, 2, 0, 0, 0, 0)], + }, + VertexProgram = default, + VertexStage = stage, + StaticParameters = lookup.StaticParameters, + }; + Fill(lookup, 0, 3); + Harness.Measure("5c", "MetalRenderHost.PipelineStateKeyOf (descriptor key build + XxHash3), 1 color target + blend", 1, 6331, i => + sink += MetalRenderHost.PipelineStateKeyOf(description, new UInt128(0, (ulong)(i % 100)), new UInt128(0, (ulong)(i % 50)), false, false, 0).Length, + note: "V2/MetalRenderHost.Pipelines.cs:1160; only paid on a pipeline-cache miss (not per draw)"); + var metalKeys = new Dictionary(); + var probes = new MetalPipelineStateKey[200]; + for (var n = 0; n < 200; n++) + { + probes[n] = MetalRenderHost.PipelineStateKeyOf(description, new UInt128(0, (ulong)(n % 100)), new UInt128(0, (ulong)(n / 4)), false, false, 0); + metalKeys[probes[n]] = n; + } + + Harness.Measure("5d", "Dictionary probe, 200 entries (hash cached, byte compare)", 1, 6331, i => + sink += metalKeys.TryGetValue(probes[i % 200], out var v) ? v : 0, note: "V2/MetalPipelineStateKey.cs:27"); + GC.KeepAlive(sink); + } +} diff --git a/tools/SharpEmu.Tools.PerfAudit/Benches.Materialization.cs b/tools/SharpEmu.Tools.PerfAudit/Benches.Materialization.cs new file mode 100644 index 000000000..a397dde84 --- /dev/null +++ b/tools/SharpEmu.Tools.PerfAudit/Benches.Materialization.cs @@ -0,0 +1,202 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using SharpEmu.Gpu.Metal.Tests.Scenes; +using SharpEmu.PerfAudit.Materialization; +using SharpEmu.ShaderCompiler; +using SharpEmu.ShaderCompiler.Resources; +using static SharpEmu.PerfAudit.Materialization.ResourceTestProgram; + +namespace SharpEmu.PerfAudit; + +/// +/// ResourceMaterializer / ResourceMaterializationCache over a synthetic shader that reads 8 T# + 2 S# + 4 V# +/// through a descriptor table in guest memory (user data = table pointer), the shape a PS5 title binds. +/// +internal static unsafe class MaterializationBenches +{ + private const int Textures = 8, Samplers = 2, Buffers = 4; + private const ulong TableStride = 512; + private const ulong TextureRegion = NativeGuestMemory.Base + 0x100_0000; + private const ulong BufferRegion = NativeGuestMemory.Base + 0x200_0000; + private const ulong TableRegion = NativeGuestMemory.Base + 0x300_0000; + + public static ShaderResourcePlan BuildPlan() + { + var program = new List(); + uint pc = 0; + // s[8+8i : 15+8i] = T#i, s[72+4j : 75+4j] = S#j, s[80+4k : 83+4k] = V#k, all loaded through the pointer in s[0:1]. + for (uint i = 0; i < Textures; i++) { program.Add(ScalarLoad(pc, 0, 8 + 8 * i, 8, (int)(i * 32))); pc += 8; } + for (uint j = 0; j < Samplers; j++) { program.Add(ScalarLoad(pc, 0, 72 + 4 * j, 4, 256 + (int)(j * 16))); pc += 8; } + for (uint k = 0; k < Buffers; k++) { program.Add(ScalarLoad(pc, 0, 80 + 4 * k, 4, 288 + (int)(k * 16))); pc += 8; } + for (uint i = 0; i < Textures; i++) { program.Add(Image(pc, "ImageSample", 8 + 8 * i, 72 + 4 * (i % Samplers))); pc += 8; } + for (uint k = 0; k < Buffers; k++) { program.Add(BufferLoad(pc, 80 + 4 * k, 0, 4)); pc += 8; } + program.Add(EndProgram(pc)); + return Extract(Program([.. program]), userDataBase: 0, userDataCount: 2, stage: ShaderStage.Pixel); + } + + /// Writes one descriptor table; makes its texture/buffer addresses distinct. + private static void WriteTable(NativeGuestMemory memory, ulong table, int salt) + { + for (var i = 0; i < Textures; i++) + { + var words = Descriptors.Texture(TextureRegion + ((ulong)salt * 16 + (ulong)i) * 0x1_0000, 56, 256, 256); + memory.WriteWords(table + (ulong)i * 32, words); + } + + for (var j = 0; j < Samplers; j++) memory.WriteWords(table + 256 + (ulong)j * 16, Descriptors.Sampler()); + for (var k = 0; k < Buffers; k++) + memory.WriteWords(table + 288 + (ulong)k * 16, Descriptors.Buffer(BufferRegion + ((ulong)salt * 8 + (ulong)k) * 0x1000, 16, 256, format: 77)); + } + + public static void Run() + { + Console.WriteLine("--- 3. Resource materialization (ResourceMaterializer / ResourceMaterializationCache) ---"); + var plan = BuildPlan(); + Console.WriteLine($"plan: images={plan.Info.Images.Count} buffers={plan.Info.Buffers.Count} samplers={plan.Info.Samplers.Count} descriptorSources={plan.DescriptorSources.Count} memoryOps={plan.Memory.Count}"); + using var memory = new NativeGuestMemory(64 << 20); + const int Pool = 4096; + for (var t = 0; t < Pool; t++) WriteTable(memory, TableRegion + (ulong)t * TableStride, t); + + bool ReadWord(ulong address, out uint word) + { + Span bytes = stackalloc byte[4]; + if (!memory.TryRead(address, bytes)) { word = 0; return false; } + word = System.Buffers.Binary.BinaryPrimitives.ReadUInt32LittleEndian(bytes); + return true; + } + + bool ReadResident(ulong address, Span destination, bool clean) => memory.TryRead(address, destination); + + var inputs = new ResourceRuntimeInputs { ReadMemory = ReadWord, ReadCleanMemory = ReadWord, ReadResidentMemory = ReadResident }; + var snapshot = new ResourceSnapshot(); + var specialization = new ResourceSpecialization(); + var table0 = TableRegion; + inputs.UserData = new uint[] { (uint)table0, (uint)(table0 >> 32) }; + const int Batch = 200; + + // 3a: the full walk, no cache, same descriptors. + Harness.Measure("3a", "FULL WALK ResourceMaterializer.Materialize (no cache), 8T+2S+4V via table", Batch, 1404, _ => + { + for (var i = 0; i < Batch; i++) + { + snapshot = new ResourceSnapshot(); specialization = new ResourceSpecialization(); + if (!ResourceMaterializer.Materialize(plan, inputs, ref snapshot, ref specialization, out var failure)) throw new InvalidOperationException(failure.ToString()); + } + }, unit: "full walk", note: "game: 1,404 full walks/frame at 92 us (detail timers on)"); + + // 3b: warm cache hit, same descriptors every draw. + var cache = new ResourceMaterializationCache(); + if (!cache.Materialize(plan, inputs, ReadResident, ref snapshot, ref specialization, out var warmFailure)) throw new InvalidOperationException(warmFailure.ToString()); + var hitsBefore = cache.Hits; + var warm = Harness.Measure("3b", "WARM cache hit, same descriptors every call (validation re-read)", Batch * 5, 13825, _ => + { + for (var i = 0; i < Batch * 5; i++) + { + if (!cache.Materialize(plan, inputs, ReadResident, ref snapshot, ref specialization, out var failure)) throw new InvalidOperationException(failure.ToString()); + } + }, unit: "cache-hit call", note: "game: ~13.8k hit calls/frame at ~4.7 us"); + Console.WriteLine($" -> cache hits {cache.Hits - hitsBefore}, misses {cache.Misses}, refreshes {cache.TableRefreshes}"); + + // 3c: cold, new table each call (new key), cache too small to retain: every call is a miss + entry build. + var cold = new ResourceMaterializationCache(generationCapacity: 8); + var cursor = 0; + Harness.Measure("3c", "COLD miss: new table address + new descriptors each call (key not found)", Batch, 1404, _ => + { + for (var i = 0; i < Batch; i++) + { + var table = TableRegion + (ulong)(cursor++ % Pool) * TableStride; + inputs.UserData = new uint[] { (uint)table, (uint)(table >> 32) }; + if (!cold.Materialize(plan, inputs, ReadResident, ref snapshot, ref specialization, out var failure)) throw new InvalidOperationException(failure.ToString()); + } + }, unit: "miss", note: "cold: includes allocating the user-data array and the entry build"); + Console.WriteLine($" -> hits {cold.Hits} misses {cold.Misses} refreshes {cold.TableRefreshes} uncacheable {cold.Uncacheable}"); + + // 3d: same key, the table's descriptors change between calls (stale same-key): tries refresh, falls back to the walk. + var stale = new ResourceMaterializationCache(); + inputs.UserData = new uint[] { (uint)table0, (uint)(table0 >> 32) }; + stale.Materialize(plan, inputs, ReadResident, ref snapshot, ref specialization, out _); + var salt = 0; + var refreshesBefore = stale.TableRefreshes; + Harness.Measure("3d", "STALE same key: descriptor words change every call (refresh or re-walk)", Batch, 1404, _ => + { + for (var i = 0; i < Batch; i++) + { + WriteTable(memory, table0, 1 + (salt++ % (Pool - 1))); + if (!stale.Materialize(plan, inputs, ReadResident, ref snapshot, ref specialization, out var failure)) throw new InvalidOperationException(failure.ToString()); + } + }, unit: "stale", note: "includes WriteTable (14 small writes) in the timed loop"); + Console.WriteLine($" -> hits {stale.Hits} misses {stale.Misses} refreshes {stale.TableRefreshes - refreshesBefore} (slot refreshes {stale.SlotRefreshes})"); + + // 3f: 50 draws per batch alternate between 8 descriptor tables at different addresses whose contents are identical: + // only the table pointer in user data differs, so every draw could reuse one materialization. + { + const int Tables = 8, Draws = 50; + var identical = new ulong[Tables]; + for (var t = 0; t < Tables; t++) + { + identical[t] = TableRegion + (ulong)(Pool + 8 + t * 3) * TableStride; + WriteTable(memory, identical[t], 7); + } + + var userData = new uint[Tables][]; + for (var t = 0; t < Tables; t++) userData[t] = new uint[] { (uint)identical[t], (uint)(identical[t] >> 32) }; + var alternating = new ResourceMaterializationCache(); + for (var i = 0; i < Draws; i++) + { + inputs.UserData = userData[i % Tables]; + if (!alternating.Materialize(plan, inputs, ReadResident, ref snapshot, ref specialization, out var failure)) throw new InvalidOperationException(failure.ToString()); + } + Console.WriteLine($" 3f first {Draws} draws (cold cache): hits {alternating.Hits} misses {alternating.Misses} refreshes {alternating.TableRefreshes} -> hit rate {100.0 * alternating.Hits / Draws:F1}%"); + var hits0 = alternating.Hits; var misses0 = alternating.Misses; var refreshes0 = alternating.TableRefreshes; + var draw = 0; + Harness.Measure("3f", "8 tables at different addresses, identical contents, 50 draws alternating (pointer-only change)", Draws, 1404, _ => + { + for (var i = 0; i < Draws; i++) + { + inputs.UserData = userData[draw++ % Tables]; + if (!alternating.Materialize(plan, inputs, ReadResident, ref snapshot, ref specialization, out var failure)) throw new InvalidOperationException(failure.ToString()); + } + }, unit: "draw"); + var calls = (alternating.Hits - hits0) + (alternating.Misses - misses0) + (alternating.TableRefreshes - refreshes0); + Console.WriteLine($" 3f steady: hits {alternating.Hits - hits0} misses {alternating.Misses - misses0} refreshes {alternating.TableRefreshes - refreshes0} -> hit rate {100.0 * (alternating.Hits - hits0) / Math.Max(1, calls):F1}%"); + inputs.UserData = new uint[] { (uint)table0, (uint)(table0 >> 32) }; + } + + // 3g: a per-frame descriptor ring. One shader draws 32 materials (distinct tables) per frame; every frame the engine + // writes each material's table again at a new ring position with the same contents, so a material's pointer differs + // from its previous frame's while its descriptors do not. Includes WriteTable per draw (subtract 3e). + { + const int Materials = 32; + const int RingFirst = 8192, RingSlots = 24576 - Materials; + var ring = new ResourceMaterializationCache(); + var frame = 0; + var ringUserData = new uint[Materials][]; + void Frame() + { + var start = (frame++ * 977) % RingSlots; + for (var m = 0; m < Materials; m++) + { + var table = TableRegion + (ulong)(RingFirst + start + m) * TableStride; + WriteTable(memory, table, 100 + m); + inputs.UserData = new uint[] { (uint)table, (uint)(table >> 32) }; + if (!ring.Materialize(plan, inputs, ReadResident, ref snapshot, ref specialization, out var failure)) throw new InvalidOperationException(failure.ToString()); + } + } + + for (var f = 0; f < 8; f++) Frame(); + Console.WriteLine($" 3g first 8 frames x {Materials} draws: hits {ring.Hits} misses {ring.Misses} refreshes {ring.TableRefreshes} -> hit rate {100.0 * ring.Hits / (8 * Materials):F1}%"); + var hits0 = ring.Hits; var misses0 = ring.Misses; var refreshes0 = ring.TableRefreshes; + Harness.Measure("3g", "descriptor ring: 32 materials/frame, same contents at a new address each frame (+WriteTable)", Materials, 1404, _ => Frame(), unit: "draw"); + var calls = (ring.Hits - hits0) + (ring.Misses - misses0) + (ring.TableRefreshes - refreshes0); + Console.WriteLine($" 3g steady: hits {ring.Hits - hits0} misses {ring.Misses - misses0} refreshes {ring.TableRefreshes - refreshes0} -> hit rate {100.0 * (ring.Hits - hits0) / Math.Max(1, calls):F1}%"); + inputs.UserData = new uint[] { (uint)table0, (uint)(table0 >> 32) }; + } + + // 3e: write-only cost of 3d for subtraction. + Harness.Measure("3e", "baseline: WriteTable only (subtract from 3d)", Batch, 1404, _ => + { + for (var i = 0; i < Batch; i++) WriteTable(memory, table0, 1 + (salt++ % (Pool - 1))); + }, unit: "write"); + } +} diff --git a/tools/SharpEmu.Tools.PerfAudit/Benches.Memory.cs b/tools/SharpEmu.Tools.PerfAudit/Benches.Memory.cs new file mode 100644 index 000000000..08cfce1d2 --- /dev/null +++ b/tools/SharpEmu.Tools.PerfAudit/Benches.Memory.cs @@ -0,0 +1,185 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using System.Buffers.Binary; +using System.Runtime.InteropServices; +using SharpEmu.Core.Cpu; +using SharpEmu.Core.Memory; +using SharpEmu.Gpu.Metal; +using SharpEmu.Gpu.Metal.V2; +using SharpEmu.Gpu.Metal.Tests.Scenes; +using SharpEmu.HLE; +using SharpEmu.HLE.GpuMemory; +using SharpEmu.HLE.Host; +using SharpEmu.Host.MacOS; + +namespace SharpEmu.PerfAudit; + +/// +/// 6. Guest-memory read layers (raw pointer -> ICpuMemory -> real PhysicalVirtualMemory -> TrackedCpuMemory -> +/// MetalRenderHost readers) and write-tracking bookkeeping (PageGuard watch add/remove, real mprotect). +/// +internal static unsafe class MemoryBenches +{ + private const ulong GuestBase = 0x2_0000_0000; + private const double WordReadsPerFrame = 13825 * 40.0; // ~40 guest words per materialization (PERF-REPORT: "a few dozen") + private const double RangeReadsPerFrame = 13825 * 6.0; + + [DllImport("libc", EntryPoint = "mmap")] private static extern nint Mmap(nint addr, nuint len, int prot, int flags, int fd, nint offset); + [DllImport("libc", EntryPoint = "mprotect")] private static extern int Mprotect(nint addr, nuint len, int prot); + + public static void Run() + { + Console.WriteLine("--- 6. Guest memory read layers + write tracking ---"); + long sink = 0; + + // 6a: raw native pointer, 32 bytes. + using var native = new NativeGuestMemory(8 << 20); + native.WriteWords(NativeGuestMemory.Base + 0x1000, Enumerable.Range(0, 1024).Select(v => (uint)v).ToArray()); + var raw = native.Raw(NativeGuestMemory.Base + 0x1000); + Harness.Measure("6a", "raw pointer: Unsafe read 4 x uint (16 B), no checks", 1, WordReadsPerFrame / 4, i => + { + var p = (uint*)(raw + ((i & 63) << 4)); + sink += p[0] + p[1] + p[2] + p[3]; + }, unit: "16B read", note: "floor: one cache line, no abstraction"); + var buffer = new byte[32]; + Harness.Measure("6b", "NativeGuestMemory.TryRead 32 B (ICpuMemory interface call, bounds + memcpy)", 1, RangeReadsPerFrame, i => + { + native.TryRead(NativeGuestMemory.Base + 0x1000 + (ulong)((i & 63) << 5), buffer); + }, unit: "32B read"); + + // Real PhysicalVirtualMemory on the macOS view host. + HostMemoryAuthority.Initialize(MacOSHost.CreateMemory); + var views = MacOSHost.CreateViewMemory(); + using var pvm = new PhysicalVirtualMemory(viewHost: views, backingBytes: 32UL * 1024 * 1024); + const ulong Region = 8UL << 20; + var hole = (Region + views.Granularity - 1) / views.Granularity * views.Granularity; + if (!pvm.TryHoldRangeAtOrAbove(GuestBase, hole, views.Granularity, out var address) || + !pvm.TryMapBacked(address, Region, 0, GuestPageProtection.Read | GuestPageProtection.Write, out var failure)) + { + throw new InvalidOperationException("PVM map failed"); + } + + using var gpuMemory = new GuestGpuMemory(pvm); + gpuMemory.Register(address, Region, GuestPageProtection.Read | GuestPageProtection.Write); + var seed = new byte[4096]; + for (var i = 0; i < seed.Length; i++) seed[i] = (byte)i; + pvm.TryWriteBacking(address, seed); + var tracked = new TrackedCpuMemory(pvm); + var word = new byte[4]; + Harness.Measure("6c", "PhysicalVirtualMemory.TryRead 32 B (backed, single-view lock-free path)", 1, RangeReadsPerFrame, i => + pvm.TryRead(address + (ulong)((i & 63) << 5), buffer), unit: "32B read", note: "Core/Memory/PhysicalVirtualMemory.cs:2330"); + Harness.Measure("6d", "PhysicalVirtualMemory.TryRead 4 B (a descriptor word)", 1, WordReadsPerFrame, i => + pvm.TryRead(address + (ulong)((i & 255) << 2), word), unit: "word"); + Harness.Measure("6e", "PhysicalVirtualMemory.TryReadBacking 32 B", 1, RangeReadsPerFrame, i => + pvm.TryReadBacking(address + (ulong)((i & 63) << 5), buffer), unit: "32B read", note: "Core/Memory/PhysicalVirtualMemory.cs:1483"); + Harness.Measure("6f", "TrackedCpuMemory.TryRead 32 B (the CPU-side wrapper)", 1, RangeReadsPerFrame, i => + tracked.TryRead(address + (ulong)((i & 63) << 5), buffer), unit: "32B read", note: "Core/Cpu/TrackedCpuMemory.cs:128 (guest CPU path; GPU uses the inner view)"); + + // MetalRenderHost reader layers over the real PVM. + var device = MetalNative.MTLCreateSystemDefaultDevice(); + var queue = MetalNative.Send(device, MetalNative.Selector("newCommandQueue")); + var relay = new MetalGpuWorkerRelay(static () => { }); + var host = new MetalRenderHost(device, queue, pvm, pvm, gpuMemory.Pages, relay); + relay.BindCurrentThread(); + Harness.Measure("6g", "MetalRenderHost.TryReadGuest 32 B (no overlay/GPU-write cover)", 1, RangeReadsPerFrame, i => + host.TryReadGuest(address + (ulong)((i & 63) << 5), buffer), unit: "32B read", note: "V2/MetalRenderHost.cs:1642 -> :1695 MayCover/MayOwn tests -> PVM.TryRead"); + Harness.Measure("6h", "MetalRenderHost.TryReadGuestWord, no word-line scope (falls to TryReadGuest)", 1, WordReadsPerFrame, i => + { + host.TryReadGuestWord(address + (ulong)((i & 255) << 2), out var w); + sink += w; + }, unit: "word", note: "V2/MetalRenderHost.Pipelines.cs:1713"); + host.BeginGuestWordReads(); + Harness.Measure("6i", "MetalRenderHost.TryReadGuestWord inside Begin/EndGuestWordReads (line-cache hit)", 1, WordReadsPerFrame, i => + { + host.TryReadGuestWord(address + (ulong)((i & 15) << 2), out var w); + sink += w; + }, unit: "word", note: "V2/MetalRenderHost.WordLines.cs:63 TryReadWordFromLine; hit path"); + host.EndGuestWordReads(); + Harness.Measure("6j", "Begin/EndGuestWordReads + 40 word reads over 4 lines (per scope, fills included)", 1, 13825, i => + { + host.BeginGuestWordReads(); + for (var w = 0; w < 40; w++) + { + host.TryReadGuestWord(address + (ulong)(((i & 63) * 256) + w * 4 + (w >> 3) * 64), out var v); + sink += v; + } + + host.EndGuestWordReads(); + }, unit: "materialization", note: "per materialization call with ~40 reads over 5 lines"); + Harness.Measure("6k", "MetalRenderHost.TryReadCleanGuestWord", 1, WordReadsPerFrame, i => + { + host.TryReadCleanGuestWord(address + (ulong)((i & 255) << 2), out var w); + sink += w; + }, unit: "word", note: "V2/MetalRenderHost.Pipelines.cs:1738 image-cache test + TryReadBacking + 2 overlay applies"); + Harness.Measure("6l", "MetalRenderHost.TryReadResidentGuestBytes 32 B", 1, RangeReadsPerFrame, i => + host.TryReadResidentGuestBytes(address + (ulong)((i & 63) << 5), buffer, true), unit: "32B read", note: "V2/MetalRenderHost.Pipelines.cs:1758"); + { + var addrs = new ulong[6]; + var lens = new int[6]; + var offs = new int[6]; + var recorded = new byte[6 * 32]; + for (var r = 0; r < 6; r++) { addrs[r] = address + (ulong)(r * 512); lens[r] = 32; offs[r] = r * 32; } + for (var r = 0; r < 6; r++) pvm.TryRead(addrs[r], recorded.AsSpan(r * 32, 32)); + Harness.Measure("6m", "MetalRenderHost.TryCompareResidentGuestBytes, 6 clustered 32 B ranges (all equal)", 1, 13825, i => + sink += host.TryCompareResidentGuestBytes(addrs, lens, offs, recorded) ? 1 : 0, unit: "validation", + note: "V2/MetalRenderHost.Pipelines.cs:1790; the warm-hit validation"); + } + + // Write tracking: PageGuard bookkeeping alone (no protection syscall), then with the real PVM address space. + var space = new SceneAddressSpace(); + using var guard = new PageGuard(space); + var watchBase = GuestBase + 0x100_0000; + Harness.Measure("6n", "PageGuard.AddWatch + RemoveWatch, 1 page, no-op address space (bookkeeping only)", 1, 1500, i => + { + var a = watchBase + (ulong)((i & 1023) << 12); + guard.AddWatch(a, 4096, false); + guard.RemoveWatch(a, 4096, false); + }, unit: "texture", note: "HLE/GpuMemory/PageGuard.cs:176/178"); + Harness.Measure("6o", "PageGuard.AddWatch + RemoveWatch, 64 pages (256 KiB), no-op address space", 1, 100, i => + { + var a = watchBase + (ulong)((i & 15) << 18); + guard.AddWatch(a, 64 * 4096, false); + guard.RemoveWatch(a, 64 * 4096, false); + }, unit: "texture", note: "bookkeeping only"); + guard.AddWatch(watchBase, 4096, false); + Harness.Measure("6p", "PageGuard.Allows (read fault check on a watched page)", 1, 1500, i => + sink += guard.Allows(watchBase + (ulong)((i & 7) << 2), FaultKind.Read) ? 1 : 0, unit: "check", note: "PageGuard.cs:185"); + guard.RemoveWatch(watchBase, 4096, false); + + // Real protection: GuestGpuMemory's page guard over the PVM, so Add/RemoveWatch really change page protection. + var realBase = address + 0x20_0000; + Harness.Measure("6q", "GuestGpuMemory.Pages AddWatch + RemoveWatch, 1 page, REAL protection change (PVM)", 1, 1500, i => + { + var a = realBase + (ulong)((i & 63) << 12); + gpuMemory.Pages.AddWatch(a, 4096, false); + gpuMemory.Pages.RemoveWatch(a, 4096, false); + }, unit: "texture", note: "write watch -> PVM.TryProtect -> host vm_protect, restored on remove"); + Harness.Measure("6r", "GuestGpuMemory.Pages AddWatch + RemoveWatch, 64 pages, REAL protection change (PVM)", 1, 100, i => + { + var a = realBase + (ulong)((i & 3) << 18); + gpuMemory.Pages.AddWatch(a, 64 * 4096, false); + gpuMemory.Pages.RemoveWatch(a, 64 * 4096, false); + }, unit: "texture", note: "one protect call per contiguous run"); + + // Syscall bound: raw mprotect round trip on an anonymous page. The SIGSEGV fault path itself (a guest write to a + // watched page -> signal handler -> PageGuard -> unprotect -> resume) cannot be driven from managed code here. + var map = Mmap(0, 1 << 20, 3, 0x1002, -1, 0); // PROT_READ|WRITE, MAP_PRIVATE|MAP_ANON + Harness.Measure("6s", "raw mprotect(RO) + mprotect(RW), 1 page (syscall bound for a watch round trip)", 1, 1500, i => + { + var a = map + ((i & 63) << 12); + Mprotect(a, 4096, 1); + Mprotect(a, 4096, 3); + }, unit: "texture", note: "lower bound for 1 watch add+remove; SIGSEGV fault delivery NOT measured"); + GC.KeepAlive(sink); + } + + private sealed class MemoryFlipTarget : IMetalFlipTarget + { + public bool HasFlipSlot() => true; + + public void PublishGuestFrame(nint texture, uint width, uint height, ulong requestId, ulong serial, bool isHdr) + { + } + } +} diff --git a/tools/SharpEmu.Tools.PerfAudit/Benches.Metal.cs b/tools/SharpEmu.Tools.PerfAudit/Benches.Metal.cs new file mode 100644 index 000000000..01f654777 --- /dev/null +++ b/tools/SharpEmu.Tools.PerfAudit/Benches.Metal.cs @@ -0,0 +1,285 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using System.Diagnostics; +using System.Runtime.InteropServices; +using SharpEmu.Gpu.Metal; +using SharpEmu.Gpu.Metal.V2; +using SharpEmu.Gpu.Metal.Tests.Scenes; + +namespace SharpEmu.PerfAudit; + +/// +/// 8-raw. The floor under the emulator's encode phase: the same Objective-C message sends the emulator makes +/// (MetalNative.Send*), issued in a tight loop against a real device, plus commit and GPU wait, plus the cost of +/// the GPU-written-word wait the guest triggers when it reads a GPU-owned word. +/// +internal static unsafe class MetalRawBenches +{ + private const int Draws = 6000; + private const int Dispatches = 2700; + + private const string Source = """ + #include + using namespace metal; + struct V { float4 p [[position]]; }; + vertex V vs(uint vid [[vertex_id]], constant float4* a [[buffer(0)]]) { + float2 c[3] = { float2(-1, -1), float2(3, -1), float2(-1, 3) }; + V v; v.p = float4(c[vid] * 0.0001 + a[0].xy, 0, 1); return v; + } + fragment half4 fs(V v [[stage_in]], constant float4* b [[buffer(0)]]) { return half4(b[0]); } + kernel void k(device uint* o [[buffer(0)]], uint t [[thread_position_in_grid]]) { o[t] = t; } + """; + + private static nint S(string name) => MetalNative.Selector(name); + + public static void Run() + { + Console.WriteLine("--- 8-raw. Raw Metal encode / commit / wait floor, GPU-written-word wait ---"); + var device = MetalNative.MTLCreateSystemDefaultDevice(); + var queue = MetalNative.Send(device, S("newCommandQueue")); + + // Timer cost: the phase profile (RenderPhaseProfile) takes two of these per scope. + long sinkTicks = 0; + Harness.Measure("8t", "Stopwatch.GetTimestamp() (what each RenderPhaseProfile scope pays twice)", 1, 6331 * 60.0, i => sinkTicks += Stopwatch.GetTimestamp(), + unit: "call", note: "frame count is a guess: ~60 profiled scope boundaries per draw with the phase profile on"); + + var pool = MetalNative.objc_autoreleasePoolPush(); + var options = MetalNative.Send(MetalNative.Send(MetalNative.Class("MTLCompileOptions"), S("alloc")), S("init")); + MetalNative.SetMslLanguageVersion(options); + nint error = 0; + var library = MetalNative.Send(device, S("newLibraryWithSource:options:error:"), MetalNative.NsString(Source), options, ref error); + if (library == 0) throw new InvalidOperationException(MetalNative.DescribeError(error)); + var fn = (string n) => MetalNative.Send(library, S("newFunctionWithName:"), MetalNative.NsString(n)); + + var descriptor = MetalNative.Send(MetalNative.Send(MetalNative.Class("MTLRenderPipelineDescriptor"), S("alloc")), S("init")); + MetalNative.SendVoid(descriptor, S("setVertexFunction:"), fn("vs")); + MetalNative.SendVoid(descriptor, S("setFragmentFunction:"), fn("fs")); + var colorAttachment = MetalNative.SendAtIndex(MetalNative.Send(descriptor, S("colorAttachments")), S("objectAtIndexedSubscript:"), 0); + const nuint Rgba8Unorm = 70; + MetalV2Native.SendVoidValue(colorAttachment, S("setPixelFormat:"), Rgba8Unorm); + var renderState = MetalNative.Send(device, S("newRenderPipelineStateWithDescriptor:error:"), descriptor, ref error); + if (renderState == 0) throw new InvalidOperationException(MetalNative.DescribeError(error)); + var computeState = MetalNative.Send(device, S("newComputePipelineStateWithFunction:error:"), fn("k"), ref error); + if (computeState == 0) throw new InvalidOperationException(MetalNative.DescribeError(error)); + + // The default usage is shader-read only; a render target needs RenderTarget (4). + var textureDescriptor = MetalNative.SendTextureDescriptor(MetalNative.Class("MTLTextureDescriptor"), S("texture2DDescriptorWithPixelFormat:width:height:mipmapped:"), Rgba8Unorm, 256, 256, false); + MetalV2Native.SendVoidValue(textureDescriptor, S("setUsage:"), 4); + MetalV2Native.SendVoidValue(textureDescriptor, S("setStorageMode:"), 2); + var texture = MetalNative.Send(device, S("newTextureWithDescriptor:"), textureDescriptor); + + var buffers = new nint[20]; + for (var i = 0; i < buffers.Length; i++) buffers[i] = MetalNative.SendNewBuffer(device, S("newBufferWithLength:options:"), 4096, 0); + MetalNative.objc_autoreleasePoolPop(pool); + + var selSetVertexBuffer = S("setVertexBuffer:offset:atIndex:"); + var selSetFragmentBuffer = S("setFragmentBuffer:offset:atIndex:"); + var selSetComputeBuffer = S("setBuffer:offset:atIndex:"); + var selDraw = S("drawPrimitives:vertexStart:vertexCount:"); + + void RenderRun(string id, string name, int buffersPerDraw, bool pipelinePerDraw, int runs = 7) + { + var encode = new List(); + var total = new List(); + var commitWait = new List(); + var bytes = new List(); + for (var run = 0; run < runs + 2; run++) + { + var scope = MetalNative.objc_autoreleasePoolPush(); + var allocBefore = GC.GetAllocatedBytesForCurrentThread(); + var t0 = Stopwatch.GetTimestamp(); + var commandBuffer = MetalNative.Send(queue, S("commandBuffer")); + var pass = MetalNative.Send(MetalNative.Class("MTLRenderPassDescriptor"), S("renderPassDescriptor")); + var attachment = MetalNative.SendAtIndex(MetalNative.Send(pass, S("colorAttachments")), S("objectAtIndexedSubscript:"), 0); + MetalNative.SendVoid(attachment, S("setTexture:"), texture); + MetalV2Native.SendVoidValue(attachment, S("setLoadAction:"), 2); + MetalV2Native.SendVoidValue(attachment, S("setStoreAction:"), 1); + var encoder = MetalNative.Send(commandBuffer, S("renderCommandEncoderWithDescriptor:"), pass); + var t1 = Stopwatch.GetTimestamp(); + var selSetPipeline = S("setRenderPipelineState:"); + MetalNative.SendVoid(encoder, selSetPipeline, renderState); + for (var draw = 0; draw < Draws; draw++) + { + if (pipelinePerDraw) MetalNative.SendVoid(encoder, selSetPipeline, renderState); + for (var b = 0; b < buffersPerDraw; b++) + { + MetalNative.SendSetBuffer(encoder, (b & 1) == 0 ? selSetVertexBuffer : selSetFragmentBuffer, buffers[(draw + b) % buffers.Length], (nuint)((draw & 15) * 256), (nuint)(b >> 1)); + } + + MetalNative.SendDrawPrimitives(encoder, selDraw, 3, 0, 3); + } + + var t2 = Stopwatch.GetTimestamp(); + MetalNative.SendVoid(encoder, S("endEncoding")); + MetalNative.SendVoid(commandBuffer, S("commit")); + var t3 = Stopwatch.GetTimestamp(); + MetalNative.SendVoid(commandBuffer, S("waitUntilCompleted")); + var t4 = Stopwatch.GetTimestamp(); + var allocAfter = GC.GetAllocatedBytesForCurrentThread(); + MetalNative.objc_autoreleasePoolPop(scope); + if (run < 2) continue; // warm: first runs pay lazy pipeline/driver setup + var ns = 1e9 / Stopwatch.Frequency; + encode.Add((t2 - t1) * ns / Draws); + total.Add((t3 - t0) * ns / Draws); + commitWait.Add((t4 - t3) * ns / Draws); + bytes.Add((allocAfter - allocBefore) / (double)Draws); + } + + Harness.Record(id, name + " [encode loop only]", encode, bytes, 6331, note: $"{buffersPerDraw} setBuffer + 1 draw per item, one encoder, {Draws} draws"); + Harness.Record(id + "e", name + " [encoder create + end + commit]", total, bytes, 6331, note: "adds pass setup, endEncoding, commit (no wait)"); + Harness.Record(id + "w", name + " [GPU wait: waitUntilCompleted after commit]", commitWait, bytes, 6331, note: "GPU execution of 6000 tiny triangles"); + } + + RenderRun("8r1", "raw Metal: 1 draw only (pipeline set once)", 0, false); + RenderRun("8r2", "raw Metal: 2 setBuffer + 1 draw", 2, false); + RenderRun("8r3", "raw Metal: 10 setBuffer + 1 draw", 10, false); + RenderRun("8r4", "raw Metal: 20 setBuffer + 1 draw (heavy bind)", 20, false); + RenderRun("8r5", "raw Metal: 10 setBuffer + setRenderPipelineState + 1 draw", 10, true); + + // Compute: 2,700 dispatches per frame. + { + var encode = new List(); + var wait = new List(); + var bytes = new List(); + for (var run = 0; run < 9; run++) + { + var scope = MetalNative.objc_autoreleasePoolPush(); + var allocBefore = GC.GetAllocatedBytesForCurrentThread(); + var commandBuffer = MetalNative.Send(queue, S("commandBuffer")); + var encoder = MetalNative.Send(commandBuffer, S("computeCommandEncoder")); + var t1 = Stopwatch.GetTimestamp(); + MetalNative.SendVoid(encoder, S("setComputePipelineState:"), computeState); + for (var d = 0; d < Dispatches; d++) + { + MetalNative.SendSetBuffer(encoder, selSetComputeBuffer, buffers[d % buffers.Length], (nuint)((d & 15) * 256), 0); + MetalNative.SendSetBuffer(encoder, selSetComputeBuffer, buffers[(d + 1) % buffers.Length], 0, 1); + MetalNative.SendDispatch(encoder, S("dispatchThreadgroups:threadsPerThreadgroup:"), + new MtlSize { Width = 1, Height = 1, Depth = 1 }, new MtlSize { Width = 64, Height = 1, Depth = 1 }); + } + + var t2 = Stopwatch.GetTimestamp(); + MetalNative.SendVoid(encoder, S("endEncoding")); + MetalNative.SendVoid(commandBuffer, S("commit")); + MetalNative.SendVoid(commandBuffer, S("waitUntilCompleted")); + var t4 = Stopwatch.GetTimestamp(); + var allocAfter = GC.GetAllocatedBytesForCurrentThread(); + MetalNative.objc_autoreleasePoolPop(scope); + if (run < 2) continue; + var ns = 1e9 / Stopwatch.Frequency; + encode.Add((t2 - t1) * ns / Dispatches); + wait.Add((t4 - t2) * ns / Dispatches); + bytes.Add((allocAfter - allocBefore) / (double)Dispatches); + } + + Harness.Record("8c1", "raw Metal compute: 2 setBuffer + 1 dispatchThreadgroups [encode loop only]", encode, bytes, 2700, "dispatch"); + Harness.Record("8c2", "raw Metal compute: endEncoding + commit + GPU wait (2700 dispatches, serial)", wait, bytes, 2700, "dispatch", "serial dispatch type: Metal inserts a barrier between dependent dispatches"); + } + + try { GpuWordWait(); } + catch (InvalidOperationException exception) { Console.WriteLine($"8g SKIPPED on {System.Runtime.InteropServices.RuntimeInformation.ProcessArchitecture}: {exception.Message}"); } + GC.KeepAlive(sinkTicks); + } + + /// + /// A guest read of a word the GPU is about to write: the host commits the open encoder and waits for the command buffer + /// (MetalRenderHost.TryReadGuest -> TryReadGuestCore -> WaitForGuestBytes). Measured through the real rig. + /// + internal static void GpuWordWait() + { + var device = MetalNative.MTLCreateSystemDefaultDevice(); + using var rig = new SceneRig(device); + const uint Groups = 1; + var buffer = rig.Memory.Allocate(Groups * 64 * 4, 256); + var stream = rig.Dispatch(new ScenePm4(), ScenePrograms.StoreGlobalId, Descriptors.Buffer(buffer, 4, Groups * 64, format: 20), Groups).ToArray(); + var address = rig.Memory.Upload(stream, 256); + var host = rig.Host; + ulong submission = 1000; + var wordBytes = new byte[4]; + var enqueueToRead = new List(); + var readOnly = new List(); + var cpuEncode = new List(); + var ns = 1e9 / Stopwatch.Frequency; + // Compile the program first: a submission parked on an async pipeline compile never wrote the word (arm64 read 0). + host.CommandStream.EnqueueGraphics(address, (uint)stream.Length, submission++, null); + DrainStream(host); + for (var i = 0; i < 400; i++) + { + host.CommandStream.EnqueueGraphics(address, (uint)stream.Length, submission++, null); + var t0 = Stopwatch.GetTimestamp(); + host.RunPendingCommands(); + host.RunCommandStreamSlices(64); + var t1 = Stopwatch.GetTimestamp(); + if (!host.TryReadGuest(buffer + 4, wordBytes)) throw new InvalidOperationException("read failed"); + var t2 = Stopwatch.GetTimestamp(); + if (BitConverter.ToUInt32(wordBytes) != (1 ^ 0x5A5A)) throw new InvalidOperationException($"word {BitConverter.ToUInt32(wordBytes):X}"); + if (i < 100) continue; // pipeline compile / warmup + cpuEncode.Add((t1 - t0) * ns); + readOnly.Add((t2 - t1) * ns); + enqueueToRead.Add((t2 - t0) * ns); + } + + Harness.Record("8g1", "one tiny dispatch: CPU parse+encode (RunPendingCommands + RunCommandStreamSlices)", cpuEncode, [0], 20, "wait", "per frame figure assumes ~20 GPU-word waits per frame (guesstimate; PERF-REPORT: 65-76 ms/frame GPU wait)"); + Harness.Record("8g2", "guest word read that must wait: commit + GPU execute + return (TryReadGuest)", readOnly, [0], 20, "wait"); + Harness.Record("8g3", "dispatch submit -> GPU-written word visible (encode + commit + wait)", enqueueToRead, [0], 20, "wait"); + GpuWordWaitBehindLongWork(rig, host, buffer, address, stream.Length, ref submission); + var probe = new byte[4]; + Harness.Measure("8g4", "same word read again with nothing pending (no wait)", 1, 20, _ => host.TryReadGuest(buffer + 8, probe), unit: "read"); + } + + /// + /// 8g5: the word is written by an early submission and the read comes after a later, long submission was recorded + /// into the same coalesced command buffer. The read needs only the early writer; a whole-buffer wait also waits for + /// the long work. + /// + private static void GpuWordWaitBehindLongWork(SceneRig rig, MetalRenderHost host, ulong buffer, ulong address, int streamDwords, ref ulong submission) + { + const uint LongGroups = 32768; // 2M threads, 8 MB of stores per dispatch + var big = rig.Memory.Allocate(LongGroups * 64 * 4, 256); + var longPm4 = new ScenePm4(); + for (var i = 0; i < 6; i++) longPm4 = rig.Dispatch(longPm4, ScenePrograms.StoreGlobalId, Descriptors.Buffer(big, 4, LongGroups * 64, format: 20), LongGroups); + var longStream = longPm4.ToArray(); + var longAddress = rig.Memory.Upload(longStream, 256); + host.CommandStream.EnqueueGraphics(longAddress, (uint)longStream.Length, submission++, null); + DrainStream(host); + var wordBytes = new byte[4]; + var readOnly = new List(); + var longOnly = new List(); + var ns = 1e9 / Stopwatch.Frequency; + for (var i = 0; i < 60; i++) + { + host.CommandStream.EnqueueGraphics(address, (uint)streamDwords, submission++, null); + host.CommandStream.EnqueueGraphics(longAddress, (uint)longStream.Length, submission++, null); + host.RunPendingCommands(); + host.RunCommandStreamSlices(64); + var t1 = Stopwatch.GetTimestamp(); + if (!host.TryReadGuest(buffer + 4, wordBytes)) throw new InvalidOperationException("read failed"); + var t2 = Stopwatch.GetTimestamp(); + if (BitConverter.ToUInt32(wordBytes) != (1 ^ 0x5A5A)) throw new InvalidOperationException($"word {BitConverter.ToUInt32(wordBytes):X}"); + host.FlushAndWaitForGpu(); + var t3 = Stopwatch.GetTimestamp(); + if (i < 10) continue; + readOnly.Add((t2 - t1) * ns); + longOnly.Add((t3 - t1) * ns); + } + + Harness.Record("8g5", "word written by an early submission, read after a long later submission in the same buffer (TryReadGuest)", readOnly, [0], 20, "wait"); + Harness.Record("8g6", "same, read + drain of the long work (reference: whole-buffer time)", longOnly, [0], 20, "wait"); + } + + private static void DrainStream(MetalRenderHost host) + { + var deadline = Stopwatch.StartNew(); + while (host.CommandStream.HasPending && deadline.Elapsed.TotalSeconds < 60) + { + host.RunPendingCommands(); + host.RunCommandStreamSlices(64); + host.CommitCoalescedFlush(); + host.PollGpuCompletions(); + host.CommandStream.RetryBlocked(); + Thread.Sleep(1); + } + + host.FlushAndWaitForGpu(); + } +} diff --git a/tools/SharpEmu.Tools.PerfAudit/Benches.Misc.cs b/tools/SharpEmu.Tools.PerfAudit/Benches.Misc.cs new file mode 100644 index 000000000..be575c51a --- /dev/null +++ b/tools/SharpEmu.Tools.PerfAudit/Benches.Misc.cs @@ -0,0 +1,27 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using System.Diagnostics; + +namespace SharpEmu.PerfAudit; + +/// Primitive costs under the host the emulator runs on (Rosetta x64 on Apple silicon): lock, Interlocked, timestamp. +internal static class MiscBenches +{ + public static void Run() + { + Console.WriteLine("--- 9. Primitive costs ---"); + var thin = new object(); + var inflated = new object(); + Monitor.Enter(inflated); Monitor.Wait(inflated, 1); Monitor.Exit(inflated); // Wait inflates the lock into a sync block + _ = inflated.GetHashCode(); + long counter = 0; + var perDraw = 14.0 * 6331; // ~ lock acquisitions per draw is unknown; this is the 6331 x 14 sizing used for the frame column + Harness.Measure("9a", "uncontended lock(obj) { } (fresh object, thin lock)", 1, perDraw, _ => { lock (thin) { counter++; } }, unit: "lock"); + Harness.Measure("9b", "uncontended lock(obj) { } on an object whose lock was inflated (Monitor.Wait used once)", 1, perDraw, _ => { lock (inflated) { counter++; } }, unit: "lock"); + Harness.Measure("9c", "Interlocked.Increment", 1, perDraw, _ => Interlocked.Increment(ref counter), unit: "op"); + Harness.Measure("9d", "Stopwatch.GetTimestamp()", 1, perDraw, _ => counter += Stopwatch.GetTimestamp(), unit: "call"); + Harness.Measure("9e", "Environment.CurrentManagedThreadId", 1, perDraw, _ => counter += Environment.CurrentManagedThreadId, unit: "call"); + GC.KeepAlive(counter); + } +} diff --git a/tools/SharpEmu.Tools.PerfAudit/Benches.Planning.cs b/tools/SharpEmu.Tools.PerfAudit/Benches.Planning.cs new file mode 100644 index 000000000..44bba0b52 --- /dev/null +++ b/tools/SharpEmu.Tools.PerfAudit/Benches.Planning.cs @@ -0,0 +1,121 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using SharpEmu.Libs.Gpu.GpuCommands; +using SharpEmu.Libs.Gpu.GpuCommands.Registers; +using SharpEmu.Libs.Gpu.Rendering; +using SharpEmu.PerfAudit.Planning; +using static SharpEmu.PerfAudit.Planning.RenderExecutorFixtures; + +namespace SharpEmu.PerfAudit; + +internal static class PlanningBenches +{ + private const int Batch = 2000; + + public static void Run() + { + Console.WriteLine("--- 2. Draw planning (RenderExecutor over a no-op host) ---"); + var host = new RecordingRenderHost(); + var pipelines = new FakePipelineProvider(); + var executor = new RenderExecutor(host, pipelines); + var banks = Banks(); + var auto = Auto(6); + + // 2a: the executor alone, same live banks every draw. + Harness.Measure("2a", "RenderExecutor.DrawAuto, 1 color target, banks unchanged", Batch, Pm4Benches.FrameDraws, _ => + { + for (var i = 0; i < Batch; i++) executor.DrawAuto(1, banks, auto); + }, note: "host calls are no-ops; real host costs are in sections 3-5/8"); + + // 2b: the state-tracking step alone: copy-on-write snapshot after one register write. + var writes = Banks(); + Harness.Measure("2b", "RegisterBanks.PrepareContextWrite + 1 reg write + Snapshot()", Batch, Pm4Benches.FrameDraws, iteration => + { + for (var i = 0; i < Batch; i++) + { + writes.PrepareContextWrite(); + writes.Context.DepthCountControl = (uint)(i & 1); + _ = writes.Snapshot(); + } + }, note: "what a deferred draw does per draw before it is queued"); + + // 2c: a deferred draw: snapshot then plan it. + var live = Banks(); + Harness.Measure("2c", "Snapshot + RenderExecutor.DrawAuto (deferred draw, color only)", Batch, Pm4Benches.FrameDraws, _ => + { + for (var i = 0; i < Batch; i++) + { + live.PrepareContextWrite(); + live.Context.DepthCountControl = (uint)(i & 1); + var snapshot = live.Snapshot(); + executor.DrawAuto(1, snapshot, auto); + } + }); + + // 2d: indexed draw, depth target + one vertex buffer + index buffer, snapshot first. + pipelines.Graphics = Programs(vertexBuffers: [new VertexInputBuffer(VertexBase, 16, 4)]); + var depthLive = Banks(withDepth: true); + var indexed = Indexed(6, instances: 2, baseVertex: 3); + Harness.Measure("2d", "Snapshot + RenderExecutor.DrawIndexed (depth + 1 VB + IB)", Batch, Pm4Benches.FrameDraws, _ => + { + for (var i = 0; i < Batch; i++) + { + depthLive.PrepareContextWrite(); + depthLive.Context.DepthCountControl = (uint)(i & 1); + var snapshot = depthLive.Snapshot(); + executor.DrawIndexed(7, snapshot, indexed); + } + }); + + // 2f-2j: index expansion (host has no fan primitive): fan, quad list, 8-bit, custom restart, non-indexed fan. + host.DrawsTriangleFans = false; + pipelines.Graphics = Programs(vertexBuffers: [new VertexInputBuffer(VertexBase, 16, 4)]); + const int IndexCount = 1000; + var wide = new byte[IndexCount * 4]; + for (var i = 0; i < IndexCount; i++) System.Buffers.Binary.BinaryPrimitives.WriteUInt16LittleEndian(wide.AsSpan(i * 2), (ushort)(i % 997)); + host.WriteGuest(IndexBase, wide); + var fanBanks = Banks(primitiveType: 5); + Harness.Measure("2f", "DrawIndexed triangle fan, 1000 x u16 indices (CPU triangulation)", Batch / 4, Pm4Benches.FrameDraws, _ => + { + for (var i = 0; i < Batch / 4; i++) executor.DrawIndexed(7, fanBanks, Indexed(IndexCount)); + }); + var quadBanks = Banks(primitiveType: 19); + Harness.Measure("2g", "DrawIndexed legacy quad list, 1000 x u16 indices", Batch / 4, Pm4Benches.FrameDraws, _ => + { + for (var i = 0; i < Batch / 4; i++) executor.DrawIndexed(7, quadBanks, Indexed(IndexCount)); + }); + var stripBanks = Banks(primitiveType: PrimitiveTriangleStrip); + Harness.Measure("2h", "DrawIndexed triangle list, 1000 x u8 indices (widened to u16)", Batch / 4, Pm4Benches.FrameDraws, _ => + { + for (var i = 0; i < Batch / 4; i++) executor.DrawIndexed(7, Banks(), Indexed(IndexCount, indexType: 2)); + }); + stripBanks.UserConfig.PrimitiveResetControl = 1; + stripBanks.Context.PrimitiveResetIndex = 0x7FFF; + Harness.Measure("2i", "DrawIndexed strip, 1000 x u16, custom restart index (widened to u32)", Batch / 4, Pm4Benches.FrameDraws, _ => + { + for (var i = 0; i < Batch / 4; i++) executor.DrawIndexed(7, stripBanks, Indexed(IndexCount)); + }); + Harness.Measure("2j", "DrawAuto triangle fan, 3000 vertices (CPU triangulation)", Batch / 4, Pm4Benches.FrameDraws, _ => + { + for (var i = 0; i < Batch / 4; i++) executor.DrawAuto(1, fanBanks, Auto(3000)); + }); + host.DrawsTriangleFans = true; + + // 2k: Snapshot() with no register write since the last one. + RegisterBanks? sink = null; + var idle = Banks(); + Harness.Measure("2k", "RegisterBanks.Snapshot() with no write since the last one", Batch, Pm4Benches.FrameDraws, _ => + { + for (var i = 0; i < Batch; i++) sink = idle.Snapshot(); + }); + + // 2e: compute dispatch planning. + pipelines.Graphics = Programs(); + var computeBanks = Banks(); + Harness.Measure("2e", "RenderExecutor.Dispatch (direct, 64-thread group)", Batch, 2700, _ => + { + for (var i = 0; i < Batch; i++) executor.Dispatch(5, computeBanks, 4, 1, 1, 0x41); + }, unit: "dispatch"); + } +} diff --git a/tools/SharpEmu.Tools.PerfAudit/Benches.Pm4.cs b/tools/SharpEmu.Tools.PerfAudit/Benches.Pm4.cs new file mode 100644 index 000000000..d3da72aa1 --- /dev/null +++ b/tools/SharpEmu.Tools.PerfAudit/Benches.Pm4.cs @@ -0,0 +1,100 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using SharpEmu.Gpu.Metal.Tests.Scenes; +using SharpEmu.Libs.Gpu.GpuCommands; +using SharpEmu.Libs.Gpu.GpuCommands.Registers; +using SharpEmu.PerfAudit.Planning; + +namespace SharpEmu.PerfAudit; + +internal static class Pm4Benches +{ + public const int FrameDraws = 6331; + public const int Draws = 6000; + public const ulong StreamAddress = NativeGuestMemory.Base + 0x1000; + + /// + /// A DCB of draws. Each draw re-sets context registers + /// (one SET_CONTEXT_REG packet each, values vary per draw), the VS and PS user-data blocks (one SET_SH_REG packet + /// each), NUM_INSTANCES, and a DRAW_INDEX_AUTO or DRAW_INDEX_2. + /// + public static uint[] BuildStream(int draws, int contextRegisters, bool indexed, out int packets) + { + uint[] contextOffsets = + [ + Cx.PaClVportXScale, Cx.PaClVportXOffset, Cx.PaClVportYScale, Cx.PaClVportYOffset, Cx.PaClVportZScale, Cx.PaClVportZOffset, + Cx.CbTargetMask, Cx.CbShaderMask, Cx.DbDepthControl, Cx.PaSuScModeCntl, Cx.CbBlend0Control, Cx.CbColorControl, + Cx.DbShaderControl, Cx.DbRenderControl, Cx.SpiPsInputEna, Cx.SpiPsInputAddr, Cx.PaScScreenScissorTl, + Cx.PaScScreenScissorBr, Cx.PaScWindowScissorTl, Cx.CbColor0Base, Cx.PaClClipCntl, Cx.PaClVteCntl, Cx.PaScAaConfig, + Cx.SpiShaderColFormat, Cx.DbEqaa, + ]; + var pm4 = new ScenePm4(); + var count = 0; + for (var draw = 0; draw < draws; draw++) + { + for (var register = 0; register < contextRegisters; register++) + { + pm4.Context(contextOffsets[register % contextOffsets.Length], SceneState.F(draw * 0.25f + register)); + count++; + } + + // VS user data (GS user SGPRs) and PS user data: 8 words each, different buffers per draw. + var vs = new uint[8]; + var ps = new uint[8]; + for (var word = 0; word < 8; word++) + { + vs[word] = (uint)(0x1000 + draw * 64 + word); + ps[word] = (uint)(0x9000 + draw * 64 + word); + } + + pm4.Shader(Sh.UserDataGs0, vs); + pm4.Shader(Sh.UserDataPs0, ps); + pm4.NumInstances(1); + count += 3; + if (indexed) + { + pm4.IndexType(0); + pm4.DrawIndexed(NativeGuestMemory.Base + 0x40_0000, 1024, 6); + count += 2; + } + else + { + pm4.DrawAuto(6); + count++; + } + } + + packets = count; + return pm4.ToArray(); + } + + public static void Run() + { + Console.WriteLine("--- 1. PM4 interpreter (GpuCommandInterpreter.Process) ---"); + using var memory = new NativeGuestMemory(64 << 20); + foreach (var (label, contextRegisters, indexed, blocks) in new (string, int, bool, bool)[] + { + ("20 ctx regs + 2 SH user-data + NUM_INST + DRAW_INDEX_AUTO, block-read host", 20, false, true), + ("same stream, per-packet-read host (no command blocks)", 20, false, false), + ("20 ctx regs + 2 SH + NUM_INST + INDEX_TYPE + DRAW_INDEX_2, block-read host", 20, true, true), + ("5 ctx regs only (delta-light draw), block-read host", 5, false, true), + ("25 ctx regs, block-read host", 25, false, true), + }) + { + var words = BuildStream(Draws, contextRegisters, indexed, out var packets); + memory.WriteWords(StreamAddress, words); + var host = new CountingCommandHost(memory, blocks); + var interpreter = new GpuCommandInterpreter(host, 0, 0); + var id = label.StartsWith("20 ctx regs + 2 SH user-data") ? "1a" : label.StartsWith("same") ? "1b" : label.StartsWith("20 ctx regs + 2 SH + NUM_INST + INDEX") ? "1c" : label.StartsWith("5 ") ? "1d" : "1e"; + var packetsPerDraw = (double)packets / Draws; + var result = Harness.Measure(id, $"PM4 {label}", Draws, FrameDraws, _ => + { + var progress = interpreter.Process(new PacketCursorStack(), StreamAddress, (uint)words.Length); + if (progress != SubmissionProgress.Complete) throw new InvalidOperationException(progress.ToString()); + }, note: $"{packetsPerDraw:F1} packets/draw, {words.Length / (double)Draws:F1} dwords/draw, stream {words.Length * 4 / 1024} KiB"); + Console.WriteLine($" -> {result.MedianNs / packetsPerDraw:F1} ns/packet ({packetsPerDraw:F1} packets/draw), host draws seen {host.Draws}"); + Harness.Results.Add(result with { Id = id + "p", Name = $" per packet: {label}", MedianNs = result.MedianNs / packetsPerDraw, P95Ns = result.P95Ns / packetsPerDraw, MinNs = result.MinNs / packetsPerDraw, BytesPerOp = result.BytesPerOp / packetsPerDraw, OpsPerFrame = FrameDraws * packetsPerDraw, Unit = "packet" }); + } + } +} diff --git a/tools/SharpEmu.Tools.PerfAudit/Benches.Queue.cs b/tools/SharpEmu.Tools.PerfAudit/Benches.Queue.cs new file mode 100644 index 000000000..85ad99a34 --- /dev/null +++ b/tools/SharpEmu.Tools.PerfAudit/Benches.Queue.cs @@ -0,0 +1,115 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using System.Diagnostics; +using SharpEmu.Libs.Gpu.GpuCommands; + +namespace SharpEmu.PerfAudit; + +/// +/// 7. CommandStreamQueue: the guest-thread enqueue, the Monitor wake of the worker, and the worker's +/// slice loop (CommandStreamWorker.Run -> ProcessOne -> WaitForWork), with the real queue and worker +/// over a lean host so only the queue machinery is measured. +/// +internal static class QueueBenches +{ + private const double SlicesPerFrame = 83; + private const double SubmitsPerFrame = 20; + + public static void Run() + { + Console.WriteLine("--- 7. CommandStreamQueue / CommandStreamWorker ---"); + using var memory = new NativeGuestMemory(64 << 20); + var words = Pm4Benches.BuildStream(1, 5, false, out _); + memory.WriteWords(Pm4Benches.StreamAddress, words); + var dwords = (uint)words.Length; + var address = Pm4Benches.StreamAddress; + + // 7a: enqueue only, no worker. The queue is drained (discarded) every 4096 pushes so it stays bounded; + // that amortised cost (~nothing per push) is inside the figure. + { + var host = new CountingCommandHost(memory, true); + var queue = new CommandStreamQueue(host); + ulong submission = 1; + Harness.Measure("7a", "CommandStreamQueue.EnqueueGraphics (no worker, uncontended)", 1, SlicesPerFrame, i => + { + if ((i & 4095) == 4095) queue.DiscardAll(); + queue.EnqueueGraphics(address, dwords, submission++, null); + }, unit: "slice", note: "lock + new CommandSubmission + list append"); + } + + // 7b-7e share one live worker that parks in Monitor.Wait between submissions. + { + var host = new CountingCommandHost(memory, true); + var queue = new CommandStreamQueue(host); + var worker = new CommandStreamWorker(queue, host, () => false, true, "bench worker"); + worker.Start(); + ulong submission = 1; + Thread.Sleep(50); + + // 7b/7c: wake latency from a parked worker. Each sample spaces the push by ~1 ms of spinning so the worker is + // really parked (not in its run-loop between slices). + const int Samples = 3000; + var latency = new double[Samples]; + var enqueueCost = new double[Samples]; + var roundTrip = new double[Samples]; + var tickNs = 1e9 / Stopwatch.Frequency; + for (var warm = 0; warm < 200; warm++) + { + queue.EnqueueGraphics(address, dwords, submission++, null); + queue.WaitForIdle(); + } + + for (var i = 0; i < Samples; i++) + { + SpinFor(Stopwatch.Frequency / 1000); + var beginsBefore = Volatile.Read(ref host.Begins); + var t0 = Stopwatch.GetTimestamp(); + queue.EnqueueGraphics(address, dwords, submission++, null); + var t1 = Stopwatch.GetTimestamp(); + queue.WaitForIdle(); + var t2 = Stopwatch.GetTimestamp(); + if (Volatile.Read(ref host.Begins) == beginsBefore) { i--; continue; } + latency[i] = (Volatile.Read(ref host.BeginTicks) - t0) * tickNs; + enqueueCost[i] = (t1 - t0) * tickNs; + roundTrip[i] = (t2 - t0) * tickNs; + } + + Harness.Record("7b", "enqueue -> worker BeginSubmission, parked worker (wake latency)", latency, [0], SubmitsPerFrame, "submit", + "Monitor.Pulse -> worker thread scheduled; each push separated by 1 ms idle"); + Harness.Record("7c", "EnqueueGraphics call cost with a parked worker (producer side, includes worker notification)", enqueueCost, [0], SlicesPerFrame, "slice"); + Harness.Record("7d", "enqueue + WaitForIdle round trip, parked worker, 1 tiny slice", roundTrip, [0], SubmitsPerFrame, "submit"); + + // 7e: pipelined: push a frame's worth of slices, then wait; worker never parks until the end. + var perRun = new List(); + var perRunBytes = new List(); + for (var run = 0; run < 7; run++) + { + GC.Collect(); + var before = GC.GetTotalAllocatedBytes(true); + var start = Stopwatch.GetTimestamp(); + const int Frames = 200; + for (var frame = 0; frame < Frames; frame++) + { + for (var slice = 0; slice < (int)SlicesPerFrame; slice++) queue.EnqueueGraphics(address, dwords, submission++, null); + queue.WaitForIdle(); + } + + var elapsed = Stopwatch.GetElapsedTime(start).TotalNanoseconds; + var ops = Frames * SlicesPerFrame; + perRun.Add(elapsed / ops); + perRunBytes.Add((GC.GetTotalAllocatedBytes(true) - before) / ops); + } + + Harness.Record("7e", "pipelined: 83 pushes then WaitForIdle, per slice (queue + worker loop + 1 draw each, tiny host)", perRun, perRunBytes, SlicesPerFrame, "slice", + "process-wide allocations; includes the interpreter running a 4-packet stream per slice"); + worker.Stop(); + } + } + + private static void SpinFor(long ticks) + { + var end = Stopwatch.GetTimestamp() + ticks; + while (Stopwatch.GetTimestamp() < end) Thread.SpinWait(20); + } +} diff --git a/tools/SharpEmu.Tools.PerfAudit/Benches.Rig.cs b/tools/SharpEmu.Tools.PerfAudit/Benches.Rig.cs new file mode 100644 index 000000000..a62a9f1bf --- /dev/null +++ b/tools/SharpEmu.Tools.PerfAudit/Benches.Rig.cs @@ -0,0 +1,36 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using System.Diagnostics; +using SharpEmu.Gpu.Metal; +using SharpEmu.Gpu.Metal.Tests.Scenes; + +namespace SharpEmu.PerfAudit; + +internal static class RigBenches +{ + public static void Smoke() + { + Console.WriteLine("--- rig smoke ---"); + var device = MetalNative.MTLCreateSystemDefaultDevice(); + Console.WriteLine($"device=0x{device:X}"); + using var rig = new SceneRig(device); + const uint Size = 64; + var target = rig.ColorTarget(Size, Size); + var state = new SceneState(Size, Size).ColorTarget(0, target, Size, Size); + var pm4 = rig.DrawMesh(new ScenePm4(), state, + [ + SceneVertex.At(-1, -1, 0.5f, 1, 0, 0), + SceneVertex.At(1, -1, 0.5f, 1, 0, 0), + SceneVertex.At(-1, 1, 0.5f, 1, 0, 0), + ]); + var sw = Stopwatch.StartNew(); + rig.Submit(pm4); + Console.WriteLine($"first submit {sw.Elapsed.TotalMilliseconds:F1} ms"); + var pixels = rig.ReadImage(target, Size, Size); + var pixel = SceneDraws.Pixel(pixels, Size, 2, Size - 3); + Console.WriteLine($"pixel(2,{Size - 3}) = 0x{pixel:X8} (expect FF0000FF)"); + if (pixel != 0xFF0000FF) + throw new InvalidOperationException($"Synthetic triangle readback mismatch: 0x{pixel:X8}"); + } +} diff --git a/tools/SharpEmu.Tools.PerfAudit/Benches.Translator.cs b/tools/SharpEmu.Tools.PerfAudit/Benches.Translator.cs new file mode 100644 index 000000000..5c67ffabc --- /dev/null +++ b/tools/SharpEmu.Tools.PerfAudit/Benches.Translator.cs @@ -0,0 +1,91 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using System.Runtime.InteropServices; +using SharpEmu.HLE.GuestMemory; +using SharpEmu.HLE.Host; + +namespace SharpEmu.PerfAudit; + +/// k17: GuestMirrorTranslator hit path, miss rate and miss allocation over view-table layouts. +internal static unsafe class TranslatorBenches +{ + private const ulong Base = 0x20_0000_0000; + private const ulong Block = 0x10000; + private const ulong MirrorBase = 0x7100_0000_0000; + private const ulong MirrorBytes = 1UL << 36; + private const int Ops = 4096; + + private static GuestViewTable NewTable(int capacity) + { + var bytes = GuestViewTable.RegionBytes(capacity, 256); + var address = (ulong)NativeMemory.AllocZeroed((nuint)bytes); + var region = new HostSharedRegion(address, bytes, -1, writable: true, static r => NativeMemory.Free((void*)r.Address)); + return new GuestViewTable(region, MirrorBytes, 256); + } + + public static void Run() + { + Console.WriteLine("--- 10. Mirror translator (k17) ---"); + var random = new Random(17); + + // a: one 1 GiB view, random addresses. + { + using var table = NewTable(64); + table.Map(Base, 1UL << 30, 0, HostPageProtection.ReadWrite); + var addresses = Enumerable.Range(0, Ops).Select(_ => Base + (ulong)random.NextInt64(1L << 30)).ToArray(); + Case("10a", "1 GiB single view, random addresses", table, addresses); + } + + // b: 512 separate 1 MiB views (offsets do not continue), random. + { + using var table = NewTable(1024); + for (var i = 0; i < 512; i++) + table.Map(Base + (ulong)i * 2 * (1UL << 20), 1UL << 20, (ulong)i * 3 * (1UL << 20), HostPageProtection.ReadWrite); + var addresses = Enumerable.Range(0, Ops).Select(_ => Base + (ulong)random.Next(512) * 2 * (1UL << 20) + (ulong)random.Next(1 << 20)).ToArray(); + Case("10b", "512 separate 1 MiB views, random addresses", table, addresses); + } + + // c: 48 buffers of 64 KiB at a 4 MiB stride: every block index is a multiple of 64. + { + using var table = NewTable(128); + for (var i = 0; i < 48; i++) + table.Map(Base + (ulong)i * (4UL << 20), Block, (ulong)i * 2 * Block, HostPageProtection.ReadWrite); + var addresses = Enumerable.Range(0, Ops).Select(_ => Base + (ulong)random.Next(48) * (4UL << 20) + (ulong)random.Next((int)Block)).ToArray(); + Case("10c", "48 x 64 KiB views at a 4 MiB stride, random addresses", table, addresses); + } + + // d: a 64 MiB heap split by watches into alternating 16 KiB protections (4096 entries, runs cut at 129). + { + using var table = NewTable(8192); + table.Map(Base, 64UL << 20, 0, HostPageProtection.ReadWrite); + for (ulong at = 0; at < (64UL << 20); at += 0x8000) + table.Protect(Base + at, 0x4000, HostPageProtection.ReadOnly); + var addresses = Enumerable.Range(0, Ops).Select(_ => Base + (ulong)random.NextInt64(64L << 20)).ToArray(); + Case("10d", "64 MiB heap split into 4096 watch entries, random addresses", table, addresses); + var walk = Enumerable.Range(0, Ops).Select(i => Base + (ulong)i * 0x4000 % (64UL << 20)).ToArray(); + Case("10e", "same heap, sequential 16 KiB walk", table, walk); + } + } + + private static void Case(string id, string name, GuestViewTable table, ulong[] addresses) + { + var translator = new GuestMirrorTranslator(table.Region.Address, table.Region.Size, MirrorBase, MirrorBytes) { CountCacheAccesses = false }; + ulong sink = 0; + Harness.Measure(id, name, Ops, 1, _ => + { + for (var i = 0; i < addresses.Length; i++) + if (translator.TryTranslate(addresses[i], 8, out var mirror)) sink += mirror; + }, unit: "xlate"); + // Miss rate over warm passes. + var counting = new GuestMirrorTranslator(table.Region.Address, table.Region.Size, MirrorBase, MirrorBytes) { CountCacheAccesses = true }; + for (var pass = 0; pass < 3; pass++) + foreach (var address in addresses) counting.TryTranslate(address, 8, out _); + var hits0 = counting.CacheHits; var misses0 = counting.CacheMisses; + for (var pass = 0; pass < 10; pass++) + foreach (var address in addresses) counting.TryTranslate(address, 8, out _); + var hits = counting.CacheHits - hits0; var misses = counting.CacheMisses - misses0; + Console.WriteLine($" {id} warm miss rate {100.0 * misses / Math.Max(1, hits + misses):F2}% ({misses} misses / {hits + misses})"); + GC.KeepAlive(sink); + } +} diff --git a/tools/SharpEmu.Tools.PerfAudit/Benches.W8.cs b/tools/SharpEmu.Tools.PerfAudit/Benches.W8.cs new file mode 100644 index 000000000..8874fc9eb --- /dev/null +++ b/tools/SharpEmu.Tools.PerfAudit/Benches.W8.cs @@ -0,0 +1,128 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using System.Runtime.InteropServices; +using SharpEmu.Gpu.Metal.V2; +using SharpEmu.Gpu.Metal.Tests.Scenes; +using SharpEmu.HLE.GpuMemory; + +namespace SharpEmu.PerfAudit; + +/// +/// W8: a 64 MB staged-range-cache entry, one 4-byte guest store into it, then the rebind that has to make the +/// entry current again. The bind is what MetalRenderHost.BindStagedRange does minus the GPU gather: Acquire, then +/// either the partial refill or a whole read of the entry from guest memory. Watches go to a real PageGuard over a +/// no-op address space, so the watch bookkeeping is counted too (the syscall is not: 3.8-7.8 us per page in 6q). +/// +internal static unsafe class W8Benches +{ + private const ulong GuestBase = 0x4_0000_0000; + private const ulong EntryBytes = 64UL * 1024 * 1024; + + private sealed class Backend : IStagedRangeBackend, IDisposable + { + private readonly PageGuard _guard = new(new SceneAddressSpace()); + private readonly List _buffers = []; + public long AddCalls, RemoveCalls, AddPages, RemovePages; + + public (nint Handle, nint Contents, ulong GpuAddress) CreateBuffer(ulong capacity) + { + var memory = (nint)NativeMemory.AlignedAlloc((nuint)capacity, 16384); + _buffers.Add(memory); + return (memory, memory, (ulong)memory); + } + + public void ReleaseBuffer(nint handle) { } + + public void AddWatch(ulong address, ulong size) + { + AddCalls++; + AddPages += (long)((size + 4095) / 4096); + _guard.AddWatch(address, size, false); + } + + public void RemoveWatch(ulong address, ulong size) + { + RemoveCalls++; + RemovePages += (long)((size + 4095) / 4096); + _guard.RemoveWatch(address, size, false); + } + + public void Dispose() + { + foreach (var buffer in _buffers) + { + NativeMemory.AlignedFree((void*)buffer); + } + } + } + + public static void Run() + { + Console.WriteLine("--- W8. staged entry: one 4-byte store, rebind ---"); + var guest = new byte[EntryBytes]; + new Random(1).NextBytes(guest); + Scenario("w8a", partial: false, written: false, inFlight: false); + Scenario("w8b", partial: true, written: false, inFlight: false); + Scenario("w8c", partial: false, written: false, inFlight: true); + Scenario("w8d", partial: true, written: false, inFlight: true); + Scenario("w8e", partial: false, written: true, inFlight: false); + Scenario("w8f", partial: true, written: true, inFlight: false); + + void Scenario(string id, bool partial, bool written, bool inFlight) + { + using var backend = new Backend(); + var cache = new MetalStagedRangeCache(backend, 512UL * 1024 * 1024, partial); + long guestBytes = 0; + void Read(ulong address, Span destination) + { + guest.AsSpan((int)(address - GuestBase), destination.Length).CopyTo(destination); + guestBytes += destination.Length; + } + + ulong serial = 3; + void Bind() + { + // In flight: the previous submission has not completed when the next one binds. + var completed = inFlight ? serial - 2 : serial - 1; + var hit = cache.Acquire(GuestBase, EntryBytes, written, serial, completed, out var entry); + if (!hit) + { + if (!cache.RefillDirty(entry!, Read)) + { + Read(GuestBase, new Span((void*)entry!.StagingBuffer!.Contents, (int)EntryBytes)); + if (entry.Pristine.IsValid) + { + new ReadOnlySpan((void*)entry.StagingBuffer!.Contents, (int)EntryBytes) + .CopyTo(new Span((void*)entry.PristineBuffer!.Contents, (int)EntryBytes)); + } + } + } + + serial++; + } + + Bind(); + var iteration = 0; + guestBytes = 0; + backend.AddCalls = backend.RemoveCalls = backend.AddPages = backend.RemovePages = 0; + var result = Harness.Measure(id, + $"64 MB staged entry, one 4-byte store, rebind; partial={partial} written={written} in_flight={inFlight}", + 1, 1, i => + { + iteration++; + var offset = (ulong)((i * 7919L) % (long)(EntryBytes / 4096)) * 4096 + 64; + guest[(int)offset] ^= 1; + cache.OnCpuWrite(GuestBase + offset, 4); + Bind(); + }, unit: "rebind", maxInvokesPerRun: 4000); + var counters = cache.PartialCounters; + Console.WriteLine( + $"{id} per rebind: guest bytes copied={guestBytes / (double)iteration:N0} " + + $"watch add calls={backend.AddCalls / (double)iteration:N2} pages={backend.AddPages / (double)iteration:N0} " + + $"remove calls={backend.RemoveCalls / (double)iteration:N2} pages={backend.RemovePages / (double)iteration:N0} " + + $"partial refills={counters.Refills} " + + $"median={result.MedianNs / 1000:N1} us"); + } + } +} diff --git a/tools/SharpEmu.Tools.PerfAudit/Benches.WriteRanges.cs b/tools/SharpEmu.Tools.PerfAudit/Benches.WriteRanges.cs new file mode 100644 index 000000000..e72ee493f --- /dev/null +++ b/tools/SharpEmu.Tools.PerfAudit/Benches.WriteRanges.cs @@ -0,0 +1,27 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using SharpEmu.Gpu.Metal.V2; + +namespace SharpEmu.PerfAudit; + +/// 8w: MetalGpuWriteRanges.PendingSerial with 512 live records inside one 2 MiB bucket (the bucket gate passes). +internal static class WriteRangeBenches +{ + public static void Run() + { + const ulong Base = 0x2_0000_0000; + var ranges = new MetalGpuWriteRanges(); + for (var i = 0UL; i < 512; i++) + { + ranges.Note(Base + i * 4096, 256, 10 + i / 64); + } + + Harness.Measure("8w1", "PendingSerial miss, 512 records in the read's 2 MiB bucket", 1, 20000, + i => ranges.PendingSerial(Base + 1024 + (ulong)(i & 511) * 4096, 4, 0), unit: "read"); + Harness.Measure("8w2", "PendingSerial hit, 512 records in the read's 2 MiB bucket", 1, 20000, + i => ranges.PendingSerial(Base + 8 + (ulong)(i & 511) * 4096, 4, 0), unit: "read"); + Harness.Measure("8w3", "reference: unindexed walk of the same 512 records (miss)", 1, 20000, + i => ranges.UnindexedPendingSerial(Base + 1024 + (ulong)(i & 511) * 4096, 4, 0), unit: "read"); + } +} diff --git a/tools/SharpEmu.Tools.PerfAudit/FakeCpuMemory.cs b/tools/SharpEmu.Tools.PerfAudit/FakeCpuMemory.cs new file mode 100644 index 000000000..2ddd39536 --- /dev/null +++ b/tools/SharpEmu.Tools.PerfAudit/FakeCpuMemory.cs @@ -0,0 +1,101 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using SharpEmu.HLE; + +namespace SharpEmu.PerfAudit.Planning; + +// A single contiguous guest region backed by a byte[]. Enough to hand C strings and small +// structures to HLE exports under test without a live guest. +internal sealed class FakeCpuMemory : ICpuMemory, IGuestMemoryAllocator +{ + private readonly ulong _base; + private readonly byte[] _storage; + private ulong _allocBump; + + public FakeCpuMemory(ulong baseAddress, int size) + { + _base = baseAddress; + _storage = new byte[size]; + // Bump from the top so test fixtures at low offsets stay intact. + _allocBump = baseAddress + (ulong)size; + } + + public bool TryRead(ulong virtualAddress, Span destination) + { + if (!TryResolve(virtualAddress, destination.Length, out var offset)) + { + return false; + } + + _storage.AsSpan(offset, destination.Length).CopyTo(destination); + return true; + } + + public bool TryWrite(ulong virtualAddress, ReadOnlySpan source) + { + if (!TryResolve(virtualAddress, source.Length, out var offset)) + { + return false; + } + + source.CopyTo(_storage.AsSpan(offset, source.Length)); + return true; + } + + public bool CanRead(ulong address, ulong size) => + size != 0 && size <= int.MaxValue && TryResolve(address, (int)size, out _); + + public bool TryAllocateGuestMemory(ulong size, ulong alignment, out ulong address) + { + address = 0; + if (size == 0 || alignment == 0 || (alignment & (alignment - 1)) != 0) + { + return false; + } + + var alignedSize = (size + alignment - 1) & ~(alignment - 1); + if (alignedSize > _allocBump - _base) + { + return false; + } + + var next = (_allocBump - alignedSize) & ~(alignment - 1); + if (next < _base) + { + return false; + } + + _allocBump = next; + address = next; + return true; + } + + public bool TryFreeGuestMemory(ulong address) => false; + + public ulong WriteCString(ulong virtualAddress, string text) + { + var bytes = System.Text.Encoding.UTF8.GetBytes(text); + TryWrite(virtualAddress, bytes); + TryWrite(virtualAddress + (ulong)bytes.Length, stackalloc byte[] { 0 }); + return virtualAddress; + } + + private bool TryResolve(ulong virtualAddress, int length, out int offset) + { + offset = 0; + if (virtualAddress < _base) + { + return false; + } + + var relative = virtualAddress - _base; + if (relative + (ulong)length > (ulong)_storage.Length) + { + return false; + } + + offset = (int)relative; + return true; + } +} diff --git a/tools/SharpEmu.Tools.PerfAudit/Harness.cs b/tools/SharpEmu.Tools.PerfAudit/Harness.cs new file mode 100644 index 000000000..53feb5dd5 --- /dev/null +++ b/tools/SharpEmu.Tools.PerfAudit/Harness.cs @@ -0,0 +1,111 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using System.Diagnostics; +using System.Globalization; +using System.Text; + +namespace SharpEmu.PerfAudit; + +/// One measured benchmark: per-op time, per-op allocation, and an illustrative scaled workload cost. +internal sealed record BenchResult( + string Id, string Name, double MedianNs, double P95Ns, double MinNs, double BytesPerOp, double OpsPerFrame, string Unit, string Note) +{ + public const double BudgetNsPerItem = 1800.0; + public double FrameMs => MedianNs * OpsPerFrame / 1e6; + public double BudgetPct => MedianNs / BudgetNsPerItem * 100.0; +} + +/// +/// Hand-rolled Stopwatch harness: untimed warmup, timed runs, median and p95 of +/// the per-run ns/op, and managed allocation measured on the benchmarking thread (or process-wide +/// for the cross-thread cases). +/// +internal static class Harness +{ + public static int Runs = 5; + public static double TargetRunMs = 150; + public static double WarmupMs = 700; + public static readonly List Results = []; + + /// operations the body performs per call (ns/op = elapsed / calls / ops). + /// illustrative operation count, not a measured game frame (6331 draws, 2700 dispatches, ...). + /// the measured work; receives a running iteration index. + /// count every thread's allocations (for benchmarks that cross threads). + public static BenchResult Measure(string id, string name, double opsPerInvoke, double opsPerFrame, Action body, + string unit = "draw", string note = "", int warmupInvokes = 3, int maxInvokesPerRun = int.MaxValue, + bool processWideAllocations = false, Action? beforeEachRun = null) + { + // Warm until both the minimum invoke count and WarmupMs have passed, so tiered JIT / PGO promotion has settled + // (the shipped app runs R2R then tier-1 after a long session; a cold tier-0 loop would overstate every number). + var warm = Stopwatch.StartNew(); + for (var i = 0; i < warmupInvokes || warm.Elapsed.TotalMilliseconds < WarmupMs; i++) body(i); + // Calibrate how many invokes make a ~TargetRunMs run. + var sw = Stopwatch.StartNew(); + var calibrationInvokes = 0; + while (sw.Elapsed.TotalMilliseconds < 30 && calibrationInvokes < maxInvokesPerRun) + { + body(calibrationInvokes++); + } + + var perInvokeMs = sw.Elapsed.TotalMilliseconds / Math.Max(calibrationInvokes, 1); + var invokes = (int)Math.Clamp(TargetRunMs / Math.Max(perInvokeMs, 1e-6), 1, maxInvokesPerRun); + var times = new double[Runs]; + var bytes = new double[Runs]; + var index = 0; + for (var run = 0; run < Runs; run++) + { + beforeEachRun?.Invoke(); + GC.Collect(); + GC.WaitForPendingFinalizers(); + var allocatedBefore = processWideAllocations ? GC.GetTotalAllocatedBytes(true) : GC.GetAllocatedBytesForCurrentThread(); + var start = Stopwatch.GetTimestamp(); + for (var i = 0; i < invokes; i++) body(index++); + var elapsed = Stopwatch.GetElapsedTime(start); + var allocatedAfter = processWideAllocations ? GC.GetTotalAllocatedBytes(true) : GC.GetAllocatedBytesForCurrentThread(); + var ops = (double)invokes * opsPerInvoke; + times[run] = elapsed.TotalNanoseconds / ops; + bytes[run] = (allocatedAfter - allocatedBefore) / ops; + } + + Array.Sort(times); + Array.Sort(bytes); + var median = times[Runs / 2]; + var p95 = times[(int)Math.Ceiling(0.95 * Runs) - 1]; + var result = new BenchResult(id, name, median, p95, times[0], bytes[Runs / 2], opsPerFrame, unit, note); + Results.Add(result); + Console.WriteLine(Format(result)); + return result; + } + + /// Records a result computed elsewhere (a long-running scenario measured once per run). + public static BenchResult Record(string id, string name, IReadOnlyList nsPerOp, IReadOnlyList bytesPerOp, + double opsPerFrame, string unit = "draw", string note = "") + { + var times = nsPerOp.OrderBy(v => v).ToArray(); + var bytes = bytesPerOp.OrderBy(v => v).ToArray(); + var result = new BenchResult(id, name, times[times.Length / 2], times[(int)Math.Ceiling(0.95 * times.Length) - 1], times[0], + bytes[bytes.Length / 2], opsPerFrame, unit, note); + Results.Add(result); + Console.WriteLine(Format(result)); + return result; + } + + public static string Format(BenchResult r) => + string.Create(CultureInfo.InvariantCulture, + $"{r.Id,-6} {r.Name,-62} {r.MedianNs,12:N1} ns/op p95 {r.P95Ns,12:N1} {r.BytesPerOp,10:N1} B/op scaled@{r.OpsPerFrame:N0}/{r.Unit}: {r.FrameMs,8:N2} ms ({r.BudgetPct,7:N0}% of 1.8us)"); + + public static string Table() + { + var sb = new StringBuilder(); + sb.AppendLine("| ID | Benchmark | median ns/op | p95 ns/op | B/op | illustrative ops | scaled ms | % of 1.8 us/item |"); + sb.AppendLine("|---|---|---:|---:|---:|---:|---:|---:|"); + foreach (var r in Results) + { + sb.AppendLine(string.Create(CultureInfo.InvariantCulture, + $"| {r.Id} | {r.Name} | {r.MedianNs:N1} | {r.P95Ns:N1} | {r.BytesPerOp:N1} | {r.OpsPerFrame:N0} ({r.Unit}) | {r.FrameMs:N2} | {r.BudgetPct:N0}% |")); + } + + return sb.ToString(); + } +} diff --git a/tools/SharpEmu.Tools.PerfAudit/LeanHosts.cs b/tools/SharpEmu.Tools.PerfAudit/LeanHosts.cs new file mode 100644 index 000000000..ca9964f18 --- /dev/null +++ b/tools/SharpEmu.Tools.PerfAudit/LeanHosts.cs @@ -0,0 +1,86 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using System.Runtime.InteropServices; +using SharpEmu.HLE; +using SharpEmu.Libs.Gpu.GpuCommands; + +namespace SharpEmu.PerfAudit; + +/// One native block seen as guest memory at ; no tracking, no wrappers. +internal sealed unsafe class NativeGuestMemory : ICpuMemory, IDisposable +{ + public const ulong Base = 0x2_0000_0000; + private readonly byte* _block; + private readonly ulong _size; + + public NativeGuestMemory(ulong size) + { + _size = size; + _block = (byte*)NativeMemory.AlignedAlloc((nuint)size, 16384); + NativeMemory.Clear(_block, (nuint)size); + } + + public byte* Raw(ulong address) => _block + (address - Base); + public ulong Size => _size; + public void Dispose() => NativeMemory.AlignedFree(_block); + + public bool TryRead(ulong address, Span destination) + { + if (address < Base || address - Base + (ulong)destination.Length > _size) return false; + new ReadOnlySpan(_block + (address - Base), destination.Length).CopyTo(destination); + return true; + } + + public bool TryWrite(ulong address, ReadOnlySpan source) + { + if (address < Base || address - Base + (ulong)source.Length > _size) return false; + source.CopyTo(new Span(_block + (address - Base), source.Length)); + return true; + } + + public bool CanRead(ulong address, ulong size) => address >= Base && address - Base + size <= _size; + + public void WriteWords(ulong address, ReadOnlySpan words) => TryWrite(address, MemoryMarshal.AsBytes(words)); +} + +/// +/// A command-stream host that does nothing: every host call is a counter bump. Isolates the PM4 +/// interpreter from the host. With blocks on it serves command blocks like the Metal host does. +/// +internal sealed class CountingCommandHost(NativeGuestMemory memory, bool blocks) : ICommandStreamHost +{ + public long Draws, Dispatches, PendingRuns, BeginTicks, Begins; + public ICpuMemory Memory => memory; + public bool SupportsCommandBlocks => blocks; + public bool TryReadGuest(ulong address, Span destination) => memory.TryRead(address, destination); + + public bool TryReadCommandBlock(ulong address, Span block, out CommandReadToken token) + { + token = default; + return memory.TryRead(address, block); + } + + public bool IsCommandBlockCurrent(in CommandReadToken token) => true; + public void RunPendingCommands() => PendingRuns++; + public void BeginSubmission(int queueId, ulong submissionId, object? geometrySnapshots) { BeginTicks = System.Diagnostics.Stopwatch.GetTimestamp(); Begins++; } + public void Flush() { } + public void FlushAndWait() { } + public void SynchronizeGpu() { } + public void RunGarbageCollector() { } + public void EmitGlobalBarrier() { } + public void FillBuffer(ulong address, ulong size, uint value, bool isGds) { } + public void CopyBuffer(ulong destination, ulong source, ulong size, bool destinationIsGds, bool sourceIsGds) { } + public void ReadGds(Span destination, uint wordOffset, uint wordCount) { } + public void RecordEndOfPipe(in EndOfPipeWrite write) { } + public void TriggerInterrupt(int eventId, uint contextId) { } + public bool HasFlipSlot() => true; + public ulong PrepareFlip(int handle, int index, int flipMode, long flipArgument) => 1; + public bool IsFlipDone(int handle, int index) => true; + public void PrepareCpuFlip(int handle, int index, ulong requestId) { } + public void DrawIndexed(ulong submitId, in DrawIndexedArguments arguments) => Draws++; + public void DrawAuto(ulong submitId, in DrawAutoArguments arguments) => Draws++; + public void DispatchDirect(ulong submitId, uint groupsX, uint groupsY, uint groupsZ, uint dispatchInitiator, ulong indirectArgumentsAddress = 0) => Dispatches++; + public void OnQueueReset(int queueId) { } + public Exception Fatal(string message) => new InvalidOperationException(message); +} diff --git a/tools/SharpEmu.Tools.PerfAudit/PlanningSupport.cs b/tools/SharpEmu.Tools.PerfAudit/PlanningSupport.cs new file mode 100644 index 000000000..3cd8c7a8d --- /dev/null +++ b/tools/SharpEmu.Tools.PerfAudit/PlanningSupport.cs @@ -0,0 +1,812 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using SharpEmu.Libs.Gpu.Abstractions; +using SharpEmu.Libs.Gpu.Buffers; +using SharpEmu.Libs.Gpu.GpuCommands; +using SharpEmu.Libs.Gpu.GpuCommands.Registers; +using SharpEmu.Libs.Gpu.Images; +using SharpEmu.Libs.Gpu.Pipelines; +using SharpEmu.Libs.Gpu.Rendering; +using SharpEmu.Libs.Gpu.Scheduling; +using ResourceSnapshot = SharpEmu.ShaderCompiler.Resources.ResourceSnapshot; + +namespace SharpEmu.PerfAudit.Planning; + +internal sealed class RenderExecutorFatalException(string message) : Exception(message); + +internal sealed class RecordedBindings(ShaderStageResources stage, int id) : IPreparedBindings +{ + public ShaderStageResources Stage => stage; + + public int Id => id; +} + +// Accepts every image format the depth builder asks about. +internal sealed class AcceptingFormatSupport : IImageFormatSupport +{ + public bool TryGetImageFormatProperties(GpuFormat format, GpuImageType type, GpuImageTiling tiling, GpuImageUsage usage, GpuImageCreateFlags flags, out GpuImageFormatProperties properties) + { + properties = new GpuImageFormatProperties { SampleCounts = GpuSampleCount.Count1Bit | GpuSampleCount.Count2Bit | GpuSampleCount.Count4Bit | GpuSampleCount.Count8Bit }; + return true; + } +} + +// Records every host call in order and fakes the buffer and image caches. +internal sealed class RecordingRenderHost : IRenderHost +{ + private sealed class Preparation(RecordingRenderHost owner) : IResourcePreparation + { + private bool _disposed; + + public void Dispose() + { + if (_disposed) + { + return; + } + + _disposed = true; + owner.PreparationDepth--; + owner.Calls.Add("preparation_end"); + } + } + + public const ulong MemoryBase = 0x1_0000_0000; + public const int MemorySize = 0x100_0000; + public const ulong PageSize = 0x1000; + public const ulong NullHandle = 1; + public const ulong TransientHandle = 0x50; + + private sealed class Allocation + { + public ulong Handle; + public ulong Start; + public ulong End; + public bool Deleted; + } + + private readonly List _allocations = new(); + private readonly HashSet _dirtyPages = new(); + private readonly Dictionary _images = new(); + private ulong _nextHandle = 0x100; + private uint _nextImage = 1; + private int _nextBindings = 1; + private ulong _transientOffset; + + public FakeCpuMemory GuestMemory { get; } = new(MemoryBase, MemorySize); + + public NoCalls Calls { get; } = new(); + + public long DrawTargetCount; + + public void TraceDrawTarget(string disposition, ulong address, ulong pixelShaderAddress, ulong vertexShaderAddress) => + DrawTargetCount++; + + public long BegunRenderingCount; + + public long DynamicStateCount; + + public int GuestReads { get; private set; } + + public bool Recording { get; set; } = true; + + public RenderHostLimits Limits { get; set; } = new(16384, 8192, 16384, 16384); + + public bool SupportsNativeRectangles { get; set; } + + public bool SupportsIndirectMeshDraw { get; set; } + + public IndirectMeshDrawRequest? IndirectMeshRequest { get; private set; } + + public IIndirectMeshDraw PrepareIndirectMeshDraw(in IndirectMeshDrawRequest request) + { + IndirectMeshRequest = request; + Calls.Add("acquire_indirect_mesh"); + return new IndirectMesh(this); + } + + private sealed class IndirectMesh(RecordingRenderHost owner) : IIndirectMeshDraw + { + public ulong ParametersAddress => 0x9000; + public void Prepare() => owner.Calls.Add("prepare_indirect_mesh"); + public void Draw() => owner.Calls.Add("draw_indirect_mesh"); + public void Dispose() => owner.Calls.Add("dispose_indirect_mesh"); + } + + public IImageFormatSupport FormatSupport { get; } = new AcceptingFormatSupport(); + + public bool IsRecording => Recording; + + public GpuImageLayout ColorLayout { get; set; } = GpuImageLayout.ColorAttachmentOptimal; + + public Dictionary ImageSamples { get; } = new(); + + public Dictionary ColorMetadataClears { get; } = new(); + + public bool DepthMetadataClear { get; set; } + + public HashSet MetadataAddresses { get; } = new(); + + // The registered size of a metadata surface, where a test gives one. + public Dictionary MetadataSizes { get; } = new(); + + public HashSet ClearableMetadata { get; } = new(); + + public HashSet RegisteredDcc { get; } = new(); + + // Metadata the host knows and fills itself although its surface is not registered yet. + public HashSet HostFillableDcc { get; } = new(); + + public HashSet ClearableImages { get; } = new(); + + public Func? ClampOverride { get; set; } + + public int PreparationDepth { get; private set; } + + public int PreparationsOpened { get; private set; } + + // The next scheduler-touching operation needs a ring wrap; it is refused while a preparation is open. + public bool PendingRingWrap { get; set; } + + private void TryWrapRing(string operation) + { + if (!PendingRingWrap) + { + return; + } + + PendingRingWrap = false; + if (PreparationDepth != 0) + { + Calls.Add($"wrap_refused {operation}"); + return; + } + + LastTransient = null; + Calls.Add($"wrap {operation}"); + } + + public IResourcePreparation BeginPreparation() + { + PreparationDepth++; + PreparationsOpened++; + Calls.Add("preparation_begin"); + return new Preparation(this); + } + + public BufferBinding NullBuffer => new(NullHandle, 0); + + public uint[]? MeshIndices { get; private set; } + + public ulong UploadMeshVertexIndices(ReadOnlySpan indices) + { + MeshIndices = indices.ToArray(); + Calls.Add("upload_mesh_indices"); + return 0x8000; + } + + public void RunPendingOperations() => Calls.Add("pending"); + + public void SetDebugInformation(RecordedOperation operation, ulong submitId, uint argument0, uint argument1, uint argument2, uint argument3, ulong argument4) => + Calls.Add($"debug {operation} {submitId} {argument0:X} {argument1:X} {argument2:X} {argument3:X} {argument4:X}"); + + public bool TryReadGuest(ulong address, Span destination) + { + GuestReads++; + return GuestMemory.TryRead(address, destination); + } + + public ulong ClampMappedSize(ulong address, ulong size) => ClampOverride?.Invoke(address, size) ?? size; + + public ResourceSlotIdentifier FindImage(ref ImageRequest request, bool exactFormat) + { + var address = request.Description.Data.Address; + if (!_images.TryGetValue(address, out var image)) + { + image = new ResourceSlotIdentifier(_nextImage++, 1); + _images[address] = image; + } + + Calls.Add($"find_image {address:X} {request.Role} exact={exactFormat} -> {image.Index}"); + return image; + } + + public ResourceSlotIdentifier ImageOf(ulong address) => _images[address]; + + public void BindRenderTarget(ResourceSlotIdentifier image) => Calls.Add($"bind_target {image.Index}"); + + public void ResetBindings() => Calls.Add("reset_bindings"); + + public bool SupportsPipelineSuspend { get; set; } + + public void NotePipelineNotReady() => Calls.Add("pipeline_not_ready"); + + public bool LeavesOutDrawsWhileCompiling { get; set; } + + // The host's say on the targets: true stands in for targets rebuilt every frame. + public bool TargetsRebuiltEveryFrame { get; set; } = true; + + public bool MayLeaveOutWhileCompiling(in RenderingState rendering) + { + Calls.Add("may_leave_out"); + return TargetsRebuiltEveryFrame; + } + + public void AwaitProgramsBeforePreparation(ShaderProgram first, ShaderProgram second) => Calls.Add("await_programs"); + + public ColorAttachmentAcquisition AcquireColorAttachment(in ColorTargetState target) + { + var address = target.Resolution.BaseAddress; + TryWrapRing("acquire_color"); + Calls.Add($"acquire_color {target.Slot} {address:X} image={target.Image.Index}"); + var metadataClear = ColorMetadataClears.TryGetValue(address, out var clearValue); + return new ColorAttachmentAcquisition( + target.Image, + new GpuImageHandle(0x1000 + target.Image.Index), + ColorLayout, + ImageSamples.TryGetValue(address, out var samples) ? samples : target.Resolution.Samples, + metadataClear, + clearValue); + } + + public DepthAttachmentAcquisition AcquireDepthAttachment(in DepthAttachmentState depth) + { + var target = depth.Target.Target; + Calls.Add($"acquire_depth {target.DepthAddress:X} image={depth.Image.Index} clear={depth.Target.State.DepthClearEnabled}"); + return new DepthAttachmentAcquisition( + new GpuImageHandle(0x2000 + depth.Image.Index), + ImageSamples.TryGetValue(target.DepthAddress, out var samples) ? samples : target.Samples, + DepthMetadataClear); + } + + public void TransitionDepthAttachment(in DepthAttachmentState depth, GpuImageLayout layout, GpuImageAspect writeAspects) => + Calls.Add($"transition_depth {depth.Image.Index} {layout} {writeAspects} loadClear={depth.LoadClear}"); + + // Marks the pages the guest wrote so the next obtain re-uploads them. + public void WriteGuest(ulong address, ReadOnlySpan data) + { + for (var page = address & ~(PageSize - 1); page < address + (ulong)data.Length; page += PageSize) + { + _dirtyPages.Add(page); + } + + Calls.Add($"cpu_write {address:X} {data.Length:X}"); + } + + public BufferBinding ObtainBuffer(ulong address, ulong size, bool isWritten) + { + TryWrapRing("obtain"); + var end = address + size; + var owner = _allocations.Find(a => !a.Deleted && a.Start <= address && end <= a.End); + if (owner is null) + { + var start = address; + var stop = end; + foreach (var overlapped in _allocations.Where(a => !a.Deleted && a.Start < end && address < a.End).ToList()) + { + start = Math.Min(start, overlapped.Start); + stop = Math.Max(stop, overlapped.End); + } + + owner = new Allocation { Handle = _nextHandle++, Start = start, End = stop }; + foreach (var overlapped in _allocations.Where(a => !a.Deleted && a.Start < end && address < a.End).ToList()) + { + Calls.Add($"merge {overlapped.Handle:X}->{owner.Handle:X}"); + overlapped.Deleted = true; + } + + _allocations.Add(owner); + } + + for (var page = address & ~(PageSize - 1); page < end; page += PageSize) + { + if (_dirtyPages.Remove(page)) + { + Calls.Add($"upload {owner.Handle:X} {page:X}"); + } + } + + Calls.Add($"obtain {address:X} {size:X} written={isWritten} -> {owner.Handle:X}:{address - owner.Start:X}"); + return new BufferBinding(owner.Handle, address - owner.Start); + } + + public BufferBinding UploadTransient(ReadOnlySpan data, uint alignment) + { + var offset = _transientOffset; + _transientOffset += ((ulong)data.Length + alignment - 1) & ~(alignment - 1ul); + Calls.Add($"upload_transient {data.Length:X} align={alignment} -> {TransientHandle:X}:{offset:X}"); + + return new BufferBinding(TransientHandle, offset); + } + + public byte[]? LastTransient { get; private set; } + + public bool DrawsTriangleFans { get; set; } = true; + + public void BindVertexBuffers(ReadOnlySpan bindings, VertexInputInfo input) => + Calls.Add($"bind_vertex {string.Join(",", bindings.ToArray().Select(b => $"{b.Handle:X}:{b.Offset:X}"))}"); + + public void BindIndexBuffer(BufferBinding binding, GpuIndexType type) => Calls.Add($"bind_index {binding.Handle:X}:{binding.Offset:X} {type}"); + + public bool FailPrepareBindings { get; set; } + + public Func? PreparationFailure { get; set; } + + public IPreparedBindings PrepareBindings(ShaderStageResources stage) + { + if (PreparationFailure?.Invoke(stage) is { } failure) + { + throw failure; + } + if (FailPrepareBindings) + { + throw Fatal($"The bindings cannot be prepared: stage={stage.Program?.Stage}."); + } + + var id = _nextBindings++; + Calls.Add($"prepare_bindings {stage.Program?.Stage} -> {id}"); + return new RecordedBindings(stage, id); + } + + public void PrepareDeviceAddresses() => Calls.Add("prepare_device_addresses"); + + public void BindResources(IPreparedBindings prepared) => Calls.Add($"bind_resources {((RecordedBindings)prepared).Id}"); + + public void CommitBindings(GpuBindPoint bindPoint, in PipelineHandle pipeline, ReadOnlySpan stages) => + Calls.Add($"commit {bindPoint} {pipeline.Pipeline:X} [{string.Join(",", stages.ToArray().Select(s => ((RecordedBindings)s).Id))}]"); + + public void SetDynamicState(in DynamicDrawState state) + { + DynamicStateCount++; + Calls.Add("dynamic_state"); + } + + public void BeginRendering(in RenderingState state) + { + BegunRenderingCount++; + _renderingOpen = true; + Calls.Add($"begin_rendering {state.Width}x{state.Height}x{state.Layers} colors={state.ColorAttachmentCount} samples={state.Samples}"); + } + + private bool _renderingOpen; + + public void EndRendering() + { + Calls.Add("end_rendering"); + if (_renderingOpen) + { + _renderingOpen = false; + RenderPassSerial++; + } + } + + public bool DefersFragmentWritePassEnd { get; set; } + + public long RenderPassSerial { get; set; } + + public void BindPipeline(GpuBindPoint bindPoint, in PipelineHandle pipeline) => Calls.Add($"bind_pipeline {bindPoint} {pipeline.Pipeline:X}"); + + public void Draw(uint vertexCount, uint instanceCount, uint firstVertex, uint firstInstance) => + Calls.Add($"draw {vertexCount} {instanceCount} {firstVertex} {firstInstance}"); + + public void DrawIndexed(uint indexCount, uint instanceCount, uint firstIndex, int vertexOffset, uint firstInstance) => + Calls.Add($"draw_indexed {indexCount} {instanceCount} {firstIndex} {vertexOffset} {firstInstance}"); + + public void PrepareIndirectDraw(in PipelineHandle pipeline, ulong argumentsAddress, BufferBinding arguments, bool indexed, uint indexLimit) => + Calls.Add($"prepare_indirect {argumentsAddress:X} {arguments.Handle:X}:{arguments.Offset:X} indexed={indexed} limit={indexLimit}"); + + public void DrawIndirect(BufferBinding arguments) => + Calls.Add($"draw_indirect {arguments.Handle:X}:{arguments.Offset:X}"); + + public void DrawIndexedIndirect(BufferBinding arguments) => + Calls.Add($"draw_indexed_indirect {arguments.Handle:X}:{arguments.Offset:X}"); + + /// Set to false for a backend that runs the program as a mesh pipeline. + public bool IndirectDrawArgumentsTaken { get; set; } = true; + + public bool TakesIndirectDrawArguments(ShaderProgram vertexProgram, ulong argumentsAddress, ulong argumentsSize) + { + Calls.Add($"takes_indirect {vertexProgram.Id} {argumentsAddress:X} {argumentsSize}"); + return IndirectDrawArgumentsTaken; + } + + /// Set to false for a backend that draws a rectangle list as patches. + public bool TakesIndirectRectangleListArguments { get; set; } = true; + + public bool TakesTargetlessIndirectDrawArguments { get; set; } + + public void NoteIndirectArgumentsReadOnCpu(ulong address, ulong size) => Calls.Add($"indirect_cpu {address:X} {size}"); + + public void DrawMeshTasks(uint groupCountX, uint groupCountY, uint groupCountZ) => + Calls.Add($"draw_mesh {groupCountX} {groupCountY} {groupCountZ}"); + + public void Dispatch(uint groupsX, uint groupsY, uint groupsZ) => Calls.Add($"dispatch {groupsX} {groupsY} {groupsZ}"); + + public bool TryDispatchIndirect(ulong argumentsAddress) => false; + + public void ShaderWriteBarrier(GpuPipelineStage sourceStages) => Calls.Add($"write_barrier {sourceStages}"); + + public void ShaderWriteHazardBarrier() => Calls.Add("write_hazard_barrier"); + + public void ShaderAccessBarrier() => Calls.Add("access_barrier"); + + public bool RetainTargetlessDraws { get; set; } + + public bool OrdersVertexWritesInsidePass { get; set; } + + public List<(RegisterBanks Banks, GraphicsPrograms Programs, TargetlessDrawArguments Arguments)> RetainedDraws { get; } = new(); + + public void ClearColorTargets(ReadOnlySpan targets, SolidColorClear clear) => + Calls.Add(FormattableString.Invariant( + $"clear_targets {string.Join(",", targets.ToArray().Select(t => t.Slot))} {clear.Red},{clear.Green},{clear.Blue},{clear.Alpha}")); + + + public bool TryRetainTargetlessDraw(RegisterBanks banks, GraphicsPrograms programs, in TargetlessDrawArguments arguments) + { + Calls.Add($"retain_targetless {arguments.SubmitId}"); + if (!RetainTargetlessDraws) + { + return false; + } + + RetainedDraws.Add((banks, programs, arguments)); + return true; + } + + public void MarkGpuWritten(ResourceSlotIdentifier image) => Calls.Add($"mark_written {image.Index}"); + + public void ResolveImage(ResourceSlotIdentifier source, uint sourceMip, uint sourceLayer, ResourceSlotIdentifier destination, uint destinationMip, uint destinationLayer) => + Calls.Add($"resolve {source.Index}:{sourceMip}:{sourceLayer} -> {destination.Index}:{destinationMip}:{destinationLayer}"); + + public void CopyDepthStencilImage(ResourceSlotIdentifier source, ResourceSlotIdentifier destination, in SubresourceRange range, in GpuExtent3D extent, GpuImageAspect aspects) => + Calls.Add($"copy_depth_stencil {source.Index} -> {destination.Index} {aspects}"); + + public bool IsMetadata(ulong address) + { + Calls.Add($"is_metadata {address:X}"); + return MetadataAddresses.Contains(address); + } + + public bool CoversMetadata(ulong address, ulong size) + { + Calls.Add($"covers_metadata {address:X} {size:X}"); + return MetadataAddresses.Contains(address) && + (!MetadataSizes.TryGetValue(address, out var surface) || size >= surface); + } + + public bool ClearMetadata(ulong address) + { + Calls.Add($"clear_metadata {address:X}"); + return ClearableMetadata.Contains(address); + } + + public bool TryClearImageFromBuffer(ulong address, ulong size, uint packedClear) + { + Calls.Add($"clear_image {address:X} {size:X} {packedClear:X8}"); + return ClearableImages.Contains(address); + } + + public bool TryAbsorbDccFill(ulong address, ulong size, uint fillValue) + { + Calls.Add($"absorb_dcc {address:X} {size:X} {fillValue:X8}"); + return RegisteredDcc.Contains(address); + } + + public bool TryFillDccMetadata(ulong address, ulong size, uint fillValue) + { + Calls.Add($"fill_dcc {address:X} {size:X} {fillValue:X8}"); + return RegisteredDcc.Contains(address) || HostFillableDcc.Contains(address); + } + + public bool HostCopyAccepted { get; set; } = true; + + public bool TryCopyWordsOnHost(ulong destination, ulong source, ulong sourceWords, ulong words) + { + Calls.Add($"host_copy {destination:X} {source:X} {sourceWords} {words}"); + return HostCopyAccepted; + } + + public bool KernelFillAccepted { get; set; } + + public bool TryClearImageFromKernelFill(ulong address, ulong size, ReadOnlySpan pattern) + { + Calls.Add($"kernel_fill {address:X} {size:X} {string.Join(',', pattern.ToArray().Select(word => word.ToString("X8")))}"); + return KernelFillAccepted; + } + + public void NoteKernelFillFallback(string reason) => Calls.Add($"kernel_fill_fallback {reason}"); + + public Exception Fatal(string message) => new RenderExecutorFatalException(message); +} + +// Hands back the configured programs and records every pipeline request. +internal sealed class FakePipelineProvider : IShaderPipelineProvider +{ + public GraphicsPrograms Graphics { get; set; } = RenderExecutorFixtures.Programs(); + + // Stands in for a backend with mesh pipelines (Metal); Vulkan leaves it false. + public bool SupportsPrimitiveShaderMeshPipelines { get; set; } + + public bool MeshShadersSupported { get; set; } + + public uint MeshPrimitiveType { get; private set; } + + public bool MeshDepthBound { get; private set; } + + public GraphicsPrograms GetMeshGraphicsPrograms( + VertexStageRegisters vertex, + PixelStageRegisters pixel, + ShaderInterfaceRegisters shaderInterface, + ContextRegisters context, + UserConfigRegisters userConfig, + ReadOnlySpan targetExportMapping, + bool pixelActive, + bool depthBound) + { + Calls.Add($"get_mesh_programs pixelActive={pixelActive}"); + MeshPrimitiveType = userConfig.PrimitiveType; + MeshDepthBound = depthBound; + return Graphics; + } + + public ComputeProgram Compute { get; set; } = RenderExecutorFixtures.ComputeProgram(); + + public NoCalls Calls { get; } = new(); + + public List<(VertexStageRegisters Vertex, ContextRegisters Context, UserConfigRegisters UserConfig, + TargetlessDrawArguments Draw, DrawIndexedArguments? OriginalIndirect)> UnsupportedGraphics { get; } = new(); + + public void RecordUnsupportedGraphics(VertexStageRegisters vertex, ContextRegisters context, + UserConfigRegisters userConfig, in TargetlessDrawArguments draw, + DrawIndexedArguments? originalIndirectIndexedDraw = null) => + UnsupportedGraphics.Add((vertex, context, userConfig, draw, originalIndirectIndexedDraw)); + + public List PipelineRenderings { get; } = new(); + + public List<(GpuPrimitiveTopology Topology, bool Restart, bool PixelActive)> PipelineRequests { get; } = new(); + + public bool LastDispatchThreadDimensions { get; private set; } + + public List ExportMappings { get; } = new(); + + public GraphicsPrograms GetGraphicsPrograms( + VertexStageRegisters vertex, + PixelStageRegisters pixel, + ShaderInterfaceRegisters shaderInterface, + ContextRegisters context, + ReadOnlySpan targetExportMapping, + bool pixelActive, + bool depthBound, + bool depthAspectBound, + bool stencilAspectBound = false) + { + Calls.Add($"get_graphics_programs pixelActive={pixelActive}"); + + return Graphics; + } + + public PipelineHandle CreateGraphicsPipeline( + ReadOnlySpan colors, + in DepthAttachmentState depth, + VertexInputInfo vertexInput, + PixelInputInfo? pixelInput, + ContextRegisters context, + in RenderingState rendering, + GpuPrimitiveTopology topology, + bool primitiveRestartEnabled, + bool disableBlending, + ShaderProgram vertexProgram, + ShaderProgram pixelProgram) + { + Calls.Add($"create_graphics_pipeline colors={colors.Length} depth={depth.HasTarget} topology={topology} restart={primitiveRestartEnabled}"); + + + + return new PipelineHandle(0xA1, 0xB1, false); + } + + public List DisableBlendingRequests { get; } = new(); + + // How many left-out-able graphics requests report "still compiling" before one is handed over. + public int PendingGraphicsCompiles { get; set; } + + public bool TryCreateGraphicsPipeline( + ReadOnlySpan colors, + in DepthAttachmentState depth, + VertexInputInfo vertexInput, + PixelInputInfo? pixelInput, + ContextRegisters context, + in RenderingState rendering, + GpuPrimitiveTopology topology, + bool primitiveRestartEnabled, + bool disableBlending, + ShaderProgram vertexProgram, + ShaderProgram pixelProgram, + out PipelineHandle handle) + { + Calls.Add("try_create_graphics_pipeline"); + handle = default; + if (PendingGraphicsCompiles > 0) + { + PendingGraphicsCompiles--; + return false; + } + + handle = CreateGraphicsPipeline(colors, in depth, vertexInput, pixelInput, context, in rendering, topology, + primitiveRestartEnabled, disableBlending, vertexProgram, pixelProgram); + return true; + } + + public (uint X, uint Y, uint Z) LastDispatchDimensions { get; private set; } + + public ComputeProgram GetComputeProgram(ComputeStageRegisters compute, ShaderInterfaceRegisters shaderInterface, uint dispatchInitiator, uint dimensionX, uint dimensionY, uint dimensionZ) + { + var dispatchThreadDimensions = (dispatchInitiator & (1u << 5)) != 0; + Calls.Add($"get_compute_program threadDimensions={dispatchThreadDimensions}"); + LastDispatchThreadDimensions = dispatchThreadDimensions; + LastDispatchDimensions = (dimensionX, dimensionY, dimensionZ); + if (dispatchThreadDimensions) + { + Compute.Input.DispatchThreadsX = dimensionX; + Compute.Input.DispatchThreadsY = dimensionY; + Compute.Input.DispatchThreadsZ = dimensionZ; + } + + return Compute; + } + + public PipelineHandle CreateComputePipeline(ComputeInputInfo input, ShaderProgram program) + { + Calls.Add("create_compute_pipeline"); + return new PipelineHandle(0xC1, 0xD1, false); + } + + public Task? PendingComputeCompletion { get; set; } + public Action? CompletionRequested { get; set; } + public int ComputePipelineAttempts { get; private set; } + + public Task? GetComputePipelineCompletion(ShaderProgram program) + { + CompletionRequested?.Invoke(); + return PendingComputeCompletion; + } + + public bool TryCreateComputePipeline(ComputeInputInfo input, ShaderProgram program, out PipelineHandle handle) + { + ComputePipelineAttempts++; + if (PendingComputeCompletion is { } completion && + (ComputePipelineAttempts == 1 || !completion.IsCompletedSuccessfully)) + { + handle = default; + return false; + } + handle = CreateComputePipeline(input, program); + return true; + } +} + +// Register banks and programs for a draw that renders into one 64x64 color target. +internal static class RenderExecutorFixtures +{ + public const ulong ColorBase = RecordingRenderHost.MemoryBase; + public const ulong SecondColorBase = RecordingRenderHost.MemoryBase + 0x10_0000; + public const ulong DepthBase = RecordingRenderHost.MemoryBase + 0x20_0000; + public const ulong StencilBase = RecordingRenderHost.MemoryBase + 0x30_0000; + public const ulong VertexBase = RecordingRenderHost.MemoryBase + 0x40_0000; + public const ulong IndexBase = RecordingRenderHost.MemoryBase + 0x50_0000; + public const ulong VertexShader = 0x2_0000_0000; + public const ulong PixelShader = 0x2_0001_0000; + public const ulong ComputeShader = 0x2_0002_0000; + public const uint PrimitiveTriangleList = 4; + public const uint PrimitiveTriangleStrip = 6; + public const uint PrimitiveRectangleList = 7; + + private static Exception Fatal(string message) => new RenderExecutorFatalException(message); + + public static RegisterBanks Banks(uint primitiveType = PrimitiveTriangleList, bool withDepth = false, bool withPixel = true) + { + var banks = new RegisterBanks(Fatal); + var context = banks.Context; + context.ColorTargets[0] = RegisterWords.Color(ColorBase, 64, 64); + context.RenderTargetMask = 0xF; + context.ShaderInterface.ColorShaderMask = 0xF; + context.ScreenViewport.Viewports[0] = new ViewportRegisters { XScale = 32, XOffset = 32, YScale = -32, YOffset = 32, ZScale = 0.5f, ZOffset = 0.5f, MaxDepth = 1 }; + if (withDepth) + { + context.DepthTarget = RegisterWords.Depth(DepthBase, 64, 64); + } + + banks.Shader.Vertex.ExportAddress = VertexShader; + banks.Shader.Pixel.Address = withPixel ? PixelShader : 0; + banks.Shader.Compute.Address = ComputeShader; + banks.Shader.Compute.ThreadsX = 64; + banks.Shader.Compute.ThreadsY = 1; + banks.Shader.Compute.ThreadsZ = 1; + banks.UserConfig.PrimitiveType = primitiveType; + return banks; + } + + public static ShaderProgramInfo Program(ShaderStageKind stage, BufferResourceInfo[]? buffers = null, ImageResourceInfo[]? images = null, uint userDataBase = 0, bool usesDeviceAddresses = false, bool writesDeviceAddresses = false, bool exclusiveOr = false, int vertexOffsetScalar = ShaderProgramInfo.NoScalarRegister, int instanceOffsetScalar = ShaderProgramInfo.NoScalarRegister, uint parameterExportMask = 1) => + new() + { + Stage = stage, + Hash = 0x1234_5678_9ABC_DEF0, + UserDataBase = userDataBase, + ParameterExportMask = parameterExportMask, + Buffers = buffers ?? [], + Images = images ?? [], + UsesDeviceAddresses = usesDeviceAddresses, + WritesDeviceAddresses = writesDeviceAddresses, + HasBitwiseExclusiveOr = exclusiveOr, + VertexOffsetScalarRegister = vertexOffsetScalar, + InstanceOffsetScalarRegister = instanceOffsetScalar, + }; + + public static ShaderStageResources Stage(ShaderProgramInfo program, uint[][]? buffers = null, uint[]? userData = null) => + new(program, new ResourceSnapshot { Buffers = buffers ?? [], UserData = userData ?? [] }); + + public static GraphicsPrograms Programs(VertexInputBuffer[]? vertexBuffers = null, bool fetchEmbedded = false, ShaderStageResources? vertexStage = null, ShaderStageResources? pixelStage = null, uint pixelInputs = 1, VertexPositionStream? positionStream = null, SolidColorClear? solidClear = null) => + new() + { + Vertex = new ShaderProgram(0x11), + Pixel = new ShaderProgram(0x22), + VertexInput = new VertexInputInfo + { + Buffers = vertexBuffers ?? [], + FetchEmbedded = fetchEmbedded, + Stage = vertexStage ?? Stage(Program(ShaderStageKind.Vertex)), + }, + PixelInput = new PixelInputInfo { InputCount = pixelInputs, Stage = pixelStage ?? Stage(Program(ShaderStageKind.Pixel)) }, + PositionStream = positionStream, + SolidClear = solidClear, + }; + + public static uint[] BufferDescriptor(ulong address, uint stride, uint records, uint format = BufferDescriptorWords.Format32x4UInt, bool swizzle = false, uint indexStride = 0, bool addThreadId = false) => + [ + (uint)address, + (uint)((address >> 32) & 0xFFFF) | (stride << 16) | (swizzle ? 1u << 31 : 0), + records, + 4 | (format << 12) | (indexStride << 21) | (addThreadId ? 1u << 23 : 0), + ]; + + public static ComputeProgram ComputeProgram(ShaderStageResources? stage = null, bool threadDimensions = false, uint threadsX = 64, uint threadsY = 1, uint threadsZ = 1) => + new() + { + Program = new ShaderProgram(0x33), + Input = new ComputeInputInfo + { + ThreadsX = threadsX, + ThreadsY = threadsY, + ThreadsZ = threadsZ, + DispatchThreadDimensions = threadDimensions, + GroupIdX = true, + ThreadIdCount = 1, + WaveSize = 64, + Stage = stage ?? Stage(Program(ShaderStageKind.Compute)), + }, + }; + + public static DrawIndexedArguments Indexed(uint count, uint instances = 1, uint indexType = 0, int baseVertex = 0, uint firstInstance = 0, DrawOffsetSource source = DrawOffsetSource.Packet, ulong indexAddress = IndexBase) => + new(0, 0, count, indexAddress, indexType, instances, baseVertex, firstInstance, source); + + public static DrawAutoArguments Auto(uint count, uint instances = 1, uint firstVertex = 0, uint firstInstance = 0, DrawOffsetSource source = DrawOffsetSource.Packet) => + new(0, 0, count, instances, firstVertex, firstInstance, source); +} + +/// Drops the host-call log: interpolated holes are never formatted when the handler declines. +internal sealed class NoCalls +{ + public void Add([System.Runtime.CompilerServices.InterpolatedStringHandlerArgument("")] ref NoCallHandler handler) { } + public void Add(string text) { } + public int Count => 0; +} + +[System.Runtime.CompilerServices.InterpolatedStringHandler] +internal ref struct NoCallHandler +{ + public NoCallHandler(int literalLength, int formattedCount, NoCalls owner, out bool shouldAppend) { shouldAppend = false; } + public void AppendLiteral(string value) { } + public void AppendFormatted(T value) { } + public void AppendFormatted(T value, string? format) { } +} diff --git a/tools/SharpEmu.Tools.PerfAudit/Program.cs b/tools/SharpEmu.Tools.PerfAudit/Program.cs new file mode 100644 index 000000000..51e9c0c24 --- /dev/null +++ b/tools/SharpEmu.Tools.PerfAudit/Program.cs @@ -0,0 +1,34 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using System.Runtime.InteropServices; +using SharpEmu.PerfAudit; + +Console.WriteLine($"SharpEmu per-draw microbench arch={RuntimeInformation.ProcessArchitecture} os={RuntimeInformation.OSDescription} " + + $"runtime={RuntimeInformation.FrameworkDescription} gcServer={System.Runtime.GCSettings.IsServerGC} tieredPGO={Environment.GetEnvironmentVariable("DOTNET_TieredPGO") ?? "default"}"); +Console.WriteLine($"cores={Environment.ProcessorCount} synthetic sizing only: 6331 draws + 2700 dispatches; scaled costs are not measured frame times"); +var filter = args.Length > 0 ? args[0] : "all"; +bool On(string key) => filter == "all" || filter.Split(',').Contains(key); +foreach (var a in args) if (a.StartsWith("--warmup=")) Harness.WarmupMs = double.Parse(a[9..]); +foreach (var a in args) if (a.StartsWith("--scenarios=")) E2EBenches.OnlyScenarios = a[12..].Split(','); +foreach (var a in args) if (a.StartsWith("--soak=")) E2EBenches.SoakSeconds = double.Parse(a[7..]); +if (args.Contains("--allocs")) E2EBenches.TrackAllocations = true; +if (args.Contains("--quick")) { Harness.Runs = 3; Harness.TargetRunMs = 60; Harness.WarmupMs = 250; } + +if (filter == "smoke") RigBenches.Smoke(); +if (On("pm4")) Pm4Benches.Run(); +if (On("plan")) PlanningBenches.Run(); +if (On("e2e")) E2EBenches.Run(); +if (On("mat")) MaterializationBenches.Run(); +if (On("queue")) QueueBenches.Run(); +if (On("mem")) MemoryBenches.Run(); +if (On("lookup")) LookupBenches.Run(); +if (On("metal")) MetalRawBenches.Run(); +if (On("misc")) MiscBenches.Run(); +if (On("w8")) W8Benches.Run(); +if (filter.Split(',').Contains("xlate")) TranslatorBenches.Run(); +if (filter.Split(',').Contains("gword")) MetalRawBenches.GpuWordWait(); +if (filter.Split(',').Contains("wranges")) WriteRangeBenches.Run(); + +Console.WriteLine(); +Console.WriteLine(Harness.Table()); diff --git a/tools/SharpEmu.Tools.PerfAudit/README.md b/tools/SharpEmu.Tools.PerfAudit/README.md new file mode 100644 index 000000000..11d995918 --- /dev/null +++ b/tools/SharpEmu.Tools.PerfAudit/README.md @@ -0,0 +1,57 @@ + + +# Synthetic runtime benchmark suite + +This packages the CPU/GPU audit harness previously kept under local artifacts. +It generates its own command streams, shader instructions, descriptors and buffers; +no game executable, captured shader or external asset is loaded. macOS and an x64 +.NET 10 host are required for the complete suite. Device groups use real Metal. +The executable has its own assembly name and cannot overwrite the Metal test DLL. + +```sh +~/.dotnet-x64/dotnet build tools/SharpEmu.Tools.PerfAudit -c Release -m:2 -p:UseSharedCompilation=false -nr:false +# Short runtime check (memory/lookup groups also use Metal); not a timing acceptance run. +~/.dotnet-x64/dotnet artifacts/bin/Release/net10.0/SharpEmu.Tools.PerfAudit.dll pm4,plan,mat,queue,mem,lookup,misc,w8,xlate,wranges --quick +# Fabricated triangle through the real Metal backend. +~/.dotnet-x64/dotnet artifacts/bin/Release/net10.0/SharpEmu.Tools.PerfAudit.dll smoke +# Warm translator comparison; run each source revision in alternating order. +~/.dotnet-x64/dotnet artifacts/bin/Release/net10.0/SharpEmu.Tools.PerfAudit.dll xlate --warmup=5000 +``` + +| Selector | Work measured | +|---|---| +| `pm4` | Register snapshots, command interpretation and index expansion | +| `plan` | Draw/dispatch planning with fabricated host services | +| `mat` | Resource materialization, cold/stale/hit/ring cases | +| `queue` | Command queue and worker handoffs | +| `mem` | Guest memory translation, protection and copying | +| `lookup` | Pipeline identities, keys and dictionary probes | +| `misc` | Monitor and other small runtime costs | +| `w8` | Staged range/cache behavior | +| `xlate` | Single, fragmented and strided guest view layouts | +| `wranges` | GPU-written range tracking | +| `metal` | Raw Metal binding/encoding and submission timings | +| `e2e` | Generated textured draws through the interpreter and Metal backend | +| `gword` | Synthetic GPU-word wait path | +| `smoke` | One generated triangle and pixel readback | + +`all` selects the original broad suite; `xlate`, `wranges` and `gword` are explicit +additional selectors. `e2e` accepts `--scenarios=ID,...`, `--soak=SECONDS` and +`--allocs`; inspect its source for scenario IDs. `--quick` reduces warmup and +sampling for smoke checks only. The normal harness uses five 150 ms samples and +700 ms minimum warmup; use `--warmup=5000` for comparisons. Report the process +architecture, runtime, source revisions, tiering/PGO and switches with results. + +Run one benchmark/build/test process at a time on a quiet machine; follow the +shared validation lock/queue when coordinating workers. Warm both arms, alternate +A/B/B/A, retain every sample and allocation count, and compare controls for drift. +The printed p95 is a percentile of run averages, not individual operation latency. +The scaled-cost column uses illustrative counts (6,331 draws / 2,700 dispatches); +it does not measure a game frame, predict FPS, or account for overlapping work. + +For the final cold-admission A/B experiment, use the dedicated +[materialization harness](../SharpEmu.Tools.MaterializationBench/README.md), whose +sampling and counters match the published final result. Historical measurements +were made with the corresponding historical source, not this newly packaged binary. +See [benchmark results and scope](../../docs/performance/benchmark-index.md). diff --git a/tools/SharpEmu.Tools.PerfAudit/RegisterWords.cs b/tools/SharpEmu.Tools.PerfAudit/RegisterWords.cs new file mode 100644 index 000000000..e7df1b2db --- /dev/null +++ b/tools/SharpEmu.Tools.PerfAudit/RegisterWords.cs @@ -0,0 +1,85 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using SharpEmu.Libs.Gpu.Images; + +namespace SharpEmu.PerfAudit.Planning; + +// Raw register and descriptor words for the request builders, encoded the way the guest writes them. +internal static class RegisterWords +{ + // Destination selects X=R, Y=G, Z=B, W=A. + public const uint IdentitySelect = 0xFAC; + + public static ColorTargetWords Color( + ulong address, + uint width, + uint height, + GuestTileMode tile = GuestTileMode.Linear, + uint maxMip = 0, + uint mipLevel = 0, + uint sliceMax = 0, + uint sliceStart = 0, + uint samplesLog2 = 0, + uint fragmentsLog2 = 0, + uint dimension = 1, + uint depth = 0, + ChannelLayout layout = ChannelLayout.Bits8_8_8_8, + ChannelType type = ChannelType.UNorm) + { + var view = sliceStart | (sliceMax << 13) | (mipLevel << 26); + var info = ((uint)layout << 2) | ((uint)type << 8); + var attrib = (samplesLog2 << 12) | (fragmentsLog2 << 15); + var attrib2 = (height - 1) | ((width - 1) << 14) | (maxMip << 28); + var attrib3 = depth | ((uint)tile << 14) | (dimension << 24); + return new ColorTargetWords(address, view, info, attrib, attrib2, attrib3, 0, 0, 0, 0, 0); + } + + public static DepthTargetWords Depth( + ulong depthBase, + uint width, + uint height, + GuestDepthFormat format = GuestDepthFormat.Z32Float, + ulong stencilBase = 0, + bool depthTest = true, + bool depthWrite = true, + bool depthClear = false, + uint samplesLog2 = 0) + { + var zInfo = (uint)format | (samplesLog2 << 2); + var stencilInfo = stencilBase != 0 ? 1u : 0u; + var size = (width - 1) | ((height - 1) << 16); + var renderControl = depthClear ? 1u : 0u; + var depthControl = (depthTest ? 2u : 0u) | (depthWrite ? 4u : 0u) | (7u << 4); + return new DepthTargetWords(zInfo, stencilInfo, 0, size, true, 0, renderControl, depthControl, depthBase, depthBase, stencilBase, stencilBase, 0); + } + + public static uint[] Texture( + ulong address, + GuestPixelFormat format, + uint width, + uint height, + GuestImageType type = GuestImageType.Color2D, + GuestTileMode tile = GuestTileMode.Linear, + uint baseLevel = 0, + uint lastLevel = 0, + uint maxMip = 0, + uint layers = 1, + uint baseArray = 0) + { + var shifted = address >> 8; + var lastX = width - 1; + var lastY = height - 1; + return + [ + (uint)shifted, + ((uint)(shifted >> 32) & 0xFF) | ((uint)format << 20) | ((lastX & 0x3) << 30), + ((lastX >> 2) & 0xFFF) | (lastY << 14), + IdentitySelect | (baseLevel << 12) | (lastLevel << 16) | ((uint)tile << 20) | ((uint)type << 28), + (layers - 1) | (baseArray << 16), + maxMip << 4, + 0, + 0, + ]; + } +} diff --git a/tools/SharpEmu.Tools.PerfAudit/ResourceTestProgram.cs b/tools/SharpEmu.Tools.PerfAudit/ResourceTestProgram.cs new file mode 100644 index 000000000..6810b05ef --- /dev/null +++ b/tools/SharpEmu.Tools.PerfAudit/ResourceTestProgram.cs @@ -0,0 +1,231 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using SharpEmu.ShaderCompiler; +using SharpEmu.ShaderCompiler.Resources; + +namespace SharpEmu.PerfAudit.Materialization; + +// Hand-built instruction records for the resource-plan tests, one helper per shape. +internal static class ResourceTestProgram +{ + public const uint NullOperand = 125; + public const ulong Hash = 0x1234_5678_9ABC_DEF0; + + private static Gen5Operand Literal(uint value) => new(Gen5OperandKind.LiteralConstant, value); + + private static Gen5Operand Immediate(uint value) => Gen5Operand.Source(128 + value); + + public static Gen5Operand Operand(uint value) => + value <= 64 ? Immediate(value) : Literal(value); + + public static Gen5ShaderInstruction EndProgram(uint pc) => + new(pc, Gen5ShaderEncoding.Sopp, "SEndpgm", [0xBF810000], [], [], null); + + public static Gen5ShaderInstruction Nop(uint pc) => + new(pc, Gen5ShaderEncoding.Sopp, "SNop", [0xBF800000], [], [], null); + + public static Gen5ShaderInstruction Branch(uint pc, string opcode, short wordOffset) => + new(pc, Gen5ShaderEncoding.Sopp, opcode, [unchecked((uint)(ushort)wordOffset)], [], [], null); + + public static Gen5ShaderInstruction MoveScalar(uint pc, uint destination, uint value) => + new(pc, Gen5ShaderEncoding.Sop1, "SMovB32", [0u], [Operand(value)], [Gen5Operand.Scalar(destination)], null); + + public static Gen5ShaderInstruction MoveScalarRegister(uint pc, uint destination, uint source) => + new(pc, Gen5ShaderEncoding.Sop1, "SMovB32", [0u], [Gen5Operand.Scalar(source)], [Gen5Operand.Scalar(destination)], null); + + public static Gen5ShaderInstruction Sop1(uint pc, string opcode, uint destination, Gen5Operand source) => + new(pc, Gen5ShaderEncoding.Sop1, opcode, [0u], [source], [Gen5Operand.Scalar(destination)], null); + + public static Gen5ShaderInstruction Sop2(uint pc, string opcode, uint destination, Gen5Operand source0, Gen5Operand source1) => + new(pc, Gen5ShaderEncoding.Sop2, opcode, [0u], [source0, source1], [Gen5Operand.Scalar(destination)], null); + + public static Gen5ShaderInstruction Sopc(uint pc, string opcode, Gen5Operand source0, Gen5Operand source1) => + new(pc, Gen5ShaderEncoding.Sopc, opcode, [0u], [source0, source1], [], null); + + public static Gen5ShaderInstruction Vop1(uint pc, string opcode, uint destination, Gen5Operand source) => + new(pc, Gen5ShaderEncoding.Vop1, opcode, [0u], [source], [Gen5Operand.Vector(destination)], null); + + public static Gen5ShaderInstruction Vop2(uint pc, string opcode, uint destination, Gen5Operand source0, Gen5Operand source1) => + new(pc, Gen5ShaderEncoding.Vop2, opcode, [0u], [source0, source1], [Gen5Operand.Vector(destination)], null); + + public static Gen5ShaderInstruction Vopc(uint pc, string opcode, Gen5Operand source0, uint vectorSource1) => + new(pc, Gen5ShaderEncoding.Vopc, opcode, [0u], [source0, Gen5Operand.Vector(vectorSource1)], [], null); + + public static Gen5ShaderInstruction Vop3(uint pc, string opcode, uint destination, params Gen5Operand[] sources) => + new(pc, Gen5ShaderEncoding.Vop3, opcode, [0u, 0u], sources, [Gen5Operand.Vector(destination)], + new Gen5Vop3Control(0, 0, 0, false, 0, null)); + + public static Gen5ShaderInstruction ReadFirstLane(uint pc, uint scalarDestination, uint vectorSource) => + new(pc, Gen5ShaderEncoding.Vop1, "VReadfirstlaneB32", [0u], [Gen5Operand.Vector(vectorSource)], [Gen5Operand.Scalar(scalarDestination)], null); + + public static Gen5ShaderInstruction WriteLane(uint pc, uint vectorRegister, uint scalarRegister, uint lane) => + new(pc, Gen5ShaderEncoding.Vop3, "VWritelaneB32", [0u, 0u], + [Gen5Operand.Scalar(scalarRegister), Immediate(lane), Gen5Operand.Scalar(0)], [Gen5Operand.Vector(vectorRegister)], null); + + public static Gen5ShaderInstruction ReadLane(uint pc, uint scalarRegister, uint vectorRegister, uint lane) => + new(pc, Gen5ShaderEncoding.Vop3, "VReadlaneB32", [0u, 0u], + [Gen5Operand.Vector(vectorRegister), Immediate(lane), Gen5Operand.Scalar(0)], [Gen5Operand.Scalar(scalarRegister)], null); + + // s_load_dwordxN through the address pair at baseRegister. + public static Gen5ShaderInstruction ScalarLoad(uint pc, uint baseRegister, uint destination, uint count = 1, int immediateOffset = 0, uint? dynamicOffsetRegister = null) => + new(pc, Gen5ShaderEncoding.Smem, ScalarLoadName("SLoadDword", count), [0u, 0u], + [Gen5Operand.Scalar(baseRegister), dynamicOffsetRegister is { } register ? Gen5Operand.Scalar(register) : Gen5Operand.Source(NullOperand)], + Enumerable.Range((int)destination, (int)count).Select(index => Gen5Operand.Scalar((uint)index)).ToArray(), + new Gen5ScalarMemoryControl(count, immediateOffset, dynamicOffsetRegister)); + + // s_buffer_load_dwordxN through the descriptor at resourceRegister. + public static Gen5ShaderInstruction ScalarBufferLoad(uint pc, uint resourceRegister, uint destination, uint count = 1, int immediateOffset = 0, uint? dynamicOffsetRegister = null) => + new(pc, Gen5ShaderEncoding.Smem, ScalarLoadName("SBufferLoadDword", count), [0u, 0u], + [Gen5Operand.Scalar(resourceRegister), dynamicOffsetRegister is { } register ? Gen5Operand.Scalar(register) : Gen5Operand.Source(NullOperand)], + Enumerable.Range((int)destination, (int)count).Select(index => Gen5Operand.Scalar((uint)index)).ToArray(), + new Gen5ScalarMemoryControl(count, immediateOffset, dynamicOffsetRegister)); + + private static string ScalarLoadName(string prefix, uint count) => count == 1 ? prefix : $"{prefix}x{count}"; + + public static Gen5ShaderInstruction BufferAccess(uint pc, string opcode, uint resourceRegister, int offset = 0, uint dwords = 1, uint vectorData = 4, bool indexEnabled = false, bool offsetEnabled = false, uint vectorAddress = 0) + { + var isStore = opcode.Contains("Store", StringComparison.Ordinal) || opcode.Contains("Atomic", StringComparison.Ordinal); + var data = Enumerable.Range((int)vectorData, (int)dwords).Select(index => Gen5Operand.Vector((uint)index)).ToArray(); + return new Gen5ShaderInstruction(pc, Gen5ShaderEncoding.Mubuf, opcode, [0u, 0u], + isStore ? [Gen5Operand.Vector(vectorAddress), Gen5Operand.Scalar(resourceRegister), Gen5Operand.Source(NullOperand), .. data] + : [Gen5Operand.Vector(vectorAddress), Gen5Operand.Scalar(resourceRegister), Gen5Operand.Source(NullOperand)], + isStore ? [] : data, + new Gen5BufferMemoryControl(dwords, vectorAddress, vectorData, resourceRegister, offset, indexEnabled, offsetEnabled, Glc: false, Slc: false)); + } + + public static Gen5ShaderInstruction BufferLoad(uint pc, uint resourceRegister, int offset = 0, uint dwords = 1, bool formatted = false) => + BufferAccess(pc, formatted ? FormatName("BufferLoadFormat", dwords) : DwordName("BufferLoadDword", dwords), resourceRegister, offset, dwords); + + public static Gen5ShaderInstruction BufferStore(uint pc, uint resourceRegister, int offset = 0, uint dwords = 1, bool formatted = false) => + BufferAccess(pc, formatted ? FormatName("BufferStoreFormat", dwords) : DwordName("BufferStoreDword", dwords), resourceRegister, offset, dwords); + + public static Gen5ShaderInstruction BufferAtomicAdd(uint pc, uint resourceRegister, int offset = 0) => + BufferAccess(pc, "BufferAtomicAdd", resourceRegister, offset); + + private static string FormatName(string prefix, uint dwords) => prefix + dwords switch { 1 => "X", 2 => "Xy", 3 => "Xyz", _ => "Xyzw" }; + + private static string DwordName(string prefix, uint dwords) => dwords == 1 ? prefix : $"{prefix}x{dwords}"; + + // global_load/store through the scalar address pair, or a flat access through v[address:address+1]. + public static Gen5ShaderInstruction GlobalAccess(uint pc, string opcode, uint scalarAddress, int offset = 0, uint vectorAddress = 0, uint dwords = 1) + { + var flat = opcode.StartsWith("Flat", StringComparison.Ordinal); + var isStore = opcode.Contains("Store", StringComparison.Ordinal); + Gen5Operand[] addresses = flat + ? [Gen5Operand.Vector(vectorAddress), Gen5Operand.Vector(vectorAddress + 1)] + : [Gen5Operand.Vector(vectorAddress), Gen5Operand.Scalar(scalarAddress)]; + var data = Enumerable.Range(8, (int)dwords).Select(index => Gen5Operand.Vector((uint)index)).ToArray(); + return new Gen5ShaderInstruction(pc, Gen5ShaderEncoding.Flat, opcode, [0u, 0u], + isStore ? [.. addresses, .. data] : addresses, + isStore ? [] : data, + new Gen5GlobalMemoryControl(dwords, vectorAddress, 8, 8, flat ? uint.MaxValue : scalarAddress, offset, Glc: false, Slc: false, flat)); + } + + public static Gen5ShaderInstruction Image(uint pc, string opcode, uint resourceRegister, uint samplerRegister = 0, uint dimension = 1, uint dmask = 0xF, uint vectorAddress = 0, bool r128 = false) + { + var addressRegisters = new uint[] { vectorAddress, vectorAddress + 1, vectorAddress + 2 }; + var control = new Gen5ImageControl(dmask, vectorAddress, addressRegisters, 4, resourceRegister, samplerRegister, dimension, + dimension is 4 or 5 or 7, Glc: false, Slc: false, A16: false, D16: false); + var sources = new List { Gen5Operand.Vector(vectorAddress), Gen5Operand.Scalar(resourceRegister), Gen5Operand.Scalar(samplerRegister) }; + var destinations = opcode.StartsWith("ImageStore", StringComparison.Ordinal) ? Array.Empty() : [Gen5Operand.Vector(4)]; + return new Gen5ShaderInstruction(pc, Gen5ShaderEncoding.Mimg, opcode, [r128 ? 1u << 15 : 0u, 0u], sources, destinations, control); + } + + public static Gen5ShaderInstruction DataShareWrite(uint pc, bool gds) => + new(pc, Gen5ShaderEncoding.Ds, "DsWriteB32", [gds ? 1u << 17 : 0u, 0u], [Gen5Operand.Vector(0), Gen5Operand.Vector(1)], [], + new Gen5DataShareControl(0, 0, gds)); + + // Any DS instruction: sources first (address, data...), then the vector destinations. + public static Gen5ShaderInstruction DataShare(uint pc, string opcode, bool gds, Gen5Operand[] sources, uint[] destinations, uint offset0 = 0, uint offset1 = 0) => + new(pc, Gen5ShaderEncoding.Ds, opcode, [gds ? 1u << 17 : 0u, 0u], sources, destinations.Select(Gen5Operand.Vector).ToArray(), + new Gen5DataShareControl(offset0, offset1, gds)); + + // A global or flat access with explicit data registers: a load writes destination, + // a store or atomic reads source (an atomic with glc also writes destination). + public static Gen5ShaderInstruction GlobalMemory(uint pc, string opcode, uint scalarAddress, uint vectorAddress, uint destination, uint source, int offset = 0, uint dwords = 1, bool glc = false) + { + var flat = opcode.StartsWith("Flat", StringComparison.Ordinal); + var noScalarAddress = flat || scalarAddress >= NullOperand; + var isStore = opcode.Contains("Store", StringComparison.Ordinal) || opcode.Contains("Atomic", StringComparison.Ordinal); + Gen5Operand[] addresses = noScalarAddress + ? [Gen5Operand.Vector(vectorAddress), Gen5Operand.Vector(vectorAddress + 1)] + : [Gen5Operand.Vector(vectorAddress), Gen5Operand.Scalar(scalarAddress)]; + var data = Enumerable.Range((int)source, (int)dwords).Select(index => Gen5Operand.Vector((uint)index)).ToArray(); + var results = Enumerable.Range((int)destination, (int)dwords).Select(index => Gen5Operand.Vector((uint)index)).ToArray(); + return new Gen5ShaderInstruction(pc, Gen5ShaderEncoding.Flat, opcode, [0u, 0u], + isStore ? [.. addresses, .. data] : addresses, + isStore && !glc ? [] : results, + new Gen5GlobalMemoryControl(dwords, vectorAddress, source, destination, noScalarAddress ? NullOperand : scalarAddress, offset, Glc: glc, Slc: false, flat)); + } + + public static Gen5ShaderInstruction MoveVector(uint pc, uint destination, uint value) => + Vop1(pc, "VMovB32", destination, Operand(value)); + + public static Gen5ShaderInstruction MoveVectorFromScalar(uint pc, uint destination, uint scalarRegister) => + Vop1(pc, "VMovB32", destination, Gen5Operand.Scalar(scalarRegister)); + + public static Gen5ShaderProgram Program(params Gen5ShaderInstruction[] instructions) => new(0, instructions); + + public static ShaderResourcePlan Extract(Gen5ShaderProgram program, uint userDataBase = 0, uint userDataCount = 64, ShaderStage stage = ShaderStage.Compute, + uint waveSize = 64) => + ShaderResourcePlan.Extract(program, stage, Hash, userDataBase, userDataCount, waveSize: waveSize); + + // The plan, its default specialization applied, and the layout of one program. + public static (ShaderResourcePlan Plan, SpecializedResourceInfo Resources, BindingLayout Layout) Prepare( + Gen5ShaderProgram program, ShaderStage stage = ShaderStage.Compute, uint userDataBase = 0, uint userDataCount = 64, uint pushDataStartDword = 0, + bool usesDispatchThreadLimits = false) + { + var plan = ShaderResourcePlan.Extract(program, stage, Hash, userDataBase, userDataCount); + var resources = ResourceMaterializer.ApplyTo(plan, ResourceSpecialization.Default(plan.Info)); + var layout = BindingLayout.Allocate( + resources.Info, + BindingLayout.CollectUserDataRegisters(program, userDataBase, userDataCount, plan.Graph.ExcludedUserDataRegisters), + BindingLayout.UsesGlobalDataShare(program), + ShaderCompileRequest.RequiresFlattenedTable(plan, resources), + BindingLayout.ReadsShaderBase(program), + pushDataStartDword, + usesDispatchThreadLimits, + usesMeshDrawParameters: stage == ShaderStage.Mesh); + return (plan, resources, layout); + } + + public static ShaderCompileRequest Request(Gen5ShaderProgram program, ShaderStage stage = ShaderStage.Compute, uint userDataBase = 0, uint userDataCount = 64, uint pushDataStartDword = 0) + { + var (plan, resources, layout) = Prepare(program, stage, userDataBase, userDataCount, pushDataStartDword); + return new ShaderCompileRequest(plan, resources, layout) + { + PixelOutputs = stage == ShaderStage.Pixel ? [new Gen5PixelOutputBinding(0, 0, Gen5PixelOutputKind.Float)] : [], + }; + } + + public static ResourceRuntimeInputs Inputs(uint[] userData, GuestWordReader? readMemory = null, GuestWordReader? readCleanMemory = null, ulong shaderBase = 0) => + new() { UserData = userData, ReadMemory = readMemory, ReadCleanMemory = readCleanMemory, ShaderBase = shaderBase }; +} + +// A small dword-addressed memory image whose reads can be counted and refused. +internal sealed class TestWordMemory +{ + public ulong Base { get; init; } = 0x1000; + public uint[] Words { get; init; } = new uint[8]; + public uint Reads { get; set; } + public uint FailAfter { get; set; } = uint.MaxValue; + public ulong FailAddress { get; set; } = ulong.MaxValue; + public bool RequireAlignment { get; init; } + + public bool Read(ulong address, out uint value) + { + value = 0; + if (address < Base || address - Base >= (ulong)Words.Length * sizeof(uint) || Reads >= FailAfter || + address == FailAddress || (RequireAlignment && (address & 3) != 0)) + { + return false; + } + + value = Words[(address - Base) / sizeof(uint)]; + Reads++; + return true; + } + + public ref uint At(ulong address) => ref Words[(address - Base) / sizeof(uint)]; +} diff --git a/tools/SharpEmu.Tools.PerfAudit/Rig/SceneDraws.cs b/tools/SharpEmu.Tools.PerfAudit/Rig/SceneDraws.cs new file mode 100644 index 000000000..248761703 --- /dev/null +++ b/tools/SharpEmu.Tools.PerfAudit/Rig/SceneDraws.cs @@ -0,0 +1,164 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using System.Runtime.InteropServices; + +namespace SharpEmu.Gpu.Metal.Tests.Scenes; + +/// One vertex of a scene mesh: clip-space position and a PARAM0 vector. +[StructLayout(LayoutKind.Sequential)] +internal readonly record struct SceneVertex(float X, float Y, float Z, float W, float R, float G, float B, float A) +{ + public static SceneVertex At(float x, float y, float z, float r, float g, float b, float a = 1f) => new(x, y, z, 1f, r, g, b, a); +} + +/// The draws every scene repeats: a colour target, a mesh and its fetch programs. +internal static class SceneDraws +{ + /// + /// A linear colour target in guest memory, cleared to bytes. Rows + /// sit at the 256-byte linear pitch; read it back with . + /// + public static ulong ColorTarget(this SceneRig rig, uint width, uint height, uint bytesPerPixel = 4, byte fill = 0) + { + var size = (ulong)LinearPitchBytes(width, bytesPerPixel) * height; + var address = rig.Memory.Allocate(size, 0x10000); + if (fill != 0) + { + var bytes = new byte[size]; + Array.Fill(bytes, fill); + rig.Memory.Write(address, bytes); + } + + return address; + } + + /// + /// Uploads the mesh as two 16-byte-record buffers and returns the vertex program's + /// user data: the position V# then the PARAM0 V#. + /// + public static uint[] UploadMesh(this SceneRig rig, IReadOnlyList vertices) + { + var positions = new float[vertices.Count * 4]; + var parameters = new float[vertices.Count * 4]; + for (var index = 0; index < vertices.Count; index++) + { + var vertex = vertices[index]; + positions[(index * 4) + 0] = vertex.X; + positions[(index * 4) + 1] = vertex.Y; + positions[(index * 4) + 2] = vertex.Z; + positions[(index * 4) + 3] = vertex.W; + parameters[(index * 4) + 0] = vertex.R; + parameters[(index * 4) + 1] = vertex.G; + parameters[(index * 4) + 2] = vertex.B; + parameters[(index * 4) + 3] = vertex.A; + } + + var positionAddress = rig.Memory.Upload(positions); + var parameterAddress = rig.Memory.Upload(parameters); + return + [ + .. Descriptors.Buffer(positionAddress, 16, (uint)vertices.Count), + .. Descriptors.Buffer(parameterAddress, 16, (uint)vertices.Count), + ]; + } + + /// A full-viewport quad as two triangles at depth . + public static SceneVertex[] Quad(float z, float r, float g, float b, float a = 1f, + float left = -1f, float bottom = -1f, float right = 1f, float top = 1f) => + [ + SceneVertex.At(left, bottom, z, r, g, b, a), + SceneVertex.At(right, bottom, z, r, g, b, a), + SceneVertex.At(left, top, z, r, g, b, a), + SceneVertex.At(left, top, z, r, g, b, a), + SceneVertex.At(right, bottom, z, r, g, b, a), + SceneVertex.At(right, top, z, r, g, b, a), + ]; + + /// Draws with the fetch and interpolated-colour programs. + public static ScenePm4 DrawMesh(this SceneRig rig, ScenePm4 pm4, SceneState state, IReadOnlyList vertices, + uint[]? pixelProgram = null, uint[]? pixelUserData = null, bool predicated = false) + { + var vertex = rig.Program(ScenePrograms.FetchVertex); + var pixel = rig.Program(pixelProgram ?? ScenePrograms.InterpolatedColor); + state.VertexShader(vertex, rig.UploadMesh(vertices)); + state.PixelShader(pixel, pixelUserData ?? []); + state.Emit(pm4); + pm4.NumInstances(1); + pm4.DrawAuto((uint)vertices.Count, predicated); + return pm4; + } + + /// + /// A linear image in guest memory: rows padded to the 256-byte linear pitch the + /// hardware (and the image layout) uses. holds tightly packed rows. + /// + public static ulong LinearImage(this SceneRig rig, uint width, uint height, uint bytesPerTexel, ReadOnlySpan rows) + { + var pitch = LinearPitchBytes(width, bytesPerTexel); + var bytes = new byte[pitch * height]; + for (var y = 0; y < height; y++) + { + rows.Slice((int)(y * width * bytesPerTexel), (int)(width * bytesPerTexel)).CopyTo(bytes.AsSpan((int)(y * pitch))); + } + + var address = rig.Memory.Allocate((ulong)bytes.Length, 0x10000); + rig.Memory.Write(address, bytes); + return address; + } + + /// A linear image's texels as tightly packed rows (the 256-byte pitch padding removed). + public static byte[] ReadImage(this SceneRig rig, ulong address, uint width, uint height, uint bytesPerTexel = 4) + { + var pitch = LinearPitchBytes(width, bytesPerTexel); + var padded = rig.Read(address, (int)(pitch * height)); + var rowBytes = (int)(width * bytesPerTexel); + var packed = new byte[rowBytes * height]; + for (var y = 0; y < height; y++) + { + padded.AsSpan((int)(y * pitch), rowBytes).CopyTo(packed.AsSpan(y * rowBytes)); + } + + return packed; + } + + public static T[] ReadImage(this SceneRig rig, ulong address, uint width, uint height) where T : unmanaged => + System.Runtime.InteropServices.MemoryMarshal.Cast( + rig.ReadImage(address, width, height, (uint)System.Runtime.CompilerServices.Unsafe.SizeOf())).ToArray(); + + public static uint LinearPitchBytes(uint width, uint bytesPerTexel) => + (width * bytesPerTexel + 255) / 256 * 256; + + /// A full-viewport quad whose PARAM0.xy is the texture coordinate (0,0 at the top-left). + public static SceneVertex[] TexturedQuad(float z = 0.5f) => + [ + SceneVertex.At(-1, -1, z, 0, 1, 0, 1), + SceneVertex.At(1, -1, z, 1, 1, 0, 1), + SceneVertex.At(-1, 1, z, 0, 0, 0, 1), + SceneVertex.At(-1, 1, z, 0, 0, 0, 1), + SceneVertex.At(1, -1, z, 1, 1, 0, 1), + SceneVertex.At(1, 1, z, 1, 0, 0, 1), + ]; + + /// Dispatches with the given user data. + public static ScenePm4 Dispatch(this SceneRig rig, ScenePm4 pm4, uint[] program, uint[] userData, uint groupsX, + uint groupsY = 1, uint groupsZ = 1, uint threadsX = 64, uint threadsY = 1, uint threadsZ = 1, uint ldsBytes = 0, + bool groupIdY = false, bool groupIdZ = false, uint threadIdComponents = 0) + { + var state = new SceneState(1, 1); + state.Context.Clear(); + state.Uconfig.Clear(); + state.ComputeShader(rig.Program(program), threadsX, threadsY, threadsZ, userData, true, groupIdY, groupIdZ, + threadIdComponents, ldsBytes); + state.Emit(pm4); + pm4.DispatchDirect(groupsX, groupsY, groupsZ, 1); + return pm4; + } + + public static uint Pixel(byte[] rgba, uint width, uint x, uint y) => + BitConverter.ToUInt32(rgba, (int)(((y * width) + x) * 4)); + + public static string Hex(uint value) => $"0x{value:X8}"; +} + +/// The device gate every scene runs behind. diff --git a/tools/SharpEmu.Tools.PerfAudit/Rig/SceneMemory.cs b/tools/SharpEmu.Tools.PerfAudit/Rig/SceneMemory.cs new file mode 100644 index 000000000..6b5fbec9f --- /dev/null +++ b/tools/SharpEmu.Tools.PerfAudit/Rig/SceneMemory.cs @@ -0,0 +1,135 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using System.Buffers.Binary; +using System.Runtime.InteropServices; +using SharpEmu.HLE; +using SharpEmu.HLE.GpuMemory; +using SharpEmu.HLE.Host; + +namespace SharpEmu.Gpu.Metal.Tests.Scenes; + +/// The guest address space a scene's page guard protects: everything succeeds. +internal sealed class SceneAddressSpace : IGuestAddressSpace +{ + public ulong ProtectionPageSize => 4096; + public bool TryProtect(ulong address, ulong size, GuestPageProtection protection) => true; + public ulong AllocateAt(ulong desiredAddress, ulong size, bool executable = true, bool allowAlternative = true) => throw new NotSupportedException(); + public bool TryBackFixedRange(ulong address, ulong size, bool executable) => throw new NotSupportedException(); + public bool TryAllocateAtOrAbove(ulong desiredAddress, ulong size, bool executable, ulong alignment, out ulong actualAddress) => throw new NotSupportedException(); + public bool TryEnsureRangeCommitted(ulong address, ulong size) => throw new NotSupportedException(); + public bool TryAllocateGuestMemory(ulong size, ulong alignment, out ulong address) => throw new NotSupportedException(); + public bool TryFreeGuestMemory(ulong address) => throw new NotSupportedException(); +} + +/// +/// One page-aligned native block seen as guest memory at : the GPU and +/// the CPU share it exactly as they share the guest's direct memory. +/// +internal sealed unsafe class SceneMemory : IGuestBackedSpace, ICpuMemory, IDisposable +{ + public const ulong Base = 0x2_0000_0000; + public const ulong Size = 64 * 1024 * 1024; + private readonly void* _block = NativeMemory.AlignedAlloc((nuint)Size, 16384); + private ulong _next = Base; + + public SceneMemory() => NativeMemory.Clear(_block, (nuint)Size); + + public void Dispose() => NativeMemory.AlignedFree(_block); + + /// Hands out guest memory from a bump allocator. + public ulong Allocate(ulong size, ulong alignment = 256) + { + var address = (_next + alignment - 1) & ~(alignment - 1); + if (address + size > Base + Size) + { + throw new InvalidOperationException($"Scene memory is exhausted: size=0x{size:X}."); + } + + _next = address + size; + return address; + } + + public ulong Upload(ReadOnlySpan data, ulong alignment = 256) + { + var address = Allocate((ulong)Math.Max(data.Length, 4), alignment); + Write(address, data); + return address; + } + + public ulong Upload(ReadOnlySpan data, ulong alignment = 256) where T : unmanaged => + Upload(MemoryMarshal.AsBytes(data), alignment); + + public void Write(ulong address, ReadOnlySpan data) + { + if (!TryWrite(address, data)) + { + throw new InvalidOperationException($"Scene memory write is out of range: 0x{address:X16}+0x{data.Length:X}."); + } + } + + public void Write(ulong address, ReadOnlySpan data) where T : unmanaged => Write(address, MemoryMarshal.AsBytes(data)); + + public void Write32(ulong address, uint value) + { + Span bytes = stackalloc byte[4]; + BinaryPrimitives.WriteUInt32LittleEndian(bytes, value); + Write(address, bytes); + } + + public void Write64(ulong address, ulong value) + { + Span bytes = stackalloc byte[8]; + BinaryPrimitives.WriteUInt64LittleEndian(bytes, value); + Write(address, bytes); + } + + /// The bytes as memory holds them now, without any GPU synchronization. + public byte[] ReadRaw(ulong address, int length) + { + var bytes = new byte[length]; + if (!TryRead(address, bytes)) + { + throw new InvalidOperationException($"Scene memory read is out of range: 0x{address:X16}+0x{length:X}."); + } + + return bytes; + } + + private static bool Contains(ulong address, ulong length) => address >= Base && address - Base + length <= Size; + + public bool TryHoldRange(ulong address, ulong size) => false; + public bool TryHoldRangeAtOrAbove(ulong searchStart, ulong size, ulong alignment, out ulong address) { address = 0; return false; } + public bool TryMapBacked(ulong address, ulong size, ulong backingOffset, GuestPageProtection protection, out HostViewFailure failure) { failure = HostViewFailure.None; return false; } + public bool TryUnmapBacked(ulong address, ulong size) => false; + public bool TryClearBacking(ulong offset, ulong size) => false; + public bool IsBackedView(ulong address) => Contains(address, 1); + public bool IsBackedRange(ulong address, ulong size) => Contains(address, size); + public (ulong Base, ulong Size) BackingAliasRange => ((ulong)_block, Size); + + public bool TryResolveBackingAlias(ulong address, ulong size, out ulong alias) + { + alias = Contains(address, size) ? (ulong)_block + (address - Base) : 0; + return alias != 0; + } + + public bool TryWriteBacking(ulong address, ReadOnlySpan data) => TryWrite(address, data); + + public bool TryReadBacking(ulong address, Span data) => TryRead(address, data); + + public bool TryRead(ulong virtualAddress, Span destination) + { + if (!Contains(virtualAddress, (ulong)destination.Length)) return false; + new ReadOnlySpan((byte*)_block + (virtualAddress - Base), destination.Length).CopyTo(destination); + return true; + } + + public bool TryWrite(ulong virtualAddress, ReadOnlySpan source) + { + if (!Contains(virtualAddress, (ulong)source.Length)) return false; + source.CopyTo(new Span((byte*)_block + (virtualAddress - Base), source.Length)); + return true; + } + + public bool CanRead(ulong address, ulong size) => Contains(address, size); +} diff --git a/tools/SharpEmu.Tools.PerfAudit/Rig/ScenePm4.cs b/tools/SharpEmu.Tools.PerfAudit/Rig/ScenePm4.cs new file mode 100644 index 000000000..458829edb --- /dev/null +++ b/tools/SharpEmu.Tools.PerfAudit/Rig/ScenePm4.cs @@ -0,0 +1,102 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +namespace SharpEmu.Gpu.Metal.Tests.Scenes; + +/// +/// Builds a raw PM4 type-3 stream the way the guest's command buffer holds it. Register +/// offsets are dword offsets inside their bank: context registers from 0xA000, shader +/// registers from 0x2C00 and user-config registers from 0xC000 (the mm* address / 4). +/// +internal sealed class ScenePm4 +{ + public const uint OpNop = 0x10; + public const uint OpIndexBufferSize = 0x13; + public const uint OpDispatchDirect = 0x15; + public const uint OpDispatchIndirect = 0x16; + public const uint OpSetPredication = 0x20; + public const uint OpDrawIndirect = 0x24; + public const uint OpDrawIndexIndirect = 0x25; + public const uint OpIndexBase = 0x26; + public const uint OpDrawIndex2 = 0x27; + public const uint OpIndexType = 0x2A; + public const uint OpDrawIndexAuto = 0x2D; + public const uint OpNumInstances = 0x2F; + public const uint OpSetBase = 0x11; + public const uint OpWriteData = 0x37; + public const uint OpEventWrite = 0x46; + public const uint OpAcquireMemory = 0x58; + public const uint OpSetContextReg = 0x69; + public const uint OpSetShReg = 0x76; + public const uint OpSetUconfigReg = 0x79; + + // VGT_EVENT_TYPE values (mesa sid.h V_028A90_*). + public const uint EventPsPartialFlush = 0x10; + public const uint EventCacheFlushAndInvalidate = 0x16; + public const uint EventCsPartialFlush = 0x07; + public const uint EventPixelPipeStatControl = 0x38; + public const uint EventPixelPipeStatDump = 0x39; + public const uint EventPixelPipeStatReset = 0x3A; + + private readonly List _words = []; + + public int Count => _words.Count; + + public uint[] ToArray() => [.. _words]; + + public static uint Header(uint dwords, uint opcode, bool predicated = false) => + 0xC000_0000u | (((dwords - 2u) & 0x3FFFu) << 16) | ((opcode & 0xFFu) << 8) | (predicated ? 1u : 0u); + + public ScenePm4 Packet(uint opcode, params uint[] payload) => Packet(opcode, false, payload); + + public ScenePm4 Packet(uint opcode, bool predicated, params uint[] payload) + { + _words.Add(Header((uint)payload.Length + 1, opcode, predicated)); + _words.AddRange(payload); + return this; + } + + public ScenePm4 Context(uint offset, params uint[] values) => Packet(OpSetContextReg, [offset, .. values]); + + public ScenePm4 Shader(uint offset, params uint[] values) => Packet(OpSetShReg, [offset, .. values]); + + public ScenePm4 Uconfig(uint offset, params uint[] values) => Packet(OpSetUconfigReg, [offset, .. values]); + + public ScenePm4 ContextFloat(uint offset, params float[] values) => + Context(offset, values.Select(BitConverter.SingleToUInt32Bits).ToArray()); + + /// DRAW_INDEX_AUTO: count, then VGT_DRAW_INITIATOR (auto-index source 2). + public ScenePm4 DrawAuto(uint vertexCount, bool predicated = false) => + Packet(OpDrawIndexAuto, predicated, vertexCount, 2u); + + public ScenePm4 NumInstances(uint count) => Packet(OpNumInstances, count); + + /// INDEX_TYPE: 0 = 16-bit, 1 = 32-bit. + public ScenePm4 IndexType(uint type) => Packet(OpIndexType, type); + + /// DRAW_INDEX_2: max size, index base lo/hi, count, initiator (DMA source 0). + public ScenePm4 DrawIndexed(ulong indexAddress, uint maxIndices, uint count, bool predicated = false) => + Packet(OpDrawIndex2, predicated, maxIndices, (uint)indexAddress, (uint)(indexAddress >> 32), count, 0u); + + public ScenePm4 Event(uint eventType, uint eventIndex = 0) => Packet(OpEventWrite, eventType | (eventIndex << 8)); + + public ScenePm4 EventWithAddress(uint eventType, uint eventIndex, ulong address) => + Packet(OpEventWrite, eventType | (eventIndex << 8), (uint)address, (uint)(address >> 32)); + + /// + /// SET_PREDICATION (gfx9+ layout): flags, address lo, address hi. Flags: PRED_BOOL bit 8, + /// HINT bit 12, PRED_OP bits 16..18, CONTINUE bit 31. + /// + public ScenePm4 SetPredication(uint operation, bool drawIfVisible, bool noWaitHint, bool continueBit, ulong address) => + Packet(OpSetPredication, + (operation << 16) | (drawIfVisible ? 1u << 8 : 0u) | (noWaitHint ? 1u << 12 : 0u) | (continueBit ? 1u << 31 : 0u), + (uint)address, (uint)(address >> 32)); + + public ScenePm4 DispatchDirect(uint x, uint y, uint z, uint initiator = 1u) => Packet(OpDispatchDirect, x, y, z, initiator); + + public ScenePm4 Raw(params uint[] words) + { + _words.AddRange(words); + return this; + } +} diff --git a/tools/SharpEmu.Tools.PerfAudit/Rig/ScenePrograms.cs b/tools/SharpEmu.Tools.PerfAudit/Rig/ScenePrograms.cs new file mode 100644 index 000000000..a8ded1561 --- /dev/null +++ b/tools/SharpEmu.Tools.PerfAudit/Rig/ScenePrograms.cs @@ -0,0 +1,74 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +namespace SharpEmu.Gpu.Metal.Tests.Scenes; + +/// +/// The scenes' RDNA2 programs, assembled with +/// llvm-mc -triple=amdgcn -mcpu=gfx1030 -show-encoding; each word carries its +/// source line. Vertex programs run on the ES/GS (NGG-merged) stage, so the vertex id is +/// v5, the instance id v8 and user SGPRs start at s8. Pixel programs take user SGPRs from +/// s0, the primitive mask after them, and PERSP_CENTER barycentrics in v0/v1. +/// +internal static class ScenePrograms +{ + /// + /// Position from the V# in s[8:11] and PARAM0 from the V# in s[12:15], both 16-byte + /// records indexed by the vertex id. + /// + public static readonly uint[] FetchVertex = + [ + 0xE0382000, 0x80020005, // buffer_load_dwordx4 v[0:3], v5, s[8:11], 0 idxen + 0xE0382000, 0x80030A05, // buffer_load_dwordx4 v[10:13], v5, s[12:15], 0 idxen + 0xBF8C3F70, // s_waitcnt vmcnt(0) + 0xF80008CF, 0x03020100, // exp pos0 v0, v1, v2, v3 done + 0xF800020F, 0x0D0C0B0A, // exp param0 v10, v11, v12, v13 + 0xBF810000, // s_endpgm + ]; + + /// MRT0 = interpolated PARAM0 (no user SGPRs, so the primitive mask is s0). + public static readonly uint[] InterpolatedColor = + [ + 0xBEFC0300, // s_mov_b32 m0, s0 + 0xC8080000, // v_interp_p1_f32_e32 v2, v0, attr0.x + 0xC8090001, // v_interp_p2_f32_e32 v2, v1, attr0.x + 0xC80C0100, // v_interp_p1_f32_e32 v3, v0, attr0.y + 0xC80D0101, // v_interp_p2_f32_e32 v3, v1, attr0.y + 0xC8100200, // v_interp_p1_f32_e32 v4, v0, attr0.z + 0xC8110201, // v_interp_p2_f32_e32 v4, v1, attr0.z + 0xC8140300, // v_interp_p1_f32_e32 v5, v0, attr0.w + 0xC8150301, // v_interp_p2_f32_e32 v5, v1, attr0.w + 0xF800180F, 0x05040302, // exp mrt0 v2, v3, v4, v5 done vm + 0xBF810000, // s_endpgm + ]; + + /// + /// MRT0 = the 2D image T# s[0:7] sampled through S# s[8:11] at the interpolated + /// PARAM0.xy (twelve user SGPRs, so the primitive mask is s12). + /// + public static readonly uint[] SampledTexture = + [ + 0xBEFC030C, // s_mov_b32 m0, s12 + 0xC8080000, // v_interp_p1_f32_e32 v2, v0, attr0.x + 0xC8090001, // v_interp_p2_f32_e32 v2, v1, attr0.x + 0xC80C0100, // v_interp_p1_f32_e32 v3, v0, attr0.y + 0xC80D0101, // v_interp_p2_f32_e32 v3, v1, attr0.y + 0xF0800F08, 0x00400402, // image_sample v[4:7], v[2:3], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D + 0xBF8C3F70, // s_waitcnt vmcnt(0) + 0xF800180F, 0x07060504, // exp mrt0 v4, v5, v6, v7 done vm + 0xBF810000, // s_endpgm + ]; + + /// + /// 64-thread groups; thread g writes (g ^ 0x5A5A) to record g of the V# in s[0:3] + /// (the group id X is s4). + /// + public static readonly uint[] StoreGlobalId = + [ + 0x8F058604, // s_lshl_b32 s5, s4, 6 + 0x4A020005, // v_add_nc_u32_e32 v1, s5, v0 + 0x3A0402FF, 0x00005A5A, // v_xor_b32_e32 v2, 0x5a5a, v1 + 0xE0702000, 0x80000201, // buffer_store_dword v2, v1, s[0:3], 0 idxen + 0xBF810000, // s_endpgm + ]; +} diff --git a/tools/SharpEmu.Tools.PerfAudit/Rig/SceneRig.cs b/tools/SharpEmu.Tools.PerfAudit/Rig/SceneRig.cs new file mode 100644 index 000000000..b66a08d45 --- /dev/null +++ b/tools/SharpEmu.Tools.PerfAudit/Rig/SceneRig.cs @@ -0,0 +1,247 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using System.Collections.Concurrent; +using System.Reflection; +using System.Runtime.InteropServices; +using SharpEmu.Gpu.Metal.V2; +using SharpEmu.HLE.GpuMemory; +using SharpEmu.Libs.Agc; +using SharpEmu.Libs.Gpu; +using SharpEmu.Libs.Gpu.GpuCommands; + +namespace SharpEmu.Gpu.Metal.Tests.Scenes; + +/// +/// A whole emulated GPU for one scene: guest memory, the Metal v2 render host with its +/// command stream and draw translation, and the v2 backend as the process's guest GPU. +/// A scene writes its programs, buffers and textures into guest memory, submits a PM4 +/// stream exactly as the guest's command buffer would carry it, and reads its targets back +/// through the host's synchronized guest reads. +/// +internal sealed unsafe class SceneRig : IDisposable +{ + private const BindingFlags Statics = BindingFlags.Static | BindingFlags.NonPublic; + + private static readonly FieldInfo HeaderTable = typeof(AgcExports).GetField("_shaderHeadersByCode", Statics)!; + private static readonly FieldInfo FatalHook = typeof(SharpEmu.Libs.Gpu.Scheduling.GpuFatal).GetField("OnFatal", Statics)!; + private static readonly FieldInfo[] BackendFields = + new[] { "_factory", "_backendName", "_instance" }.Select(name => typeof(GuestGpu).GetField(name, Statics)!).ToArray(); + + private readonly nint _queue; + private readonly SceneAddressSpace _space = new(); + private readonly PageGuard _pages; + // One registry for every live rig: a scene may hold two rigs at once, the hook takes + // one manager, and every rig's memory sits at the same guest span. + private static readonly object RegistryGate = new(); + private static GuestGpuMemory? _registry; + private static int _registryUsers; + private readonly MetalGpuWorkerRelay _relay = new(static () => { }); + private readonly object?[] _savedBackend; + private readonly object? _savedFatal; + private readonly List _registeredShaders = []; + private readonly Dictionary _programs = new(ReferenceEqualityComparer.Instance); + private ulong _submissionId = 1; + + /// The registered copy of a program, placed on first use. + public ulong Program(uint[] code) + { + if (!_programs.TryGetValue(code, out var address)) + { + address = RegisterShader(code); + _programs[code] = address; + } + + return address; + } + + public SceneRig(nint device) + { + _savedBackend = BackendFields.Select(field => field.GetValue(null)).ToArray(); + // A loud refusal ends the process in the emulator; here it fails the scene instead. + _savedFatal = FatalHook.GetValue(null); + FatalHook.SetValue(null, (Action)(message => + { + lock (Fatals) + { + Fatals.Add(message); + } + + throw new SceneFatalException(message); + })); + BackendFields[0].SetValue(null, (Func)(() => new MetalV2Backend())); + BackendFields[1].SetValue(null, "Metal"); + BackendFields[2].SetValue(null, new MetalV2Backend()); + _queue = MetalNative.Send(device, MetalNative.Selector("newCommandQueue")); + _pages = new PageGuard(_space); + // The guest GPU memory registry the emulator attaches at boot: the device-address + // page table a shader dereferences raw guest pointers through is built from its + // spans, so the scene's block is registered as the guest's mapped memory. + lock (RegistryGate) + { + if (_registryUsers++ == 0) + { + _registry = new GuestGpuMemory(new SceneAddressSpace()); + _registry.Register(SceneMemory.Base, SceneMemory.Size, SharpEmu.HLE.GuestPageProtection.Read | SharpEmu.HLE.GuestPageProtection.Write); + GuestGpuMemoryHook.Attach(_registry); + } + } + Host = new MetalRenderHost(device, _queue, Memory, Memory, _pages, _relay); + _relay.BindCurrentThread(); + Host.InitializeCommandStream(new FlipTarget()); + } + + public SceneMemory Memory { get; } = new(); + + /// Every GPU fatal raised while this rig ran (each also threw ). + public List Fatals { get; } = []; + + public MetalRenderHost Host { get; } + + private static ConcurrentDictionary Headers => (ConcurrentDictionary)HeaderTable.GetValue(null)!; + + /// + /// Places a program in guest memory and registers its AGC shader header, as + /// sceAgcCreateShader leaves it: code size, user-data block and input semantics. + /// + public ulong RegisterShader(uint[] code, uint[]? inputSemantics = null) + { + var codeAddress = Memory.Upload(code, 256); + // The user-data block: no direct resources (offset table null, count 0 at +0x2C). + var userData = Memory.Upload(new byte[0x40], 16); + var semantics = inputSemantics is { Length: > 0 } ? Memory.Upload(inputSemantics, 16) : 0UL; + var header = new byte[0x60]; + var words = MemoryMarshal.Cast(header.AsSpan()); + words[0] = 0x34333231; + words[1] = 0x18; + BitConverter.TryWriteBytes(header.AsSpan(0x08), userData); + BitConverter.TryWriteBytes(header.AsSpan(0x10), codeAddress); + BitConverter.TryWriteBytes(header.AsSpan(0x30), semantics); + BitConverter.TryWriteBytes(header.AsSpan(0x44), (uint)(code.Length * 4)); + BitConverter.TryWriteBytes(header.AsSpan(0x50), (ushort)(inputSemantics?.Length ?? 0)); + var headerAddress = Memory.Upload(header, 16); + Headers[codeAddress] = headerAddress; + _registeredShaders.Add(codeAddress); + return codeAddress; + } + + /// Writes the stream into guest memory, runs it on the graphics queue to completion and waits for the GPU. + public void Submit(ScenePm4 stream) => Submit(stream.ToArray()); + + /// Runs on the graphics queue (0) or a compute queue (0x20 to 0x57). + public void Submit(uint[] words, int queue = 0) + { + Enqueue(words, queue); + Drain(); + } + + /// Hands to the command stream without running it. + public void Enqueue(uint[] words, int queue = 0) + { + var address = Memory.Upload(words, 256); + if (queue == 0) + { + Host.CommandStream.EnqueueGraphics(address, (uint)words.Length, _submissionId++, null); + } + else + { + Host.CommandStream.EnqueueCompute((uint)queue, address, (uint)words.Length, _submissionId++, null); + } + } + + /// The relay the rig's host posts GPU-queue work through, for driving the worker's own pass. + internal MetalGpuWorkerRelay Relay => _relay; + + /// Runs the worker loop until no submission is runnable, then waits for the GPU. + public void Drain() + { + var deadline = DateTime.UtcNow.AddSeconds(30); + while (true) + { + Host.RunPendingCommands(); + Host.RunCommandStreamSlices(64); + if (Host.CommandStream.HasUnblockedPending || _relay.HasPendingCommands) + { + continue; + } + + Host.CommitCoalescedFlush(); + Host.FlushAndWaitForGpu(); + Host.PollGpuCompletions(); + if (!Host.CommandStream.HasPending) + { + return; + } + + if (DateTime.UtcNow > deadline) + { + throw new TimeoutException("The scene's command stream stayed blocked."); + } + + Host.CommandStream.RetryBlocked(); + } + } + + /// Guest bytes after every GPU write to them landed (the host's synchronized read). + public byte[] Read(ulong address, int length) + { + var bytes = new byte[length]; + if (!Host.TryReadGuest(address, bytes)) + { + throw new InvalidOperationException($"The scene read failed: 0x{address:X16}+0x{length:X}."); + } + + return bytes; + } + + public T[] Read(ulong address, int count) where T : unmanaged => + MemoryMarshal.Cast(Read(address, count * Marshal.SizeOf())).ToArray(); + + public void Dispose() + { + try + { + Host.ImageCache.StopWatching(); + Host.Dispose(); + } + finally + { + foreach (var code in _registeredShaders) + { + Headers.TryRemove(code, out _); + } + + FatalHook.SetValue(null, _savedFatal); + for (var index = 0; index < BackendFields.Length; index++) + { + BackendFields[index].SetValue(null, _savedBackend[index]); + } + + lock (RegistryGate) + { + if (--_registryUsers == 0) + { + GuestGpuMemoryHook.Attach(null); + _registry!.Dispose(); + _registry = null; + } + } + + _pages.Dispose(); + Memory.Dispose(); + MetalNative.SendVoid(_queue, MetalNative.Selector("release")); + } + } + + private sealed class FlipTarget : IMetalFlipTarget + { + public bool HasFlipSlot() => true; + + public void PublishGuestFrame(nint texture, uint width, uint height, ulong requestId, ulong serial, bool isHdr) + { + } + } +} + +/// A GPU fatal (a loud refusal) raised by the emulated GPU while a scene ran. +internal sealed class SceneFatalException(string message) : Exception(message); diff --git a/tools/SharpEmu.Tools.PerfAudit/Rig/SceneState.cs b/tools/SharpEmu.Tools.PerfAudit/Rig/SceneState.cs new file mode 100644 index 000000000..872de33dc --- /dev/null +++ b/tools/SharpEmu.Tools.PerfAudit/Rig/SceneState.cs @@ -0,0 +1,397 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +namespace SharpEmu.Gpu.Metal.Tests.Scenes; + +/// Context register dword offsets (mm* / 4 - 0xA000), gfx10.3. +internal static class Cx +{ + public const uint DbRenderControl = 0x000; + public const uint DbCountControl = 0x001; + public const uint DbDepthView = 0x002; + public const uint DbRenderOverride = 0x003; + public const uint DbHtileDataBase = 0x005; + public const uint DbDepthSizeXy = 0x007; + public const uint DbDepthBoundsMin = 0x008; + public const uint DbDepthBoundsMax = 0x009; + public const uint DbStencilClear = 0x00A; + public const uint DbDepthClear = 0x00B; + public const uint PaScScreenScissorTl = 0x00C; + public const uint PaScScreenScissorBr = 0x00D; + public const uint DbZInfo = 0x010; + public const uint DbStencilInfo = 0x011; + public const uint DbZReadBase = 0x012; + public const uint DbStencilReadBase = 0x013; + public const uint DbZWriteBase = 0x014; + public const uint DbStencilWriteBase = 0x015; + public const uint DbZReadBaseHi = 0x01A; + public const uint DbStencilReadBaseHi = 0x01B; + public const uint DbZWriteBaseHi = 0x01C; + public const uint DbStencilWriteBaseHi = 0x01D; + public const uint PaScWindowOffset = 0x080; + public const uint PaScWindowScissorTl = 0x081; + public const uint PaScWindowScissorBr = 0x082; + public const uint CbTargetMask = 0x08E; + public const uint CbShaderMask = 0x08F; + public const uint PaScGenericScissorTl = 0x090; + public const uint PaScGenericScissorBr = 0x091; + public const uint PaScVportScissor0Tl = 0x094; + public const uint PaScVportScissor0Br = 0x095; + public const uint PaScVportZMin0 = 0x0B4; + public const uint PaScVportZMax0 = 0x0B5; + public const uint CbBlendRed = 0x105; + public const uint DbStencilControl = 0x10B; + public const uint DbStencilRefMask = 0x10C; + public const uint DbStencilRefMaskBf = 0x10D; + public const uint PaClVportXScale = 0x10F; + public const uint PaClVportXOffset = 0x110; + public const uint PaClVportYScale = 0x111; + public const uint PaClVportYOffset = 0x112; + public const uint PaClVportZScale = 0x113; + public const uint PaClVportZOffset = 0x114; + public const uint PaClUcp0X = 0x16F; + public const uint SpiPsInputCntl0 = 0x191; + public const uint SpiVsOutConfig = 0x1B1; + public const uint SpiPsInputEna = 0x1B3; + public const uint SpiPsInputAddr = 0x1B4; + public const uint SpiPsInControl = 0x1B6; + public const uint SpiBarycCntl = 0x1B8; + public const uint SpiShaderPosFormat = 0x1C3; + public const uint SpiShaderZFormat = 0x1C4; + public const uint SpiShaderColFormat = 0x1C5; + public const uint CbBlend0Control = 0x1E0; + public const uint DbDepthControl = 0x200; + public const uint DbEqaa = 0x201; + public const uint CbColorControl = 0x202; + public const uint DbShaderControl = 0x203; + public const uint PaClClipCntl = 0x204; + public const uint PaSuScModeCntl = 0x205; + public const uint PaClVteCntl = 0x206; + public const uint PaClVsOutCntl = 0x207; + public const uint PaSuLineCntl = 0x282; + public const uint PaScModeCntl0 = 0x292; + public const uint VgtShaderStagesEn = 0x2D5; + public const uint DbAlphaToMask = 0x2DC; + public const uint PaSuPolyOffsetDbFmtCntl = 0x2DE; + public const uint PaSuPolyOffsetClamp = 0x2DF; + public const uint PaSuPolyOffsetFrontScale = 0x2E0; + public const uint PaSuPolyOffsetFrontOffset = 0x2E1; + public const uint PaSuPolyOffsetBackScale = 0x2E2; + public const uint PaSuPolyOffsetBackOffset = 0x2E3; + public const uint PaScAaConfig = 0x2F8; + public const uint PaClGbVertClipAdj = 0x2FA; + public const uint PaClGbVertDiscAdj = 0x2FB; + public const uint PaClGbHorzClipAdj = 0x2FC; + public const uint PaClGbHorzDiscAdj = 0x2FD; + public const uint CbColor0Base = 0x318; + public const uint CbColor0View = 0x31B; + public const uint CbColor0Info = 0x31C; + public const uint CbColor0Attrib = 0x31D; + public const uint CbColor0BaseExt = 0x390; + public const uint CbColor0Attrib2 = 0x3B0; + public const uint CbColor0Attrib3 = 0x3B8; + public const uint CbColorSlotStride = 15; +} + +/// Shader register dword offsets (mm* / 4 - 0x2C00). +internal static class Sh +{ + public const uint PgmLoPs = 0x008; + public const uint PgmHiPs = 0x009; + public const uint PgmRsrc1Ps = 0x00A; + public const uint PgmRsrc2Ps = 0x00B; + public const uint UserDataPs0 = 0x00C; + public const uint PgmRsrc1Gs = 0x08A; + public const uint PgmRsrc2Gs = 0x08B; + public const uint UserDataGs0 = 0x08C; + public const uint PgmLoEs = 0x0C8; + public const uint PgmHiEs = 0x0C9; + public const uint ComputePgmLo = 0x20C; + public const uint ComputePgmHi = 0x20D; + public const uint ComputePgmRsrc1 = 0x212; + public const uint ComputePgmRsrc2 = 0x213; + public const uint ComputeNumThreadX = 0x207; + public const uint ComputeNumThreadY = 0x208; + public const uint ComputeNumThreadZ = 0x209; + public const uint ComputeUserData0 = 0x240; +} + +/// User-config register dword offsets (mm* / 4 - 0xC000). +internal static class Uc +{ + public const uint VgtPrimitiveType = 0x242; + public const uint VgtIndexType = 0x243; +} + +/// VGT_PRIMITIVE_TYPE values. +internal static class Prim +{ + public const uint PointList = 1; + public const uint LineList = 2; + public const uint LineStrip = 3; + public const uint TriangleList = 4; + public const uint TriangleFan = 5; + public const uint TriangleStrip = 6; + public const uint RectangleList = 17; +} + +/// Compare functions as DB_DEPTH_CONTROL and the S# encode them. +internal enum CompareFunc : uint +{ + Never = 0, + Less = 1, + Equal = 2, + LessEqual = 3, + Greater = 4, + NotEqual = 5, + GreaterEqual = 6, + Always = 7, +} + +/// +/// The register state of one draw, emitted as one SET_*_REG packet per register so every +/// write goes through the interpreter's own writer for it. Defaults are a minimal valid +/// pipeline: no depth, no culling, D3D clip space, one 32-bit-per-channel colour export. +/// +internal sealed class SceneState +{ + public SortedDictionary Context { get; } = new(); + public SortedDictionary Shader { get; } = new(); + public SortedDictionary Uconfig { get; } = new(); + + public SceneState(uint width, uint height) + { + Context[Cx.VgtShaderStagesEn] = 0; + Context[Cx.CbTargetMask] = 0xF; + Context[Cx.CbShaderMask] = 0xF; + Context[Cx.SpiShaderColFormat] = 9; + Context[Cx.SpiShaderZFormat] = 0; + Context[Cx.SpiShaderPosFormat] = 4; + Context[Cx.SpiPsInputEna] = 0x2; + Context[Cx.SpiPsInputAddr] = 0x2; + Context[Cx.SpiPsInControl] = 1; + Context[Cx.SpiPsInputCntl0] = 0; + Context[Cx.SpiVsOutConfig] = 0; + Context[Cx.PaClClipCntl] = 1u << 19; + Context[Cx.PaSuScModeCntl] = 0; + Context[Cx.PaClVteCntl] = 0x43F; + Context[Cx.DbDepthControl] = 0; + Context[Cx.DbRenderControl] = 0; + Context[Cx.DbShaderControl] = 0x10; + Context[Cx.CbColorControl] = (1u << 4) | (0xCCu << 16); + Context[Cx.CbBlend0Control] = 0; + Context[Cx.PaScAaConfig] = 0; + Context[Cx.DbEqaa] = 0; + Uconfig[Uc.VgtPrimitiveType] = Prim.TriangleList; + Viewport(0, 0, width, height); + ScreenScissor(0, 0, (int)width, (int)height); + } + + public static uint F(float value) => BitConverter.SingleToUInt32Bits(value); + + /// A D3D-style viewport: NDC y up maps to the top row, z [0,1] to [minZ, maxZ]. + public SceneState Viewport(float x, float y, float width, float height, float minZ = 0f, float maxZ = 1f) + { + Context[Cx.PaClVportXScale] = F(width / 2); + Context[Cx.PaClVportXOffset] = F(x + (width / 2)); + Context[Cx.PaClVportYScale] = F(-height / 2); + Context[Cx.PaClVportYOffset] = F(y + (height / 2)); + Context[Cx.PaClVportZScale] = F(maxZ - minZ); + Context[Cx.PaClVportZOffset] = F(minZ); + Context[Cx.PaScVportZMin0] = F(Math.Min(minZ, maxZ)); + Context[Cx.PaScVportZMax0] = F(Math.Max(minZ, maxZ)); + return this; + } + + public SceneState ScreenScissor(int left, int top, int right, int bottom) + { + Context[Cx.PaScScreenScissorTl] = (uint)(left & 0xFFFF) | ((uint)(top & 0xFFFF) << 16); + Context[Cx.PaScScreenScissorBr] = (uint)(right & 0xFFFF) | ((uint)(bottom & 0xFFFF) << 16); + return this; + } + + /// A colour target in slot . Format and number type as CB_COLOR_INFO encodes them. + public SceneState ColorTarget(uint slot, ulong address, uint width, uint height, uint format = 10, uint numberType = 0, + uint tileMode = 0, uint compSwap = 0, uint samplesLog2 = 0, uint lastSlice = 0, uint mipLevel = 0, uint maxMip = 0, + bool blendBypass = false, bool blendClamp = false) + { + var block = slot * Cx.CbColorSlotStride; + Context[Cx.CbColor0Base + block] = (uint)(address >> 8); + Context[Cx.CbColor0BaseExt + slot] = (uint)(address >> 40); + Context[Cx.CbColor0View + block] = (lastSlice << 13) | (mipLevel << 26); + Context[Cx.CbColor0Info + block] = (format << 2) | (numberType << 8) | (compSwap << 11) | + (blendClamp ? 1u << 15 : 0u) | (blendBypass ? 1u << 16 : 0u); + Context[Cx.CbColor0Attrib + block] = (samplesLog2 << 12) | (samplesLog2 << 15); + Context[Cx.CbColor0Attrib2 + slot] = (height - 1) | ((width - 1) << 14) | (maxMip << 28); + Context[Cx.CbColor0Attrib3 + slot] = (tileMode << 14) | (1u << 24); + return this; + } + + /// + /// A depth (and optional stencil) target. is DB_Z_INFO.FORMAT: + /// 1 = Z16, 3 = Z32_FLOAT. Bases must be 64 KiB aligned; the layout is 64KB_Z_X (SW mode 24). + /// + public SceneState DepthTarget(ulong depthAddress, uint width, uint height, uint zFormat, ulong stencilAddress = 0) + { + Context[Cx.DbZInfo] = zFormat | (24u << 4); + Context[Cx.DbStencilInfo] = stencilAddress != 0 ? 1u | (24u << 4) : 0u; + Context[Cx.DbZReadBase] = (uint)(depthAddress >> 8); + Context[Cx.DbZWriteBase] = (uint)(depthAddress >> 8); + Context[Cx.DbZReadBaseHi] = (uint)(depthAddress >> 40); + Context[Cx.DbZWriteBaseHi] = (uint)(depthAddress >> 40); + Context[Cx.DbStencilReadBase] = (uint)(stencilAddress >> 8); + Context[Cx.DbStencilWriteBase] = (uint)(stencilAddress >> 8); + Context[Cx.DbStencilReadBaseHi] = (uint)(stencilAddress >> 40); + Context[Cx.DbStencilWriteBaseHi] = (uint)(stencilAddress >> 40); + Context[Cx.DbDepthSizeXy] = (width - 1) | ((height - 1) << 16); + Context[Cx.DbDepthView] = 0; + Context[Cx.PaSuPolyOffsetDbFmtCntl] = zFormat == 3 ? (1u << 8) | (unchecked((uint)-23) & 0xFF) : unchecked((uint)-16) & 0xFF; + return this; + } + + public SceneState DepthControl(bool test, bool write, CompareFunc func, bool bounds = false, bool stencil = false, + CompareFunc stencilFunc = CompareFunc.Always, bool backFace = false, CompareFunc stencilFuncBack = CompareFunc.Always) + { + Context[Cx.DbDepthControl] = (stencil ? 1u : 0u) | (test ? 2u : 0u) | (write ? 4u : 0u) | (bounds ? 8u : 0u) | + ((uint)func << 4) | (backFace ? 1u << 7 : 0u) | ((uint)stencilFunc << 8) | ((uint)stencilFuncBack << 20); + return this; + } + + /// The vertex program at the ES address with its user data in GS user SGPRs (s8 onward). + public SceneState VertexShader(ulong address, params uint[] userData) + { + Shader[Sh.PgmLoEs] = (uint)(address >> 8); + Shader[Sh.PgmHiEs] = (uint)(address >> 40); + // VGPRS = 63 blocks of 4 (wave32 granule 8: generous), SGPRS generous. + Shader[Sh.PgmRsrc1Gs] = 0x3F | (0xF << 6); + Shader[Sh.PgmRsrc2Gs] = ((uint)userData.Length & 0x1F) << 1 | ((uint)userData.Length >> 5) << 27; + for (var index = 0u; index < userData.Length; index++) + { + Shader[Sh.UserDataGs0 + index] = userData[index]; + } + + return this; + } + + /// The pixel program with its user data in s0 onward; the primitive mask follows it. + public SceneState PixelShader(ulong address, params uint[] userData) + { + Shader[Sh.PgmLoPs] = (uint)(address >> 8); + Shader[Sh.PgmHiPs] = (uint)(address >> 40); + Shader[Sh.PgmRsrc1Ps] = 0x3F | (0xF << 6); + Shader[Sh.PgmRsrc2Ps] = ((uint)userData.Length & 0x1F) << 1 | ((uint)userData.Length >> 5) << 27; + for (var index = 0u; index < userData.Length; index++) + { + Shader[Sh.UserDataPs0 + index] = userData[index]; + } + + return this; + } + + /// + /// The compute program with its user data in s0 onward. Thread-group ids X/Y/Z follow + /// the user data when enabled; local ids arrive in v0 (and v1/v2 for 2D/3D groups). + /// + public SceneState ComputeShader(ulong address, uint threadsX, uint threadsY, uint threadsZ, uint[] userData, + bool groupIdX = true, bool groupIdY = false, bool groupIdZ = false, uint threadIdComponents = 0, uint ldsBytes = 0) + { + Shader[Sh.ComputePgmLo] = (uint)(address >> 8); + Shader[Sh.ComputePgmHi] = (uint)(address >> 40); + Shader[Sh.ComputePgmRsrc1] = 0x3F | (0xF << 6); + Shader[Sh.ComputePgmRsrc2] = (((uint)userData.Length & 0x1F) << 1) | (groupIdX ? 1u << 7 : 0u) | + (groupIdY ? 1u << 8 : 0u) | (groupIdZ ? 1u << 9 : 0u) | (threadIdComponents << 11) | + (((ldsBytes + 511) / 512) << 15); + Shader[Sh.ComputeNumThreadX] = threadsX; + Shader[Sh.ComputeNumThreadY] = threadsY; + Shader[Sh.ComputeNumThreadZ] = threadsZ; + for (var index = 0u; index < userData.Length; index++) + { + Shader[Sh.ComputeUserData0 + index] = userData[index]; + } + + return this; + } + + /// Writes every register, one packet each, in bank order. + public ScenePm4 Emit(ScenePm4 pm4) + { + foreach (var (offset, value) in Uconfig) + { + pm4.Uconfig(offset, value); + } + + foreach (var (offset, value) in Shader) + { + pm4.Shader(offset, value); + } + + foreach (var (offset, value) in Context) + { + pm4.Context(offset, value); + } + + return pm4; + } +} + +/// Resource descriptor words, encoded as the gfx10 hardware reads them. +internal static class Descriptors +{ + public const uint IdentitySelect = 0xFAC; + + /// A V# for structured (idxen) loads: bytes per record. + public static uint[] Buffer(ulong address, uint stride, uint records, uint format = 77, uint oobSelect = 0) => + [ + (uint)address, + ((uint)(address >> 32) & 0xFFFF) | (stride << 16), + records, + IdentitySelect | (format << 12) | (1u << 24) | (oobSelect << 28), + ]; + + /// + /// A T# over a linear or tiled image. 9 = 2D, 13 = 2D array, 11 = cube, 10 = 3D. Word 5 MAX_MIP is + /// the resource's mip count minus one, independent of the view's LAST_LEVEL; defaults to LAST_LEVEL. + /// + public static uint[] Texture(ulong address, uint format, uint width, uint height, uint type = 9, uint tileMode = 0, + uint baseLevel = 0, uint lastLevel = 0, uint depthOrLastLayer = 0, uint baseArray = 0, uint dstSelect = IdentitySelect, + uint minLodFixed = 0, uint? maxMip = null) + { + var shifted = address >> 8; + var lastX = width - 1; + var lastY = height - 1; + return + [ + (uint)shifted, + ((uint)(shifted >> 32) & 0xFF) | (minLodFixed << 8) | (format << 20) | ((lastX & 0x3) << 30), + ((lastX >> 2) & 0xFFF) | (lastY << 14), + dstSelect | (baseLevel << 12) | (lastLevel << 16) | (tileMode << 20) | (type << 28), + depthOrLastLayer | (baseArray << 16), + (maxMip ?? lastLevel) << 4, + 0, + 0, + ]; + } + + /// + /// An S#. Clamp 0 = wrap, 1 = mirror, 2 = clamp-to-last, 4 = clamp-half-border, 6 = clamp-to-border; filters 0 = point, 1 = bilinear; + /// mip filter 0 = none, 1 = point, 2 = linear. LOD fields are unsigned 4.8 fixed point; bias is signed 6.8. + /// + public static uint[] Sampler(uint clampX = 2, uint clampY = 2, uint clampZ = 2, CompareFunc depthCompare = CompareFunc.Never, + uint magFilter = 0, uint minFilter = 0, uint mipFilter = 0, float minLod = 0, float maxLod = 15, float lodBias = 0, + uint borderColorType = 0, uint maxAniso = 0, bool forceUnnormalized = false, uint zFilter = 0) + { + static uint Fixed(float value, int bits) => (uint)Math.Clamp((int)MathF.Round(value * 256f), 0, (1 << bits) - 1); + var bias = (uint)((int)MathF.Round(lodBias * 256f) & 0x3FFF); + return + [ + clampX | (clampY << 3) | (clampZ << 6) | (maxAniso << 9) | ((uint)depthCompare << 12) | (forceUnnormalized ? 1u << 15 : 0u), + Fixed(minLod, 12) | (Fixed(maxLod, 12) << 12), + bias | (magFilter << 20) | (minFilter << 22) | (zFilter << 24) | (mipFilter << 26), + borderColorType << 30, + ]; + } + + public static uint Lo(ulong address) => (uint)address; + + public static uint Hi(ulong address) => (uint)(address >> 32); +} diff --git a/tools/SharpEmu.Tools.PerfAudit/SharpEmu.Tools.PerfAudit.csproj b/tools/SharpEmu.Tools.PerfAudit/SharpEmu.Tools.PerfAudit.csproj new file mode 100644 index 000000000..6c46cdeb0 --- /dev/null +++ b/tools/SharpEmu.Tools.PerfAudit/SharpEmu.Tools.PerfAudit.csproj @@ -0,0 +1,25 @@ + + + + + Exe + SharpEmu.Tools.PerfAudit + SharpEmu.PerfAudit + true + false + true + true + false + false + $(NoWarn);CS1591;CS8981 + + + + + + + + + + From a6b2978dfcb697ae8c97994ab8e883c2423834b5 Mon Sep 17 00:00:00 2001 From: kuba Date: Mon, 26 Oct 2026 17:21:55 +0100 Subject: [PATCH 30/82] [Tests] Allow the Metal performance audit benchmark friend Scope the PerfAudit exception to the Metal package and name the architecture contract to include approved offline benchmarks. Validated all 75 ArchitectureRules cases in Release with zero failures or skips. --- .../ExtractedPackageContractTests.cs | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/tests/SharpEmu.Libs.Tests/ArchitectureRules/ExtractedPackageContractTests.cs b/tests/SharpEmu.Libs.Tests/ArchitectureRules/ExtractedPackageContractTests.cs index 0772485f3..5f06c41a7 100644 --- a/tests/SharpEmu.Libs.Tests/ArchitectureRules/ExtractedPackageContractTests.cs +++ b/tests/SharpEmu.Libs.Tests/ArchitectureRules/ExtractedPackageContractTests.cs @@ -73,15 +73,22 @@ public void SharedAssemblyFriendsAreLimitedToExistingCoreAndTests(string project [Theory] [MemberData(nameof(Packages))] - public void ExtractedPackagesGrantInternalAccessOnlyToTests(string directory, string assembly) + public void ExtractedPackagesGrantInternalAccessOnlyToTestsAndApprovedBenchmarks(string directory, string assembly) { var project = XDocument.Load(RepoPath($"{directory}/{assembly}.csproj")); - // Friends are test assemblies only: the shared Libs suite, or the - // package's own dedicated test project (e.g. SharpEmu.Gpu.Metal.Tests), plus the - // offline shader-compile benchmark, which times internal compiler phases. + // Keep friend access limited to tests and named offline benchmarks. + // PerfAudit exercises Metal's internal resource paths; it does not need + // friend access to the other extracted packages. + var allowedFriends = new List + { + "SharpEmu.Libs.Tests", $"{assembly}.Tests", $"{assembly}.LeakTests", + "SharpEmu.Tools.ShaderCompileBench", + }; + if (assembly == "SharpEmu.Gpu.Metal") + allowedFriends.Add("SharpEmu.Tools.PerfAudit"); + foreach (var friend in project.Descendants("InternalsVisibleTo")) - Assert.Contains(friend.Attribute("Include")!.Value, - new[] { "SharpEmu.Libs.Tests", $"{assembly}.Tests", $"{assembly}.LeakTests", "SharpEmu.Tools.ShaderCompileBench" }); + Assert.Contains(friend.Attribute("Include")!.Value, allowedFriends); } [Theory] From 3cf2d655e485aea8faf55ce767b6bdfba192756f Mon Sep 17 00:00:00 2001 From: kuba Date: Mon, 26 Oct 2026 19:33:23 +0100 Subject: [PATCH 31/82] [GPU] Accelerate portable dword vertex fetches --- docs/performance/benchmark-index.md | 7 ++ docs/performance/combined-audit.md | 4 +- docs/performance/shadow-mesh-fetch.md | 118 ++++++++++++++++++ .../Gen5MslCodegenOptions.cs | 10 +- .../Gen5MslTranslator.cs | 43 +++++++ .../MslRuntimeFormatLoadTests.Benchmark.cs | 84 +++++++++++++ .../MslRuntimeFormatLoadTests.cs | 116 +++++++++++++++++ .../Program.cs | 10 +- 8 files changed, 387 insertions(+), 5 deletions(-) create mode 100644 docs/performance/shadow-mesh-fetch.md create mode 100644 tests/SharpEmu.ShaderCompiler.Metal.Tests/MslRuntimeFormatLoadTests.Benchmark.cs create mode 100644 tests/SharpEmu.ShaderCompiler.Metal.Tests/MslRuntimeFormatLoadTests.cs diff --git a/docs/performance/benchmark-index.md b/docs/performance/benchmark-index.md index 2b887fda6..5d1b66241 100644 --- a/docs/performance/benchmark-index.md +++ b/docs/performance/benchmark-index.md @@ -102,3 +102,10 @@ a mismatch exits with an error. REUSE passes for all 2,816 files and whitespace checks pass. The emulator algorithms are unchanged by this packaging follow-up; the Metal project only grants the uniquely named tool access to its internals. Full-suite/CI counts above belong to the original combined implementation. + +## Portable vertex-fetch follow-up + +See [shadow mesh fetch measurements](shadow-mesh-fetch.md) for the subsequent runtime +localization, generic formatted-load optimization, descriptor/bounds tests, synthetic +GPU benchmark commands and fallback costs. This follow-up has no validated gameplay +FPS result. diff --git a/docs/performance/combined-audit.md b/docs/performance/combined-audit.md index d3c425cac..27c0a9913 100644 --- a/docs/performance/combined-audit.md +++ b/docs/performance/combined-audit.md @@ -5,7 +5,9 @@ SPDX-License-Identifier: GPL-2.0-or-later --> This integrates the round-one audit (`cde04b713e60de7ab02dfa092e8f91d086df0c83`) and the valid round-two changes. Validation uses fabricated guest programs, -synthetic descriptors and real Metal device tests. No commercial game was run. +synthetic descriptors and real Metal device tests. No commercial game was run for +that integration. A later user-authorized diagnostic and the resulting generic +[portable vertex-fetch follow-up](shadow-mesh-fetch.md) are documented separately. Microkernel, byte-work and microbenchmark results do not establish game performance. See the [change and benchmark index](benchmark-index.md) for round-one coverage, diff --git a/docs/performance/shadow-mesh-fetch.md b/docs/performance/shadow-mesh-fetch.md new file mode 100644 index 000000000..5c81c67ad --- /dev/null +++ b/docs/performance/shadow-mesh-fetch.md @@ -0,0 +1,118 @@ + + +# Portable vertex-fetch follow-up + +The user-authorized October 26 diagnostic reproduced 2.49–2.50 guest FPS with +scripted input. Five six-layer depth/shadow encoders averaged 55.9–56.7 ms each +(281.3 ms summed means). Metal's vertex/tiling phase accounted for 99.7% of their +aggregate measured duration; even the 32×32×6 target took about 56 ms. These are +encoder observations, not an exclusive frame-time decomposition. The exact +visual gameplay scene was not confirmed. + +The three mesh shaders in those encoders contain 32, 34 and 35 dynamic formatted +buffer-fetch sites. Sites in different branches need not all execute. Offline +translation matched their recorded MSL keys exactly: + +| Guest shader hash | Recorded MSL library key | Dynamic fetch sites | +| --- | --- | ---: | +| `76DB3753289971F0` | `A66DB853C407D44A` | 32 | +| `B482CD3A01A37F0F` | `1E6B8DB38CF962DD` | 34 | +| `9FC48AA9F987C1A4` | `4E13425202FEDA43` | 35 | + +No game code, captured MSL, descriptors or raw runtime logs are checked in. + +## Change + +Portable buffer descriptors retain their run-time formats and strides. Previously, +formatted loads decoded four component layouts and numeric conversions even for +identity-swizzled 32-bit float/integer streams. A guarded path now loads the required +dwords together when the **current** descriptor proves that shape and the full requested +range is in bounds. Float3/integer3 fetched as XYZW supplies the appropriate W=1. +When that full run would cross the buffer end, or other components are missing, the +general decoder checks only the selected components and supplies the missing values. +Unaligned accesses use the existing word-assembly helper. + +Other formats/swizzles, out-of-bounds elements, placed/swizzled addresses, device +address descriptors and D16 packing retain the general decoder. EXEC masking and +GLC coherent reads remain in effect. `SHARPEMU_MSL_RUNTIME_DWORD_FORMAT=0` restores +the old decoder path for comparison. No shader hash or title controls this behavior. + +The offline shader compilation tool also now applies the runtime's translation +switches before decoding portable specializations. Without this, erased format +and stride fields were baked into regenerated shaders: only 53/498 library keys +matched the capture. With runtime switches, 497/497 generated keys matched (two +recorded variants collapse to one source). This is an offline-tool correction; +the runtime already applied those switches. + +## Reproduction + +Run the synthetic tests and opt-in benchmark serially on a Metal-capable Mac: + +```sh +SHARPEMU_TEST_REQUIRE_DEVICE=1 dotnet test tests/SharpEmu.ShaderCompiler.Metal.Tests \ + -c Release --filter MslRuntimeFormatLoadTests \ + --settings docs/performance/shader-serial.runsettings --blame-hang-timeout 120s + +SHARPEMU_TEST_REQUIRE_DEVICE=1 SHARPEMU_GPU_BENCH=1 \ + dotnet test tests/SharpEmu.ShaderCompiler.Metal.Tests -c Release \ + --filter MslRuntimeFormatLoadTests.BenchmarkDynamicFormats \ + --settings docs/performance/shader-serial.runsettings --blame-hang-timeout 120s \ + --logger 'console;verbosity=detailed' +``` + +The benchmark translates a fabricated guest program both ways, checks equal output, +and times 1,024 groups × 32 threads × 256 formatted loads using Metal command-buffer +GPU timestamps. Input addresses vary in the loop; each thread writes a separate +output outside the input range. Four outer rounds alternate before/after ordering; +each arm reports the best of five GPU samples after one warmup. Other apps stay open. +These cache-resident compute-kernel timings isolate fetch overhead. They do not +measure mesh-stage occupancy, a complete shadow pass, or game FPS. + +Correctness checks reuse the same compiled shaders while changing all 128 descriptor +format codes, then cover each supported dword format with alternate swizzles and +unaligned/end-of-buffer ranges. Five test cases cover X/XY/XYZ/XYZW and coherent XYZW: +1,600 descriptor/bounds combinations, compared with the old decoder, plus independent +CPU expectations for optimized layouts and their missing components. + +## Measured GPU results + +Apple M4, macOS 27.2 (26B5112a), x64 .NET 10.0.3 host; GPU timestamps measure device +execution. Each value is the median of four alternating-arm round minima, as described +above. These are synthetic operation measurements, **not game FPS estimates**. + +| Workload | Decoder ms | Fast path enabled ms | Change | +| --- | ---: | ---: | ---: | +| 32x4-float-identity | 2.0751 | 0.1396 | -93.27% | +| 32x4-uint-identity | 1.7670 | 0.1425 | -91.94% | +| 32x3-float-missing-w | 1.8795 | 0.1423 | -92.43% | +| 32x1-float-missing-yzw | 1.4884 | 1.5353 | +3.15% | +| 16x4-float-fallback | 2.1888 | 2.2610 | +3.30% | +| 32x4-float-swizzle-fallback | 2.0894 | 2.1261 | +1.76% | + +The supported paths improve 12.4–14.9× in this cache-resident benchmark. Fallback +cases cost 1.8–3.3% more; this is an accepted conditional-path cost, not a universal +speedup. Two broader missing-component implementations were rejected because their +fallback costs increased 26–27%. Those implementations are absent from the final diff. + +## Validation + +- Release Metal shader suite: **3,302 passed, 0 failed, 0 reported skips** with a required + Metal device and serial execution. The benchmark is opt-in and returns without timing + when its environment flag is absent; its measured run is listed separately below. +- Focused correctness plus opt-in benchmark: **6 passed, 0 failed**, including 1,600 + changing descriptor/bounds combinations and six alternating-arm timing workloads. +- Offline retranslation: **896 records, 0 failures, 0 unstable outputs**, 497 distinct + MSL sources. The three regenerated shadow mesh libraries all compile with the OS Metal + compiler (MSL 3.2, relaxed arithmetic, precise floating-point functions). This is library + compilation, not a draw replay or full pipeline validation. +- REUSE lint: compliant, 2,819/2,819 files have copyright and license information. + +## Acceptance limits + +The runtime capture identifies an expensive path. This patch accelerates a synthetic +operation present in that path; it has not yet established how frequently the game's +live descriptors take it or a gameplay FPS improvement. No additional game run was +performed for the implementation. The retained original runtime payload has hashes +but no verified source commit; matching the three generated MSL bodies establishes +shader-source correspondence, not whole-binary provenance. diff --git a/src/SharpEmu.ShaderCompiler.Metal/Gen5MslCodegenOptions.cs b/src/SharpEmu.ShaderCompiler.Metal/Gen5MslCodegenOptions.cs index bd73b4910..c4eff0293 100644 --- a/src/SharpEmu.ShaderCompiler.Metal/Gen5MslCodegenOptions.cs +++ b/src/SharpEmu.ShaderCompiler.Metal/Gen5MslCodegenOptions.cs @@ -97,6 +97,10 @@ namespace SharpEmu.ShaderCompiler.Metal; /// threads the dispatch launches rather than the device's 1024. Only a size the dispatch is /// known to use (more than one thread) is declared. SHARPEMU_MSL_TGHINT=0 leaves it /// out. +/// Read identity-swizzled 32-bit formatted buffer +/// components as one checked run when the current descriptor permits it. Other formats, +/// swizzles and bounds retain the general decoder. SHARPEMU_MSL_RUNTIME_DWORD_FORMAT=0 +/// disables this run-time fast path. /// Lower V_MIN/MAX/MIN3/MAX3/MED3_F32 outside IEEE mode to float /// compares and selects on the operand bits, where that gives the bit-level helpers' answer for /// every input: against an inline constant, and between two registers while FLOAT_MODE flushes @@ -139,7 +143,8 @@ public sealed record Gen5MslCodegenOptions( bool NativeFloatMinMax = true, bool ConstantTableSlots = true, bool FastReciprocals = true, - bool ContractPixelCompute = false) + bool ContractPixelCompute = false, + bool RuntimeDwordFormatLoads = true) { public static Gen5MslCodegenOptions FromEnvironment { get; } = new( StructuredControlFlow: Environment.GetEnvironmentVariable("SHARPEMU_MSL_STRUCTURED") != "0", @@ -166,5 +171,6 @@ public sealed record Gen5MslCodegenOptions( NativeFloatMinMax: Environment.GetEnvironmentVariable("SHARPEMU_MSL_FMINMAX") != "0", ConstantTableSlots: Environment.GetEnvironmentVariable("SHARPEMU_MSL_TABLEPAD") != "0", FastReciprocals: Environment.GetEnvironmentVariable("SHARPEMU_MSL_FASTRCP") != "0", - ContractPixelCompute: Environment.GetEnvironmentVariable("SHARPEMU_MSL_CONTRACT") == "1"); + ContractPixelCompute: Environment.GetEnvironmentVariable("SHARPEMU_MSL_CONTRACT") == "1", + RuntimeDwordFormatLoads: Environment.GetEnvironmentVariable("SHARPEMU_MSL_RUNTIME_DWORD_FORMAT") != "0"); } diff --git a/src/SharpEmu.ShaderCompiler.Metal/Gen5MslTranslator.cs b/src/SharpEmu.ShaderCompiler.Metal/Gen5MslTranslator.cs index b3034b2f2..0048ef01b 100644 --- a/src/SharpEmu.ShaderCompiler.Metal/Gen5MslTranslator.cs +++ b/src/SharpEmu.ShaderCompiler.Metal/Gen5MslTranslator.cs @@ -2686,6 +2686,44 @@ private void EmitDecodedBufferFormatLoad( var entry = Temp("uint", $"sharpemu_gfx10_formats[({word3} >> 12) & 0x7Fu]"); var dataFormat = Temp("uint", $"{entry} & 0xFFu"); var numberFormat = Temp("uint", $"({entry} >> 8) & 0xFFu"); + // Portable vertex layouts keep word 3 dynamic. The common whole-dword case + // needs neither four layout lookups nor four numeric conversions. Check the + // current word, including every selected swizzle, on each invocation: this + // must not specialize a descriptor that can change between draws. + var dwordRun = _options.RuntimeDwordFormatLoads && !d16.Enabled && + byteAddress.Contiguous && memory.LoadWords is not null && + !TryParseUIntLiteral(descriptorWord3, out _); + if (dwordRun) + { + var formats = componentCount switch + { + 1 => $"{dataFormat} == 4u || {dataFormat} == 11u || {dataFormat} == 13u || {dataFormat} == 14u", + 2 => $"{dataFormat} == 11u || {dataFormat} == 13u || {dataFormat} == 14u", + _ => $"{dataFormat} == 13u || {dataFormat} == 14u", + }; + var selectorMask = (1u << (int)(componentCount * 3)) - 1; + // A float3 XYZW fetch may read one extra dword only when it is in + // bounds; the final short element retains the general decoder. + var inRange = memory.ElementInBounds(byteAddress.Base, $"{componentCount * sizeof(uint)}u"); + Line($"if (({formats}) && ({numberFormat} == 4u || {numberFormat} == 5u || {numberFormat} == 7u) && " + + $"({word3} & 0x{selectorMask:X}u) == 0x{0xFACu & selectorMask:X}u && {inRange})"); + Line("{"); + _indent++; + var words = Temp("uint4", memory.LoadWords!(componentCount, byteAddress.Base)); + if (componentCount == 4) + { + Line($"{words}.w = {dataFormat} == 13u ? sharpemu_format_one({numberFormat}) : {words}.w;"); + } + for (uint component = 0; component < componentCount; component++) + { + StoreVector(vectorData + component, $"{words}[{component}]"); + } + _indent--; + Line("}"); + Line("else"); + Line("{"); + _indent++; + } // A format the table cannot decode reads every destination as zero there. var valid = memory.ZeroInvalidFormat ? Temp("bool", $"{dataFormat} != 0u") : null; var canonical = new string[4]; @@ -2731,6 +2769,11 @@ private void EmitDecodedBufferFormatLoad( } StoreFormatLoadComponents(vectorData, values, d16, numberFormat); + if (dwordRun) + { + _indent--; + Line("}"); + } } private static bool TryParseUIntLiteral(string text, out uint value) diff --git a/tests/SharpEmu.ShaderCompiler.Metal.Tests/MslRuntimeFormatLoadTests.Benchmark.cs b/tests/SharpEmu.ShaderCompiler.Metal.Tests/MslRuntimeFormatLoadTests.Benchmark.cs new file mode 100644 index 000000000..119cc74ef --- /dev/null +++ b/tests/SharpEmu.ShaderCompiler.Metal.Tests/MslRuntimeFormatLoadTests.Benchmark.cs @@ -0,0 +1,84 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using Xunit; + +namespace SharpEmu.ShaderCompiler.Metal.Tests; + +public sealed partial class MslRuntimeFormatLoadTests +{ + // Only synthetic instructions and data. Adjacent input records vary each iteration; + // every lane writes its own output, outside the input area, including across groups. + private static Gen5ComputeFixture BenchmarkFixture() + { + var words = new List + { + 0x34120084, // v_lshlrev_b32 v9, 4, v0 + 0x7E020280, // v_mov_b32 v1, 0 + 0xBE8003FF, 256, // s_mov_b32 s0, 256 + }; + var loop = words.Count; + words.AddRange([ + 0xE00C1400, 0x80020409, // buffer_load_format_xyzw v[4:7], v9, s[8:11], offset:1024 offen + 0xBF8C0070, + 0x4A020304, 0x4A020305, 0x4A020306, 0x4A020307, // sum all four words + 0x4A121290, // v_add_nc_u32 v9, 16, v9 + 0x361212FF, 0x3F0, // v_and_b32 v9, 0x3f0, v9 + 0x81808100, // s_sub_i32 s0, s0, 1 + 0xBF078000, // s_cmp_lg_u32 s0, 0 + ]); + words.Add(0xBF850000 | (ushort)(short)(loop - words.Count - 1)); + words.AddRange([ + 0x34140082, // v_lshlrev_b32 v10, 2, v0 + 0x7E120210, // v_mov_b32 v9, s16 (group x) + 0x34121287, // v_lshlrev_b32 v9, 7, v9 + 0x4A121509, // v_add_nc_u32 v9, v9, v10 + 0x4A1212FF, 4096, // v_add_nc_u32 v9, 4096, v9 + 0xE0701000, 0x80030109, // buffer_store_dword v1, v9, s[12:15] offen + 0xBF810000, + ]); + return new("runtime-format-benchmark", words.ToArray(), 12, 4096 + 1024 * 128); + } + + [Fact] + public void BenchmarkDynamicFormats() + { + if (Environment.GetEnvironmentVariable("SHARPEMU_GPU_BENCH") != "1") return; + if (!RequireDevice()) return; + var fixture = BenchmarkFixture(); + var (before, layout) = Compile(fixture, false, 32); + var (after, _) = Compile(fixture, true, 32); + var data = Enumerable.Range(0, fixture.StoreBackingBytes).Select(i => (byte)(i * 37 + 11)).ToArray(); + foreach (var (format, swizzle, name) in new (uint, uint, string)[] + { + (77, 0xFAC, "32x4-float-identity"), + (75, 0xFAC, "32x4-uint-identity"), + (74, 0xFAC, "32x3-float-missing-w"), + (22, 0xFAC, "32x1-float-missing-yzw"), + (71, 0xFAC, "16x4-float-fallback"), + (77, 0x977, "32x4-float-swizzle-fallback"), + }) + { + var push = Push(layout, (format << 12) | swizzle, 0); + // Prove equal outputs and that the fixture really touched all output lanes. + Assert.True(MetalNative.TryCompileLibrary(before.Source, out var oldLibrary, out var oldError), oldError); + Assert.True(MetalNative.TryCompileLibrary(after.Source, out var newLibrary, out var newError), newError); + Assert.True(MetalNative.TryExecuteWithArgumentBuffer(oldLibrary, before.EntryPoint, data, push, + before.ArgumentLayout!, 32, out var expected, out var oldRunError), oldRunError); + Assert.True(MetalNative.TryExecuteWithArgumentBuffer(newLibrary, after.EntryPoint, data, push, + after.ArgumentLayout!, 32, out var actual, out var newRunError), newRunError); + Assert.Equal(expected, actual); + Assert.False(actual.AsSpan(4096, 128).SequenceEqual(data.AsSpan(4096, 128))); + for (var round = 0; round < 4; round++) + { + foreach (var fast in round % 2 == 0 ? new[] { false, true } : new[] { true, false }) + { + Assert.True(MetalNative.TryBenchmark(fast ? after : before, data, push, 1024, 32, 1, + out var result, out var error), error); + Assert.False(result.StepCapHit); + output.WriteLine(FormattableString.Invariant($"{name} round={round} fast={fast} gpu_ms={result.GpuMilliseconds:F6}")); + } + } + } + } +} diff --git a/tests/SharpEmu.ShaderCompiler.Metal.Tests/MslRuntimeFormatLoadTests.cs b/tests/SharpEmu.ShaderCompiler.Metal.Tests/MslRuntimeFormatLoadTests.cs new file mode 100644 index 000000000..ed60921c6 --- /dev/null +++ b/tests/SharpEmu.ShaderCompiler.Metal.Tests/MslRuntimeFormatLoadTests.cs @@ -0,0 +1,116 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using System.Buffers.Binary; +using SharpEmu.ShaderCompiler.Resources; +using Xunit; +using Xunit.Abstractions; + +namespace SharpEmu.ShaderCompiler.Metal.Tests; + +public sealed partial class MslRuntimeFormatLoadTests(ITestOutputHelper output) +{ + private const int BufferBytes = 128; + + // Independent read and write descriptors, as for a vertex stream and an output buffer. + // s0 is the byte offset, supplied at execution rather than translation time. + private static Gen5ComputeFixture Fixture(uint components, bool glc = false) => new( + "runtime-format-load", + [ + 0x7E020200, // v_mov_b32 v1, s0 + 0xE0001000 | ((components - 1) << 18) | (glc ? 1u << 14 : 0), 0x80020401, + 0xE0700000, 0x80030400, + 0xE0700004, 0x80030500, + 0xE0700008, 0x80030600, + 0xE070000C, 0x80030700, + 0xBF810000, + ], 12, BufferBytes); + + private static (Gen5MslShader Shader, BindingLayout Layout) Compile( + Gen5ComputeFixture fixture, bool fast, uint threads = 1) + { + var program = Gen5ComputeFixtures.DecodeOrThrow(fixture); + var plan = ShaderResourcePlan.Extract(program, ShaderStage.Compute, 0x1234, 0, 16); + var resources = ResourceMaterializer.ApplyTo(plan, ResourceSpecialization.Default(plan.Info)); + var layout = BindingLayout.Allocate(resources.Info, + BindingLayout.CollectUserDataRegisters(program, 0, 16), + false, false, false, portableBuffers: true); + var request = new ShaderCompileRequest(plan, resources, layout) + { + LocalSizeX = threads, + ComputeSystemRegisters = new Gen5ComputeSystemRegisters(16, null, null, null), + }; + Assert.True(Gen5MslTranslator.TryCompileProgram(request, out var shader, out var error, + new Gen5MslCodegenOptions(RuntimeDwordFormatLoads: fast, ForceRegisterInitialization: true)), error); + return (shader, layout); + } + + private static byte[] Push(BindingLayout layout, uint word3, uint offset) + { + var push = new byte[PushData.ByteSize]; + for (var i = 0; i < layout.UserDataRegisters.Count; i++) + { + if (layout.UserDataRegisters[i] == 0) + BinaryPrimitives.WriteUInt32LittleEndian(push.AsSpan(i * 4), offset); + } + for (uint i = 0; i < layout.MemoryOffsetCount; i++) + BinaryPrimitives.WriteUInt32LittleEndian(push.AsSpan((int)(layout.FirstBufferWordDword() + i * 2 + 1) * 4), word3); + return push; + } + + private bool RequireDevice() + { + if (MetalNative.IsAvailable) return true; + Assert.False(Environment.GetEnvironmentVariable("SHARPEMU_TEST_REQUIRE_DEVICE") == "1", + "A Metal device is required for runtime-format validation."); + output.WriteLine("No Metal device; runtime-format execution skipped."); + return false; + } + + [Theory] + [InlineData(1u, false)] + [InlineData(2u, false)] + [InlineData(3u, false)] + [InlineData(4u, false)] + [InlineData(4u, true)] + public void ChangingDescriptorsAndBoundsMatchTheGeneralDecoder(uint components, bool glc) + { + var (fast, layout) = Compile(Fixture(components, glc), true); + var (slow, _) = Compile(Fixture(components, glc), false); + Assert.Contains($"sharpemu_load_words{(glc ? "_coherent" : "")}<{components}>", fast.Source); + Assert.NotEqual(fast.Source, slow.Source); + if (!RequireDevice()) return; + Assert.True(MetalNative.TryCompileLibrary(fast.Source, out var fastLibrary, out var fastError), fastError); + Assert.True(MetalNative.TryCompileLibrary(slow.Source, out var slowLibrary, out var slowError), slowError); + // Same two compiled libraries, changing the descriptor each time. All 128 + // unified-format codes and nonidentity/constant selectors exercise fallback. + var cases = Enumerable.Range(0, 128).Select(format => ((uint)format, 0xFACu, 32u)) + .Concat(from format in new uint[] { 20, 21, 22, 62, 63, 64, 72, 73, 74, 75, 76, 77 } + from swizzle in new uint[] { 0xFAC, 0x977, 0x204, 0xFFF } + from offset in new uint[] { 33, 112, 113, 124 } + select (format, swizzle, offset)); + foreach (var (format, swizzle, offset) in cases) + { + var data = Enumerable.Range(0, BufferBytes).Select(i => (byte)(i * 37 + 11)).ToArray(); + var push = Push(layout, (format << 12) | swizzle, offset); + Assert.True(MetalNative.TryExecuteWithArgumentBuffer(slowLibrary, slow.EntryPoint, data, push, + slow.ArgumentLayout!, 1, out var expected, out var slowRunError), slowRunError); + Assert.True(MetalNative.TryExecuteWithArgumentBuffer(fastLibrary, fast.EntryPoint, data, push, + fast.ArgumentLayout!, 1, out var actual, out var fastRunError), fastRunError); + Assert.True(expected.AsSpan().SequenceEqual(actual), + $"components={components} glc={glc} format={format} swizzle=0x{swizzle:X} offset={offset}"); + // Independent reference for all optimized formats, including missing components. + var present = format switch { >= 20 and <= 22 => 1u, >= 62 and <= 64 => 2u, + >= 72 and <= 74 => 3u, >= 75 and <= 77 => 4u, _ => 0u }; + if (present != 0 && swizzle == 0xFAC) + { + var valid = offset <= BufferBytes - Math.Min(components, present) * 4; + var one = format is 22 or 64 or 74 or 77 ? 0x3F800000u : 1u; + for (var c = 0; c < components; c++) + Assert.Equal(!valid ? 0u : c >= present ? (c == 3 ? one : 0u) : + BinaryPrimitives.ReadUInt32LittleEndian(data.AsSpan((int)offset + c * 4)), + BinaryPrimitives.ReadUInt32LittleEndian(actual.AsSpan(c * 4))); + } + } + } +} diff --git a/tools/SharpEmu.Tools.ShaderCompileBench/Program.cs b/tools/SharpEmu.Tools.ShaderCompileBench/Program.cs index ff86e5acc..e048a6848 100644 --- a/tools/SharpEmu.Tools.ShaderCompileBench/Program.cs +++ b/tools/SharpEmu.Tools.ShaderCompileBench/Program.cs @@ -91,6 +91,7 @@ private static int Main(string[] argv) var items = list.LoadedComputes(); Console.Error.WriteLine($"computes={items.Count}"); MetalGpuBackend.Register(); + MetalRenderHost.ApplyTranslationSwitches(); var compiler = SharpEmu.Libs.Gpu.GuestGpu.Current; // The learned device data loads change the MSL; seed them as a start of the title would. if (Arg("loads", "") is { Length: > 0 } loads) @@ -225,8 +226,13 @@ private static int GraphicsMatch(string dir) { MetalGpuBackend.Register(); var compiler = SharpEmu.Libs.Gpu.GuestGpu.Current; - // The runtime turns mip statistics on with its command stream (MetalRenderHost.LodStatistics). - SharpEmu.ShaderCompiler.Resources.ResourceMaterializer.RecordMipStatistics = Arg("mipstats", "1") == "1"; + // A portable specialization has erased its stride/format. Reproduce the runtime + // switches, or those erased values get baked into the offline translation. + MetalRenderHost.ApplyTranslationSwitches(); + if (_args.ContainsKey("mipstats")) + { + SharpEmu.ShaderCompiler.Resources.ResourceMaterializer.RecordMipStatistics = Arg("mipstats", "1") == "1"; + } if (Arg("loads", "") is { Length: > 0 } loads) { var learned = MetalDeviceDataLoadsStore.Load(Path.Combine(dir, loads), out _); From 52f07c7b37570adaad7317b8196a86f9bb4d6548 Mon Sep 17 00:00:00 2001 From: kuba Date: Mon, 26 Oct 2026 20:21:56 +0100 Subject: [PATCH 32/82] [GPU] Record matched Silent Hill shadow-pass runtime results --- docs/performance/benchmark-index.md | 10 +-- docs/performance/combined-audit.md | 3 +- docs/performance/shadow-mesh-fetch.md | 94 ++++++++++++++++++++++++--- 3 files changed, 94 insertions(+), 13 deletions(-) diff --git a/docs/performance/benchmark-index.md b/docs/performance/benchmark-index.md index 5d1b66241..c64fa606c 100644 --- a/docs/performance/benchmark-index.md +++ b/docs/performance/benchmark-index.md @@ -89,8 +89,8 @@ stalls remain unresolved; the full serial run passes unchanged assertions. The original combined validation totals are **16,741 passed, 0 failed, 13 existing skips**, plus REUSE, script checks and successful macOS CI at `18323b718`. -No commercial game was run. Benchmark smoke checks for this packaging change are -reported separately from those full-suite results. +No commercial game was run for that original integration. Benchmark smoke checks +for this packaging change are reported separately from those full-suite results. ## Benchmark packaging validation @@ -107,5 +107,7 @@ Full-suite/CI counts above belong to the original combined implementation. See [shadow mesh fetch measurements](shadow-mesh-fetch.md) for the subsequent runtime localization, generic formatted-load optimization, descriptor/bounds tests, synthetic -GPU benchmark commands and fallback costs. This follow-up has no validated gameplay -FPS result. +GPU benchmark commands and fallback costs. The subsequent matched Silent Hill +rooftop/title comparison observes 2.49 → 3.98 median guest FPS and 56.509 → 26.753 ms +mean shadow encoders. Its conditions, failed captures and remaining bottlenecks are +listed in that report; this is not a full-game gameplay benchmark. diff --git a/docs/performance/combined-audit.md b/docs/performance/combined-audit.md index 27c0a9913..c3ec9ace9 100644 --- a/docs/performance/combined-audit.md +++ b/docs/performance/combined-audit.md @@ -7,7 +7,8 @@ This integrates the round-one audit (`cde04b713e60de7ab02dfa092e8f91d086df0c83`) and the valid round-two changes. Validation uses fabricated guest programs, synthetic descriptors and real Metal device tests. No commercial game was run for that integration. A later user-authorized diagnostic and the resulting generic -[portable vertex-fetch follow-up](shadow-mesh-fetch.md) are documented separately. +[portable vertex-fetch follow-up](shadow-mesh-fetch.md) are documented separately, +including the later matched title-scene runtime comparison. Microkernel, byte-work and microbenchmark results do not establish game performance. See the [change and benchmark index](benchmark-index.md) for round-one coverage, diff --git a/docs/performance/shadow-mesh-fetch.md b/docs/performance/shadow-mesh-fetch.md index 5c81c67ad..7788a5646 100644 --- a/docs/performance/shadow-mesh-fetch.md +++ b/docs/performance/shadow-mesh-fetch.md @@ -7,8 +7,8 @@ The user-authorized October 26 diagnostic reproduced 2.49–2.50 guest FPS with scripted input. Five six-layer depth/shadow encoders averaged 55.9–56.7 ms each (281.3 ms summed means). Metal's vertex/tiling phase accounted for 99.7% of their aggregate measured duration; even the 32×32×6 target took about 56 ms. These are -encoder observations, not an exclusive frame-time decomposition. The exact -visual gameplay scene was not confirmed. +encoder observations, not an exclusive frame-time decomposition. The initial capture did not establish the visual scene. The later comparison below +confirms the rooftop/title scene with screenshots. The three mesh shaders in those encoders contain 32, 34 and 35 dynamic formatted buffer-fetch sites. Sites in different branches need not all execute. Offline @@ -108,11 +108,89 @@ fallback costs increased 26–27%. Those implementations are absent from the fin compilation, not a draw replay or full pipeline validation. - REUSE lint: compliant, 2,819/2,819 files have copyright and license information. +## Silent Hill runtime comparison + +A subsequent authorized comparison used only **SILENT HILL: The Short Message** +(PPSA10112), at 1920×1080, render scale 1, VSync on, HDR auto and strict compute on. +Screenshots confirm the rooftop/title menu. This is not a traversal of playable levels. +Both processes were freshly published in Release from clean commit +`3cf2d655e485aea8faf55ce767b6bdfba192756f`: self-contained x64 emulator and native +ARM64 GPU host, .NET SDK 10.0.103/runtime 10.0.3, ReadyToRun disabled in both arms. +Source provenance and both payloads' SHA256 manifests were recorded before launch. + +The same executable and independent APFS copies of one saved user/cache state were +used sequentially. The sole difference between final arms was +`SHARPEMU_MSL_RUNTIME_DWORD_FORMAT=0` versus `1`. Both used +`SHARPEMU_SHADER_PREWARM=0` to exclude background precompilation from the steady +comparison. Other apps stayed open; system memory pressure/paging remained present. +Input repeated Enter every six seconds until `UI_TitleMovie01` appeared, then stopped. +The local harness used the shared game lock, captured pass timings/memory/screenshots, +and stopped 60 seconds after the shadow workload began presenting. Native samples +were taken after the measured interval. Both final processes exited **0** after SIGTERM. + +Each measurement covers **70 ≤ elapsed seconds < 100** from process launch, after +initial rendering and before thread sampling. FPS statistics use 15 two-second guest +presentation reports per arm, not host window refreshes. Counts at interval boundaries +are partial frames. One interleaved/truncated baseline pass line was excluded and +recorded; no optimized-arm pass line was excluded. No shader library, pipeline +compilation or library wait time was reported in either measured interval. + +| Measurement | Fast path disabled | Fast path enabled | +| --- | ---: | ---: | +| Guest FPS, median | 2.49 | 3.98 | +| Guest FPS, minimum–maximum report | 1.99–2.50 | 3.00–4.00 | +| Complete parsed pass records | 10,311 | 16,636 | +| Observed frame IDs (partial boundaries) | 70 | 114 | +| Five target shadow encoders, observations | 350 | 565 | +| Target shadow encoder duration, weighted mean | 56.509 ms | 26.753 ms | +| 512×512×6, 84 draws, mean | 56.299 ms | 26.692 ms | +| 512×512×6, 102 draws, mean | 57.173 ms | 27.141 ms | +| Two 256×256×6 encoders, pooled mean | 56.539 ms | 26.694 ms | +| 32×32×6, 102 draws, mean | 55.995 ms | 26.544 ms | + +The five shadow encoders retain the same shader identities, draw counts and logical +mesh threadgroups. Their mean duration falls **52.7%**, while the observed median +scene FPS rises about **60%**. The optimized encoders still spend about 99% of their +measured time in Metal's vertex/tiling phase. Summed encoder means are not an exclusive +frame-time decomposition. This paired observation is not a statistical confidence +interval or a general gameplay FPS claim. + +A separate pair with normal prewarming enabled observed 2.49 versus 3.98 median FPS +and 56.638 versus 26.723 ms mean shadow encoders. Its control retained 9.489 seconds +of background library work (zero reported blocking library waits); it is corroborating +evidence, not the primary compilation-free comparison. + +The remaining large encoder uses a **72×41×96** layered target and mesh shader +`80BB12DEB34BFB24` / pixel shader `B56A794C42C12B7B`. Its 46-draw variant averages +41.263 ms in the control and 42.340 ms with the fast path, across 69 and 97 observations +respectively. This path has no demonstrated improvement. Its vertex/tiling work and +the remaining roughly 27 ms shadow encoders explain why the scene is still slow. +No shader/title-specific bypass or quality reduction was introduced. + +Local evidence identifiers (raw captures contain game data and are not committed): + +- Control: `20261026-201309-silent hill short message.zar-dword-off-steady`. +- Optimized: `20261026-201514-silent hill short message.zar-dword-on-steady`. +- Default-prewarm pair: `20261026-200501-…-dword-on-3` and + `20261026-200737-…-dword-off-2`. +- Aggregate JSON: `artifacts/runtime-diagnosis-20261026/runtime-comparison-steady.json`; + each run retains `game.tlog`, environment/provenance, screenshots, memory telemetry, + bounded pass analysis and actual process exit status. + +Earlier attempts are explicitly excluded: an initial cold optimized run had prolonged +pipeline compilation and a shutdown abort; a control ended with `ENOSPC` and a shutdown +abort; a retry stopped at the new 1 GiB free-space guard. The abort stack shows an APR +worker reading a ZArchive decompressor after disposal. That lifetime defect is recorded, +not fixed by this shader change. Idle generated A/B artifacts were relocated with all +5,775 file hashes verified; idle compiler cache and this run's rebuildable intermediates +were cleared before the completed pairs. Original source, saves and logs were retained. + ## Acceptance limits -The runtime capture identifies an expensive path. This patch accelerates a synthetic -operation present in that path; it has not yet established how frequently the game's -live descriptors take it or a gameplay FPS improvement. No additional game run was -performed for the implementation. The retained original runtime payload has hashes -but no verified source commit; matching the three generated MSL bodies establishes -shader-source correspondence, not whole-binary provenance. +The runtime result now demonstrates a substantial improvement in the captured title +scene, but roughly four FPS is still poor performance. The 96-layer pass, remaining +shadow cost, host memory pressure and observed shutdown lifetime defect need further +work. No full-game visual/correctness or gameplay-performance claim is made. The +original runtime payload had no verified source commit; the new comparisons above use +one fully identified build in both arms. The existing GUI was not replaced by this +isolated diagnostic build. From 35eb5deee3f0161848fc2f9a39732a3dfd244681 Mon Sep 17 00:00:00 2001 From: kuba Date: Tue, 27 Oct 2026 18:05:55 +0100 Subject: [PATCH 33/82] [GPU] Add native occlusion and preserve measured Silent Hill performance work --- docs/performance/native-occlusion-runtime.md | 1018 +++++++++++++++++ .../Native/DirectExecutionBackend.Imports.cs | 36 +- src/SharpEmu.Core/Runtime/SharpEmuRuntime.cs | 6 + .../RemoteGuestAddressSpace.cs | 32 +- .../GuestMemory/GuestMirrorTranslator.cs | 15 +- .../GuestMemory/GuestViewTable.cs | 10 + src/SharpEmu.HLE/Host/WindowHostInput.cs | 33 +- .../Agc/AgcExports.CommandStream.cs | 3 + .../Agc/AgcExports.GeometryRegisters.cs | 4 +- src/SharpEmu.Libs/Agc/AgcExports.cs | 2 +- .../Ampr/AmprExports.Submission.cs | 256 +++-- src/SharpEmu.Libs/Ampr/AmprExports.cs | 23 +- .../Gpu/GpuCommands/CommandRegisterBanks.cs | 2 +- .../Gpu/GpuCommands/CommandStreamQueue.cs | 27 +- .../DeferredCommandThread.Recycling.cs | 82 ++ .../Gpu/GpuCommands/DeferredCommandThread.cs | 68 +- .../Gpu/GpuCommands/DenseShaderRegisterMap.cs | 232 ++++ .../GpuCommandInterpreter.Registers.cs | 18 +- .../Gpu/GpuCommands/GpuResourceOrigin.cs | 99 ++ .../RegisterWriteTable.ScalarReuse.cs | 82 ++ .../Registers/RegisterWriteTable.cs | 16 +- .../Gpu/Images/ImageDescription.cs | 21 + .../ImageRequestBuilders.ColorTarget.cs | 43 +- .../ImageRequestBuilders.DepthTarget.cs | 28 +- .../Images/ImageRequestBuilders.Texture.cs | 7 +- .../Gpu/Pipelines/ShaderProgramCache.cs | 7 + .../Gpu/Pipelines/VertexInputResolver.cs | 13 +- .../VideoOut/RenderPhaseProfile.cs | 3 + src/SharpEmu.Logging/MemoryTelemetry.cs | 59 +- src/SharpEmu.Logging/SharpEmu.Logging.csproj | 3 + .../Gen5MslCodegenOptions.cs | 54 +- .../Gen5MslShader.cs | 31 +- .../Gen5MslTranslator.ExecGuard.cs | 20 + .../Gen5MslTranslator.Mesh.cs | 72 +- .../Gen5MslTranslator.MeshHotLds.cs | 143 +++ .../Gen5MslTranslator.MeshLdsForwarding.cs | 50 + .../Gen5MslTranslator.PackedFormats.cs | 65 ++ .../Gen5MslTranslator.Resources.cs | 28 +- .../Gen5MslTranslator.ScalarConstants.cs | 37 + .../Gen5MslTranslator.ScratchSlots.cs | 84 ++ .../Gen5MslTranslator.Structured.cs | 20 +- .../Gen5MslTranslator.cs | 29 +- .../Templates/scalar_constant_prelude.msl | 39 + .../Ir/Gen5ExecFullAnalysis.cs | 20 +- .../Ir/Gen5VectorObservationAnalysis.cs | 5 +- ...ResourceMaterializationCache.Validation.cs | 32 +- .../Resources/ResourceMaterializationCache.cs | 47 +- .../ResourceMaterializationProfile.cs | 46 +- .../V2/MetalCommandStream.cs | 127 +- .../V2/MetalGraphicsBindingMemo.cs | 33 + .../SharpEmu.Gpu.Metal/V2/MetalGuestMemory.cs | 10 +- .../V2/MetalLibraryCompilePool.cs | 11 + .../V2/MetalPageRangeFilter.cs | 67 ++ .../V2/MetalPendingWriteOverlay.cs | 15 +- .../V2/MetalPresentationCache.cs | 24 + .../V2/MetalRenderHost.CommandThread.cs | 8 +- .../V2/MetalRenderHost.Descriptors.cs | 74 +- .../V2/MetalRenderHost.Occlusion.cs | 310 +++++ .../V2/MetalRenderHost.OcclusionAudit.cs | 140 +++ .../V2/MetalRenderHost.Pipelines.cs | 23 +- .../V2/MetalRenderHost.Profile.cs | 16 +- .../V2/MetalRenderHost.Rendering.cs | 9 + .../V2/MetalRenderHost.StagedStorage.cs | 5 +- .../SharpEmu.Gpu.Metal/V2/MetalRenderHost.cs | 42 +- .../V2/MetalSamplerViewClasses.cs | 31 + .../SharpEmu.Gpu.Metal/V2/MetalV2Native.cs | 4 + .../V2/MetalV2Presenter.HostMovie.cs | 2 +- .../SharpEmu.Gpu.Metal/V2/MetalV2Presenter.cs | 91 +- .../MetalAsyncPipelineTests.cs | 9 +- .../MetalCompileSchedulerTests.cs | 29 + .../MetalDepthOnlyJoinTests.cs | 15 +- .../MetalEndOfPipeOrderTests.cs | 26 +- .../MetalGraphicsBindingMemoTests.cs | 58 + .../MetalPageRangeFilterTests.cs | 78 ++ .../MetalPairLockTableTests.cs | 130 +++ .../MetalPendingWriteOverlayTests.cs | 54 + .../MetalPresentationCacheTests.cs | 81 ++ .../MetalSamplerViewClassesTests.cs | 91 ++ .../MetalShaderSourceReleaseTests.cs | 31 + .../Scenes/BindingReuseSceneTests.cs | 63 + .../Scenes/GpuWordNoWaitSceneTests.cs | 9 +- .../Scenes/GpuWordWaitSceneTests.cs | 9 +- .../Scenes/NativeOcclusionSceneTests.cs | 289 +++++ .../Scenes/OcclusionSceneTests.cs | 37 + .../Scenes/QueuedCommandWriteSceneTests.cs | 62 + .../Scenes/QueuedQwordWriteSceneTests.cs | 212 ++++ .../Scenes/SceneRig.cs | 34 +- .../Agc/AgcGeometrySnapshotValidationTests.cs | 60 + .../Ampr/AprSessionLifetimeTests.cs | 336 ++++++ .../Audio/AudioOut2QueueDispatchTests.cs | 117 ++ .../Audio/AudioOut2QueueTests.cs | 2 +- .../Cpu/HostShutdownLeafDispatchTests.cs | 146 +++ .../GpuCommands/CommandStreamQueueTests.cs | 44 + .../DeferredCommandItemReuseTests.cs | 265 +++++ .../GpuCommands/DeferredCommandThreadTests.cs | 78 ++ .../DenseShaderRegisterMapTests.cs | 148 +++ ...uCommandInterpreterPacketSemanticsTests.cs | 12 + .../GpuCommandInterpreterRegisterTests.cs | 15 + .../Gpu/GpuCommands/GpuResourceOriginTests.cs | 73 ++ .../Registers/IdenticalScalarWriteTests.cs | 193 ++++ .../Registers/RegisterWriteTableTests.cs | 48 + .../Gpu/Images/ColorTargetMemoTests.cs | 80 ++ .../Gpu/Images/DepthTargetMemoTests.cs | 110 ++ .../Gpu/Images/MipLevelLayoutsTests.cs | 76 ++ .../Images/NativeColorClearRequestTests.cs | 15 +- .../ShaderPipelineLookupCacheTests.cs | 33 + .../Logging/MemoryTelemetryTests.cs | 58 + .../Memory/GuestMemory/GuestViewTableTests.cs | 23 + .../Pad/HostWindowInputTests.cs | 32 + .../Gen5NggFixtures.cs | 200 ++++ .../Goldens/exec-store-layout.msl | 5 +- .../MetalNative.Mesh.cs | 9 +- .../MslConstantExecRestoreTests.cs | 66 ++ .../MslLiteralFormatLoadTests.cs | 47 +- .../MslStaticScratchSlotTests.cs | 132 +++ .../MslUnobservedAluTests.cs | 70 ++ .../NggMeshIndirectLaunchTests.cs | 5 + .../NggMeshNarrowWave64Tests.cs | 17 +- .../NggMeshRuntimeTests.cs | 508 +++++++- .../Gen5ExecFullAnalysisTests.cs | 65 ++ .../Gen5VectorObservationAnalysisTests.cs | 22 + ...urceMaterializationExactComparisonTests.cs | 237 ++++ .../ResourceMaterializationProfileTests.cs | 17 + .../Benches.Materialization.cs | 45 +- .../README.md | 16 + .../LabelFilterBenches.cs | 60 + tools/SharpEmu.Tools.PerfAudit/Program.cs | 2 + tools/SharpEmu.Tools.PerfAudit/README.md | 12 + .../ShaderRegisterMapBenches.cs | 71 ++ .../Observation.cs | 76 ++ .../Program.cs | 5 + 131 files changed, 8705 insertions(+), 375 deletions(-) create mode 100644 docs/performance/native-occlusion-runtime.md create mode 100644 src/SharpEmu.Libs/Gpu/GpuCommands/DeferredCommandThread.Recycling.cs create mode 100644 src/SharpEmu.Libs/Gpu/GpuCommands/DenseShaderRegisterMap.cs create mode 100644 src/SharpEmu.Libs/Gpu/GpuCommands/GpuResourceOrigin.cs create mode 100644 src/SharpEmu.Libs/Gpu/GpuCommands/Registers/RegisterWriteTable.ScalarReuse.cs create mode 100644 src/SharpEmu.ShaderCompiler.Metal/Gen5MslTranslator.MeshHotLds.cs create mode 100644 src/SharpEmu.ShaderCompiler.Metal/Gen5MslTranslator.MeshLdsForwarding.cs create mode 100644 src/SharpEmu.ShaderCompiler.Metal/Gen5MslTranslator.PackedFormats.cs create mode 100644 src/SharpEmu.ShaderCompiler.Metal/Gen5MslTranslator.ScalarConstants.cs create mode 100644 src/SharpEmu.ShaderCompiler.Metal/Gen5MslTranslator.ScratchSlots.cs create mode 100644 src/SharpEmu.ShaderCompiler.Metal/Templates/scalar_constant_prelude.msl create mode 100644 src/graphics/SharpEmu.Gpu.Metal/V2/MetalGraphicsBindingMemo.cs create mode 100644 src/graphics/SharpEmu.Gpu.Metal/V2/MetalPageRangeFilter.cs create mode 100644 src/graphics/SharpEmu.Gpu.Metal/V2/MetalPresentationCache.cs create mode 100644 src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.Occlusion.cs create mode 100644 src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.OcclusionAudit.cs create mode 100644 src/graphics/SharpEmu.Gpu.Metal/V2/MetalSamplerViewClasses.cs create mode 100644 tests/SharpEmu.Gpu.Metal.Tests/MetalGraphicsBindingMemoTests.cs create mode 100644 tests/SharpEmu.Gpu.Metal.Tests/MetalPageRangeFilterTests.cs create mode 100644 tests/SharpEmu.Gpu.Metal.Tests/MetalPairLockTableTests.cs create mode 100644 tests/SharpEmu.Gpu.Metal.Tests/MetalPresentationCacheTests.cs create mode 100644 tests/SharpEmu.Gpu.Metal.Tests/MetalSamplerViewClassesTests.cs create mode 100644 tests/SharpEmu.Gpu.Metal.Tests/Scenes/NativeOcclusionSceneTests.cs create mode 100644 tests/SharpEmu.Gpu.Metal.Tests/Scenes/QueuedCommandWriteSceneTests.cs create mode 100644 tests/SharpEmu.Gpu.Metal.Tests/Scenes/QueuedQwordWriteSceneTests.cs create mode 100644 tests/SharpEmu.Libs.Tests/Ampr/AprSessionLifetimeTests.cs create mode 100644 tests/SharpEmu.Libs.Tests/Audio/AudioOut2QueueDispatchTests.cs create mode 100644 tests/SharpEmu.Libs.Tests/Cpu/HostShutdownLeafDispatchTests.cs create mode 100644 tests/SharpEmu.Libs.Tests/Gpu/GpuCommands/DeferredCommandItemReuseTests.cs create mode 100644 tests/SharpEmu.Libs.Tests/Gpu/GpuCommands/DeferredCommandThreadTests.cs create mode 100644 tests/SharpEmu.Libs.Tests/Gpu/GpuCommands/DenseShaderRegisterMapTests.cs create mode 100644 tests/SharpEmu.Libs.Tests/Gpu/GpuCommands/GpuResourceOriginTests.cs create mode 100644 tests/SharpEmu.Libs.Tests/Gpu/GpuCommands/Registers/IdenticalScalarWriteTests.cs create mode 100644 tests/SharpEmu.Libs.Tests/Gpu/Images/ColorTargetMemoTests.cs create mode 100644 tests/SharpEmu.Libs.Tests/Gpu/Images/DepthTargetMemoTests.cs create mode 100644 tests/SharpEmu.Libs.Tests/Gpu/Images/MipLevelLayoutsTests.cs create mode 100644 tests/SharpEmu.Libs.Tests/Logging/MemoryTelemetryTests.cs create mode 100644 tests/SharpEmu.ShaderCompiler.Metal.Tests/MslConstantExecRestoreTests.cs create mode 100644 tests/SharpEmu.ShaderCompiler.Metal.Tests/MslStaticScratchSlotTests.cs create mode 100644 tests/SharpEmu.ShaderCompiler.Metal.Tests/MslUnobservedAluTests.cs create mode 100644 tests/SharpEmu.ShaderCompiler.Tests/Resources/ResourceMaterializationExactComparisonTests.cs create mode 100644 tools/SharpEmu.Tools.PerfAudit/LabelFilterBenches.cs create mode 100644 tools/SharpEmu.Tools.PerfAudit/ShaderRegisterMapBenches.cs create mode 100644 tools/SharpEmu.Tools.ShaderCompileBench/Observation.cs diff --git a/docs/performance/native-occlusion-runtime.md b/docs/performance/native-occlusion-runtime.md new file mode 100644 index 000000000..198b9328e --- /dev/null +++ b/docs/performance/native-occlusion-runtime.md @@ -0,0 +1,1018 @@ + + +# Native occlusion and Silent Hill room measurements + +Native Metal visibility counters replace the conservative always-visible query +results when `SHARPEMU_METAL_V2_NATIVE_OCCLUSION=1`. In a paired, same-binary +Silent Hill first-room comparison, median guest FPS increased from 4.485 to +13.43. This does not establish 30 FPS, full-game performance, or performance in +the user's original 2-FPS scene. + +## Query behavior + +Counting draws accumulate samples in distinct visibility intervals. A GPU prefix +sum preserves the counter value at each dump; ordered GPU copies publish these +snapshots with the guest ready bit. Empty intervals, sparse instance masks, +resets, multiple queues, pending predication, and query-memory reuse retain +stream ordering. GPU-produced values are never prematurely read into the CPU +label replay cache. + +Each batch has 4,096 intervals. At most 256 batches are retained (16 MiB for +counts and snapshots). Between passes, exhaustion waits for retired storage or +commits and waits for the current buffer. Unfinished storage is not reused. +The initial experiment aborted on budget exhaustion; that failed run is excluded +from performance results and has a bounded-backpressure regression test. + +The latest implementation initializes only exposed slots, caches visibility mode +and offset within each encoder, and skips a reduction only when no counting draw +or referenced snapshot requires it. A disabled pass preserves an existing +nonzero counter. Sparse dumps excluding block zero still publish ready-only +values without requiring a batch snapshot. + +## Measured comparison + +Apple M4; x64/Rosetta emulator with native arm64 GPU host; Release, self-contained +.NET 10; 1920×1080, scale 1, VSync enabled and existing rendering settings retained. +Each run uses a private copy of the same warm user/cache seed. Other applications +remain open. Host memory pressure and background load are not eliminated. +Only Silent Hill: The Short Message (PPSA10112) was launched. + +The phone was closed after visual confirmation. The stationary view faces the +same first-room shelf and lamp. Samples are two-second guest-presentation reports +in the 375–410 second interval after launch. Host presentation Hz is a separate +counter and must not be substituted for guest FPS. + +| Build/arm | Reports | Median guest FPS | Report range | Interpretation | +| --- | ---: | ---: | ---: | --- | +| v2, native queries disabled | 18 | 4.485 | 3.50–5.00 | Same-binary control | +| v2, native queries enabled, serial resolution | 18 | 13.43 | 10.45–14.44 | 2.994× paired median ratio | +| v3, command thread and parallel resolution | 18 | 11.455 | 9.46–11.95 | Slower than serial; different code and mode | +| v4, also queue CP writes | 18 | 11.725 | 9.46–13.00 | No established improvement | +| v5, also request native command-thread QoS | 17 | 11.98 | 10.95–13.00 | QoS applied; no established improvement | + +These are observational comparisons, not confidence intervals. Only the first +two rows change one switch in the same executable. Their independently reported +draw and frame windows suggest roughly 9,492 versus 3,366 recorded draws per +frame; those ratios are approximate, not exact per-frame accounting. The five +successful runs completed their 480-second limit and controlled shutdown with +exit 0. A post-measurement camera turn in v5 changed the view and rendered new +frames; its approximately 23-FPS wall view is not the stationary benchmark. + +## Experimental switches and correctness fixes + +The following performance switches remain opt-in: + +- `SHARPEMU_METAL_V2_NATIVE_OCCLUSION=1`: native sample counts and ordered snapshots. +- `SHARPEMU_METAL_V2_QUEUE_CP_WRITES=1`: asynchronously enqueue byte and eligible qword CP writes when the + existing command thread is active; pending-range reads and prefetch gates keep + consumers behind those writes. +- `SHARPEMU_METAL_V2_COMMAND_THREAD_QOS=1`: create the command producer using the + existing native helper-thread QoS policy, with managed fallback. +- `SHARPEMU_METAL_V2_REUSE_PRESENTED_FRAME=1`: suppress repeated presentation of an + unchanged frame while preserving movie/CPU-frame updates, output changes, + restore handling and animated overlays. Gameplay acceptance remains incomplete. + +Overlapping pending-label writes previously left stale cached qword answers. +The corrected shared queue invalidates conflicting cached widths and falls back +to ordered backend reads when a complete answer is unavailable. The same device +predication test failed in both synchronous and queued modes before the fix. + +The command-thread coalescing path now waits for ordered producer-idle work or +normal budgets instead of committing merely because the producer's queue has +become empty while a removed slice is still executing. Test drains use a trailing +producer barrier, rather than mistaking queue removal for completion. + +APR session teardown now stops admission, cancels queued commands and joins +executing workers before archive unmount. Admitted submission and result writes +hold lifetime leases without holding the global queue lock during guest access. +Stopped root-memory ownership also rejects newly created wrappers. Cached host +files remain alive while readers hold leases, including during cache eviction. +Address waits and chunked reads cooperate with cancellation. + +## Validation and unresolved evidence + +Latest source validation, before the v6 runtime publication: + +- Full Libs: 8,971 passed, 0 failed, 5 existing Opus skips, 1m18s. +- Full Metal, default mode with required device: 1,606 passed, 0 failed, + 7 existing skips, 1m24s. +- Native-query device cases with command thread, QoS, parallel resolution and + queued CP writes: 13 passed, 0 failed, 1s. +- APR suite: 57 passed, 0 failed, 753ms, including five new lifetime cases. +- Command-thread lifecycle tests: four passed with QoS requested and four with + native QoS disabled. v5 full threaded Metal: 1,604 passed, 0 failed, 7 skips. + +The first new admission test waited for a guest read that cached capture never +performed. It failed while 56 APR tests passed. The corrected test parks the +actual admitted submission-ID write; it does not weaken the shutdown assertion. + +A presentation-reuse run aborted during startup, before any room measurement: +pixel shader `17174605ABA538FD` loaded a descriptor 72 bytes beyond a mapped guest +region. An earlier unrelated experimental run failed at the same shader offset +and a different mapping boundary. The mapping failure remains unresolved; no +causal attribution to presentation reuse is established. Failure-time user-data +logging now preserves the actual failing draw's inputs. The later archive-close +exception exposed the separately repaired APR ownership race. + +An artifact-only single-thread compute copy experiment reduced CPU encoding time +but increased GPU time. At 128 tiny copies, CPU encoding changed from +62.5–65.2 µs to 9–10.8 µs while GPU time rose from 10.6–11.1 µs to 26.6–27.1 µs. +It is not integrated and is not an FPS improvement. + +## CPU lookup follow-up + +A second v7 diagnostic run used CPU phase timing only, disabled native automatic +sampling, and completed its 600-second bound cleanly. In the fixed 475–510-second +first-room interval, 17 presentation reports had a median of 12.0 FPS (9.96–13.5). +This is an instrumented diagnostic, not a new uninstrumented performance record. +Across 422 reported frames, deferred-label ordering consumed 5.38 ms per frame; +resource materialization consumed 6.79 ms and draw-resource preparation 5.79 ms. + +The pending-copy list now has two bounded negative page filters: one for guest +addresses and one for native buffer/offset pairs. A missing page avoids scanning +the list; possible matches retain the exact overlap check. Large or wrapping +ranges force fallback, and draining the list clears both filters. The switch +`SHARPEMU_METAL_V2_LABEL_PAGE_FILTER=0` restores the original lookup. CPU profiling +reports query, rejection, scan, inspected-entry and peak-held-copy counts. + +A synthetic ARM64 Release benchmark measured a binding in a gap between two +clusters of held copies. Four alternating trials, 100,000 lookups per trial, +tiering disabled; all measured lookups allocated zero bytes: + +| Held copies | Original median | Filter median | +| --- | ---: | ---: | +| 16 | 8.27 ns | 3.81 ns | +| 256 | 142.30 ns | 3.84 ns | +| 4,096 | 1,944.69 ns | 3.78 ns | + +A first-entry match costs about 2.2–2.4 ns more with the filter. Same-page gaps +still require the exact scan. Filter insertion and reset cost about 6 ns per +copy in the 4,096-copy case. These are synthetic lookup measurements, not FPS. + +Sampler numeric-class preparation also changes from a per-sampler scan of all +sampled-image pairs to one pass over the pairs. Classification is recomputed +from the current resolved image formats on every preparation; mixed, invalid, +and unpaired cases retain their behavior. + +Validation: 78 focused device/unit cases passed with the filter enabled, and +78 passed with it disabled and CPU profiling enabled. The full Metal suite +passed 1,617 cases with 7 existing skips. Five render-phase tests passed. +Subsequent opt-in materialization diagnostics passed 66 cache/profile cases, +including moved descriptors and in-place comparison. The v8 CPU-profiled run completed its 600-second bound cleanly. In the same +475–510-second room interval, 17 reports had a median of 13.44 FPS (12.49–13.5). +Deferred-label ordering fell from 5.38 to 0.89 ms per reported frame. The filter +rejected 5,347,658 queries beyond the coarse address-hull check; 625,835 exact +scans remained, with a peak of 16,620 held copies. This is a profiled combined +filter/sampler comparison; an uninstrumented v8 acceptance run remains pending. + +## Reproducibility + +All runtime publications retain their source snapshot, tracked binary diff, +untracked-source archive and payload hashes in the local diagnostic artifacts. +They share base revision `52f07c7b37570adaad7317b8196a86f9bb4d6548`; they are dirty +source snapshots, not separate clean commits. Tracked source-diff SHA-256: + +| Publication | SHA-256 | +| --- | --- | +| v2 | `c436a870becdc70703e5b1c29c399cdf146ab0be11119daebdd96dd9c0b39587` | +| v3 | `138d695e9425b452c1b67625a6a377b8d7989a1bfa8e217d6d276d9047a8db54` | +| v4 | `f24dcf90c9d0d797fd7e0346573d7d685b0013f22c97d4d17cd829eb9104f24a` | +| v5 | `1d7372c56af1bb810d4fd75b2bdee2c6243f9bed3c87c9c2b4f974190b707656` | +| v6 | `6cc88fe55291141199cbec17e62ef80dcbf288fbf1b226049a65db5f187acc65` | +| v7 | `efdf79607bd5046d1ef71b74f0e04f9b39fc934fc964f3d8bb4916f9ea9ce0f4` | +| v8 | `2325d12d7defa5ab7fafcd05c425871163a2062e2eb3123ec6ea563217a30f3a` | + +The v6 publication adds the query-work reductions and APR lifetime fix. Its +profiled run stopped cleanly at 497 seconds. Automatic native process sampling +and memory inspection started near 487 seconds, contaminating the planned +475–510-second interval; that interval is excluded from FPS comparisons. A +post-hoc diagnostic interval at 440–475 seconds attributes 12.23 ms per frame +to storage-buffer resolution, the largest exclusive CPU phase. Its sampled +GPU encoder durations sum to 16.98 ms per frame, with 17.01 ms of gaps; these +are not exclusive wall times and cannot be added to CPU phases. This identifies +the next investigation, not an uninstrumented performance improvement. Commercial assets, shader +captures, screenshots and raw game logs are not included in this repository. + +### Unprofiled v8 retry limitation + +The next unprofiled v8 run failed before gameplay at 59.977 seconds, so it adds +no FPS result. The shader's captured pointer again lay 232 bytes before the +end of the mapped region, with the requested descriptor read 72 bytes beyond +it. The complete mapping snapshot and actual user-data words were retained. +This recurring failure predates the label filter and native-query changes; its +cause remains unresolved. The best qualified unprofiled stationary-room result +remains the v2 measurement above. + +### v10 diagnostic and register-map follow-up + +The v10 CPU/materialization diagnostic completed its 600-second limit and +controlled shutdown with exit 0. No descriptor failure recurred; its cause is +still unresolved. The matching stationary room at 475–510 seconds produced +17 reports with median 12.95 FPS (10.46–13.0). The phone was closed at 401.6 +seconds, later than the prior runs' approximately 380 seconds. Detailed profiling +and bounded pointer-origin history were enabled, so this is not an unprofiled +FPS acceptance result. + +Across 458 reported frames, resource materialization cost 7.455 ms/frame. +Sampled snapshot preparation estimates approximately 1.559 ms/frame, versus +0.417 ms/frame for hit validation. The cumulative counter interval contained +1,744,075 normal hits, 197 recalled hits, 1,724,144 rebased hits, and 150,881 +recorded ranges. Many hits therefore have no ranges to compare. Sampled and +nested timings are diagnostic estimates and must not be summed as exact +exclusive frame accounting. + +The dense shader register map preserves written-zero presence, arbitrary +capture keys, reset behavior and AGC's shared map identity. Full Release Libs +validation passed 8,986 tests with zero failures and five existing Opus skips. +A native ARM64 interpreter ABBA comparison changed only `SharpEmu.Libs.dll`: + +| PM4 case | Control ns/draw, two trials | Candidate ns/draw, two trials | Mean change | +| --- | --- | --- | ---: | +| 20 context registers, automatic draw | 615.0 / 639.4 | 592.7 / 615.1 | -3.71% | +| Same stream, individual packet reads | 616.3 / 658.6 | 642.4 / 627.0 | -0.43% | +| Indexed draw | 631.9 / 657.1 | 656.7 / 632.9 | +0.05% | +| Five context registers | 248.0 / 262.1 | 246.3 / 249.7 | -2.76% | +| 25 context registers | 723.7 / 763.7 | 748.3 / 742.3 | +0.22% | + +All cases reported 0.1 B/draw. Each case used five seconds of warmup and five +150-ms samples, with tiering disabled. Host drift and two trials per arm limit +inference; these numbers establish no game-FPS gain. A separate register-batch +experiment showed no improvement and was reverted before this comparison. + +The subsequent v10 command-thread/serial-resolution run stopped at 238 seconds +because the harness's free-space guard tripped. It produced no stationary-room +result and cannot establish a threading performance regression or deadlock. + +### Additional candidate validation + +Exact resident comparison now covers recalled and rebased ranges using each +range's existing address calculation. Refusal retains the original reader and +base-word probe path. Empty-range hits avoid the comparer. Successful head +lookup also avoids repeating the identical user-data check; other variants +retain their checks. The materialization suites passed 107 tests both normally +and with in-place verification enabled. A first failing test was corrected to +honor the existing immutable-user-data input contract; its probe assertion was +retained. The synthetic comparisons below do not establish an FPS improvement. + +Queued qword writes use a conservative pending destination hull only when its +arithmetic is valid. Disabled and wrapping cases retain synchronous behavior. +Twenty-two focused device tests passed, including actual producer-thread +ordering, sparse destinations, indirect packets and predication. + +`SHARPEMU_GPU_RESOURCE_ORIGIN=1` retains at most 128 historical PS s28/s29 writes +and reports their packet/table origins on materialization failure. Correlation +with captured inputs and a source-word reread are diagnostics, not proof of +current ownership or permission to bypass a failed descriptor read. Forty-five +focused provenance/interpreter tests passed with the recorder enabled. + +### Resident-comparison benchmark follow-up + +Native ARM64 ABBA runs changed only `SharpEmu.ShaderCompiler.dll`; the tool and +all other libraries were byte-identical between arms. Tiering was disabled, +with five seconds of warmup and five 150-ms samples per case. The first run +used a harness that converted a capturing reader delegate on every call: + +| Case | Control ns/op (two trials) | Candidate ns/op (two trials) | +| --- | --- | --- | +| Full walk | 1503.8 / 1522.0 | 1490.4 / 1535.6 | +| Warm hit | 130.5 / 130.2 | 122.6 / 122.4 | +| Cold miss | 4066.4 / 4064.7 | 4209.8 / 4046.9 | +| Changed descriptors | 4841.3 / 4846.0 | 4903.3 / 4851.4 | +| Eight returning tables | 231.0 / 242.2 | 156.8 / 159.0 | +| Moving descriptor ring | 389.9 / 402.5 | 313.5 / 317.8 | +| Table-write control | 63.4 / 64.0 | 63.3 / 63.7 | + +The tool now retains that delegate, matching the renderer. A repeat confirms +zero allocated bytes per warm hit and returning-table hit in **both** arms; +the earlier 64 B/op was harness overhead, not an emulator allocation saving. +Cold misses report approximately 6315–6326 B/op after this harness correction. +The corrected run's timing was unstable: unchanged full-walk and table-write +controls shifted by approximately 25% and 36% between arm means. Warm hits were +140.2/139.7 ns for control and 161.6/143.6 ns for candidate; returning-table hits +were 379.1/241.0 versus 253.5/202.7 ns. These repeats do not support a reliable +speedup estimate. The comparer uses fixed synthetic native memory, without +Metal ownership checks. Neither run establishes a game-FPS improvement. + +### Graphics binding candidate and runtime exclusions + +`SHARPEMU_METAL_V2_GRAPHICS_BINDING_REUSE=1` opts into skipping an identical +native argument/push buffer binding within a graphics encoder. Matching includes +physical vertex/mesh/fragment stage, selector, slot, buffer and offset. Encoder +reset clears the cache even if a native pointer is reused. Compute bindings, +resource acquisition, uploads and residency declarations retain their behavior. +Cumulative attempted/skipped counters are printed per physical stage. + +Six focused tests passed with zero failures or skips: three memo tests, the +existing binding-reuse scene, and control/candidate device scenes with repeated +and changed textures on one target plus an encoder restart. No real MTL mesh +stage-switch device scene was added; physical-stage isolation has pure-test +coverage. Combined Release validation passed: Metal 1,632/0/7 with binding reuse enabled; Libs 8,992/0/5; ShaderCompiler 2,192/0/1; ShaderCompiler.Metal 3,588/0/0 (passed/failed/skipped). Device suites required a device, and Libs used serial run settings. These four suites total 16,404 passed, zero failed, 13 existing skips. This run preceded the shutdown-import fix described below. The v12 runtime measurement below enables binding reuse; a same-binary disabled control remains pending. + +The first v11 presentation-reuse run completed 600 seconds and controlled +shutdown, but missed phone dismissal. Its 9.975 FPS median at 475–510 seconds +is from a ceiling/phone view and is excluded from stationary-room comparisons. +Host presentation tracked guest FPS rather than repeatedly presenting at 120 Hz. +A second run stopped near 190 seconds when the disk guard detected less than +1 GiB free. A null-destination memcpy fault followed the shutdown request on the +guest RHI thread; it is a separate shutdown defect, not evidence of a gameplay +crash or a presentation-reuse regression. Neither run changes the best qualified +unprofiled room result of 13.43 FPS. The 30 FPS target remains unmet. + +### Managed import shutdown safe points + +The managed import dispatcher checked for shutdown after several leaf paths had +already returned. A stop request could therefore leave a guest thread executing +imports and returning to guest code during teardown. The dispatcher now checks +before selecting a leaf, and checks again after a leaf body for a stop arriving +during that call. The late check preserves pending blocks, entry exits, yields +and context transfers. Existing return-slot validation and executor lifetime +fences remain in place. + +Sixteen tests covering four leaf paths, main and worker contexts, and stop +requests before or during an import all failed against the unfixed dispatcher. +With the fix, 95 focused shutdown/import tests passed with no failures or skips. +The full serial Libs Release rerun passed 9,009 tests with zero failures and five +existing Opus skips (1 minute 14 seconds). Leaf dispatch was explicitly enabled +for these runs. The fix is included in the immutable v12 runtime, whose x64 CLI +and native arm64 GPU host both published successfully; 471 payload hashes and +the source snapshot were retained. + +This is a bounded managed-import fix: native constant thunks and failed return +redirects are outside its scope. It does not establish the cause of the separate +Demon's Souls read-only-destination copy fault after GPU-host termination, and +it makes no FPS claim. + +### v12 observed room performance + +The 600-second v12 run completed its planned stop with process exit 0 after four +seconds. Phone dismissal at 380.43 seconds was visually gated; screenshots at +407 and 497 seconds show the same shelf/lamp view used for the earlier room +measurements. Resolution remained 1920x1080. Native occlusion, label filtering, +presentation reuse and graphics binding reuse were enabled; command-thread and +parallel resolution were disabled, as were heavy CPU/GPU/memory profiles. + +| Measurement, 475–510 seconds | Observed result | +| --- | --- | +| Guest FPS, 17 reports | Median **15.43**, range **11.97–15.99** | +| Host presentation rate | Median 15.43 Hz | +| Eligible graphics bindings, first-to-last PERF report | 5,401,490 attempts; 1,710,231 skipped (**31.66%**) over 32.616 seconds | +| Native binding skips by physical stage | Vertex 1,281,050; fragment 429,181; mesh 0 | +| Host memory pressure | Warn throughout all seven samples; compression and swapping active | + +The earlier unprofiled v2 room median was 13.43 FPS. Different source versions, +measurement times and host conditions prevent attributing the difference to +binding reuse. This is an observed combined-build result, not an isolated cache +speedup. The higher FPS after the camera turn at 555 seconds belongs to a simpler +wall view and is excluded. **The 30 FPS target is not met.** + +A separate libproc sample at 530–540 seconds recorded approximately 8.00 CPU +seconds on the Metal render worker, 9.02 on the guest audio event thread, and +5.71 on the guest RHI thread. These are thread CPU-time deltas, not stack-profile +attribution or an explanation of every lost frame. GPU interval durations are +not additive to them. The next comparison must use the same v12 binary with +binding reuse disabled before assigning a performance gain to that cache. + +No resource-materialization fatal or unsolicited CPU trap appeared. The planned +shutdown logged its expected host-stop redirect/CPU_TRAP result; this does not +certify complete guest teardown. Peak reported footprint was 9,777 MiB and RSS +2,679 MiB; alias-accounted footprint is not unique physical memory. REUSE lint +also passed for all 2,856 files after the source and documentation changes. + + +### Audio queue query dispatch candidate + +The binding-disabled v12 control exited normally after its 600-second bound, +but screenshots through 598 seconds show the phone overlay still open. It is +excluded from the shelf/lamp gameplay comparison. No isolated binding-reuse +speedup is established by that run. + +A guest audio event-thread sample led to the queue-level query import. Both +normal and low-latency queue-level exports read register arguments and context +counters, write optional 32-bit outputs, and do not park the guest or invoke +guest callbacks. `SHARPEMU_AUDIO_OUT2_QUEUE_LEAF=1` routes these two queries +through the existing trivial dispatcher. It remains **off by default**. The +queue implementation, clock, device buffering, and push/advance behavior are +unchanged; this change does not insert sleeps into audio polling. + +Fourteen real-dispatch comparisons exercise both exports with empty/full queues, +optional and aliased outputs, invalid pointers and missing contexts. The enabled +and disabled focused runs each passed 74 tests with no failures or skips. The +fixed-tier enabled queue-only run passed 21 tests with no failures or skips. + +A synthetic measurement invokes the real managed dispatcher and queue export +through a cached delegate, with 20,000 warm-up calls per path followed by eight +alternating 200,000-call batches. `DOTNET_TieredCompilation=0` holds compilation +mode constant for this measurement; these figures are not game frame timings. + +| Mode | Full gateway median | Classified query median | Allocations per query | +| --- | --- | --- | --- | +| Flag off | 365.04 ns | 368.20 ns | 40 B on both paths | +| Flag on | 369.41 ns | 84.50 ns | Full: approximately 40 B; fast: 0 B | + +The within-process enabled ratio is about 4.37x. Earlier tiered timings varied +while code was recompiling and are not used for this conclusion. A cheaper poll +can allow more polling rather than reduce total thread CPU time; an FPS gain +requires a visually matched runtime comparison. Raw logs and benchmark JSON +are retained under the private runtime-diagnosis artifact directory. + +The enabled full serial Release Libs rerun passed **9,024 tests, zero failures, +and five existing Opus skips** in 1 minute 14 seconds. The immutable v13 x64 CLI +and native arm64 GPU host both published successfully; 471 payload hashes and +the exact dirty-source snapshot were retained. REUSE passed all 2,857 files. + + +### v13 audio candidate runtime result + +The enabled v13 run reached the same shelf/lamp view, with the phone overlay +visually confirmed and dismissed at 380.6 seconds. The 475–510 second window +contained 18 guest-presentation reports: **13.955 FPS median**, range +**11.99–14.95**. It completed its 600-second limit and planned shutdown with +process exit 0 after four seconds. **30 FPS remains unachieved.** + +The previous v12 observation was 15.43 FPS. The only production source change +between these snapshots is the audio dispatch switch, but this is not a paired +same-binary control: host conditions and menu timing differ. All seven memory +samples again reported warning pressure; system-wide swap activity was higher +than in the v12 window. This result establishes neither a gain nor an isolated +regression. Keep the audio switch off by default. + +During the separate 530–540 second sample, the audio event thread used 9.61 CPU +seconds, the RHI thread 6.35 and the Metal render worker 9.52. The audio thread +therefore remained busy despite the cheaper individual query. The sample does +not count polling frequency or prove where all that CPU time was spent. + +A ten-second EventPipe sampled-thread-time trace was collected afterward, +outside both measurement windows. It repeatedly attributes stacks to shader +program/resource preparation, argument-buffer writes and completion-serial +reads. Its inclusive stack durations overlap, and its leaf attribution is +predominantly `UNMANAGED_CODE_TIME`; they must not be presented as exclusive CPU +percentages. Together with the busy render worker, this supports testing the +existing command-thread/parallel-resolution path with the newer queued qword +writes. Earlier v3–v5 negative results predate those queued-write fixes. + + +### v13 command-overlap failure + +The v13 run with command-thread execution, parallel resolution and queued CP +writes enabled terminated after 103 seconds, before gameplay. Pixel-shader +resource materialization attempted a 16-byte read 72 bytes beyond the end of a +complete mapped range. The descriptor base was 232 bytes before that boundary; +the shader load added 304 bytes. The bounded provenance log shows that both +pointer words in the submitted direct-register packet matched the captured +shader inputs and were unchanged when reread at failure. This narrows the +investigation to the supplied pointer, its lifetime and the associated shader +state; it does not establish ownership or correct shader selection. + +The same boundary signature occurred in earlier serial runs. It is therefore +not established as an overlap regression, but the overlap candidate still +fails runtime acceptance. No gameplay or FPS comparison can be claimed from +this run. The failure remains explicit; unmapped reads are not replaced with +zeroes or fabricated mappings. + + +### Pair-lock table publication + +The sampled v13 stacks frequently contained monitor entry under argument-buffer +writes. The pair-lock table getter acquired a monitor on every read even after +its sole initialization. It now reads the published GPU address atomically; +creation still takes the existing lock and publishes only after zeroing the +table and committing residency. Its lifetime still ends at quiescent disposal. + +A synthetic device test starts eight simultaneous first readers, verifies one +zeroed table with one resident allocation, and confirms later reads and a new +submission preserve its sequence words. It passes on x64 and arm64, including +both explicit-resource and residency-set modes. The final arm64 residency-mode +runs each passed one test. The full x64 Metal suite passed **1,634 tests, zero +failures, seven existing skips** before the test-only residency assertion was +expanded. An initial benchmark invocation used a relative output path and +failed to write its result; rerunning with an absolute path passed. An initial +residency-set assertion incorrectly inspected the explicit-resource list; +the corrected test checks the Metal residency set itself. + +Nine one-million-read batches with tiering disabled measured these medians: + +| Architecture | Original monitor getter | Published-address getter | Allocations | +| --- | --- | --- | --- | +| Native arm64 | 4.49 ns | 1.71 ns | 0 B/read | +| x64 through Rosetta | 9.66 ns | 1.92 ns | 0 B/read | + +This is a small lookup improvement, not evidence that monitor contention +explains the slow frames. The uncontrolled inclusive sampled-stack durations +cannot be equated to these timings. A new immutable v14 runtime will measure +the same room and capture actual contention events outside the FPS window. +No FPS gain from this change or achievement of 30 FPS is established yet. + + +### v14 matched-room result and profiler limitation + +The immutable v14 run reached the shelf/lamp room and completed its 600-second +limit, followed by planned termination after four seconds with process exit 0. +The phone overlay was visibly confirmed and dismissed at 431.0 seconds. The +475–510 second window contains 18 reports: **11.96 FPS median**, range +**6.97–14.50**. **30 FPS is not achieved.** The v13 observation was 13.955 FPS; +these are separate runs, not a controlled isolated performance regression. + +All seven system-memory samples reported warning pressure. Their interval +totals included 26,795 swap-in pages, 3,824 swap-out pages, 1,286,241 compressed +pages and 1,238,361 decompressed pages. These are system-wide counts with +window-edge overlap, not emulator-specific paging attribution. Median reported +GPU busy time was 956.5 ms per roughly 2.03-second interval, host wait was zero, +and worker idle time was 385 ms. No resource-materialization fatal or +unsolicited CPU trap occurred; the post-stop CPU_TRAP is not a certification +of complete guest teardown. Peak reported footprint was 9,794 MiB and RSS +3,038 MiB; shared-alias footprint is not unique physical memory. + +The separate 530–540 second thread measurement recorded 7.32 CPU seconds on +the GPU worker, 8.39 on the guest audio event thread and 5.43 on the RHI thread. +The subsequent CLR trace captured 353 contention starts and 353 stops; the GPU +worker accumulated only **13.96 ms** of reported monitor contention. Its sampled +stacks nevertheless attributed **1,680 ms** to monitor entry. Those sampled +durations are not evidence of equivalent lock wait or exclusive CPU cost. + +The trace also recorded 4,537 paired `SuspendOther` intervals totaling 3,946 ms, +72 GC suspension intervals totaling 338.8 ms, and one GC-preparation interval +of 1.89 ms. All `SuspendOther` events came from the same thread. Sampling-related runtime +suspension is a likely explanation, but a no-sampler control is needed to +establish that attribution. +The later native sample coincided with a separate slowdown to 0–1.49 FPS and +a worker waiting for work, so it is not representative of the earlier FPS +window. Further measurement must avoid treating this heavily perturbed sample +as the steady bottleneck. + +The only production source change from v13 is the pair-lock table getter. Its +small synthetic improvement remains valid; no game FPS improvement is +established. Both runtime publishes passed, all 471 payload hashes verified, +and REUSE passed all 2,858 files. The build's tracked-source diff SHA-256 is +`4260b73e0534a1c51f37388fef54cdc7cd3175ce2043fa68fb2915df4c845adc`. +The descriptor boundary failure remains unresolved. All prior commits and WIP +are preserved; this continuation made no commit or push. + + +### v14 overlap run without CLR stack sampling + +The same v14 payload completed a second 600-second run with command threading, +parallel resource resolution, and queued command-processor writes enabled. +The phone was closed at 379.7 seconds; the shelf/lamp gameplay view was confirmed +in `shot-0427.png`, with no movement input. The 475–510 second window contains +17 reports: **13.46 FPS median, 11.97–15.45 range**. This remains below 30 FPS. +Different host conditions prevent attributing the difference from serial v14 +to overlap alone. No fatal or unsolicited CPU trap appeared before the planned +stop; the process exited 0 four seconds later. The known intermittent descriptor +boundary failure is still unresolved. + +Median counters per two-second interval were 92,300 draws, 3,323 passes, 778.6 ms +GPU busy, zero host wait, 277 ms worker idle, 535.2 ms command-thread synchronous +wait, and 759.5 ms queue-space wait. These include overlapping threads and must +not be summed into a frame duration. Seven system-memory samples included two +warning-pressure readings and five normal readings, 21,902 swap-in pages and +zero swap-out pages. Compression/decompression totals were 558,265/437,067 pages; +these are system-wide intervals with window-edge overlap. Peak reported +footprint/RSS was 9,864/6,146 MiB; aliases prevent interpreting footprint as +unique physical memory. + +A separate ten-second trace beginning at 550 seconds enabled runtime GC and +contention events, with no CLR stack-sampling provider. It recorded 20,568 +allocation ticks totaling 2,220,640,488 bytes, with the largest sampled type +attribution being `UserScalarRegisters` (291,706,192 bytes), followed by +`VertexInputInfo` (165,494,368) and `ResourceSnapshot` (155,113,072). These type +figures are sampled attribution, not exact per-type totals. There were 262 GC +suspensions totaling 924.1 ms and one preparation suspension of 1.7 ms; no +`SuspendOther` interval appeared. This strengthens the concern that the prior +stack-sampled capture was intrusive, but it is not a controlled same-workload +comparison because overlap also changed. + +The next experiment targets unchanged scalar packets before copy-on-write. +It compares values, kind markers and initialized extent, preserves validation +and marker consumption, and remains opt-in until runtime measurement. + + +### Exact scalar-packet reuse candidate (v15) + +`SHARPEMU_REGISTER_IDENTICAL_SCALARS=1` enables a pre-copy comparison for direct +pixel, geometry, hull, and compute user-scalar packets. Identical values alone +are insufficient: every kind marker must match and the stored count must cover +the packet. A successful match consumes the packet marker and preserves the +snapshot. Other shader writes, legacy per-entry marker behavior, and invalid +packets retain their original writers. This is default-off. Optional +`SHARPEMU_PROFILE_IDENTICAL_SCALARS=1` reports cumulative checked/reused packet +counts per writer thread every 262,144 eligible packets; reuse counts do not +measure bytes saved when a bank was already writable. + +The focused register suites passed 78/0/0 on both x64 and native arm64 with the +feature enabled and disabled. The full x64 Libs suite with reuse enabled passed +**9,039 tests, zero failures, five existing Opus skips**. An initial test-source +build failed because `Random` resolved to a test namespace; qualifying +`System.Random` fixed it. Tests cover retained snapshots, independent mixed-write +state, changed kinds, zero writes that extend the count, packet boundaries, +empty-packet failure semantics, and legacy markers. + +Seven 100,000-operation batches per case, with tiering disabled, measured a +packet write followed by a snapshot. Both arms had matching checksums. Medians: + +| Host | 32-word pattern | Baseline ns/op | Reuse ns/op | Baseline B/op | Reuse B/op | +| --- | --- | ---: | ---: | ---: | ---: | +| Native arm64 | Identical | 100.55 | 18.61 | 400 | 0 | +| Native arm64 | Always changing | 101.40 | 103.08 | 400 | 400 | +| Native arm64 | Seven of eight change | 102.08 | 93.80 | 400 | 350 | +| x64 / Rosetta | Identical | 164.92 | 26.54 | 400 | 0 | +| x64 / Rosetta | Always changing | 164.85 | 178.94 | 400 | 400 | +| x64 / Rosetta | Seven of eight change | 167.57 | 160.03 | 400 | 350 | + +The comparison adds cost when values change, particularly under Rosetta. These +synthetic savings do not establish a gameplay improvement. The next room run +uses native arm64 GPU processing and the same overlap settings as v14. +The immutable v15 tracked-source diff SHA-256 is +`3e93b3efa1114171d16c88fd03ffb9ad4770fd70f855d33c8138a6a7afe8531b`; +both Release publishes passed and all 471 payload hashes verified. + +The full x64 Metal device suite in the same overlap mode, with scalar reuse +enabled, passed **1,634 tests, zero failures, seven existing skips**. REUSE +passed all 2,860 files. Provenance comparison confirms all earlier production +patches and untracked source hashes are preserved; the v15 production delta +is limited to `RegisterWriteTable.cs` and its new scalar-reuse partial file. +No commit was rewritten, created, or pushed in this continuation. + + +### v15 room verdict: reject enabling scalar reuse by default + +Run `20261027-145800-silent hill short message.zar-room-scalar-reuse-v15-on-1` +reached the matching shelf/lamp view (`shot-0471.png`). The phone was closed at +432.75 seconds. In the 475–510 second window, 18 reports measured **12.995 FPS +median, range 9.48–15.99**. The preceding v14 overlap observation was 13.46 FPS; +these are not sufficient for an isolated regression claim. **30 FPS remains +unachieved.** + +The candidate reused only **18,994 of 2,359,296 eligible scalar packets +(0.805%)** between its 475.352 and 509.634 second reports. Its large synthetic +best-case saving does not apply to most writes in this scene. Keep the feature +**disabled by default**; further scalar work must reduce the cost of writes +that actually change state, rather than relying on identical values. + +All seven memory samples in the FPS window showed warning pressure. System-wide +interval totals were 9,551 swap-in pages, 2,992 swap-out pages, 760,250 compressed +pages, and 636,199 decompressed pages. These include window-edge overlap and +cannot be attributed solely to the emulator. Median counters per two-second +interval were 91,157 draws, 3,279 passes, 750.2 ms GPU busy, zero host wait, +357 ms worker idle, 519.0 ms command-thread synchronous wait, and 721.1 ms +queue-space wait. These overlapping counters are not additive frame times. + +The 530–540 second CPU measurement recorded 9.73 CPU seconds on the guest audio +event thread, 6.76 on the RHI thread, 7.46 on the GPU worker, 4.22 on program +resolution, and 3.43 on the command stream. The separate trace beginning at +550.02 seconds used the same GC/contention providers as the v14 overlap trace, +with no CLR stack sampler. It recorded 20,507 allocation ticks totaling +**2,194,898,680 bytes**. Sampled type attribution remained led by +`UserScalarRegisters` (316,529,552 bytes), `ResourceSnapshot` (165,137,720), and +`VertexInputInfo` (162,523,712). These are statistical type attribution, not exact +per-type totals. The near-unchanged overall allocation volume supports the +low-reuse-rate verdict; it does not establish a small allocation improvement. + +The trace contained 262 GC suspensions totaling 1,127.15 ms, median 2.88 ms, +maximum 226.83 ms, plus one 2.32 ms preparation suspension. There were no +`SuspendOther` intervals. The long outlier and different memory pressure +prevent assigning this timing difference to scalar reuse. Runtime contention +starts/stops were paired at 4,194 each. + +A late native game sample at 575.21–580.04 seconds captured only three stack +observations per named thread and coincided with two zero-FPS reports. The +subsequent GPU sample overlapped recovery. Neither sample is accepted as a +representative steady-state CPU profile; both are outside the FPS window. +The game reached the 600-second limit, then exited 0 after the planned stop. +No fatal or unsolicited CPU trap occurred. The first shutdown CPU_TRAP was at +600.727 seconds. Peak reported footprint/RSS was 9,806/6,283 MiB. Both game +processes and all capture helpers are gone. + +The next allocation investigation should address objects created for changing +draws: snapshot state, resource snapshots and deferred work lifetimes. Queue +items cannot simply be recycled when `Done` is set: synchronous producers may +still read their result or failure, and pending writes/ranges may retain item +references. Those ownership constraints must be proved before pooling. + + +### v16: bounded reuse of asynchronous command items + +The v15 GC trace attributed about 139 MB of its sampled allocation volume to +`DeferredCommandThread.Item`. The v16 candidate adds an opt-in, 64-entry return +ring (`SHARPEMU_COMMAND_ITEM_REUSE=1`, off by default). The render consumer returns +only asynchronous items after execution, pending-write/range removal and its last +item read. The interpreter producer acquires them for subsequent work. Synchronous +callbacks keep their original allocation and result ownership; failures are not +returned to the ring. Cached items clear retained payload references. + +The two writers use separate cursor cache lines and cache the opposite cursor +until their current batch is exhausted. An initial version that read the opposite +cursor for every item increased the native two-thread median from about 540 to +591 ns/item; it was superseded before runtime testing. This is why a same-thread +allocation benchmark alone is insufficient for this queue. + +The final synthetic comparison disables tiering, alternates baseline/candidate +order, uses eight 200,000-item trials per same-thread batch size, and six +100,000-item trials with real producer/consumer threads. Median results: + +| Architecture | Queue mode | Baseline ns/item | Reuse ns/item | Baseline B/item | Reuse B/item | +|---|---|---:|---:|---:|---:| +| Native arm64 | Same thread, batch 1 | 30.28 | 26.08 | 80 | 0 | +| Native arm64 | Same thread, batch 16 | 19.54 | 16.04 | 80 | 0 | +| Native arm64 | Same thread, batch 64 | 18.79 | 15.44 | 80 | 0 | +| Native arm64 | Two threads, batch 64 | 389.97 | 354.52 | 80.0008 | 0.0260 | +| Rosetta x64 | Same thread, batch 1 | 58.92 | 52.86 | 80 | 0 | +| Rosetta x64 | Same thread, batch 16 | 38.63 | 33.61 | 80 | 0 | +| Rosetta x64 | Same thread, batch 64 | 41.47 | 32.74 | 80 | 0 | +| Rosetta x64 | Two threads, batch 64 | 506.83 | 437.00 | 80.0008 | 0.0136 | + +The focused deferred-command selection passed **12 tests, zero failures, zero +skips**, including mixed work ordering, pending write overlays, copied write +payloads, ring wrapping on two threads, synchronous exception identity, +asynchronous failure handling and bounded retention. Native arm64 and Rosetta +benchmark runs passed. These synthetic results establish reduced queue-item +allocation; they do not establish a gameplay FPS improvement. + +Full integration with item reuse enabled passed **9,047 Libs tests, zero +failures, five existing skips**, then **1,634 Metal device tests, zero failures, +seven existing skips**. Metal used the command-thread, parallel-resolution and +queued-command-write settings intended for the room run. The commands and +complete output are preserved in `queue-recycle-v16-libs-on.log` and +`queue-recycle-v16-metal-on.log` under the private diagnosis artifact directory. +The v16 tracked-source diff SHA-256 is +`57374ddf9e0258002e70130c91acd5c873bda2bb054dd6069b3e10ebebeae77b`. +Comparison against v15 confirms all earlier production patches and untracked +source hashes are preserved; only `DeferredCommandThread.cs` and its new +recycling partial changed in production code. HEAD remains `52f07c7b3`. + + +The first v16 run, `20261027-154622-silent hill short message.zar-room-queue-recycle-v16-on-1`, +reached the shelf/lamp view at 468 seconds, but its camera moved away during the +475–510 second measurement window. That window is **rejected for matched FPS +comparison**. Its 23.45 median and 34.39 maximum do not establish the 30 FPS +goal or an improvement over the prior room results. The later GC/CPU capture +also covers a different view and is not a matched allocation comparison. +The trace recorded 875,663,072 allocation-tick bytes and 98 GC suspensions +(457.40 ms total), but workload differences prevent attributing the reduction +to item reuse. The run reached its 600-second limit and exited 0 after the +planned stop; peak footprint/RSS was 9,837/5,490 MiB. + +### v17: isolate scripted input for repeatable runtime measurements + +Inspection showed `WindowHostInput` merges scripted keys with physical keyboard +and controller input. `SHARPEMU_INPUT_SCRIPTED_ONLY=1`, together with +`SHARPEMU_INPUT_FILE`, now permits controlled runs to consume only scripted keys. +The mode also suppresses physical controller discovery and outputs. Without both +settings, interactive input follows the existing path. This is a measurement +reliability change, not an FPS optimization. The camera movement's exact source +was not recorded; the first v16 run remains rejected regardless of that source. + +The input selection passed all **82 pad/input tests, zero failures, zero skips**, +including existing physical-input forwarding and the new exclusive-mode check. +Both v17 Release publishes passed; all 471 payload hashes verified. Its +tracked-source diff SHA-256 is +`6937a2a83673811c1e9b58a2dedaabdc68fd6eb50e6839d48925170b7924e468`. +The production delta from v16 is solely `WindowHostInput.cs`; the renderer and +queue candidate are unchanged. Existing commits and earlier production patches +remain preserved. + + +The isolated-input run +`20261027-160143-silent hill short message.zar-room-input-isolated-v17-on-1` +reached the phone later, which was closed at 485.05 seconds. Therefore the +original 475–510 second window is rejected: it includes the phone and transition. +Screenshots 0499, 0510 and 0520 confirm the stationary shelf/lamp view. A later +**500–525 second observation measured 4.5 FPS median, range 0–10.99, 13 reports**. +It is not an identical elapsed-time comparison with v14/v15. **30 FPS remains +unachieved; queue reuse is not accepted as a gameplay improvement and remains +disabled by default.** + +All five memory samples in that later window reported warning pressure. +System-wide interval totals were 18,520 swap-in pages, 31,740 swap-out pages, +2,045,519 compressions and 1,905,837 decompressions. These overlapping intervals +cannot identify emulator-only traffic. Median two-second counters were 20,916 +draws, 768 passes, 404.4 ms GPU busy, zero host wait, 701 ms worker idle, +492.9 ms command-thread synchronous wait and 651.6 ms queue-space wait. These +are overlapping intervals, not additive frame times. + +The later 530–540 second CPU sample recorded 7.10 CPU seconds on the audio event +thread, 4.03 on RHI, 4.61 on the GPU worker, 2.36 on program resolution and 2.05 +on the command stream. The separate GC/contention capture at 550.02–560.93 +seconds recorded 11,503 allocation ticks totaling 1,232,101,880 bytes. +`Item` accounted for only 12 ticks / 1,291,376 sampled bytes, consistent with +queue-item reuse working, but total allocation and GC figures must be normalized +to comparable work before inferring performance. There were 147 GC suspensions +totaling 659.68 ms (median 4.08, maximum 12.23), one 2.53 ms preparation +suspension, no unpaired starts, and 4,247 paired contention starts/stops. + +The run reached its 600-second limit, then exited 0 after the planned stop. +The first CPU_TRAP was at 600.338 seconds during shutdown; no earlier fatal or +resource-materialization failure appeared. Peak footprint/RSS was 9,888/4,298 +MiB. The next required comparison is the **same immutable v17 build** with +scripted-only input retained and `SHARPEMU_COMMAND_ITEM_REUSE=0`. This must +precede attributing the slowdown to either item reuse or host pressure. + + +### 2026-10-27: reject unmatched v17 controls; host scheduling confounder + +The queue-reuse-off v17 control at `20261027-161900...room-input-isolated-v17-off-1` +ended normally at 600 seconds but remained on the Continue screen (`shot-0597.png`). +Its CPU/GC captures are not gameplay comparison evidence. The repeat at +`20261027-163109...room-phase-v17-off-2` used the same 471 verified v17 payloads, +scripted-only input, queue reuse off, unchanged 1080p settings, and render phase +profiling. It ended normally after its 900-second bound but remained in the +opening cinematic (`shot-final.png`). Neither establishes a queue-reuse FPS +verdict or progress to 30 FPS. + +A read-only host census found an unrelated lint process using approximately +4795 MiB of per-process footprint and 237.9% CPU during startup. Later kernel +thread snapshots at elapsed 842.79–852.97 seconds found maximum scheduler priority +4 for all 92 observed emulator and all 50 GPU-host threads; all 139 observed +GeForce NOW threads had maximum priority 63. `ps` reported process priority 97 +for GeForce NOW and 4 for both emulator processes. Both emulator processes had +successfully acquired their macOS activity assertions, so missing activity +assertions are not supported as the explanation. + +The system's `gamepolicyd` log recorded Game Mode enabled/status on at local +16:36:37, alongside the full-screen gaming session. Apple documents that Game Mode +prioritizes the selected game's CPU/GPU access and reduces background activity: +https://support.apple.com/en-ie/105118 . This is a strong scheduling confounder, +not yet a measured causal FPS improvement. No other application was stopped or +its settings changed. The user explicitly declined a temporary Game Mode-off comparison; GeForce +NOW and its settings must remain unchanged. Continue independent code and +synthetic work, and qualify runtime results under the observed scheduling limits. + +Raw evidence remains in ignored run artifacts: `scheduler-priorities.txt`, +`thread-cpu-deltas-0842.json`, `host-footprints-0172.json`, +`host-footprints-0469.json`, `unqualified-room-result.json`, and +`runtime-diagnosis-20261026/gamepolicy-live-20261027.log`. Per-process footprints +can double-count shared mappings and must not be summed as unique physical RAM. + + +### Depth-target memo collision recovery + +`ImageRequestBuilders.DepthTarget` hashed `ZReadBase ^ ZWriteBase` at the same +shift. Normal equal read/write bases therefore removed all address information; +alternating surfaces with equal depth control replaced the same cache entry. +The new regression reproduced 516 format queries for four warmed targets plus +512 lookups, where only the initial four queries were necessary. + +The cache now mixes different address bits and keeps four exact states in each +of eight buckets. Full `DepthTargetWords` equality and device identity still +guard every hit. The uncached trace path, the `WorkerScratch` fallback, copy-mode +requests, and copy-on-write mip layouts retain their existing behavior. This is +the only new production delta in this continuation; all earlier source changes +and HEAD `52f07c7b37570adaad7317b8196a86f9bb4d6548` remain preserved. + +The isolated synthetic comparison used identical test files and dependencies, +swapping only `SharpEmu.Libs.dll` between the verified immutable v17 publication +and the candidate. Seven batches of 100,000 calls per case ran with tiering off. + +| Runtime / targets | Before wall ns | After wall ns | Before bytes | After bytes | +| --- | ---: | ---: | ---: | ---: | +| Native ARM64 / one repeated | 55.15 | 37.06 | 0 | 0 | +| Native ARM64 / four alternating | 174.88 | 51.93 | 136 | 0 | +| Rosetta x64 / one repeated | 47.14 | 50.91 | 0 | 0 | +| Rosetta x64 / four alternating | 351.10 | 63.78 | 136 | 0 | + +Native per-thread CPU medians for four alternating targets were 171.11 to +47.47 ns/op. The raw Rosetta CPU-clock readings did not qualify as nanoseconds +and are excluded; subsequent benchmark code emits CPU time only on native ARM64. +The x64 repeated-target wall case increased by 3.77 ns, approximately 8%. +Scheduling and frequency variation remain limitations. These are cache lookup +measurements, not gameplay FPS or proof that depth requests explain the previous +2 FPS report. Exact allocations and format-query counts confirm the eliminated +rebuilds independently of timing. + +Validation: 78 focused image/request/memo tests passed. The original regression +failed before the fix and passed afterward; it covers different target addresses, +four states sharing one address, device identity, changed dimensions and clear +state, inactive targets, and independent copied layouts. Pipeline-lookahead +rechecks passed 4/4 on both reference and candidate. REUSE passed 2863/2863. + +Full Libs validation did **not** complete: the 600-second bounded run recorded +one pipeline-lookahead failure and four isolated native-return worker timeouts +(30 seconds each) before its outer timeout. There is no full-suite total or green +integration claim. All descendant test processes were verified gone afterward. +The four native-return cases have not been compared against the reference in +this continuation; the passing isolated lookahead reruns do not erase its full-run +failure. The candidate has not been republished into a game bundle or measured +in the stationary room. The 30 FPS goal remains unachieved. + +Private evidence: `depth-memo-preservation.json`, `depth-memo-before-test.log`, +`depth-memo-after-test.log`, `depth-memo-cpu-comparison-provenance.json`, +`depth-memo-cpu-summary.json`, `depth-memo-full-libs-result.json`, and +`depth-memo-reuse.log` under the runtime-diagnosis artifact directory. + +### Native-worker follow-up with GeForce NOW unchanged + +The four native-worker cases that timed out in the full Libs attempt were rerun +serially on x64, using the same isolated test payload and swapping only the Libs +assembly. The immutable v17 reference passed **4/4**, and the depth-cache candidate +passed **4/4**, with no failures or skips. Whole invocations took 94.87 and 90.81 +seconds respectively; individual parent tests took 15–20 seconds against their +unchanged 30-second child-process deadline. `DOTNET_HOST_PATH` and `DOTNET_ROOT` +explicitly selected the x64 runtime for both arms. + +A direct candidate continuation worker then passed **1/1**. Its adapter reported +about 3 seconds in the test body and 15.80 seconds for the whole test invocation; +discovery completed at adapter timestamp 7.32 seconds. This demonstrates material +runner/startup overhead. It does not establish that startup alone caused the +earlier timeouts, nor exclude a full-suite order or resource interaction. The +earlier incomplete full run and its five observed failures remain recorded. + +No emulator code, timeout, rendering option, GeForce NOW setting, commit, or game +bundle changed during this follow-up. No game was launched. The 30 FPS goal is +still unachieved. Evidence: `depth-memo-native-recheck.json`, +`depth-memo-native-reference.log`, `depth-memo-native-candidate.log`, and +`depth-memo-native-direct-worker.log` in the runtime-diagnosis artifact directory. + + +### Current combined source: full Libs recovery and v18 diagnostic + +The subsequent full x64 Libs rerun completed: **9,052 passed, zero failed, +five existing Opus skips** (9,057 total; 269.61 seconds for the invocation). +It used the unchanged depth-cache candidate, explicit x64 `DOTNET_HOST_PATH` +and `DOTNET_ROOT`, serial runsettings and a 120-second hang detector. The +previous timed-out invocation and its five failures remain historical evidence; +the successful rerun does not establish their cause. + +The tested Libs SHA-256 is +`552394e3993e9ff7572c3d0060a67676fa9938f89ef0e1ca072c972d4e6a071f`; +the test assembly is +`3ca3a592afc07d5673ac329122252172f2e8edf3037d41144939820f4d4b9b36`. +Release self-contained x64 CLI and native arm64 GPU-host publications both +succeeded. All 471 v18 payload hashes were checked before launch. This build +includes the depth-cache change and uses the same source snapshot as the tests. + +Run `20261027-173845-silent hill short message.zar-room-phase-v18-1` +reached the first room. Screenshots at 478 and 508 seconds confirm the stationary +shelf/lamp view with the phone closed. Rendering remained 1920×1080, scale 1, +VSync on. Native occlusion, command threading, parallel resolution and queued CP +writes were enabled; command-item reuse and identical-scalar reuse were disabled. +Scripted-only input prevented external controller movement. GeForce NOW and all +its settings remained unchanged. + +In the 475–510-second window, **17 guest-presentation reports had a median of +6.98 FPS, range 0.0–12.48**. This instrumented observation is slower than the +roughly 14-FPS observations above; it is not an isolated depth-cache regression +or an acceptance result. Six of seven nearby memory samples reported warning +pressure. System-wide interval deltas included 30,245 swapped-in and 62,172 +swapped-out pages, plus 1,763,427 compressions and 1,613,994 decompressions. +Those counters include other processes and interval-boundary overlap. + +Fifteen complete render-profile intervals contained 229 frames over 31.8 +reported seconds. A report ending inside the window but beginning before it +was excluded rather than attributed to the whole window. Exclusive render-thread +wall time per reported frame was: + +| Phase | ms/frame | +| --- | ---: | +| Program-prefetch wait | 36.13 | +| Idle | 33.12 | +| Draw-resource preparation | 8.84 | +| Command stream | 6.07 | +| Graphics-state application | 5.35 | + +These include instrumentation and scheduling delay. They are not CPU utilization; +background program-resolution time overlaps prefetch wait and must not be added. +A separate thread CPU sample at 580.94–590.94 seconds observed 7.95 CPU-seconds +in the guest audio-event thread, 4.71 in RHI, 5.98 in the GPU worker, 2.81 in +program resolution and 2.50 in the command thread. These later samples are not +aligned with the FPS window. All sampled emulator/GPU-host threads had maximum +scheduler priority 63, so the earlier priority-4 observation did not persist. + +The run stopped at its requested marker around 717 seconds; process and harness +exit codes were zero. The CPU_TRAP result occurred after the logged host-stop +redirect, not before the measurement. Guest teardown remains unverified. Peak +main-process footprint/RSS was 9,856/3,780 MiB. Shared mapping charges mean that +main-process and GPU-host footprints must not be summed as unique physical RAM. +No 30-FPS result is established. Next work should separate memory residency and +background program-resolution costs while keeping other applications unchanged. + +Private evidence: `depth-memo-full-libs-recheck.json` and its TRX, v18 build +logs/source snapshot/payload manifest, plus `room-fps-475-510.json`, +`render-phases-475-510.json`, `room-memory-window.json` and the thread-CPU +capture in the run directory. Raw logs, runtime bundles and game-derived assets +are not part of this PR. + + +### PR publication validation + +The accumulated source package was rechecked before committing it to PR #23: + +| Suite | Passed | Failed | Existing skips | +| --- | ---: | ---: | ---: | +| Libs, x64, serial | 9,052 | 0 | 5 | +| Gpu.Metal, required device, command overlap and queued CP writes | 1,634 | 0 | 7 | +| ShaderCompiler, x64, serial | 2,192 | 0 | 1 | +| ShaderCompiler.Metal, required device, serial | 3,588 | 0 | 0 | +| Total for these four suites | 16,466 | 0 | 13 | + +The last three invocations rebuilt their Release dependencies. All used +`docs/performance/shader-serial.runsettings` and `--blame-hang-timeout 120s`; +`SHARPEMU_TEST_REQUIRE_DEVICE=1` was scoped to the two Metal suites. Commands, +TRX results and elapsed times are retained in `pr23-current-validation.json` +and `pr23-current-results` under the private diagnostic directory. The Libs +rerun provenance is recorded above. This total does not reclassify older failed +or incomplete attempts, and is not a rerun of every project in the solution. +REUSE passed **2,863/2,863** files. The staged whitespace check found and removed +one trailing blank line from the new scalar-constant MSL template; that final +formatting-only difference was made after the suite build. Main remained clean. +The publication contains source, tests, benchmark tools and this report, without +raw commercial captures, game binaries or local runtime bundles. diff --git a/src/SharpEmu.Core/Cpu/Native/DirectExecutionBackend.Imports.cs b/src/SharpEmu.Core/Cpu/Native/DirectExecutionBackend.Imports.cs index 64e502604..c0be8e4f1 100644 --- a/src/SharpEmu.Core/Cpu/Native/DirectExecutionBackend.Imports.cs +++ b/src/SharpEmu.Core/Cpu/Native/DirectExecutionBackend.Imports.cs @@ -508,6 +508,13 @@ private unsafe ulong DispatchImportCore(int importIndex, nint argPackPtr) LastError = $"Import dispatch index out of range: {importIndex}"; return 18446744071562199042uL; } + // Every managed import path, including leaves, is a shutdown safe point. + // The full-path check below remains for a request racing this earlier check. + if (Volatile.Read(ref _forcedGuestExit) && TryEndGuestSliceForShutdown(argPackPtr)) + { + cpuContext[CpuRegister.Rax] = 1uL; + return 1uL; + } // By reference: the entry is 48 bytes of per-import classification that // nothing here mutates, and copying it was pure cost on every guest call. ref readonly ImportStubEntry importStubEntry = ref _importEntries[importIndex]; @@ -576,7 +583,7 @@ private unsafe ulong DispatchImportCore(int importIndex, nint argPackPtr) in importStubEntry, num, leafArg0, leafArg1, leafArg2, leafArg3, leafReturnRip, builderResult); } - return builderResult; + return CompleteLeafImportShutdown(cpuContext, argPackPtr, builderResult); } // The leaf paths skip the rest of the gateway entirely, so they mark @@ -599,7 +606,7 @@ private unsafe ulong DispatchImportCore(int importIndex, nint argPackPtr) in importStubEntry, num, leafArg0, leafArg1, leafArg2, leafArg3, leafReturnRip, hotMemoryResult); } - return hotMemoryResult; + return CompleteLeafImportShutdown(cpuContext, argPackPtr, hotMemoryResult); } } @@ -614,7 +621,7 @@ private unsafe ulong DispatchImportCore(int importIndex, nint argPackPtr) in importStubEntry, num, leafArg0, leafArg1, leafArg2, leafArg3, leafReturnRip, trivialResult); } - return trivialResult; + return CompleteLeafImportShutdown(cpuContext, argPackPtr, trivialResult); } if (importStubEntry.IsLeaf) @@ -635,7 +642,7 @@ private unsafe ulong DispatchImportCore(int importIndex, nint argPackPtr) in importStubEntry, num, leafArg0, leafArg1, leafArg2, leafArg3, leafReturnRip, leafResult); } - return leafResult; + return CompleteLeafImportShutdown(cpuContext, argPackPtr, leafResult); } } @@ -2154,7 +2161,14 @@ importStubEntry.Export is not { } export || // Every entry below was checked for those four properties - a poll that // takes a host lock still qualifies, because the lock is the emulator's own // and cannot park the guest thread. + // Queue-level reads only inspect the context/device counters and write two register- + // addressed outputs. They neither park the guest nor invoke guest callbacks. Keep the + // audio poll fast path opt-in until its runtime comparison is complete. + private static readonly bool _audioQueueLeafEnabled = + Environment.GetEnvironmentVariable("SHARPEMU_AUDIO_OUT2_QUEUE_LEAF") == "1"; + internal static bool IsTrivialLeafImport(string nid) => + (_audioQueueLeafEnabled && nid is "R7d0F1g2qsU" or "Q8DZkKQ-SYc") || nid is "0-KXaS70xy4" or // pthread_getspecific "eoht7mQOCmo" or // scePthreadGetspecific @@ -2518,6 +2532,20 @@ private void TraceImportFrameChain(CpuContext context, long dispatchIndex) } } + // A stop can arrive while a leaf body runs. Only ordinary guest returns may + // be redirected here: preserve already prepared exits, blocks and context transfers. + // Failed redirects retain the existing behavior; this is not a quiescence fence. + private unsafe ulong CompleteLeafImportShutdown(CpuContext context, nint argPackPtr, ulong result) + { + if (!Volatile.Read(ref _forcedGuestExit)) return result; + var state = GuestThreadExecution.State; + if (ActiveGuestThreadYieldRequested || state.HasPendingDispatchOutcome || state.PreparedContextTransferStub != 0) + return result; + if (!TryEndGuestSliceForShutdown(argPackPtr)) return result; + context[CpuRegister.Rax] = 1uL; + return 1uL; + } + // Returns the import straight to the host entry stub, which ends this guest slice. private unsafe bool TryEndGuestSliceForShutdown(nint argPackPtr) { diff --git a/src/SharpEmu.Core/Runtime/SharpEmuRuntime.cs b/src/SharpEmu.Core/Runtime/SharpEmuRuntime.cs index cc00c1d11..43c26c738 100644 --- a/src/SharpEmu.Core/Runtime/SharpEmuRuntime.cs +++ b/src/SharpEmu.Core/Runtime/SharpEmuRuntime.cs @@ -12,6 +12,7 @@ using SharpEmu.HLE.GpuMemory; using SharpEmu.HLE.GuestMemory; using SharpEmu.Libs.VideoOut; +using SharpEmu.Libs.Ampr; using SharpEmu.Libs.Kernel; using SharpEmu.Libs.AppContent; using SharpEmu.Libs.SaveData; @@ -159,6 +160,7 @@ public SelfImage LoadImage(string ebootPath) public OrbisGen2Result Run(string ebootPath) { using var archiveMount = MountArchive(ref ebootPath); + using var aprSession = _aprSession = AmprExports.BeginRuntimeSession(_virtualMemory); var normalizedEbootPath = Path.GetFullPath(ebootPath); using var app0Binding = BindApp0Root(normalizedEbootPath); Console.Error.WriteLine($"[RUNTIME] Loading: {ebootPath}"); @@ -1506,6 +1508,8 @@ public OrbisGen2Result DispatchHleCall(string nid, CpuContext context) return _moduleManager.Dispatch(nid, context); } + private IDisposable? _aprSession; + public void Dispose() { if (_disposed) @@ -1531,6 +1535,8 @@ public void Dispose() return; } + _aprSession?.Dispose(); + if (_gpuMemory != null) { if (GuestGpuMemoryHook.TryTakeShutdownSummary(out var gpuSummary)) diff --git a/src/SharpEmu.GpuSplit/RemoteGuestAddressSpace.cs b/src/SharpEmu.GpuSplit/RemoteGuestAddressSpace.cs index 2b7097d82..4b72ce3f2 100644 --- a/src/SharpEmu.GpuSplit/RemoteGuestAddressSpace.cs +++ b/src/SharpEmu.GpuSplit/RemoteGuestAddressSpace.cs @@ -147,7 +147,7 @@ public bool TryCompare(ulong virtualAddress, ReadOnlySpan expected, out bo { if (!TryPiece(virtualAddress + done, size - done, out var pointer, out var take)) { - NoteMiss(virtualAddress, size, "compare"); + NoteMiss(virtualAddress, size, "compare", virtualAddress + done); return false; } @@ -475,7 +475,7 @@ private bool Copy(ulong address, ulong size, byte* bytes, bool toGuest) { if (!TryPiece(address + done, size - done, out var pointer, out var take)) { - NoteMiss(address, size, toGuest ? "write" : "read"); + NoteMiss(address, size, toGuest ? "write" : "read", address + done); return false; } @@ -494,14 +494,38 @@ private bool Copy(ulong address, ulong size, byte* bytes, bool toGuest) return true; } - private void NoteMiss(ulong address, ulong size, string access) + private void NoteMiss(ulong address, ulong size, string access, ulong? failedAddress = null) { Interlocked.Increment(ref _misses); if (Interlocked.Increment(ref _missReports) <= MaxMissReports) { Console.Error.WriteLine( $"[GPU-SPLIT][WARN] GPU {access} of guest 0x{address:X16}+0x{size:X} failed: not in the shared backing " + - "(memory private to the guest process)."); + "(unmapped, private, or outside the mirrored range)."); + if (Environment.GetEnvironmentVariable("SHARPEMU_SPLIT_MISS_DETAILS") == "1") + { + var failed = failedAddress ?? address; + var snapshot = _translator.Snapshot(); + var before = snapshot.LastOrDefault(entry => entry.Address <= failed); + var after = snapshot.FirstOrDefault(entry => entry.Address > failed); + var sequence = _translator.ChangeSequence; + Console.Error.WriteLine($"[GPU-SPLIT][MISS] first=0x{failed:X16} generation={_translator.MappingGeneration} " + + $"complete={_translator.IsComplete} mirror_bytes=0x{_mirrorBytes:X} changes={sequence} " + + $"before=[0x{before.Address:X16},0x{before.End:X16}) offset=0x{before.Offset:X} " + + $"after=[0x{after.Address:X16},0x{after.End:X16}) offset=0x{after.Offset:X}"); + var first = sequence > 65536 ? sequence - 65536 : 0; + var reported = 0; + for (var number = sequence; number > first && reported < 16;) + { + if (_translator.TryReadChange(--number, out var change) && change.Kind != GuestViewLogKind.Protect && + failed >= change.Address && failed - change.Address < change.Size) + { + Console.Error.WriteLine($"[GPU-SPLIT][MISS-CHANGE] n={number} kind={change.Kind} " + + $"address=0x{change.Address:X16} size=0x{change.Size:X} offset=0x{change.Offset:X}"); + reported++; + } + } + } } } } diff --git a/src/SharpEmu.HLE/GuestMemory/GuestMirrorTranslator.cs b/src/SharpEmu.HLE/GuestMemory/GuestMirrorTranslator.cs index 3443a72e3..eaa99de74 100644 --- a/src/SharpEmu.HLE/GuestMemory/GuestMirrorTranslator.cs +++ b/src/SharpEmu.HLE/GuestMemory/GuestMirrorTranslator.cs @@ -47,6 +47,8 @@ public sealed unsafe class GuestMirrorTranslator private readonly GuestViewTableHeader* _header; private readonly GuestViewTableEntry* _entries; private readonly int _capacity; + private readonly GuestViewLogRecord* _log; + private readonly int _logCapacity; private readonly Slot[] _cache = new Slot[CacheSlots]; private long _hits; private long _misses; @@ -81,7 +83,7 @@ private struct LastRange /// public GuestMirrorTranslator(ulong tableAddress, ulong tableBytes, ulong mirrorBase, ulong mirrorBytes) { - GuestViewTableReader.Open(tableAddress, tableBytes, out _header, out _entries, out _capacity, out _, out _); + GuestViewTableReader.Open(tableAddress, tableBytes, out _header, out _entries, out _capacity, out _log, out _logCapacity); MirrorBase = mirrorBase; MirrorBytes = Math.Min(mirrorBytes, _header->BackingSize); } @@ -102,6 +104,17 @@ public GuestMirrorTranslator(HostSharedRegion table, HostBackingObject mirror) public ulong MappingGeneration => Volatile.Read(ref _header->MappingGeneration); + /// The sequence after the last published mapping/protection change, for diagnostics. + public ulong ChangeSequence => Volatile.Read(ref _header->LogSequence); + + /// A published change while the owner's ring still retains it. Reading it + /// does not advance the replica or alter translation caches. + public bool TryReadChange(ulong number, out GuestViewLogEntry entry) + { + entry = default; + return number < ChangeSequence && GuestViewTableReader.TryReadLog(_log, _logCapacity, number, out entry); + } + /// /// Whether translations count their cache hits and misses (, /// ). Off unless SHARPEMU_SPLIT_STATS=1: the worker, the program diff --git a/src/SharpEmu.HLE/GuestMemory/GuestViewTable.cs b/src/SharpEmu.HLE/GuestMemory/GuestViewTable.cs index 370ead4c7..d402f0875 100644 --- a/src/SharpEmu.HLE/GuestMemory/GuestViewTable.cs +++ b/src/SharpEmu.HLE/GuestMemory/GuestViewTable.cs @@ -86,6 +86,7 @@ public sealed unsafe class GuestViewTable : IDisposable private readonly GuestViewLogRecord* _log; private readonly object _gate = new(); private int _overflowReports; + private readonly bool _traceMappings = Environment.GetEnvironmentVariable("SHARPEMU_SPLIT_MISS_DETAILS") == "1"; /// /// Builds an empty table over a writable shared region, with a log of @@ -261,6 +262,15 @@ private void ChangeMapping(ulong cutAddress, ulong cutSize, GuestViewTableEntry? if (!fits) NoteOverflowLocked(); } finally { EndWrite(); } + // Log after publishing the even sequence so diagnostic I/O does not hold readers + // in the seqlock retry loop. Mapping-only history survives protection-log churn. + if (_traceMappings) + { + if (cutSize != 0) + Console.Error.WriteLine($"[GPU-VIEW][UNMAP] generation={MappingGeneration} address=0x{cutAddress:X16} size=0x{cutSize:X}"); + if (added is { } traced) + Console.Error.WriteLine($"[GPU-VIEW][MAP] generation={MappingGeneration} address=0x{traced.Address:X16} size=0x{traced.Size:X} offset=0x{traced.Offset:X}"); + } } } diff --git a/src/SharpEmu.HLE/Host/WindowHostInput.cs b/src/SharpEmu.HLE/Host/WindowHostInput.cs index 8568160cb..b840c58dc 100644 --- a/src/SharpEmu.HLE/Host/WindowHostInput.cs +++ b/src/SharpEmu.HLE/Host/WindowHostInput.cs @@ -13,38 +13,53 @@ namespace SharpEmu.HLE.Host; /// public sealed class WindowHostInput : IHostInput { + private readonly bool _scriptedOnly; + + public WindowHostInput() + : this(ScriptedKeyInput.IsEnabled && + Environment.GetEnvironmentVariable("SHARPEMU_INPUT_SCRIPTED_ONLY") == "1") + { + } + + internal WindowHostInput(bool scriptedOnly) => _scriptedOnly = scriptedOnly; + + // A controlled run must not mix file input with keys/sticks from a focused window. + // Without both an input file and the explicit opt-in, physical input stays available. + private IHostWindowInputSource? PhysicalInput => + _scriptedOnly ? null : HostWindowInputSource.Current; + public void EnsureStarted() { // SDL owns device discovery and pumps it on the window thread. } public int GetGamepadStates(Span destination) => - HostWindowInputSource.Current?.GetGamepadStates(destination) ?? 0; + PhysicalInput?.GetGamepadStates(destination) ?? 0; public string? DescribeConnectedGamepad() => - HostWindowInputSource.Current?.DescribeConnectedGamepad(); + PhysicalInput?.DescribeConnectedGamepad(); public void SetRumble(byte largeMotor, byte smallMotor) => - HostWindowInputSource.Current?.SetRumble(largeMotor, smallMotor); + PhysicalInput?.SetRumble(largeMotor, smallMotor); public void SetTriggerRumble(byte? leftTrigger, byte? rightTrigger) => - HostWindowInputSource.Current?.SetTriggerRumble(leftTrigger, rightTrigger); + PhysicalInput?.SetTriggerRumble(leftTrigger, rightTrigger); public void SetAdaptiveTriggerEffect( HostAdaptiveTriggerEffect? leftTrigger, HostAdaptiveTriggerEffect? rightTrigger) => - HostWindowInputSource.Current?.SetAdaptiveTriggerEffect(leftTrigger, rightTrigger); + PhysicalInput?.SetAdaptiveTriggerEffect(leftTrigger, rightTrigger); public void SetLightbar(byte red, byte green, byte blue) => - HostWindowInputSource.Current?.SetLightbar(red, green, blue); + PhysicalInput?.SetLightbar(red, green, blue); - public void ResetLightbar() => HostWindowInputSource.Current?.ResetLightbar(); + public void ResetLightbar() => PhysicalInput?.ResetLightbar(); public bool IsHostWindowFocused() => - (HostWindowInputSource.Current?.HasKeyboardFocus ?? false) || + (PhysicalInput?.HasKeyboardFocus ?? false) || (ScriptedKeyInput.IsEnabled && ScriptedKeyInput.AnyHeld); public bool IsKeyDown(int virtualKey) => (ScriptedKeyInput.IsEnabled && ScriptedKeyInput.IsHeld(virtualKey)) || - (HostWindowInputSource.Current?.IsKeyDown(virtualKey) ?? false); + (PhysicalInput?.IsKeyDown(virtualKey) ?? false); } diff --git a/src/SharpEmu.Libs/Agc/AgcExports.CommandStream.cs b/src/SharpEmu.Libs/Agc/AgcExports.CommandStream.cs index b5151e668..14d6d7b7f 100644 --- a/src/SharpEmu.Libs/Agc/AgcExports.CommandStream.cs +++ b/src/SharpEmu.Libs/Agc/AgcExports.CommandStream.cs @@ -60,6 +60,9 @@ internal enum GeometrySnapshotDecision internal static IReadOnlyDictionary GetGeometryCaptureRegistersForTests(ICpuMemory memory) => new Dictionary(_submittedGpuStates.GetValue(CanonicalMemory(memory), static _ => new SubmittedGpuState()).GeometryCapture.ShRegisters); + internal static DenseShaderRegisterMap GetExecutedShaderRegistersForTests(ICpuMemory memory) => + _submittedGpuStates.GetValue(CanonicalMemory(memory), static _ => new SubmittedGpuState()).Graphics.ShRegisters; + // Builds the snapshots of one draw as the prepass would, for tests of the execution-time check. internal static object CreateGeometrySnapshotsForTests( ulong packetAddress, diff --git a/src/SharpEmu.Libs/Agc/AgcExports.GeometryRegisters.cs b/src/SharpEmu.Libs/Agc/AgcExports.GeometryRegisters.cs index 2d315293d..7c90dcd2f 100644 --- a/src/SharpEmu.Libs/Agc/AgcExports.GeometryRegisters.cs +++ b/src/SharpEmu.Libs/Agc/AgcExports.GeometryRegisters.cs @@ -29,7 +29,7 @@ private static void ApplySubmittedRegisters( startRegister &= 0x0FFF_FFFFu; } - var directDestination = operation switch + IDictionary directDestination = operation switch { ItSetShReg => state.ShRegisters, ItSetContextReg => state.CxRegisters, @@ -66,7 +66,7 @@ private static void ApplySubmittedRegisters( return; } - Dictionary destination; + IDictionary destination; uint registerCount; ulong registersAddress; uint indirectRegister; diff --git a/src/SharpEmu.Libs/Agc/AgcExports.cs b/src/SharpEmu.Libs/Agc/AgcExports.cs index aaf079375..0f98b15f5 100644 --- a/src/SharpEmu.Libs/Agc/AgcExports.cs +++ b/src/SharpEmu.Libs/Agc/AgcExports.cs @@ -238,7 +238,7 @@ private sealed class SubmittedDcbState private RegisterBanks? _interpreterBanks; public Dictionary CxRegisters { get; private set; } = new(); - public Dictionary ShRegisters { get; private set; } = new(); + public DenseShaderRegisterMap ShRegisters { get; private set; } = new(); public Dictionary UcRegisters { get; private set; } = new(); // With an interpreter attached the banks and the composite extent are its own. diff --git a/src/SharpEmu.Libs/Ampr/AmprExports.Submission.cs b/src/SharpEmu.Libs/Ampr/AmprExports.Submission.cs index 18c765650..ae3a50c37 100644 --- a/src/SharpEmu.Libs/Ampr/AmprExports.Submission.cs +++ b/src/SharpEmu.Libs/Ampr/AmprExports.Submission.cs @@ -6,6 +6,7 @@ using System.Buffers.Binary; using System.Diagnostics; using System.Threading; +using System.Runtime.CompilerServices; namespace SharpEmu.Libs.Ampr; @@ -24,6 +25,7 @@ public static partial class AmprExports private sealed class AprSubmission { + public required AprSession Session { get; init; } public required uint Id { get; init; } public required CapturedCommandBuffer Captured { get; init; } public required ICpuMemory Memory { get; init; } @@ -50,8 +52,103 @@ private sealed class AprWorker private static readonly object _aprGate = new(); private static readonly Dictionary _aprSubmissions = new(); private static readonly Queue _aprCompletedOrder = new(); - private static readonly AprWorker[] _aprWorkers = - [new AprWorker(), new AprWorker(), new AprWorker(), new AprWorker()]; + private sealed class AprSession : IDisposable + { + internal readonly AprWorker[] Workers = [new(), new(), new(), new()]; + internal volatile bool Stopping; + // Admission and collection leases are guarded by _aprGate. + internal int ActiveProducers; + private readonly object _stopGate = new(); + private bool _stopped; + + public void Dispose() + { + // Serialize repeated Dispose calls through the entire join, not just the stop flag. + lock (_stopGate) + { + if (_stopped) return; + lock (_aprGate) + { + Stopping = true; + foreach (var worker in Workers) + { + while (worker.Pending.TryDequeue(out var pending)) + { + pending.ExecutionResult = AprStoppedResult; + pending.Done = true; + } + } + Monitor.PulseAll(_aprGate); + while (ActiveProducers != 0) Monitor.Wait(_aprGate); + } + foreach (var worker in Workers) + { + worker.Thread?.Join(); + } + lock (_aprGate) + { + foreach (var id in _aprSubmissions.Where(pair => ReferenceEquals(pair.Value.Session, this)) + .Select(pair => pair.Key).ToArray()) + _aprSubmissions.Remove(id); + var retained = _aprCompletedOrder.Where(item => !ReferenceEquals(item.Session, this)).ToArray(); + _aprCompletedOrder.Clear(); + foreach (var item in retained) _aprCompletedOrder.Enqueue(item); + } + // Cache leases protect readers belonging to other independent memories. + EvictAllCachedHostFiles(); + lock (_aprGate) + { + if (ReferenceEquals(_runtimeAprSession, this)) _runtimeAprSession = null; + } + _stopped = true; + } + } + } + + private const int AprStoppedResult = (int)OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND; + private static readonly ConditionalWeakTable _aprSessions = new(); + // Preserve the process-wide worker pool for direct library callers that do not own a runtime. + private static readonly AprSession _standaloneAprSession = new(); + private static AprSession? _runtimeAprSession; + [ThreadStatic] private static AprSession? _executingAprSession; + private static bool AprExecutionStopping => _executingAprSession?.Stopping == true; + + /// + /// Owns APR submissions for a runtime's root memory, including future tracked wrappers. + /// The stopped root association survives disposal, so late wrappers cannot enter the fallback pool. + /// Dispose after guest dispatch exits and before unmounting app0 or releasing guest memory. + /// Direct library callers retain the shared standalone worker pool. + /// + public static IDisposable BeginRuntimeSession(ICpuMemory memory) + { + lock (_aprGate) + { + if (_runtimeAprSession is not null) + throw new InvalidOperationException("An APR runtime session is already active."); + var session = new AprSession(); + var owner = AprMemoryOwner(memory); + _aprSessions.Remove(owner); + _aprSessions.Add(owner, session); + return _runtimeAprSession = session; + } + } + + private static ICpuMemory AprMemoryOwner(ICpuMemory memory) + { + while (memory is ICpuMemoryWrapper wrapper && !ReferenceEquals(memory, wrapper.Inner)) + memory = wrapper.Inner; + return memory; + } + + private static void ReleaseAprProducer(AprSession session) + { + lock (_aprGate) + { + session.ActiveProducers--; + Monitor.PulseAll(_aprGate); + } + } + private static int _nextAprSubmissionId; private static long _aprSequence; @@ -71,81 +168,101 @@ internal static int SubmitCommandBuffer( out uint submissionId) { submissionId = 0; - var captureResult = CaptureCommandBuffer(ctx, commandBuffer, out var captured); - if (captureResult != (int)OrbisGen2Result.ORBIS_GEN2_OK) + // Resolve wrappers before admission; no guest byte access occurs under the gate. + var owner = AprMemoryOwner(ctx.Memory); + AprSession session; + lock (_aprGate) { - return captureResult; + session = _aprSessions.GetValue(owner, static _ => _standaloneAprSession); + if (session.Stopping) return AprStoppedResult; + session.ActiveProducers++; } - - submissionId = unchecked((uint)Interlocked.Increment(ref _nextAprSubmissionId)); - if (submissionId == 0) + try { - submissionId = unchecked((uint)Interlocked.Increment(ref _nextAprSubmissionId)); - } + submissionId = 0; + var captureResult = CaptureCommandBuffer(ctx, commandBuffer, out var captured); + if (captureResult != (int)OrbisGen2Result.ORBIS_GEN2_OK) + { + return captureResult; + } - if (outIdAddress != 0 && !ctx.TryWriteUInt32(outIdAddress, submissionId)) - { - return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; - } + submissionId = unchecked((uint)Interlocked.Increment(ref _nextAprSubmissionId)); + if (submissionId == 0) + { + submissionId = unchecked((uint)Interlocked.Increment(ref _nextAprSubmissionId)); + } - var submission = new AprSubmission - { - Id = submissionId, - Captured = captured!, - Memory = ctx.Memory, - Generation = ctx.TargetGeneration, - ResultAddress = resultAddress, - Retained = outIdAddress != 0, - Sequence = Interlocked.Increment(ref _aprSequence), - DsCaller = DsTrace.CurrentRa1, - DsReturnAddress = dsReturnAddress, - SubmitTimestamp = Stopwatch.GetTimestamp(), - }; - if (submission.Retained) - { - lock (_aprGate) + if (outIdAddress != 0 && !ctx.TryWriteUInt32(outIdAddress, submissionId)) { - _aprSubmissions[submissionId] = submission; + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; } - } - var worker = _aprWorkers[(int)Math.Min(priority, AprPriorityCount - 1)]; - lock (worker) - { - worker.Pending.Enqueue(submission); - if (worker.Thread is null) + var submission = new AprSubmission + { + Session = session, + Id = submissionId, + Captured = captured!, + Memory = ctx.Memory, + Generation = ctx.TargetGeneration, + ResultAddress = resultAddress, + Retained = outIdAddress != 0, + Sequence = Interlocked.Increment(ref _aprSequence), + DsCaller = DsTrace.CurrentRa1, + DsReturnAddress = dsReturnAddress, + SubmitTimestamp = Stopwatch.GetTimestamp(), + }; + lock (_aprGate) { - var index = Array.IndexOf(_aprWorkers, worker); - worker.Thread = new Thread(() => RunAprWorker(worker)) + if (session.Stopping) return AprStoppedResult; + if (submission.Retained) { - IsBackground = true, - Name = $"SharpEmu apr p{index}", - }; - worker.Thread.Start(); + _aprSubmissions[submissionId] = submission; + } + + var worker = session.Workers[(int)Math.Min(priority, AprPriorityCount - 1)]; + worker.Pending.Enqueue(submission); + if (worker.Thread is null) + { + var index = Array.IndexOf(session.Workers, worker); + worker.Thread = new Thread(() => RunAprWorker(session, worker)) + { + IsBackground = true, + Name = $"SharpEmu apr p{index}", + }; + worker.Thread.Start(); + } + + Monitor.PulseAll(_aprGate); + } - Monitor.Pulse(worker); + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + } + finally + { + ReleaseAprProducer(session); } - - return (int)OrbisGen2Result.ORBIS_GEN2_OK; } - private static void RunAprWorker(AprWorker worker) + private static void RunAprWorker(AprSession session, AprWorker worker) { while (true) { AprSubmission submission; - lock (worker) + lock (_aprGate) { - while (worker.Pending.Count == 0) + while (!session.Stopping && worker.Pending.Count == 0) { - Monitor.Wait(worker); + Monitor.Wait(_aprGate); } + if (session.Stopping) return; submission = worker.Pending.Dequeue(); } - CompleteAprSubmission(submission); + _executingAprSession = session; + try { CompleteAprSubmission(submission); } + finally { _executingAprSession = null; } } } @@ -159,7 +276,7 @@ private static void CompleteAprSubmission(AprSubmission submission) var ctx = new CpuContext(submission.Memory, submission.Generation); ExecuteCapturedCommandBuffer(ctx, submission.Captured, out var executionResult, out var errorOffset); - if (submission.ResultAddress != 0) + if (!submission.Session.Stopping && submission.ResultAddress != 0) { _ = TryWriteAprResult(submission.Memory, submission.ResultAddress, executionResult, errorOffset); } @@ -169,7 +286,7 @@ private static void CompleteAprSubmission(AprSubmission submission) submission.ExecutionResult = executionResult; submission.ErrorOffset = errorOffset; submission.Done = true; - if (submission.Retained) + if (!submission.Session.Stopping && submission.Retained) { _aprCompletedOrder.Enqueue(submission); // A guest that never waits must not grow the table without bound. @@ -186,7 +303,8 @@ private static void CompleteAprSubmission(AprSubmission submission) Monitor.PulseAll(_aprGate); } - _ = GuestThreadExecution.Scheduler?.WakeBlockedThreads(AprWaitWakeKey); + if (!submission.Session.Stopping) + _ = GuestThreadExecution.Scheduler?.WakeBlockedThreads(AprWaitWakeKey); if (dsApr) { DsTrace.Emit( @@ -226,6 +344,7 @@ internal static int WaitCommandBuffer(CpuContext ctx, uint submissionId, out ulo { commandBuffer = 0; AprSubmission? submission; + bool done; lock (_aprGate) { if (!_aprSubmissions.TryGetValue(submissionId, out submission)) @@ -234,12 +353,13 @@ internal static int WaitCommandBuffer(CpuContext ctx, uint submissionId, out ulo } commandBuffer = submission.Captured.CommandBuffer; - if (submission.Done) - { - return CollectAprSubmission(ctx, submission); - } + done = submission.Done; } + // Collect may write guest memory. Do not retain this outer gate while + // its own admission lease protects the lifetime of that write. + if (done) return CollectAprSubmission(ctx, submission); + if (GuestThreadExecution.RequestCurrentThreadBlock( ctx, "sceKernelAprWaitCommandBuffer", @@ -271,18 +391,24 @@ private static int CollectAprSubmission(CpuContext ctx, AprSubmission submission { _aprSubmissions.Remove(submission.Id); } - + if (submission.Session.Stopping) return AprStoppedResult; + submission.Session.ActiveProducers++; executionResult = submission.ExecutionResult; errorOffset = submission.ErrorOffset; } - - if (submission.ResultAddress != 0 && - !TryWriteAprResult(ctx.Memory, submission.ResultAddress, executionResult, errorOffset)) + try { - return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; + if (submission.ResultAddress != 0 && + !TryWriteAprResult(ctx.Memory, submission.ResultAddress, executionResult, errorOffset)) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; + } + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + } + finally + { + ReleaseAprProducer(submission.Session); } - - return (int)OrbisGen2Result.ORBIS_GEN2_OK; } private static bool TryWriteAprResult(ICpuMemory memory, ulong resultAddress, int executionResult, uint errorOffset) diff --git a/src/SharpEmu.Libs/Ampr/AmprExports.cs b/src/SharpEmu.Libs/Ampr/AmprExports.cs index 98ed9f7ac..96f021ec1 100644 --- a/src/SharpEmu.Libs/Ampr/AmprExports.cs +++ b/src/SharpEmu.Libs/Ampr/AmprExports.cs @@ -129,6 +129,9 @@ private sealed class CachedHostFile : IDisposable private readonly SafeFileHandle? _handle; private readonly ZArchiveFileStream? _archived; private long _nextSequentialOffset = -1; + // One cache ownership reference plus one reference for each active read. + private int _references = 1; + public void Acquire() => Interlocked.Increment(ref _references); public CachedHostFile(string path) { @@ -200,8 +203,11 @@ public void AdviseSequentialReadAhead(long offset, int length) public void Dispose() { - _handle?.Dispose(); - _archived?.Dispose(); + if (Interlocked.Decrement(ref _references) == 0) + { + _handle?.Dispose(); + _archived?.Dispose(); + } } } @@ -1093,6 +1099,11 @@ private static void ExecuteCapturedCommandBuffer( foreach (var commandEntry in captured.Order) { + if (AprExecutionStopping) + { + executionResult = AprStoppedResult; + return; + } switch (commandEntry.Kind) { case 0: @@ -1479,6 +1490,7 @@ private static int TryReadFileToGuestMemoryCore( const int ChunkSize = 4 * 1024 * 1024; // Rented only for a chunk that cannot be read in place. byte[]? buffer = null; + CachedHostFile? readLease = null; try { @@ -1487,6 +1499,7 @@ private static int TryReadFileToGuestMemoryCore( return openResult; } + readLease = cachedFile; if (fileOffset >= (ulong)cachedFile.Length) { return (int)OrbisGen2Result.ORBIS_GEN2_OK; @@ -1494,6 +1507,7 @@ private static int TryReadFileToGuestMemoryCore( while (bytesRead < size) { + if (AprExecutionStopping) return AprStoppedResult; if (bytesRead > ulong.MaxValue - fileOffset) { return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT; @@ -1558,6 +1572,7 @@ private static int TryReadFileToGuestMemoryCore( } finally { + readLease?.Dispose(); if (buffer is not null) { ArrayPool.Shared.Return(buffer); @@ -1603,6 +1618,7 @@ private static bool TryGetCachedHostFileCore(string hostPath, out CachedHostFile _hostFileLru.Remove(existing); _hostFileLru.AddFirst(existing); file = existing.Value.File; + file.Acquire(); return true; } } @@ -1647,6 +1663,7 @@ private static bool TryGetCachedHostFileCore(string hostPath, out CachedHostFile _hostFileLru.Remove(raced); _hostFileLru.AddFirst(raced); file = raced.Value.File; + file.Acquire(); return true; } @@ -1659,6 +1676,7 @@ private static bool TryGetCachedHostFileCore(string hostPath, out CachedHostFile var node = _hostFileLru.AddFirst(entry); _hostFileByPath[cachePath] = node; file = opened; + file.Acquire(); return true; } } @@ -1999,6 +2017,7 @@ private static int WaitForAddress(CpuContext ctx, WaitAddressCommand command) var spinWait = new SpinWait(); while (true) { + if (AprExecutionStopping) return AprStoppedResult; var observed = GuestAddressWatch.Version; if (DsTrace.Io) { diff --git a/src/SharpEmu.Libs/Gpu/GpuCommands/CommandRegisterBanks.cs b/src/SharpEmu.Libs/Gpu/GpuCommands/CommandRegisterBanks.cs index b7a9aaad7..58a2537b7 100644 --- a/src/SharpEmu.Libs/Gpu/GpuCommands/CommandRegisterBanks.cs +++ b/src/SharpEmu.Libs/Gpu/GpuCommands/CommandRegisterBanks.cs @@ -14,7 +14,7 @@ public enum ContextStateOperation : uint // Shader compilation reads raw words until its typed register connection is complete. public sealed class CommandRegisterBanks { - public Dictionary Shader { get; } = new(); + public DenseShaderRegisterMap Shader { get; } = new(); // The user-config words this queue wrote, keyed by offset from 0xC000; a register-space // WAIT_REG_MEM polls them. diff --git a/src/SharpEmu.Libs/Gpu/GpuCommands/CommandStreamQueue.cs b/src/SharpEmu.Libs/Gpu/GpuCommands/CommandStreamQueue.cs index 4bbb9f44d..73e3c3f0a 100644 --- a/src/SharpEmu.Libs/Gpu/GpuCommands/CommandStreamQueue.cs +++ b/src/SharpEmu.Libs/Gpu/GpuCommands/CommandStreamQueue.cs @@ -953,7 +953,7 @@ private void NotePendingLabel(ulong address, int size, ulong value) // wake, when there is anything to wake, follows it. lock (_labelGate) { - _pendingLabels[address] = (value, size, tick); + StorePendingLabelLocked(address, size, value, tick); } if (!_blockedHeadsPossible) @@ -987,7 +987,7 @@ private void NotePendingQwords(ulong address, int count, ulong strideBytes, ulon { for (var index = 0; index < count; index++) { - _pendingLabels[address + (ulong)index * strideBytes] = (value, sizeof(ulong), tick); + StorePendingLabelLocked(address + (ulong)index * strideBytes, sizeof(ulong), value, tick); } } @@ -1006,6 +1006,29 @@ private void NotePendingQwords(ulong address, int count, ulong strideBytes, ulon } } + private void StorePendingLabelLocked(ulong address, int size, ulong value, ulong tick) + { + // The cache answers whole dwords/qwords. A partial replacement must not leave + // an older overlapping value eligible to satisfy a wait or a predicate. + // Unaligned/other-sized stores take the backend's ordered-memory path; dropping + // cached answers is conservative and keeps every cached start dword-aligned. + if ((address & 3) != 0 || size is not (sizeof(uint) or sizeof(ulong))) + { + _pendingLabels.Clear(); + return; + } + + if (address >= sizeof(uint) && + _pendingLabels.TryGetValue(address - sizeof(uint), out var preceding) && + preceding.Size == sizeof(ulong)) + _pendingLabels.Remove(address - sizeof(uint)); + + if (size == sizeof(ulong) && address <= ulong.MaxValue - sizeof(uint)) + _pendingLabels.Remove(address + sizeof(uint)); + + _pendingLabels[address] = (value, size, tick); + } + // Caller holds _gate. private void LowerBlockedHintLocked() { diff --git a/src/SharpEmu.Libs/Gpu/GpuCommands/DeferredCommandThread.Recycling.cs b/src/SharpEmu.Libs/Gpu/GpuCommands/DeferredCommandThread.Recycling.cs new file mode 100644 index 000000000..49e24e40e --- /dev/null +++ b/src/SharpEmu.Libs/Gpu/GpuCommands/DeferredCommandThread.Recycling.cs @@ -0,0 +1,82 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using System.Runtime.InteropServices; + +namespace SharpEmu.Libs.Gpu.GpuCommands; + +public sealed partial class DeferredCommandThread +{ + private static readonly bool RecycleAsyncItemsEnabled = + Environment.GetEnvironmentVariable("SHARPEMU_COMMAND_ITEM_REUSE") == "1"; + + // A return channel, opposite to the work queue: only the render thread publishes items; + // only the interpreter takes them. The existing producer/consumer ownership also applies + // during Stop, whose runItems callback runs on the consumer thread. + private readonly Item?[]? _recycledItems; + private RecycleCursors _recycleCursors; + + // Separate the two writers' counters so returning an item does not invalidate the producer's + // write counter's cache line. The extra spacing also tolerates a non-aligned containing object. + [StructLayout(LayoutKind.Explicit, Size = 256)] + private struct RecycleCursors + { + [FieldOffset(0)] public long Read; + [FieldOffset(8)] public long ReadAvailable; + [FieldOffset(128)] public long Write; + [FieldOffset(136)] public long WriteLimit; + } + + private Item RentAsyncItem(Kind kind) + { + var slots = _recycledItems; + if (slots is not null) + { + var read = _recycleCursors.Read; + // Refresh the remote cursor only after consuming the published batch. Reading it + // on every item would repeatedly transfer the consumer's cache line between cores. + if (read == _recycleCursors.ReadAvailable) + _recycleCursors.ReadAvailable = Volatile.Read(ref _recycleCursors.Write); + if (read != _recycleCursors.ReadAvailable) + { + var index = (int)(read & (MaxOutstanding - 1)); + var item = slots[index]!; + slots[index] = null; + // Publish the empty slot after taking its item; the consumer may then refill it. + Volatile.Write(ref _recycleCursors.Read, read + 1); + item.Kind = kind; + return item; + } + } + + return new Item { Kind = kind }; + } + + private void RecycleAsyncItem(Item item) + { + var slots = _recycledItems; + if (slots is null) + return; + + var write = _recycleCursors.Write; + if (write == _recycleCursors.WriteLimit) + { + _recycleCursors.WriteLimit = Volatile.Read(ref _recycleCursors.Read) + slots.Length; + if (write == _recycleCursors.WriteLimit) + return; + } + + // Do not retain a draw's banks, closures, or write buffers while the queue is idle. + item.Work = null; + item.Action = null; + item.Data = null; + item.Failure = null; + item.IsDraw = false; + item.Address = 0; + item.RangeSize = 0; + item.JobsBefore = 0; + item.Done = false; + slots[(int)(write & (MaxOutstanding - 1))] = item; + Volatile.Write(ref _recycleCursors.Write, write + 1); + } +} diff --git a/src/SharpEmu.Libs/Gpu/GpuCommands/DeferredCommandThread.cs b/src/SharpEmu.Libs/Gpu/GpuCommands/DeferredCommandThread.cs index e59449a13..48657f7e5 100644 --- a/src/SharpEmu.Libs/Gpu/GpuCommands/DeferredCommandThread.cs +++ b/src/SharpEmu.Libs/Gpu/GpuCommands/DeferredCommandThread.cs @@ -17,7 +17,7 @@ namespace SharpEmu.Libs.Gpu.GpuCommands; // The command stream is interpreted on its own thread, which hands its work to the render thread // in stream order. Draws, dispatches, barriers, completions and the interpreter's own stores are // queued; a callback that returns a result waits for the render thread to run everything before it. -public sealed class DeferredCommandThread : AgcExports.IDeferredWorkSink +public sealed partial class DeferredCommandThread : AgcExports.IDeferredWorkSink { // The thread runs this far ahead of the render thread at most: enough to keep it busy, and // no more, since a GPU round trip (a readback, a label wait) waits for everything queued. @@ -56,6 +56,10 @@ private sealed class Item // Wakes the thread that runs the queued items (the backend's render or GPU worker thread). private readonly Action _wakeConsumer; private readonly Thread _thread; + private readonly bool _useHostQos; + private SharpEmu.HLE.Host.HostHelperThread? _helperThread; + private int _producerThreadId; + private int _started; private readonly object _lock = new(); private List _incoming = new(); private List _draining = new(); @@ -88,17 +92,33 @@ public DeferredCommandThread( ICpuMemory memory, Action runDeferred, ParallelProgramPrefetch? prefetch, - Action wakeConsumer) + Action wakeConsumer, + bool useHostQos = false, + bool? recycleAsyncItems = null) { _wakeConsumer = wakeConsumer; _queue = queue; _memory = memory; _runDeferred = runDeferred; _prefetch = prefetch; + _useHostQos = useHostQos; + if (recycleAsyncItems ?? RecycleAsyncItemsEnabled) + _recycledItems = new Item?[MaxOutstanding]; _thread = new Thread(Run) { IsBackground = true, Name = "SharpEmu command stream" }; } - public void Start() => _thread.Start(); + public void Start() + { + if (Interlocked.Exchange(ref _started, 1) != 0) + throw new ThreadStateException("The command thread has already started."); + if (_useHostQos) + _helperThread = SharpEmu.HLE.Host.HostHelperThread.Start("SharpEmu command stream", 16 * 1024 * 1024, Run); + else + _thread.Start(); + } + + /// Whether the producer was created with its requested native QoS class. + public bool HasHostQos => _helperThread?.HasQos == true; // The producer's side since the last TakeCounters: synchronous calls, their wait, the wait for // room in the queue, and the items handed over. @@ -119,7 +139,7 @@ public DeferredCommandThread( /// public Action? ProducerIdle { get; set; } - public bool IsProducerThread => Environment.CurrentManagedThreadId == _thread.ManagedThreadId; + public bool IsProducerThread => Environment.CurrentManagedThreadId == Volatile.Read(ref _producerThreadId); public long Gate => _enqueuedNonDraw; @@ -131,14 +151,24 @@ public bool ConsumerIdle set => _consumerIdle = value; } - public bool IsAlive => _thread.IsAlive; + public bool IsAlive => _helperThread is { } helper ? !helper.Join(TimeSpan.Zero) : _thread.IsAlive; // The producer side, on the command stream thread. - public void EnqueueDeferred(AgcExports.DeferredWork work, bool isDraw) => - Enqueue(new Item { Kind = Kind.Deferred, Work = work, IsDraw = isDraw }); + public void EnqueueDeferred(AgcExports.DeferredWork work, bool isDraw) + { + var item = RentAsyncItem(Kind.Deferred); + item.Work = work; + item.IsDraw = isDraw; + Enqueue(item); + } - public void Post(Action action) => Enqueue(new Item { Kind = Kind.Call, Action = action }); + public void Post(Action action) + { + var item = RentAsyncItem(Kind.Call); + item.Action = action; + Enqueue(item); + } public void Run(Action action) { @@ -159,7 +189,9 @@ public T Run(Func function) public void PostWrite(ulong address, ReadOnlySpan data) { - var item = new Item { Kind = Kind.Write, Address = address, Data = data.ToArray() }; + var item = RentAsyncItem(Kind.Write); + item.Address = address; + item.Data = data.ToArray(); lock (_writeLock) { _pendingWrites.Enqueue(item); @@ -172,7 +204,10 @@ public void PostWrite(ulong address, ReadOnlySpan data) // A fill or copy the render thread runs in order; reads of its destination wait for it. public void PostRange(Action action, ulong address, ulong size) { - var item = new Item { Kind = Kind.Call, Action = action, Address = address, RangeSize = size }; + var item = RentAsyncItem(Kind.Call); + item.Action = action; + item.Address = address; + item.RangeSize = size; if (size != 0) { lock (_writeLock) @@ -385,6 +420,11 @@ public int RunBatch(long deadline, int maxItems, Action betweenItems) } } + // Execute has removed pending write/range references, and the consumer has finished + // reading the item. Synchronous callers still own their result, so never recycle them. + if (!item.Sync) + RecycleAsyncItem(item); + betweenItems(); if (ran >= maxItems || ((ran & 31) == 0 && Stopwatch.GetTimestamp() >= deadline)) { @@ -483,10 +523,11 @@ public void Stop(Action runItems) _stopping = true; _queue.Wake(); var deadline = Stopwatch.GetTimestamp() + 5 * Stopwatch.Frequency; - while (_thread.IsAlive && Stopwatch.GetTimestamp() < deadline) + while (IsAlive && Stopwatch.GetTimestamp() < deadline) { runItems(); - _thread.Join(1); + if (_helperThread is { } helper) helper.Join(TimeSpan.FromMilliseconds(1)); + else _thread.Join(1); } runItems(); @@ -494,6 +535,9 @@ public void Stop(Action runItems) private void Run() { + // A native helper can enter before Start has returned its wrapper. Publish + // identity from the body, rather than depending on that wrapper assignment. + Volatile.Write(ref _producerThreadId, Environment.CurrentManagedThreadId); try { var lastProgress = Stopwatch.GetTimestamp(); diff --git a/src/SharpEmu.Libs/Gpu/GpuCommands/DenseShaderRegisterMap.cs b/src/SharpEmu.Libs/Gpu/GpuCommands/DenseShaderRegisterMap.cs new file mode 100644 index 000000000..3e46f2425 --- /dev/null +++ b/src/SharpEmu.Libs/Gpu/GpuCommands/DenseShaderRegisterMap.cs @@ -0,0 +1,232 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using System.Collections; + +namespace SharpEmu.Libs.Gpu.GpuCommands; + +/// Dense shader words with explicit presence and sparse capture keys. +/// Not thread safe. Clear preserves this object's identity. Enumeration follows insertion order; +/// overwrites preserve position, and removal followed by insertion appends a new position. +public sealed class DenseShaderRegisterMap : IDictionary, IReadOnlyDictionary +{ + private const int Capacity = (int)RegisterBankLayout.ShaderRegisterCount; + private readonly uint[] _values = new uint[Capacity]; + private readonly ulong[] _present = new ulong[(Capacity + 63) / 64]; + private readonly int[] _positions = new int[Capacity]; + private readonly List _order = new(); + private Dictionary? _overflow; + private int _version; + private KeyCollection? _keys; + private ValueCollection? _valueCollection; + + public int Count { get; private set; } + public bool IsReadOnly => false; + public ICollection Keys => _keys ??= new KeyCollection(this); + public ICollection Values => _valueCollection ??= new ValueCollection(this); + IEnumerable IReadOnlyDictionary.Keys => Keys; + IEnumerable IReadOnlyDictionary.Values => Values; + + public uint this[uint key] + { + get => TryGetValue(key, out var value) ? value : throw new KeyNotFoundException(); + set + { + if (key < Capacity) + { + if (!ContainsDense(key)) + { + _positions[key] = Append(key); + _present[key >> 6] |= 1UL << (int)(key & 63); + } + _values[key] = value; + } + else + { + _overflow ??= new(); + var position = _overflow.TryGetValue(key, out var previous) ? previous.Position : Append(key); + _overflow[key] = (value, position); + } + } + } + + private bool ContainsDense(uint key) => (_present[key >> 6] & (1UL << (int)(key & 63))) != 0; + + private int Append(uint key) + { + // Only insertion invalidates enumerators. Compact removal tombstones here, + // so repeated remove/reinsert does not grow the order list without a bound. + if (_order.Count > Math.Max(Capacity, (long)Count * 2)) + { + var kept = 0; + for (var index = 0; index < _order.Count; index++) + { + var existing = _order[index]; + if (!TryGetValue(existing, out var value)) continue; + var oldPosition = existing < Capacity ? _positions[existing] : _overflow![existing].Position; + if (oldPosition != index) continue; + _order[kept] = existing; + if (existing < Capacity) _positions[existing] = kept; + else _overflow![existing] = (value, kept); + kept++; + } + _order.RemoveRange(kept, _order.Count - kept); + } + var position = _order.Count; + _order.Add(key); + Count++; + _version++; + return position; + } + + public bool ContainsKey(uint key) => key < Capacity ? ContainsDense(key) : _overflow?.ContainsKey(key) == true; + + public bool TryGetValue(uint key, out uint value) + { + if (key < Capacity && ContainsDense(key)) + { + value = _values[key]; + return true; + } + if (key >= Capacity && _overflow is not null && _overflow.TryGetValue(key, out var entry)) + { + value = entry.Value; + return true; + } + value = 0; + return false; + } + + public void Add(uint key, uint value) + { + if (ContainsKey(key)) throw new ArgumentException("The key already exists.", nameof(key)); + this[key] = value; + } + + public bool Remove(uint key) + { + if (key < Capacity) + { + if (!ContainsDense(key)) return false; + _present[key >> 6] &= ~(1UL << (int)(key & 63)); + } + else if (_overflow is null || !_overflow.Remove(key)) return false; + Count--; + return true; + } + + public void Clear() + { + Array.Clear(_present); + _overflow?.Clear(); + _order.Clear(); + Count = 0; + } + + public void Add(KeyValuePair item) => Add(item.Key, item.Value); + public bool Contains(KeyValuePair item) => TryGetValue(item.Key, out var value) && value == item.Value; + public bool Remove(KeyValuePair item) => Contains(item) && Remove(item.Key); + + public void CopyTo(KeyValuePair[] array, int arrayIndex) + { + ValidateCopy(array, arrayIndex, Count); + foreach (var item in this) array[arrayIndex++] = item; + } + + private static void ValidateCopy(T[] array, int index, int count) + { + ArgumentNullException.ThrowIfNull(array); + if ((uint)index > array.Length) throw new ArgumentOutOfRangeException(nameof(index)); + if (array.Length - index < count) throw new ArgumentException("The destination array is too small."); + } + + public Enumerator GetEnumerator() => new(this); + IEnumerator> IEnumerable>.GetEnumerator() => GetEnumerator(); + IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); + + public struct Enumerator : IEnumerator> + { + private readonly DenseShaderRegisterMap _owner; + private readonly int _version; + private int _index; + public KeyValuePair Current { get; private set; } + object IEnumerator.Current => _index == 0 || _index > _owner._order.Count + ? throw new InvalidOperationException("The enumerator is not positioned on an entry.") : Current; + + internal Enumerator(DenseShaderRegisterMap owner) + { + _owner = owner; + _version = owner._version; + _index = 0; + Current = default; + } + + public bool MoveNext() + { + if (_version != _owner._version) throw new InvalidOperationException("The collection was modified."); + while (_index < _owner._order.Count) + { + var position = _index++; + var key = _owner._order[position]; + if (!_owner.TryGetValue(key, out var value)) continue; + var currentPosition = key < Capacity ? _owner._positions[key] : _owner._overflow![key].Position; + if (position != currentPosition) continue; + Current = new(key, value); + return true; + } + _index = _owner._order.Count + 1; + Current = default; + return false; + } + + public void Reset() + { + if (_version != _owner._version) throw new InvalidOperationException("The collection was modified."); + _index = 0; + Current = default; + } + public void Dispose() { } + } + + private abstract class ReadOnlyCollection(DenseShaderRegisterMap owner) : ICollection + { + protected readonly DenseShaderRegisterMap Owner = owner; + public int Count => Owner.Count; + public bool IsReadOnly => true; + public void Add(uint item) => throw new NotSupportedException(); + public void Clear() => throw new NotSupportedException(); + public bool Remove(uint item) => throw new NotSupportedException(); + public abstract bool Contains(uint item); + public abstract IEnumerator GetEnumerator(); + IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); + public void CopyTo(uint[] array, int arrayIndex) + { + ValidateCopy(array, arrayIndex, Count); + foreach (var item in this) array[arrayIndex++] = item; + } + } + + private sealed class KeyCollection(DenseShaderRegisterMap owner) : ReadOnlyCollection(owner) + { + public override bool Contains(uint item) => Owner.ContainsKey(item); + public override IEnumerator GetEnumerator() => Project(Owner.GetEnumerator()); + private static IEnumerator Project(Enumerator source) + { + while (source.MoveNext()) yield return source.Current.Key; + } + } + + private sealed class ValueCollection(DenseShaderRegisterMap owner) : ReadOnlyCollection(owner) + { + public override bool Contains(uint item) + { + foreach (var entry in Owner) if (entry.Value == item) return true; + return false; + } + public override IEnumerator GetEnumerator() => Project(Owner.GetEnumerator()); + private static IEnumerator Project(Enumerator source) + { + while (source.MoveNext()) yield return source.Current.Value; + } + } +} diff --git a/src/SharpEmu.Libs/Gpu/GpuCommands/GpuCommandInterpreter.Registers.cs b/src/SharpEmu.Libs/Gpu/GpuCommands/GpuCommandInterpreter.Registers.cs index 031e36434..85fa6747e 100644 --- a/src/SharpEmu.Libs/Gpu/GpuCommands/GpuCommandInterpreter.Registers.cs +++ b/src/SharpEmu.Libs/Gpu/GpuCommands/GpuCommandInterpreter.Registers.cs @@ -154,7 +154,20 @@ internal uint SetShaderRegisterPacket(in PacketContext packet, ReadOnlySpan= consumed) continue; + var entry = register - offset; + GpuResourceOrigin.Record(QueueId, SubmitId, in packet, false, + packet.PacketAddress + 8 + (ulong)entry * sizeof(uint), register, register, + payload[1 + (int)entry], entry, count, TypedRegisters); + } + } + return consumed + 1; } internal uint SetUserConfigRegisterPacket(in PacketContext packet, ReadOnlySpan payload) @@ -320,6 +333,9 @@ private void ApplyRegisterTable(RegisterBank bank, ulong tableAddress, uint coun Registers.Shader[offset] = value; RegisterWriteTable.WriteShaderEntry(TypedRegisters, offset, value, tableAddress); + if (GpuResourceOrigin.Enabled && offset >= GpuResourceOrigin.PointerLow && offset <= GpuResourceOrigin.PointerHigh) + GpuResourceOrigin.Record(QueueId, SubmitId, in packet, true, + tableAddress + (ulong)index * entryBytes + sizeof(uint), offset, rawOffset, value, index, count, TypedRegisters); break; case RegisterBank.UserConfig: if (offset == RegisterBankLayout.UserConfigNop) diff --git a/src/SharpEmu.Libs/Gpu/GpuCommands/GpuResourceOrigin.cs b/src/SharpEmu.Libs/Gpu/GpuCommands/GpuResourceOrigin.cs new file mode 100644 index 000000000..33d76d51a --- /dev/null +++ b/src/SharpEmu.Libs/Gpu/GpuCommands/GpuResourceOrigin.cs @@ -0,0 +1,99 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using System.Buffers.Binary; +using SharpEmu.Libs.Gpu.GpuCommands.Registers; +using SharpEmu.Libs.Gpu.Pipelines; +using SharpEmu.ShaderCompiler.Resources; + +namespace SharpEmu.Libs.Gpu.GpuCommands; + +// Historical packet provenance, never a cache dependency or an authorization to read a descriptor. +internal static class GpuResourceOrigin +{ + internal static readonly bool Enabled = Environment.GetEnvironmentVariable("SHARPEMU_GPU_RESOURCE_ORIGIN") == "1"; + private static readonly GpuResourceOriginRing? History = Enabled ? new(128) : null; + internal const uint PointerLow = ShaderRegisterOffset.SpiShaderUserDataPs0 + 28; + internal const uint PointerHigh = PointerLow + 1; + + internal static void Record(int queue, ulong submit, in PacketContext packet, bool table, + ulong sourceAddress, uint offset, uint rawOffset, uint value, uint entry, uint count, RegisterBanks banks) + { + if (!Enabled || offset < PointerLow || offset > PointerHigh) return; + History!.Add(Capture(queue, submit, in packet, table, sourceAddress, offset, rawOffset, value, entry, count, banks)); + } + + internal static GpuResourceOriginWrite Capture(int queue, ulong submit, in PacketContext packet, bool table, + ulong sourceAddress, uint offset, uint rawOffset, uint value, uint entry, uint count, RegisterBanks banks) + { + var pixel = banks.Shader.Pixel; + return new(0, queue, submit, packet.PacketAddress, packet.Header, table, sourceAddress, + offset, rawOffset, value, entry, count, pixel.Address, pixel.UserScalars.Values[28], pixel.UserScalars.Values[29]); + } + + internal static void ReportFailure(ShaderSource source, ResidentGuestBytesReader reader, TextWriter output) + { + if (!Enabled || source.Stage != ShaderStage.Pixel) return; + var lowIndex = 28L - source.UserDataBase; + var captured = lowIndex >= 0 && lowIndex + 1 < source.UserData.Length; + var low = captured ? source.UserData[(int)lowIndex] : 0; + var high = captured ? source.UserData[(int)lowIndex + 1] : 0; + output.WriteLine($"[GPU][RESOURCE_ORIGIN] historical=1 not_current_proof=1 capacity=128 hash=0x{source.Hash:X16} " + + $"shader=0x{source.Address:X16} captured_pointer_known={captured} captured_pointer=0x{((ulong)high << 32 | low):X16}"); + Span bytes = stackalloc byte[sizeof(uint)]; + foreach (var write in History!.Snapshot()) + { + var readable = reader(write.SourceAddress, bytes, false); + var current = readable ? BinaryPrimitives.ReadUInt32LittleEndian(bytes) : 0; + var matches = captured && write.ShaderAddress == source.Address && write.PointerLow == low && write.PointerHigh == high; + output.WriteLine($"[GPU][RESOURCE_ORIGIN] historical=1 seq={write.Sequence} queue={write.Queue} submit={write.Submit} " + + $"packet=0x{write.PacketAddress:X16} header=0x{write.Header:X8} source={(write.Table ? "table" : "direct")} " + + $"value_address=0x{write.SourceAddress:X16} offset=0x{write.Offset:X4} raw_offset=0x{write.RawOffset:X8} " + + $"value=0x{write.Value:X8} entry={write.Entry} count={write.Count} shader=0x{write.ShaderAddress:X16} " + + $"pointer=0x{((ulong)write.PointerHigh << 32 | write.PointerLow):X16} matches_captured_inputs={matches} " + + $"reread_readable={readable} reread_value=0x{current:X8} reread_changed={(readable && current != write.Value)}"); + } + } +} + +internal readonly record struct GpuResourceOriginWrite(long Sequence, int Queue, ulong Submit, + ulong PacketAddress, uint Header, bool Table, ulong SourceAddress, uint Offset, uint RawOffset, + uint Value, uint Entry, uint Count, ulong ShaderAddress, uint PointerLow, uint PointerHigh); + +// Pure bounded storage so wraparound and ordering can be checked without a GPU or process flags. +internal sealed class GpuResourceOriginRing +{ + private readonly object _gate = new(); + private readonly GpuResourceOriginWrite[] _records; + private int _next; + private int _count; + private long _sequence; + + internal GpuResourceOriginRing(int capacity) + { + if (capacity <= 0) throw new ArgumentOutOfRangeException(nameof(capacity)); + _records = new GpuResourceOriginWrite[capacity]; + } + + internal void Add(in GpuResourceOriginWrite record) + { + lock (_gate) + { + _records[_next] = record with { Sequence = ++_sequence }; + _next = (_next + 1) % _records.Length; + _count = Math.Min(_count + 1, _records.Length); + } + } + + internal GpuResourceOriginWrite[] Snapshot() + { + lock (_gate) + { + var result = new GpuResourceOriginWrite[_count]; + var first = (_next - _count + _records.Length) % _records.Length; + for (var index = 0; index < result.Length; index++) + result[index] = _records[(first + index) % _records.Length]; + return result; + } + } +} diff --git a/src/SharpEmu.Libs/Gpu/GpuCommands/Registers/RegisterWriteTable.ScalarReuse.cs b/src/SharpEmu.Libs/Gpu/GpuCommands/Registers/RegisterWriteTable.ScalarReuse.cs new file mode 100644 index 000000000..ec1edf5d8 --- /dev/null +++ b/src/SharpEmu.Libs/Gpu/GpuCommands/Registers/RegisterWriteTable.ScalarReuse.cs @@ -0,0 +1,82 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using static SharpEmu.Libs.Gpu.GpuCommands.Registers.ShaderRegisterOffset; + +namespace SharpEmu.Libs.Gpu.GpuCommands.Registers; + +public static partial class RegisterWriteTable +{ + // Experimental until matched gameplay measurements establish the cost of the comparison. + private static readonly bool ScalarReuseEnabled = + Environment.GetEnvironmentVariable("SHARPEMU_REGISTER_IDENTICAL_SCALARS") == "1"; + private static readonly bool ScalarReuseProfileEnabled = + Environment.GetEnvironmentVariable("SHARPEMU_PROFILE_IDENTICAL_SCALARS") == "1"; + + [ThreadStatic] private static long _scalarPacketsChecked; + [ThreadStatic] private static long _scalarPacketsReused; + + private static bool TryReuseScalarPacket(RegisterBanks banks, uint offset, ReadOnlySpan values) + { + // Only the four direct scalar writers apply one marker to an entire packet. Legacy + // vertex/export packets use per-entry writers, which clear it after the first word. + UserScalarRegisters target; + uint slot; + uint capacity = 32; + if (offset - SpiShaderUserDataPs0 < 32) + { + slot = offset - SpiShaderUserDataPs0; + target = banks.Shader.Pixel.UserScalars; + } + else if (offset - SpiShaderUserDataGs0 < 32) + { + slot = offset - SpiShaderUserDataGs0; + target = banks.Shader.Vertex.GeometryUserScalars; + } + else if (offset - SpiShaderUserDataHs0 < 32) + { + slot = offset - SpiShaderUserDataHs0; + target = banks.Shader.Vertex.HullUserScalars; + } + else if (offset - ComputeUserData0 < 16) + { + slot = offset - ComputeUserData0; + target = banks.Shader.Compute.UserScalars; + capacity = 16; + } + else + { + return false; + } + + // Leave invalid packets to the original writer so its failure and marker semantics stay + // intact. Zero words can still initialize previously unwritten slots: Count must match too. + if (values.IsEmpty || (uint)values.Length > capacity - slot) + return false; + + var unchanged = slot + (uint)values.Length <= target.Count && + ((ReadOnlySpan)target.Values).Slice((int)slot, values.Length).SequenceEqual(values); + if (unchanged) + { + var marker = banks.UserDataMarker; + for (var index = 0; index < values.Length; index++) + { + if (target.Kinds[(int)slot + index] != marker) + { + unchanged = false; + break; + } + } + } + + if (ScalarReuseProfileEnabled) + { + _scalarPacketsChecked++; + if (unchanged) _scalarPacketsReused++; + if ((_scalarPacketsChecked & 0x3FFFF) == 0) + Console.Error.WriteLine($"[REGISTER-SCALAR-REUSE] thread={Environment.CurrentManagedThreadId} checked={_scalarPacketsChecked} reused={_scalarPacketsReused}"); + } + + return unchanged; + } +} diff --git a/src/SharpEmu.Libs/Gpu/GpuCommands/Registers/RegisterWriteTable.cs b/src/SharpEmu.Libs/Gpu/GpuCommands/Registers/RegisterWriteTable.cs index 9ce68813e..b5c80129b 100644 --- a/src/SharpEmu.Libs/Gpu/GpuCommands/Registers/RegisterWriteTable.cs +++ b/src/SharpEmu.Libs/Gpu/GpuCommands/Registers/RegisterWriteTable.cs @@ -10,7 +10,7 @@ namespace SharpEmu.Libs.Gpu.GpuCommands.Registers; public delegate void RegisterWriter(RegisterBanks banks, uint offset, uint value); // Two tiers per bank: a packet tries its direct writer, then needs an indirect writer for every value. -public static class RegisterWriteTable +public static partial class RegisterWriteTable { public static readonly RegisterPacketWriter?[] Context = new RegisterPacketWriter?[RegisterBankLayout.ContextRegisterCount]; public static readonly RegisterPacketWriter?[] Shader = new RegisterPacketWriter?[RegisterBankLayout.ShaderRegisterCount]; @@ -44,8 +44,18 @@ public static uint WriteContextPacket(RegisterBanks banks, in PacketContext pack // A shader packet without values has no writer; the other banks accept it. public static uint WriteShaderPacket(RegisterBanks banks, in PacketContext packet, uint offset, ReadOnlySpan values) { - banks.PrepareShaderWrite(offset, (uint)values.Length); - var consumed = WritePacket(banks, in packet, offset, values, Shader, ShaderIndirect, "shader", emptyIsHandled: false); + uint consumed; + if (ScalarReuseEnabled && TryReuseScalarPacket(banks, offset, values)) + { + // The packet still consumes its marker, even when its stored state is unchanged. + banks.UserDataMarker = UserScalarKind.Unknown; + consumed = (uint)values.Length; + } + else + { + banks.PrepareShaderWrite(offset, (uint)values.Length); + consumed = WritePacket(banks, in packet, offset, values, Shader, ShaderIndirect, "shader", emptyIsHandled: false); + } if (Rendering.RenderTrace.Enabled) { for (var index = 0; index < values.Length && index < consumed; index++) diff --git a/src/SharpEmu.Libs/Gpu/Images/ImageDescription.cs b/src/SharpEmu.Libs/Gpu/Images/ImageDescription.cs index cad55f8d1..189411abd 100644 --- a/src/SharpEmu.Libs/Gpu/Images/ImageDescription.cs +++ b/src/SharpEmu.Libs/Gpu/Images/ImageDescription.cs @@ -77,6 +77,27 @@ public struct MipLevelLayouts { private MipLevelLayout[]? _levels; + // Build a chain with one owned array instead of copying an increasingly long array for + // every level. Existing value copies and the caller's input remain independent. As with + // individual indexer writes, levels after the supplied prefix retain their old values. + public void SetPrefix(ReadOnlySpan values) + { + if (values.Length > ImageDescription.MaxLevels) + { + throw new ArgumentOutOfRangeException(nameof(values)); + } + + if (values.IsEmpty) + { + return; + } + + var levels = new MipLevelLayout[Math.Max(values.Length, _levels?.Length ?? 0)]; + _levels?.CopyTo(levels, 0); + values.CopyTo(levels); + _levels = levels; + } + public MipLevelLayout this[int index] { readonly get => _levels is { } levels && (uint)index < (uint)levels.Length ? levels[index] : default; diff --git a/src/SharpEmu.Libs/Gpu/Images/ImageRequestBuilders.ColorTarget.cs b/src/SharpEmu.Libs/Gpu/Images/ImageRequestBuilders.ColorTarget.cs index 035794e2d..389459590 100644 --- a/src/SharpEmu.Libs/Gpu/Images/ImageRequestBuilders.ColorTarget.cs +++ b/src/SharpEmu.Libs/Gpu/Images/ImageRequestBuilders.ColorTarget.cs @@ -91,17 +91,21 @@ private struct ColorTargetMemo { public bool Valid; public ColorTargetWords Words; - public uint TargetMask; + public bool Enabled; public uint DrawLayerOffset; - public bool IgnoreTargetMask; + public bool NativeClearMetadata; public ColorTargetResolution? Result; } private const int TargetMemoCount = 8; + private const int TargetMemoWays = 4; [ThreadStatic] private static ColorTargetMemo[]? _colorTargetMemos; + [ThreadStatic] + private static byte[]? _colorTargetMemoNext; + private static int TargetMemoSlot(ulong address) => (int)((address >> 12) ^ (address >> 20)) & (TargetMemoCount - 1); // Builds the request for a bound color target. Null when the slot carries no target. @@ -112,22 +116,35 @@ private struct ColorTargetMemo return ColorTargetCore(in words, targetMask, drawLayerOffset, ignoreTargetMask); } - var memos = _colorTargetMemos ??= new ColorTargetMemo[TargetMemoCount]; - ref var memo = ref memos[TargetMemoSlot(words.BaseAddress)]; - if (memo.Valid && memo.TargetMask == targetMask && memo.DrawLayerOffset == drawLayerOffset && - memo.IgnoreTargetMask == ignoreTargetMask && memo.Words.Equals(words)) + // The mask only decides whether a target exists; channel write masks are applied by + // the draw. Keep equivalent masks together, including an ignored zero mask. Multiple + // attachments often share the address hash, so retain four requests in each bucket. + var enabled = words.BaseAddress != 0 && (ignoreTargetMask || (targetMask & 0xF) != 0); + var nativeClearMetadata = NativeColorClearMetadata; + var memos = _colorTargetMemos ??= new ColorTargetMemo[TargetMemoCount * TargetMemoWays]; + var bucket = TargetMemoSlot(words.BaseAddress); + var start = bucket * TargetMemoWays; + for (var way = 0; way < TargetMemoWays; way++) { - return memo.Result; + ref var held = ref memos[start + way]; + if (held.Valid && held.Enabled == enabled && held.DrawLayerOffset == drawLayerOffset && + held.NativeClearMetadata == nativeClearMetadata && held.Words.Equals(words)) + { + return held.Result; + } } var result = ColorTargetCore(in words, targetMask, drawLayerOffset, ignoreTargetMask); - memo = new ColorTargetMemo + var next = _colorTargetMemoNext ??= new byte[TargetMemoCount]; + var slot = start + next[bucket]; + next[bucket] = (byte)((next[bucket] + 1) % TargetMemoWays); + memos[slot] = new ColorTargetMemo { Valid = true, Words = words, - TargetMask = targetMask, + Enabled = enabled, DrawLayerOffset = drawLayerOffset, - IgnoreTargetMask = ignoreTargetMask, + NativeClearMetadata = nativeClearMetadata, Result = result, }; return result; @@ -415,20 +432,22 @@ private struct ColorTargetMemo description.Metadata.DccAlphaMsb = DccAlphaOnMsb(words); } + Span layouts = stackalloc MipLevelLayout[ImageDescription.MaxLevels]; for (var level = 0; level < levels; level++) { if (volume) { var mip = volumeLayout!.Mips[level]; - description.MipLayout[level] = new MipLevelLayout { Offset = mip.Offset, Size = mip.Size, Pitch = mip.PaddedWidth, Height = mip.PaddedHeight }; + layouts[level] = new MipLevelLayout { Offset = mip.Offset, Size = mip.Size, Pitch = mip.PaddedWidth, Height = mip.PaddedHeight }; continue; } var span = mipSpans[level]; var levelOffset = span.SourceSize != 0 ? span.SourceOffset : span.Offset; var levelSize = (ulong)(span.SourceSize != 0 ? span.SourceSize : span.Size) * view.ImageLayers; - description.MipLayout[level] = new MipLevelLayout { Offset = levelOffset, Size = levelSize, Pitch = mipPadded[level].Width, Height = mipPadded[level].Height }; + layouts[level] = new MipLevelLayout { Offset = levelOffset, Size = levelSize, Pitch = mipPadded[level].Width, Height = mipPadded[level].Height }; } + description.MipLayout.SetPrefix(layouts[..(int)levels]); var viewType = is1D ? view.LayerCount == 1 ? GpuImageViewType.Type1D : GpuImageViewType.Type1DArray diff --git a/src/SharpEmu.Libs/Gpu/Images/ImageRequestBuilders.DepthTarget.cs b/src/SharpEmu.Libs/Gpu/Images/ImageRequestBuilders.DepthTarget.cs index c3b6c4772..e36a4d0ac 100644 --- a/src/SharpEmu.Libs/Gpu/Images/ImageRequestBuilders.DepthTarget.cs +++ b/src/SharpEmu.Libs/Gpu/Images/ImageRequestBuilders.DepthTarget.cs @@ -89,17 +89,27 @@ private static bool ReadsThroughWriteBase(in DepthTargetWords depthWords) => return built; } - // A frame alternates between a few depth states (passes), which a one-entry memo missed - // and refilled with a new object each time; the slots are structs, as ColorTarget's are. - var memos = _depthTargetMemos ??= new DepthTargetSlot[DepthMemoCount]; - ref var slot = ref memos[(int)((depthWords.ZReadBase >> 12) ^ (depthWords.ZWriteBase >> 12) ^ (depthWords.DepthControl * 31u)) & (DepthMemoCount - 1)]; - if (slot.Valid && ReferenceEquals(slot.Device, device) && slot.Words.Equals(depthWords)) + // Equal read/write bases are normal. XORing them at the same shift erased the address + // and made every target with the same depth control evict the preceding one. Mix distinct + // address bits, and retain four exact states per bucket for passes that share a surface. + var memos = _depthTargetMemos ??= new DepthTargetSlot[DepthMemoCount * DepthMemoWays]; + var bucket = (int)((depthWords.ZReadBase >> 16) ^ (depthWords.ZReadBase >> 24) ^ + (depthWords.ZWriteBase >> 20) ^ (depthWords.DepthControl * 31u)) & (DepthMemoCount - 1); + var start = bucket * DepthMemoWays; + for (var way = 0; way < DepthMemoWays; way++) { - return slot.Result; + ref var held = ref memos[start + way]; + if (held.Valid && ReferenceEquals(held.Device, device) && held.Words.Equals(depthWords)) + { + return held.Result; + } } var result = DepthTargetCore(in depthWords, device, copyMode: false, writeBuffer: false); - slot = new DepthTargetSlot { Valid = true, Words = depthWords, Device = device, Result = result }; + var next = _depthTargetMemoNext ??= new byte[DepthMemoCount]; + var slot = start + next[bucket]; + next[bucket] = (byte)((next[bucket] + 1) % DepthMemoWays); + memos[slot] = new DepthTargetSlot { Valid = true, Words = depthWords, Device = device, Result = result }; return result; } @@ -117,10 +127,14 @@ private struct DepthTargetSlot } private const int DepthMemoCount = 8; + private const int DepthMemoWays = 4; [ThreadStatic] private static DepthTargetSlot[]? _depthTargetMemos; + [ThreadStatic] + private static byte[]? _depthTargetMemoNext; + // Builds the source or destination image used by DB_RENDER_OVERRIDE depth/stencil copies. public static DepthTargetResolution? DepthTargetCopy(in DepthTargetWords depthWords, IImageFormatSupport device, bool writeBuffer) => DepthTargetCore(in depthWords, device, copyMode: true, writeBuffer); diff --git a/src/SharpEmu.Libs/Gpu/Images/ImageRequestBuilders.Texture.cs b/src/SharpEmu.Libs/Gpu/Images/ImageRequestBuilders.Texture.cs index 6ec578895..fa415a57d 100644 --- a/src/SharpEmu.Libs/Gpu/Images/ImageRequestBuilders.Texture.cs +++ b/src/SharpEmu.Libs/Gpu/Images/ImageRequestBuilders.Texture.cs @@ -94,6 +94,7 @@ private static TextureRequestResolution NullTextureResolution(in ShaderImageShap // Fills the mip layout of a texture from the guest tiling rules. public static void PopulateTextureMipLayout(ref ImageDescription description) { + Span layouts = stackalloc MipLevelLayout[ImageDescription.MaxLevels]; if (description.IsVolume && description.TileMode != GuestTileMode.Linear) { var surfaceDescription = new TiledSurfaceDescription( @@ -109,9 +110,10 @@ public static void PopulateTextureMipLayout(ref ImageDescription description) for (var level = 0; level < description.Resources.Levels; level++) { var mip = surface.Mips[level]; - description.MipLayout[level] = new MipLevelLayout { Offset = mip.Offset, Size = mip.Size, Pitch = mip.PaddedWidth, Height = mip.PaddedHeight }; + layouts[level] = new MipLevelLayout { Offset = mip.Offset, Size = mip.Size, Pitch = mip.PaddedWidth, Height = mip.PaddedHeight }; } + description.MipLayout.SetPrefix(layouts[..(int)description.Resources.Levels]); return; } @@ -125,7 +127,7 @@ public static void PopulateTextureMipLayout(ref ImageDescription description) var offset = span.SourceSize != 0 ? span.SourceOffset : span.Offset; ulong size = span.SourceSize != 0 ? span.SourceSize : span.Size; size *= description.IsVolume ? Math.Max(description.Extent.Depth >> level, 1) : description.Resources.Layers; - description.MipLayout[level] = new MipLevelLayout + layouts[level] = new MipLevelLayout { Offset = offset, Size = size, @@ -133,6 +135,7 @@ public static void PopulateTextureMipLayout(ref ImageDescription description) Height = padded[level].Height != 0 ? padded[level].Height : Math.Max(description.Extent.Height >> level, 1), }; } + description.MipLayout.SetPrefix(layouts[..(int)description.Resources.Levels]); } // The view follows the compiled module: a volume, a layer window to the last layer, or one layer. diff --git a/src/SharpEmu.Libs/Gpu/Pipelines/ShaderProgramCache.cs b/src/SharpEmu.Libs/Gpu/Pipelines/ShaderProgramCache.cs index 27933df80..d28d930e8 100644 --- a/src/SharpEmu.Libs/Gpu/Pipelines/ShaderProgramCache.cs +++ b/src/SharpEmu.Libs/Gpu/Pipelines/ShaderProgramCache.cs @@ -620,6 +620,13 @@ private ShaderProgram GetOrCompileCore(ShaderSource source, StageCompileOptions if (!materialized) { + // Keep the draw's pointer inputs with the failure. A later draw or a + // shader dump made at compilation may carry entirely different user data. + Console.Error.WriteLine($"[GPU][RESOURCE_FAILURE_INPUTS] stage={source.Label} hash=0x{source.Hash:X16} " + + $"shader=0x{source.Address:X16} code_bytes={source.CodeSize} user_data_base={source.UserDataBase} " + + $"user_data=[{string.Join(",", source.UserData.Select(word => $"{word:X8}"))}]"); + if (GpuCommands.GpuResourceOrigin.Enabled) + GpuCommands.GpuResourceOrigin.ReportFailure(source, _host.TryReadResidentGuestBytes, Console.Error); var message = $"The shader resources could not be materialized: stage={source.Label} hash=0x{source.Hash:X16} shader=0x{source.Address:X16} reason={materializationFailure}."; if (materializationFailure is ResourceMaterializationFailure.IncompatibleImageCandidates or ResourceMaterializationFailure.ImageCapacityExceeded) throw new ShaderProgramRejectedException(message); diff --git a/src/SharpEmu.Libs/Gpu/Pipelines/VertexInputResolver.cs b/src/SharpEmu.Libs/Gpu/Pipelines/VertexInputResolver.cs index 4a1b8e795..e07048471 100644 --- a/src/SharpEmu.Libs/Gpu/Pipelines/VertexInputResolver.cs +++ b/src/SharpEmu.Libs/Gpu/Pipelines/VertexInputResolver.cs @@ -23,12 +23,11 @@ public sealed class VertexTableMetadata // A draw's decoded vertex layout and the guest bytes it came from: the attribute table words from the lowest // to the highest semantic the program reads, and the buffer descriptors from index zero to the highest index -// an attribute names. A draw that reads the same bytes at the same tables has the same layout. -internal sealed class VertexLayoutMemo(ulong attributeTable, ulong bufferTable, byte[] attributeBytes, byte[] bufferBytes, +// an attribute names. The descriptor bytes contain absolute stream addresses: moving the tables +// without changing their bytes leaves the decoded layout unchanged. Every lookup rereads both. +internal sealed class VertexLayoutMemo(byte[] attributeBytes, byte[] bufferBytes, VertexAttributeResource[] attributes, VertexInputBuffer[] buffers) { - public readonly ulong AttributeTable = attributeTable; - public readonly ulong BufferTable = bufferTable; public readonly byte[] AttributeBytes = attributeBytes; public readonly byte[] BufferBytes = bufferBytes; public readonly VertexAttributeResource[] Attributes = attributes; @@ -379,7 +378,7 @@ private static void AddResource(ShaderInputSemantic semantic, uint attribute, Re // The tables of a draw in two guest reads, one over the attribute words the program names and one over the buffer // descriptors they index, instead of two reads per attribute (each taking the memory read lock). A draw whose - // bytes equal the last draw's at the same tables gets that draw's attributes and buffers back without decoding or + // bytes equal the last draw's gets that draw's attributes and buffers back without decoding or // allocating them; any other byte decodes afresh, so a guest that rewrites a table is seen on the next draw. // False leaves the word-by-word path to decode the draw, and to report whatever stopped this one. private static bool TryResolveLayout(CpuContext context, VertexTableMetadata metadata, ulong attributeTable, ulong bufferTable, @@ -437,7 +436,7 @@ private static bool TryResolveLayout(CpuContext context, VertexTableMetadata met } var memo = metadata.Layout; - if (memo is not null && memo.AttributeTable == attributeTable && memo.BufferTable == bufferTable && + if (memo is not null && attributeBytes.SequenceEqual(memo.AttributeBytes) && bufferBytes.SequenceEqual(memo.BufferBytes)) { attributes = memo.Attributes; @@ -465,7 +464,7 @@ private static bool TryResolveLayout(CpuContext context, VertexTableMetadata met buffers = MergeBuffers(resources, shaderAddress); attributes = resources.ToArray(); resources.Clear(); - metadata.Layout = new VertexLayoutMemo(attributeTable, bufferTable, attributeBytes.ToArray(), bufferBytes.ToArray(), attributes, buffers); + metadata.Layout = new VertexLayoutMemo(attributeBytes.ToArray(), bufferBytes.ToArray(), attributes, buffers); return true; } diff --git a/src/SharpEmu.Libs/VideoOut/RenderPhaseProfile.cs b/src/SharpEmu.Libs/VideoOut/RenderPhaseProfile.cs index aa3999408..b05f8c0cd 100644 --- a/src/SharpEmu.Libs/VideoOut/RenderPhaseProfile.cs +++ b/src/SharpEmu.Libs/VideoOut/RenderPhaseProfile.cs @@ -203,6 +203,9 @@ public enum Phase GpuWriteFlush, MappedSizeClamp, StorageBufferResolve, + StorageDeferredLabels, + StorageSharedLookup, + StorageStoreOrder, // The command stream's own time on the Metal worker, split by what it was doing. // Exclusive: nested scopes (draw translation, waits, label encoding) are carved out. CommandPacketRead, diff --git a/src/SharpEmu.Logging/MemoryTelemetry.cs b/src/SharpEmu.Logging/MemoryTelemetry.cs index 2f9223929..1aad00eae 100644 --- a/src/SharpEmu.Logging/MemoryTelemetry.cs +++ b/src/SharpEmu.Logging/MemoryTelemetry.cs @@ -134,7 +134,16 @@ private static void Run(int seconds) Thread.Sleep(TimeSpan.FromSeconds(seconds)); var elapsed = Stopwatch.GetElapsedTime(started).TotalSeconds; var pause = GC.GetTotalPauseDuration(); - Report(elapsed, pause - lastPause, pause); + try + { + Report(elapsed, pause - lastPause, pause); + } + catch (Exception exception) + { + // Even taking the provider snapshot can fail. Optional diagnostics must + // not take down the emulation process from this background thread. + WriteError(Console.Error, "report", exception); + } lastPause = pause; } } @@ -142,7 +151,6 @@ private static void Run(int seconds) private static void Report(double elapsed, TimeSpan intervalPause, TimeSpan totalPause) { var culture = CultureInfo.InvariantCulture; - Console.Error.WriteLine($"[MEM] dotnet t={elapsed.ToString("F0", culture)} {Dotnet(intervalPause, totalPause)}"); (string Section, Func Provider)[] providers; MemoryCounter[] counters; lock (Gate) @@ -151,17 +159,8 @@ private static void Report(double elapsed, TimeSpan intervalPause, TimeSpan tota counters = [.. Counters]; } - foreach (var (section, provider) in providers) - { - try - { - Console.Error.WriteLine($"[MEM] {section} {provider()}"); - } - catch (Exception exception) - { - Console.Error.WriteLine($"[MEM] {section} error={exception.GetType().Name}: {exception.Message}"); - } - } + ReportSections(Console.Error, + () => $"t={elapsed.ToString("F0", culture)} {Dotnet(intervalPause, totalPause)}", providers); if (counters.Length != 0) { @@ -176,6 +175,40 @@ private static void Report(double elapsed, TimeSpan intervalPause, TimeSpan tota } } + // The built-in process sampler has the same failure boundary as registered providers. + // Process inspection can fail as host threads/process resources change underneath it. + internal static void ReportSections(TextWriter output, Func dotnet, + IReadOnlyList<(string Section, Func Provider)> providers) + { + WriteSection(output, "dotnet", dotnet); + foreach (var (section, provider) in providers) + WriteSection(output, section, provider); + } + + private static void WriteSection(TextWriter output, string section, Func provider) + { + try + { + output.WriteLine($"[MEM] {section} {provider()}"); + } + catch (Exception exception) + { + WriteError(output, section, exception); + } + } + + private static void WriteError(TextWriter output, string section, Exception exception) + { + try + { + output.WriteLine($"[MEM] {section} error={exception.GetType().Name}: {exception.Message}"); + } + catch (Exception) + { + // A closed/failed diagnostic sink cannot be used to report its own failure. + } + } + private static string Dotnet(TimeSpan intervalPause, TimeSpan totalPause) { const double Mb = 1048576.0; diff --git a/src/SharpEmu.Logging/SharpEmu.Logging.csproj b/src/SharpEmu.Logging/SharpEmu.Logging.csproj index 5f834d873..76d2bbb44 100644 --- a/src/SharpEmu.Logging/SharpEmu.Logging.csproj +++ b/src/SharpEmu.Logging/SharpEmu.Logging.csproj @@ -7,4 +7,7 @@ SPDX-License-Identifier: GPL-2.0-or-later $(NoWarn);1591 + + + diff --git a/src/SharpEmu.ShaderCompiler.Metal/Gen5MslCodegenOptions.cs b/src/SharpEmu.ShaderCompiler.Metal/Gen5MslCodegenOptions.cs index c4eff0293..abda881da 100644 --- a/src/SharpEmu.ShaderCompiler.Metal/Gen5MslCodegenOptions.cs +++ b/src/SharpEmu.ShaderCompiler.Metal/Gen5MslCodegenOptions.cs @@ -146,6 +146,45 @@ public sealed record Gen5MslCodegenOptions( bool ContractPixelCompute = false, bool RuntimeDwordFormatLoads = true) { + /// Diagnostic threshold for moving mesh LDS from threadgroup memory to the + /// device pool, in bytes. The supported mesh allocation is at most 16 KiB. Zero + /// selects the device pool for every nonempty allocation; 16384 keeps all allocations + /// in threadgroup memory. The default preserves the existing 12 KiB policy. + public uint MeshDeviceLdsThresholdBytes { get; init; } = 12288; + + /// Experimental structured mesh exits, enabled with SHARPEMU_MSL_MESH_EXIT_JOIN=1. + /// A single hardware wave may also publish its output at a loop-limit exit. + public bool MeshExitJoin { get; init; } + + /// Experimental constant indexing of provably invariant private scratch slots. + /// Enabled with SHARPEMU_MSL_STATIC_SCRATCH=1. + public bool StaticScratchSlots { get; init; } + + /// Experimental removal of pure ALU guards when inactive values cannot + /// be observed. SHARPEMU_MSL_UNOBSERVED_ALU=1 enables this for wave32 compute + /// and single-wave mesh shaders. + public bool UnobservedAluWrites { get; init; } + + /// Experimental separation of fused geometry input and output capacities, + /// with complete wave64 execution. SHARPEMU_MSL_GEOMETRY_CAPACITY=1. + public bool GeometryOutputCapacity { get; init; } + + /// Experimental single-run loads for statically known packed formats. + /// SHARPEMU_MSL_PACKED_FORMAT_LOADS=1; partial trailing elements retain the decoder. + public bool PackedFormatLoads { get; init; } + + /// Experimental guarded forwarding of the last 128-bit LDS write in a + /// single-wave mesh subgroup. SHARPEMU_MSL_MESH_LDS_FORWARDING=1. + public bool MeshLdsForwarding { get; init; } + + /// Experimental constant-address-space bindings for scalar-only buffers + /// in mesh shaders without device-memory writes. SHARPEMU_MSL_SCALAR_CONSTANT_BUFFERS=1. + public bool ScalarConstantBuffers { get; init; } + + /// Experimental compact threadgroup bank for frequently addressed LDS + /// regions, retaining device storage for other addresses. SHARPEMU_MSL_MESH_HOT_LDS=1. + public bool MeshHotLds { get; init; } + public static Gen5MslCodegenOptions FromEnvironment { get; } = new( StructuredControlFlow: Environment.GetEnvironmentVariable("SHARPEMU_MSL_STRUCTURED") != "0", RegisterAllocation: Environment.GetEnvironmentVariable("SHARPEMU_MSL_REGALLOC") != "0", @@ -172,5 +211,18 @@ public sealed record Gen5MslCodegenOptions( ConstantTableSlots: Environment.GetEnvironmentVariable("SHARPEMU_MSL_TABLEPAD") != "0", FastReciprocals: Environment.GetEnvironmentVariable("SHARPEMU_MSL_FASTRCP") != "0", ContractPixelCompute: Environment.GetEnvironmentVariable("SHARPEMU_MSL_CONTRACT") == "1", - RuntimeDwordFormatLoads: Environment.GetEnvironmentVariable("SHARPEMU_MSL_RUNTIME_DWORD_FORMAT") != "0"); + RuntimeDwordFormatLoads: Environment.GetEnvironmentVariable("SHARPEMU_MSL_RUNTIME_DWORD_FORMAT") != "0") + { + UnobservedAluWrites = Environment.GetEnvironmentVariable("SHARPEMU_MSL_UNOBSERVED_ALU") == "1", + GeometryOutputCapacity = Environment.GetEnvironmentVariable("SHARPEMU_MSL_GEOMETRY_CAPACITY") == "1", + PackedFormatLoads = Environment.GetEnvironmentVariable("SHARPEMU_MSL_PACKED_FORMAT_LOADS") == "1", + MeshLdsForwarding = Environment.GetEnvironmentVariable("SHARPEMU_MSL_MESH_LDS_FORWARDING") == "1", + ScalarConstantBuffers = Environment.GetEnvironmentVariable("SHARPEMU_MSL_SCALAR_CONSTANT_BUFFERS") == "1", + MeshHotLds = Environment.GetEnvironmentVariable("SHARPEMU_MSL_MESH_HOT_LDS") == "1", + StaticScratchSlots = Environment.GetEnvironmentVariable("SHARPEMU_MSL_STATIC_SCRATCH") == "1", + MeshExitJoin = Environment.GetEnvironmentVariable("SHARPEMU_MSL_MESH_EXIT_JOIN") == "1", + MeshDeviceLdsThresholdBytes = uint.TryParse( + Environment.GetEnvironmentVariable("SHARPEMU_MSL_MESH_LDS_THRESHOLD"), out var threshold) + ? Math.Min(threshold, 16384u) : 12288u, + }; } diff --git a/src/SharpEmu.ShaderCompiler.Metal/Gen5MslShader.cs b/src/SharpEmu.ShaderCompiler.Metal/Gen5MslShader.cs index 913c3040b..d2ef1a191 100644 --- a/src/SharpEmu.ShaderCompiler.Metal/Gen5MslShader.cs +++ b/src/SharpEmu.ShaderCompiler.Metal/Gen5MslShader.cs @@ -54,12 +54,14 @@ public sealed record Gen5MslMeshInfo( bool DeviceLds = false, bool Wave64 = false, uint LdsBytes = 0, - bool Amplified = false) + bool Amplified = false, + uint GeometryInputVertices = 0, + uint GeometryInputPrimitives = 0) { - /// Lanes in one guest wave of this program: an amplified wave64 subgroup runs + /// Lanes in one guest wave of this program: an amplified or capacity-aware geometry wave64 subgroup runs /// its 64-lane waves as two bridged simdgroups each; everything else runs a guest wave /// as one simdgroup (). - public uint GuestWaveLanes => Amplified && Wave64 ? 64u : WaveLanes; + public uint GuestWaveLanes => (Amplified || GeometryInputVertices != 0) && Wave64 ? 64u : WaveLanes; /// The mesh slot the device-memory LDS pool is bound at when /// is set. @@ -103,12 +105,11 @@ public sealed record Gen5MslMeshInfo( /// public uint InputPrimitivesPerSubgroup(Gen5MslMeshInputTopology input) { - // A wave64 GS half numbers its ES vertices, GS primitives and emitted vertices as - // (merged-wave subgroup id << 6) + lane. A mesh wave is 32 lanes, so the second - // wave's lanes land at 64 + lane while the offsets the prologue packs count on - // from 32: a subgroup has to stay inside the first wave to line up. - var vertices = Math.Min(MaxVertices, WaveLanes); - var primitives = Math.Min(MaxPrimitives, WaveLanes); + // The legacy path models a guest wave with one simdgroup and must stay within + // its first 32 lanes. Capacity-aware geometry has separate input/output + // budgets and bridges both halves of every wave64. + var vertices = GeometryInputVertices != 0 ? GeometryInputVertices : Math.Min(MaxVertices, WaveLanes); + var primitives = GeometryInputPrimitives != 0 ? GeometryInputPrimitives : Math.Min(MaxPrimitives, WaveLanes); var byInput = input switch { Gen5MslMeshInputTopology.TriangleStrip => vertices > 2 ? vertices - 2 : 1, @@ -116,8 +117,16 @@ public uint InputPrimitivesPerSubgroup(Gen5MslMeshInputTopology input) _ => Math.Max(vertices / 3, 1), }; var emitted = Math.Max(GeometryVerticesOut, 1); - var byOutputVertices = Math.Max(vertices / emitted, 1); - var byOutputPrimitives = Math.Max(primitives / (emitted > 2 ? emitted - 2 : 1), 1); + var outputVertices = GeometryInputVertices != 0 ? MaxVertices : vertices; + var outputPrimitives = GeometryInputPrimitives != 0 ? MaxPrimitives : primitives; + var perInput = GeometryInputVertices == 0 ? (emitted > 2 ? emitted - 2 : 1) : Topology switch + { + Gen5MslMeshTopology.Point => emitted, + Gen5MslMeshTopology.Line => Math.Max(emitted, 2) - 1, + _ => Math.Max(emitted, 3) - 2, + }; + var byOutputVertices = Math.Max(outputVertices / emitted, 1); + var byOutputPrimitives = Math.Max(outputPrimitives / perInput, 1); return Math.Min(Math.Min(byInput, primitives), Math.Min(byOutputVertices, byOutputPrimitives)); } diff --git a/src/SharpEmu.ShaderCompiler.Metal/Gen5MslTranslator.ExecGuard.cs b/src/SharpEmu.ShaderCompiler.Metal/Gen5MslTranslator.ExecGuard.cs index 484e5e4e6..4d7e852ce 100644 --- a/src/SharpEmu.ShaderCompiler.Metal/Gen5MslTranslator.ExecGuard.cs +++ b/src/SharpEmu.ShaderCompiler.Metal/Gen5MslTranslator.ExecGuard.cs @@ -36,10 +36,30 @@ private sealed partial class CompilationContext /// Set while the instruction being emitted is one of those: its vector /// writes need no guard. private bool _execKnownFull; + private Ir.Gen5VectorObservationPlan? _observationPlan; + private bool _observationPlanBuilt; + private Ir.VectorRegisterSet _unobservedAluWrites; private void BeginInstruction(Gen5ShaderInstruction instruction) { _execKnownFull = IsExecKnownFull(instruction.Pc); + _unobservedAluWrites = default; + // Only pure ALU destination writes participate. Memory, exports and + // mask updates retain their guards even when their result is dead. + if (_options.UnobservedAluWrites && _options.ExecGuardElision && _request.EnableExecGuardElision && + _waveLaneCount == 32 && (_stage == Gen5MslStage.Compute || + (_stage == Gen5MslStage.Mesh && Mesh.ThreadsPerThreadgroup <= 32)) && + instruction.Encoding is (Gen5ShaderEncoding.Vop1 or Gen5ShaderEncoding.Vop2 or + Gen5ShaderEncoding.Vop3 or Gen5ShaderEncoding.Vop3p) && + instruction.Control is not (Gen5DppControl or Gen5Dpp8Control)) + { + if (!_observationPlanBuilt) + { + _observationPlan = Ir.Gen5VectorObservationAnalysis.Analyze(_request.Program, wave32: true); + _observationPlanBuilt = true; + } + _unobservedAluWrites = _observationPlan?.UnobservedWrites(instruction.Pc) ?? default; + } } private bool IsExecKnownFull(uint pc) diff --git a/src/SharpEmu.ShaderCompiler.Metal/Gen5MslTranslator.Mesh.cs b/src/SharpEmu.ShaderCompiler.Metal/Gen5MslTranslator.Mesh.cs index 7ca94f754..5589ee9d4 100644 --- a/src/SharpEmu.ShaderCompiler.Metal/Gen5MslTranslator.Mesh.cs +++ b/src/SharpEmu.ShaderCompiler.Metal/Gen5MslTranslator.Mesh.cs @@ -50,11 +50,6 @@ public static partial class Gen5MslTranslator private const uint MeshLdsDwordFloor = 512; private const uint MeshLdsDwordCeiling = 4096; - // A guest LDS above this does not stay in threadgroup memory: its occupancy cost - // (a 15.6 KB allocation leaves a handful of threadgroups per core) outweighs the cost of - // keeping the data in device memory, so it lives in a slot of a shared device pool. - private const uint MeshDeviceLdsThresholdBytes = 12288; - private const string MeshVertexStructName = "Gen5MeshVertex"; private const string MeshPrimitiveStructName = "Gen5MeshPrimitive"; private const string MeshDrawInfoStructName = "Gen5MeshDrawInfo"; @@ -79,7 +74,8 @@ request.PrimitiveShader is { } primitiveShader && // The subgroup budget the geometry-engine registers describe, clamped to what a Metal // mesh threadgroup can carry. - private static Gen5MslMeshInfo BuildMeshInfo(ShaderCompileRequest request, uint ldsDwords) + private static Gen5MslMeshInfo BuildMeshInfo(ShaderCompileRequest request, uint ldsDwords, uint deviceLdsThreshold, + bool geometryOutputCapacity) { var ngg = request.PrimitiveShader ?? default; var fallback = ngg.MaxOutputPerSubgroup != 0 ? ngg.MaxOutputPerSubgroup : MeshWaveLanes; @@ -88,7 +84,27 @@ private static Gen5MslMeshInfo BuildMeshInfo(ShaderCompileRequest request, uint var perPrimitive = Gen5NggExports.VerticesPerPrimitive(ngg.OutputPrimitiveType); if (ngg.AmplifiedVerticesOut != 0 && ngg.GeometryVerticesOut == 0) { - return BuildAmplifiedMeshInfo(ngg, ldsDwords); + return BuildAmplifiedMeshInfo(ngg, ldsDwords, deviceLdsThreshold); + } + + var geometryInputs = geometryOutputCapacity && ngg.GeometryVerticesOut != 0; + var inputVertices = geometryInputs ? vertices : 0; + var inputPrimitives = geometryInputs ? primitives : 0; + if (geometryInputs) + { + // ES vertices and GS primitives describe the input work, not the space + // GS_ALLOC_REQ can request after amplification. Keep both capacities. + var emitted = Math.Max(ngg.GeometryVerticesOut, 1u); + vertices = Clamp(ngg.MaxOutputPerSubgroup != 0 ? ngg.MaxOutputPerSubgroup : + (uint)Math.Min((ulong)inputPrimitives * emitted, MeshMaxVertices), 1, MeshMaxVertices); + var primitivesPerInput = ngg.OutputPrimitiveType switch + { + Gen5NggExports.OutputPrimitivePointList => emitted, + Gen5NggExports.OutputPrimitiveLineStrip => Math.Max(emitted, 2u) - 1u, + _ => Math.Max(emitted, 3u) - 2u, + }; + primitives = Clamp((uint)Math.Min((ulong)inputPrimitives * primitivesPerInput, MeshMaxPrimitives), + 1, MeshMaxPrimitives); } // A wave64 program with no GS half runs as a narrow wave64: one 32-lane simdgroup @@ -96,14 +112,14 @@ private static Gen5MslMeshInfo BuildMeshInfo(ShaderCompileRequest request, uint // vertex, one per primitive) has to fit in 32 lanes. The pipeline declares the full // 32 of each; how many a draw uses is per-draw state (draw-info words 0/1 for the // whole-triangle path, the meshlet header otherwise), never the declared extent. - var deviceLds = ldsDwords * sizeof(uint) > MeshDeviceLdsThresholdBytes; + var deviceLds = ldsDwords * sizeof(uint) > deviceLdsThreshold; var narrow = IsNarrowWave64(request); if (narrow) { vertices = MeshWaveLanes; primitives = MeshWaveLanes; } - else if (request.MeshPerVertexParameterMask != 0) + else if (geometryInputs || request.MeshPerVertexParameterMask != 0) { // Three vertex values per handed parameter make each primitive heavy, so the // extents shrink until the declared outputs fit what the device accepts. @@ -116,7 +132,14 @@ private static Gen5MslMeshInfo BuildMeshInfo(ShaderCompileRequest request, uint } } + // This diagnostic must not pack even one GS invocation into an output budget + // smaller than its declared maximum. Leave unsupported shapes on the old path. + if (geometryInputs && vertices < ngg.GeometryVerticesOut) + return BuildMeshInfo(request, ldsDwords, deviceLdsThreshold, geometryOutputCapacity: false); + var waves = Clamp((Math.Max(vertices, primitives) + MeshWaveLanes - 1) / MeshWaveLanes, 1, MeshMaxWaves); + if (geometryInputs && ngg.IsWave64) + waves = (waves + 1u) & ~1u; var topology = ngg.OutputPrimitiveType switch { Gen5NggExports.OutputPrimitivePointList => Gen5MslMeshTopology.Point, @@ -126,7 +149,7 @@ private static Gen5MslMeshInfo BuildMeshInfo(ShaderCompileRequest request, uint return new Gen5MslMeshInfo( ThreadsPerThreadgroup: waves * MeshWaveLanes, - WavesPerThreadgroup: waves, + WavesPerThreadgroup: geometryInputs && ngg.IsWave64 ? waves / 2u : waves, MaxVertices: vertices, MaxPrimitives: primitives, VerticesPerPrimitive: Gen5NggExports.VerticesPerPrimitive(ngg.OutputPrimitiveType), @@ -139,7 +162,9 @@ private static Gen5MslMeshInfo BuildMeshInfo(ShaderCompileRequest request, uint NarrowWave64: narrow, DeviceLds: deviceLds, Wave64: ngg.IsWave64, - LdsBytes: ldsDwords * sizeof(uint)); + LdsBytes: ldsDwords * sizeof(uint), + GeometryInputVertices: inputVertices, + GeometryInputPrimitives: inputPrimitives); } // An amplified subgroup (Prospero's mesh shader, Gen5NggStageInfo.AmplifiedVerticesOut) is @@ -147,13 +172,13 @@ private static Gen5MslMeshInfo BuildMeshInfo(ShaderCompileRequest request, uint // and PRIM_AMP_FACTOR primitives, each thread its own vertex and primitive at index // (merged-wave subgroup id << 6) + lane. So the threadgroup holds the whole output in // real guest waves: 64 lanes as two bridged simdgroups for a wave64 program. - private static Gen5MslMeshInfo BuildAmplifiedMeshInfo(Gen5NggStageInfo ngg, uint ldsDwords) + private static Gen5MslMeshInfo BuildAmplifiedMeshInfo(Gen5NggStageInfo ngg, uint ldsDwords, uint deviceLdsThreshold) { var lanes = ngg.IsWave64 ? 2 * MeshWaveLanes : MeshWaveLanes; var vertices = Clamp(ngg.AmplifiedVerticesOut, 1, MeshMaxVertices); var primitives = Clamp(ngg.AmplifiedPrimitivesOut, 1, MeshMaxPrimitives); var waves = Math.Max((Math.Max(vertices, primitives) + lanes - 1) / lanes, 1u); - var deviceLds = ldsDwords * sizeof(uint) > MeshDeviceLdsThresholdBytes; + var deviceLds = ldsDwords * sizeof(uint) > deviceLdsThreshold; return new Gen5MslMeshInfo( ThreadsPerThreadgroup: waves * lanes, WavesPerThreadgroup: waves, @@ -258,10 +283,6 @@ private sealed partial class CompilationContext /// the exact allocation is above the threadgroup-memory threshold. private bool UsesDeviceLds => _usesLds && _meshInfo is { DeviceLds: true }; - /// Dwords of one device LDS slot; a power of two so the masked LDS index - /// stays inside the slot. - private uint DeviceLdsSlotDwords => (uint)System.Numerics.BitOperations.RoundUpToPowerOf2(_meshLdsDwords); - /// The address space of a pointer to guest LDS. private string LdsAddressSpace => UsesDeviceLds ? "device" : "threadgroup"; @@ -443,6 +464,14 @@ private void EmitMeshDeclarations(StringBuilder source) source.AppendLine(" threadgroup atomic_uint sharpemu_mesh_live[2];"); source.AppendLine(" threadgroup atomic_uint sharpemu_mesh_parked[2];"); source.AppendLine(" threadgroup uint sharpemu_mesh_alloc[2];"); + if (_meshHotLds.Length != 0) + source.AppendLine($" threadgroup uint sharpemu_hot_lds[{MeshHotLdsWords}];"); + if (UseMeshLdsForwarding) + { + source.AppendLine(" uint4 sharpemu_lds_forward_values = uint4(0u);"); + source.AppendLine(" uint sharpemu_lds_forward_base = 0u;"); + source.AppendLine(" bool sharpemu_lds_forward_valid = false;"); + } source.AppendLine(" bool sharpemu_mesh_wait = false;"); source.AppendLine(" uint sharpemu_mesh_round = 0u;"); source.AppendLine($" {MeshVertexStructName} sharpemu_out = {{}};"); @@ -519,8 +548,8 @@ private void EmitMeshDeclarations(StringBuilder source) source.AppendLine( $" : sharpemu_gs_input == {strip}u ? (sharpemu_gs_verts > 2u ? sharpemu_gs_verts - 2u : 0u)"); source.AppendLine(" : sharpemu_gs_verts;"); - source.AppendLine($" sharpemu_mesh_alloc[0] = min(sharpemu_gs_verts, {mesh.MaxVertices}u);"); - source.AppendLine($" sharpemu_mesh_alloc[1] = min(sharpemu_gs_prims, {mesh.MaxPrimitives}u);"); + source.AppendLine($" sharpemu_mesh_alloc[0] = min(sharpemu_gs_verts, {(mesh.GeometryInputVertices != 0 ? mesh.GeometryInputVertices : mesh.MaxVertices)}u);"); + source.AppendLine($" sharpemu_mesh_alloc[1] = min(sharpemu_gs_prims, {(mesh.GeometryInputPrimitives != 0 ? mesh.GeometryInputPrimitives : mesh.MaxPrimitives)}u);"); } else if (mesh.Amplified) { @@ -600,9 +629,12 @@ private void EmitMeshDeclarations(StringBuilder source) source.AppendLine($" {MeshWorkgroupBarrier}"); if (UsesDeviceLds) { + // All pipelines share the same reservations and allocation. A smaller + // shader may mask fewer LDS words, but its slot must keep the fixed stride + // or slot 1 of that shader can alias slot 0 of a larger live shader. source.AppendLine( $" device uint* sharpemu_lds = (device uint*)sharpemu_lds_pool + {Gen5MslMeshInfo.LdsPoolHeaderDwords}u + " + - $"sharpemu_lds_slot * {DeviceLdsSlotDwords}u;"); + $"sharpemu_lds_slot * {Gen5MslMeshInfo.LdsPoolSlotDwords}u;"); } } diff --git a/src/SharpEmu.ShaderCompiler.Metal/Gen5MslTranslator.MeshHotLds.cs b/src/SharpEmu.ShaderCompiler.Metal/Gen5MslTranslator.MeshHotLds.cs new file mode 100644 index 000000000..21c96099b --- /dev/null +++ b/src/SharpEmu.ShaderCompiler.Metal/Gen5MslTranslator.MeshHotLds.cs @@ -0,0 +1,143 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using SharpEmu.ShaderCompiler.Resources; + +namespace SharpEmu.ShaderCompiler.Metal; + +public static partial class Gen5MslTranslator +{ + private sealed partial class CompilationContext + { + private sealed record HotLdsRegion(uint First, uint Count, uint Packed, int Weight); + private readonly HotLdsRegion[] _meshHotLds = []; + private uint MeshHotLdsWords => _meshHotLds.Aggregate(0u, (sum, region) => sum + region.Count); + + // This is an address-space partition, not a cache: a normalized guest word + // always has one physical home, regardless of the opcode/offset used to name it. + // The chosen ranges affect performance only. No address-range proof is needed + // because every other word still lives in the original full device allocation. + private HotLdsRegion[] PlanMeshHotLds() + { + if (!_options.MeshHotLds || _options.MeshLdsForwarding || + _meshInfo is not { DeviceLds: true, ThreadsPerThreadgroup: 32 } mesh || + _request.Memory.Entries.Any(entry => entry.AddressSpace is FlatAddressSpace.Shared or FlatAddressSpace.SharedOrPrivate) || + _request.Program.Instructions.Any(instruction => instruction.Control is Gen5BufferMemoryControl { Lds: true })) + return []; + + var intervals = new List(); + var words = System.Numerics.BitOperations.RoundUpToPowerOf2(_meshLdsDwords); + foreach (var instruction in _request.Program.Instructions) + { + if (instruction.Control is not Gen5DataShareControl control || control.Gds) continue; + if (instruction.Opcode is "DsSwizzleB32" or "DsBpermuteB32" or "DsNop") continue; + if (instruction.Opcode is not ("DsWriteB32" or "DsReadB32" or "DsWriteB128" or "DsReadB128")) + return []; + var width = instruction.Opcode.EndsWith("B128", StringComparison.Ordinal) ? 4u : 1u; + var first = (control.SingleOffsetBytes >> 2) & (words - 1); + intervals.Add(new(first, Math.Min(32u * width, words - first), 0, (int)width)); + } + + var merged = new List(); + foreach (var region in intervals.OrderBy(region => region.First)) + { + if (merged.Count != 0 && region.First <= merged[^1].First + merged[^1].Count) + { + var previous = merged[^1]; + merged[^1] = previous with + { + Count = Math.Max(previous.First + previous.Count, region.First + region.Count) - previous.First, + Weight = previous.Weight + region.Weight, + }; + } + else merged.Add(region); + } + + // Leave ample room for the mesh outputs and the existing rendezvous state. + var outputBytes = mesh.MaxVertices * MeshVertexBytesBound(_request) + + mesh.MaxPrimitives * MeshPrimitiveBytesBound(_request); + var budget = outputBytes + 256 >= MeshOutputBudgetBytes ? 0u : + Math.Min(3072u, MeshOutputBudgetBytes - outputBytes - 256u) / sizeof(uint); + var chosen = new List(); + uint packed = 0; + foreach (var region in merged.OrderByDescending(region => region.Weight).ThenBy(region => region.First)) + { + if (region.Count > budget - packed) continue; + chosen.Add(region with { Packed = packed }); + packed += region.Count; + } + return chosen.ToArray(); + } + + private string HotLdsContains(HotLdsRegion region, string index, uint count = 1) => + $"({index} - {region.First}u) <= {region.Count - count}u"; + + private string HotLdsIndex(HotLdsRegion region, string index) => + $"({index} - {region.First}u + {region.Packed}u)"; + + private string ReadHotOrDeviceLds(string index) + { + var value = $"sharpemu_lds[{index}]"; + foreach (var region in _meshHotLds.Reverse()) + value = $"({HotLdsContains(region, index)} ? sharpemu_hot_lds[{HotLdsIndex(region, index)}] : {value})"; + return value; + } + + private void WriteHotOrDeviceLds(string index, string value) + { + foreach (var region in _meshHotLds) + Line($"if ({HotLdsContains(region, index)}) {{ sharpemu_hot_lds[{HotLdsIndex(region, index)}] = {value}; }} else"); + Line($"{{ sharpemu_lds[{index}] = {value}; }}"); + } + + private bool TryEmitMeshHotLds(Gen5ShaderInstruction instruction, Gen5DataShareControl control) + { + if (_meshHotLds.Length == 0 || instruction.Opcode is not + ("DsWriteB32" or "DsReadB32" or "DsWriteB128" or "DsReadB128")) return false; + + var count = instruction.Opcode.EndsWith("B128", StringComparison.Ordinal) ? 4u : 1u; + var write = instruction.Opcode.StartsWith("DsWrite", StringComparison.Ordinal); + var address = Temp("uint", RawSource(instruction, 0)); + var index = Temp("uint", LdsIndex(address, control.SingleOffsetBytes)); + var nominal = (control.SingleOffsetBytes >> 2) & + (System.Numerics.BitOperations.RoundUpToPowerOf2(_meshLdsDwords) - 1); + var primary = _meshHotLds.FirstOrDefault(region => region.Count >= count && + nominal >= region.First && nominal - region.First <= region.Count - count); + BeginExecGuard(); + var values = Temp("uint4", write + ? $"uint4({string.Join(", ", Enumerable.Range(0, 4).Select(i => i < count ? RawSource(instruction, i + 1) : "0u"))})" + : "uint4(0u)"); + if (primary is not null) + { + Line($"if ({HotLdsContains(primary, index, count)})"); + Line("{"); + _indent++; + for (uint i = 0; i < count; i++) + { + var element = $"sharpemu_hot_lds[{HotLdsIndex(primary, index)} + {i}u]"; + Line(write ? $"{element} = {values}[{i}];" : $"{values}[{i}] = {element};"); + } + _indent--; + Line("} else"); + Line("{"); + _indent++; + } + for (uint i = 0; i < count; i++) + { + var componentIndex = Temp("uint", LdsIndex(address, control.SingleOffsetBytes + i * sizeof(uint))); + if (write) WriteHotOrDeviceLds(componentIndex, $"{values}[{i}]"); + else Line($"{values}[{i}] = {ReadHotOrDeviceLds(componentIndex)};"); + } + if (primary is not null) + { + _indent--; + Line("}"); + } + if (!write) + for (uint i = 0; i < count; i++) + StoreVector(instruction.Destinations[(int)i].Value, $"{values}[{i}]"); + EndExecGuard(); + return true; + } + } +} diff --git a/src/SharpEmu.ShaderCompiler.Metal/Gen5MslTranslator.MeshLdsForwarding.cs b/src/SharpEmu.ShaderCompiler.Metal/Gen5MslTranslator.MeshLdsForwarding.cs new file mode 100644 index 000000000..50f8f5406 --- /dev/null +++ b/src/SharpEmu.ShaderCompiler.Metal/Gen5MslTranslator.MeshLdsForwarding.cs @@ -0,0 +1,50 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using SharpEmu.ShaderCompiler; +using SharpEmu.ShaderCompiler.Resources; + +namespace SharpEmu.ShaderCompiler.Metal; + +public static partial class Gen5MslTranslator +{ + private sealed partial class CompilationContext + { + // One hardware wave only: every lane reaches the shuffle even when guest EXEC + // is partial. Other workgroups cannot address this group's LDS. Refuse aliases + // whose writes do not pass through the DS instruction emitter. + private bool UseMeshLdsForwarding => _options.MeshLdsForwarding && + _stage == Gen5MslStage.Mesh && Mesh.ThreadsPerThreadgroup == 32 && + !_request.Memory.Entries.Any(entry => + entry.AddressSpace is FlatAddressSpace.Shared or FlatAddressSpace.SharedOrPrivate) && + !_request.Program.Instructions.Any(instruction => instruction.Control is Gen5BufferMemoryControl { Lds: true }); + + private void RememberMeshLdsWrite(Gen5ShaderInstruction instruction, string address, uint offset) + { + if (!UseMeshLdsForwarding || instruction.Opcode != "DsWriteB128" || + (offset & 3u) != 0 || _meshLdsDwords < 128 || offset > _meshLdsDwords * 4u - 512u) return; + + // Prove the active writers own disjoint records, before forwarding anything. + // A shader using another layout still performs its original memory accesses. + var affine = Temp("bool", $"simd_all(!exec || {address} == sharpemu_lane * 16u)"); + Line($"sharpemu_lds_forward_valid = exec && {affine};"); + Line($"sharpemu_lds_forward_base = {offset}u;"); + Line($"sharpemu_lds_forward_values = uint4({RawSource(instruction, 1)}, {RawSource(instruction, 2)}, " + + $"{RawSource(instruction, 3)}, {RawSource(instruction, 4)});"); + } + + private bool TryForwardMeshLdsRead(Gen5ShaderInstruction instruction, string address, uint offset) + { + if (!UseMeshLdsForwarding || instruction.Opcode != "DsReadB128") return false; + var distance = Temp("uint", $"({address} + {offset}u) - sharpemu_lds_forward_base"); + var lane = Temp("uint", $"({distance} >> 4u) & 31u"); + var valid = Temp("bool", $"simd_shuffle((uint)sharpemu_lds_forward_valid, {lane}) != 0u && " + + $"{distance} < 512u && ({distance} & 15u) == 0u"); + var values = Temp("uint4", $"simd_shuffle(sharpemu_lds_forward_values, {lane})"); + for (uint component = 0; component < 4; component++) + StoreVector(instruction.Destinations[(int)component].Value, + $"{valid} ? {values}[{component}] : sharpemu_lds[{LdsIndex(address, offset + component * 4)}]"); + return true; + } + } +} diff --git a/src/SharpEmu.ShaderCompiler.Metal/Gen5MslTranslator.PackedFormats.cs b/src/SharpEmu.ShaderCompiler.Metal/Gen5MslTranslator.PackedFormats.cs new file mode 100644 index 000000000..e9ad2b224 --- /dev/null +++ b/src/SharpEmu.ShaderCompiler.Metal/Gen5MslTranslator.PackedFormats.cs @@ -0,0 +1,65 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using SharpEmu.ShaderCompiler; + +namespace SharpEmu.ShaderCompiler.Metal; + +public static partial class Gen5MslTranslator +{ + private sealed partial class CompilationContext + { + // A packed element's components share words. Load each word once instead of + // performing overlapping unaligned reads at every component's byte offset. + // The caller emits the original decoder in the trailing else, preserving short + // final elements whose selected bytes fit but whose rounded word span does not. + private bool BeginLiteralPackedFormatLoad(FormatMemory memory, string address, uint word3, + uint vectorData, uint componentCount) + { + if (!Gfx10UnifiedFormat.TryDecode((word3 >> 12) & 0x7Fu, out var dataFormat, out var numberFormat) || + Gfx10UnifiedFormat.HasWholeDwordComponents(dataFormat)) return false; + var present = Gfx10UnifiedFormat.ComponentCount(dataFormat); + if (present == 0) return false; + var layouts = new (uint Word, uint Bit, uint Bits)[componentCount]; + uint run = 0; + for (uint destination = 0; destination < componentCount; destination++) + { + var selector = (word3 >> (int)(destination * 3)) & 7u; + if (selector < 4 || selector - 4 >= present) continue; + if (!Gfx10UnifiedFormat.TryGetComponentLayout(dataFormat, selector - 4, + out var offset, out var bit, out var bits)) return false; + bit += (offset % 4) * 8; + if (bits == 0 || bit + bits > 32) return false; + layouts[destination] = (offset / 4, bit, bits); + run = Math.Max(run, offset / 4 + 1); + } + if (run is 0 or > 4) return false; + var inBounds = memory.ElementInBounds(address, $"{run * 4}u"); + Line($"if ({inBounds})"); + Line("{"); + _indent++; + var words = Temp("uint4", memory.LoadWords!(run, address)); + var one = $"sharpemu_format_one({numberFormat}u)"; + for (uint destination = 0; destination < componentCount; destination++) + { + var selector = (word3 >> (int)(destination * 3)) & 7u; + string value; + if (selector < 4) value = selector == 1 ? one : "0u"; + else if (selector - 4 >= present) value = selector == 7 ? one : "0u"; + else + { + var (word, bit, bits) = layouts[destination]; + var raw = Temp("uint", $"extract_bits({words}[{word}], {bit}u, {bits}u)"); + value = Temp("uint", $"sharpemu_format_convert({raw}, {bits}u, {numberFormat}u, {dataFormat}u)"); + } + StoreVector(vectorData + destination, value); + } + _indent--; + Line("}"); + Line("else"); + Line("{"); + _indent++; + return true; + } + } +} diff --git a/src/SharpEmu.ShaderCompiler.Metal/Gen5MslTranslator.Resources.cs b/src/SharpEmu.ShaderCompiler.Metal/Gen5MslTranslator.Resources.cs index 59812916f..862f04753 100644 --- a/src/SharpEmu.ShaderCompiler.Metal/Gen5MslTranslator.Resources.cs +++ b/src/SharpEmu.ShaderCompiler.Metal/Gen5MslTranslator.Resources.cs @@ -200,7 +200,11 @@ public CompilationContext(ShaderCompileRequest request, Gen5MslCodegenOptions op if (_stage == Gen5MslStage.Mesh) { _meshLdsDwords = MeshLdsDwordCount(request); - _meshInfo = BuildMeshInfo(request, _meshLdsDwords); + _meshInfo = BuildMeshInfo(request, _meshLdsDwords, Math.Min(options.MeshDeviceLdsThresholdBytes, 16384u), + options.GeometryOutputCapacity); + _meshHotLds = PlanMeshHotLds(); + if (_meshHotLds.Length != 0) + _meshInfo = _meshInfo with { ThreadgroupMemoryBytes = MeshHotLdsWords * sizeof(uint) }; } _computeLdsDwords = options.DispatchThreadgroupMemory @@ -385,6 +389,16 @@ private void EmitResourceStruct(StringBuilder source) continue; } + if (field.FieldKind == MslArgumentFieldKind.BufferPointers && + Enumerable.Range(0, (int)field.Count).Any(IsScalarConstantBuffer)) + { + // Keep the host's pointer array ABI: each pointer retains its id and + // eight-byte slot, even when adjacent pointers have different types. + for (var index = 0; index < field.Count; index++) + source.AppendLine($" {(IsScalarConstantBuffer(index) ? "constant" : "device")} uint* buffer_{index} [[id({field.FirstId + index})]];"); + continue; + } + var declaration = field.FieldKind switch { MslArgumentFieldKind.BufferPointers => $"array {field.Name}", @@ -480,9 +494,11 @@ private void EmitLayoutInitialState(StringBuilder source) var request = _request; var layout = request.Bindings; var bufferCount = request.Resources.Info.Buffers.Count; + var splitBufferFields = Enumerable.Range(0, bufferCount).Any(IsScalarConstantBuffer); for (var index = 0; index < bufferCount; index++) { - source.AppendLine($" device uint* b{index} = {ResourcesName}.buffers[{index}];"); + var field = splitBufferFields ? $"buffer_{index}" : $"buffers[{index}]"; + source.AppendLine($" {(IsScalarConstantBuffer(index) ? "constant" : "device")} uint* b{index} = {ResourcesName}.{field};"); } if (!_options.ElideByteBias) @@ -1964,7 +1980,7 @@ private bool TryEmitScratchMemory(Gen5ShaderInstruction instruction, Gen5GlobalM : $"(v[{control.SourceVectorRegister}] >> {sourceShift})"; var element = FormatUInt(byteCount == 1 ? 0xFFu : 0xFFFFu); var shift = Temp("uint", $"({byteAddress} & 3u) << 3u"); - var word = ScratchWord(byteAddress); + var word = ScratchAccessWord(control, byteAddress); Line($"{word} = ({word} & ~({element} << {shift})) | (((({source}) & {element})) << {shift});"); } else @@ -1974,7 +1990,7 @@ private bool TryEmitScratchMemory(Gen5ShaderInstruction instruction, Gen5GlobalM var value = control.SourceIsScalar ? ScalarExpression(control.SourceVectorRegister + index) : $"v[{control.SourceVectorRegister + index}]"; - Line($"{ScratchWord(byteAddress, index * sizeof(uint))} = {value};"); + Line($"{ScratchAccessWord(control, byteAddress, index * sizeof(uint))} = {value};"); } } @@ -1994,7 +2010,7 @@ private bool TryEmitScratchMemory(Gen5ShaderInstruction instruction, Gen5GlobalM var shift = Temp("uint", $"({byteAddress} & 3u) << 3u"); var loaded = Temp( "uint", - $"({ScratchWord(byteAddress)} >> {shift}) & {FormatUInt(loadBytes == 1 ? 0xFFu : 0xFFFFu)}"); + $"({ScratchAccessWord(control, byteAddress)} >> {shift}) & {FormatUInt(loadBytes == 1 ? 0xFFu : 0xFFFFu)}"); if (signExtend) { loaded = Temp("uint", $"(uint)extract_bits(as_type({loaded}), 0u, {loadBytes * 8}u)"); @@ -2011,7 +2027,7 @@ private bool TryEmitScratchMemory(Gen5ShaderInstruction instruction, Gen5GlobalM for (uint index = 0; index < control.DwordCount; index++) { - StoreVector(destination + index, ScratchWord(byteAddress, index * sizeof(uint))); + StoreVector(destination + index, ScratchAccessWord(control, byteAddress, index * sizeof(uint))); } return true; diff --git a/src/SharpEmu.ShaderCompiler.Metal/Gen5MslTranslator.ScalarConstants.cs b/src/SharpEmu.ShaderCompiler.Metal/Gen5MslTranslator.ScalarConstants.cs new file mode 100644 index 000000000..30f6553d6 --- /dev/null +++ b/src/SharpEmu.ShaderCompiler.Metal/Gen5MslTranslator.ScalarConstants.cs @@ -0,0 +1,37 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using SharpEmu.ShaderCompiler.Resources; + +namespace SharpEmu.ShaderCompiler.Metal; + +public static partial class Gen5MslTranslator +{ + private sealed partial class CompilationContext + { + private bool IsScalarConstantBuffer(int binding) + { + if (!_options.ScalarConstantBuffers || !_options.RawDwordAccess || _stage != Gen5MslStage.Mesh) + return false; + + // SMEM coherence is carried by the instruction control, not consistently + // copied into MemoryAccessInfo. Do not infer it from the resource table alone. + if (_request.Program.Instructions.Any(instruction => instruction.Control is + Gen5ScalarMemoryControl { Glc: true } or Gen5ScalarMemoryControl { Dlc: true })) + return false; + + // A store through another descriptor, a flat address or an image can alias + // this binding. Refuse the entire shader when any device resource is written. + // LDS and private scratch cannot alias a host-bound buffer. + if (_request.Memory.Entries.Any(entry => entry.Access is MemoryAccess.Write or MemoryAccess.Atomic && + entry.Kind is not (MemoryResourceKind.LocalDataShare or MemoryResourceKind.Scratch))) + return false; + + var uses = _request.Memory.Entries.Where(entry => !entry.PlanningOnly && + entry.Resource == (uint)binding && entry.Kind is + (MemoryResourceKind.ScalarBuffer or MemoryResourceKind.Buffer or MemoryResourceKind.Global or MemoryResourceKind.Flat)).ToArray(); + return uses.Length != 0 && uses.All(entry => entry.Kind == MemoryResourceKind.ScalarBuffer && + entry.Access == MemoryAccess.Read && !entry.DeviceDescriptor && !entry.Glc && !entry.Slc); + } + } +} diff --git a/src/SharpEmu.ShaderCompiler.Metal/Gen5MslTranslator.ScratchSlots.cs b/src/SharpEmu.ShaderCompiler.Metal/Gen5MslTranslator.ScratchSlots.cs new file mode 100644 index 000000000..e4da81964 --- /dev/null +++ b/src/SharpEmu.ShaderCompiler.Metal/Gen5MslTranslator.ScratchSlots.cs @@ -0,0 +1,84 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using SharpEmu.ShaderCompiler.Resources; + +namespace SharpEmu.ShaderCompiler.Metal; + +public static partial class Gen5MslTranslator +{ + private sealed partial class CompilationContext + { + private bool _staticScratchSlots; + + // Renaming a private allocation by a per-invocation cyclic permutation is + // unobservable. For base v0 = local_id.x and dword-aligned constant offsets, + // ((v0 + offset) >> 2) % N = (v0 / 4 + offset / 4) % N, provided the add + // cannot wrap. Removing the common rotation leaves constant array indices, + // allowing Metal to scalarize guest register spills. No other access may + // observe the allocation, and v0 must retain its entry value at every access. + private bool CanUseStaticScratchSlots() + { + bool Refuse(string reason) + { + if (_options.StaticScratchSlots && Environment.GetEnvironmentVariable("SHARPEMU_MSL_STATIC_SCRATCH_REPORT") == "1") + Console.Error.WriteLine($"[MSL-SCRATCH] hash={_request.Hash:X16} refused={reason}"); + return false; + } + + if (!_options.StaticScratchSlots || _stage != Gen5MslStage.Compute || + _request.Tessellation is not null || _scratchDwordCount == 0 || _localSizeX == 0 || + _request.Memory.Entries.Any(static m => + m.AddressSpace is FlatAddressSpace.Private or FlatAddressSpace.SharedOrPrivate)) + { + return Refuse("stage-or-aperture"); + } + + var instructions = _request.Program.Instructions; + var last = -1; + for (var index = 0; index < instructions.Count; index++) + { + var instruction = instructions[index]; + if (!instruction.Opcode.StartsWith("Scratch", StringComparison.Ordinal)) continue; + if (instruction.Opcode is not ("ScratchLoadDword" or "ScratchLoadDwordx2" or + "ScratchLoadDwordx3" or "ScratchLoadDwordx4" or "ScratchStoreDword" or + "ScratchStoreDwordx2" or "ScratchStoreDwordx3" or "ScratchStoreDwordx4") || + instruction.Control is not Gen5GlobalMemoryControl control || + control.ScalarAddress < 125 || control.VectorAddress != 0 || + control.DynamicOffsetRegister is not null || control.AddThreadId || + control.OffsetBytes < 0 || (control.OffsetBytes & 3) != 0 || + control.DwordCount is < 1 or > 4 || + (ulong)control.OffsetBytes + (control.DwordCount - 1) * 4 + _localSizeX - 1 > uint.MaxValue) + { + return Refuse($"address pc={instruction.Pc:X} op={instruction.Opcode} control={instruction.Control}"); + } + last = index; + } + if (last < 0) return false; + + for (var index = 0; index < instructions.Count; index++) + { + var instruction = instructions[index]; + // V_MOVRELS reads a relative source but writes its explicit destination; + // the destination check below still rejects a write to v0. + // Unknown/indirect control flow or register writes cannot prove the + // lifetime of the base. A later ordinary write is fine only when no + // path from that suffix can return to an access. + if (instruction.Opcode.Contains("Raw", StringComparison.Ordinal) || + (instruction.Opcode.StartsWith("VMovrel", StringComparison.Ordinal) && instruction.Opcode != "VMovrelsB32") || + instruction.Opcode is "VSwaprelB32" or "SSetpcB64" or "SSwappcB64" or "SRfeB64" or + "SCbranchJoin" or "SCbranchGFork" or "SCbranchIFork") return Refuse($"dynamic pc={instruction.Pc:X} op={instruction.Opcode}"); + if (index <= last && instruction.Destinations.Any(static d => + d.Kind == Gen5OperandKind.VectorRegister && d.Value == 0)) return Refuse($"base-write pc={instruction.Pc:X} op={instruction.Opcode}"); + if (index > last && Ir.Gen5IrBranchResolver.Instance.TryGetBranchTarget(instruction, out var target) && + target <= instructions[last].Pc) return Refuse($"backedge pc={instruction.Pc:X} target={target:X} last={instructions[last].Pc:X}"); + } + return true; + } + + private string ScratchAccessWord(Gen5GlobalMemoryControl control, string byteAddress, uint offsetBytes = 0) => + _staticScratchSlots + ? $"sharpemu_scratch[{((uint)control.OffsetBytes / 4 + offsetBytes / 4) % _scratchDwordCount}u]" + : ScratchWord(byteAddress, offsetBytes); + } +} diff --git a/src/SharpEmu.ShaderCompiler.Metal/Gen5MslTranslator.Structured.cs b/src/SharpEmu.ShaderCompiler.Metal/Gen5MslTranslator.Structured.cs index 4fa4b9440..8f765c82a 100644 --- a/src/SharpEmu.ShaderCompiler.Metal/Gen5MslTranslator.Structured.cs +++ b/src/SharpEmu.ShaderCompiler.Metal/Gen5MslTranslator.Structured.cs @@ -199,7 +199,12 @@ private bool TryAssembleStructuredBody( text.AppendLine(); } - EmitRegion(text, 0, NoFollow, 1); + // Mesh exits converge at the common output epilogue, which contains a + // workgroup barrier. Treat that exit as the outer region's follow: the + // assembler then encloses each remaining path in its conditional rather + // than returning from the kernel. An exit from inside a nested loop still + // has to satisfy the existing loop/region rules; otherwise we fall back. + EmitRegion(text, 0, _stage == Gen5MslStage.Mesh && _options.MeshExitJoin ? ExitTarget : NoFollow, 1); if (_structureFailed) { return false; @@ -607,6 +612,19 @@ private void AppendExit(StringBuilder text, int indent) AppendIndented(text, "return sharpemu_out;", indent); return; case Gen5MslStage.Mesh: + if (indent > 1 && _options.MeshExitJoin && Mesh.ThreadsPerThreadgroup <= MeshWaveLanes) + { + // Scalar control flow (including the loop budget) is uniform in + // this one hardware wave. Every lane reaches this copy of the + // epilogue together, including its barrier and LDS slot release. + // A multi-wave group must still reconverge at the common copy. + var meshEpilogue = new StringBuilder(); + EmitMeshEpilogue(meshEpilogue); + AppendBlockText(text, meshEpilogue.ToString(), indent - 1); + AppendIndented(text, "return;", indent); + return; + } + // The mesh epilogue publishes the whole workgroup's output after a // threadgroup barrier, so every thread has to reach it: an exit at the // end of the program falls into it, and one anywhere else would leave diff --git a/src/SharpEmu.ShaderCompiler.Metal/Gen5MslTranslator.cs b/src/SharpEmu.ShaderCompiler.Metal/Gen5MslTranslator.cs index 0048ef01b..6d26428aa 100644 --- a/src/SharpEmu.ShaderCompiler.Metal/Gen5MslTranslator.cs +++ b/src/SharpEmu.ShaderCompiler.Metal/Gen5MslTranslator.cs @@ -21,7 +21,8 @@ internal enum Gen5MslPreludes Float64 = 1 << 5, Bvh = 1 << 6, MipStatistics = 1 << 7, - All = (1 << 8) - 1, + ScalarConstants = 1 << 8, + All = (1 << 9) - 1, } // Compiles shader requests into Metal source with argument-buffer bindings. @@ -417,7 +418,7 @@ private string StepCapReportCode /// waves per threadgroup. Their exchanges meet at a pair barrier of the wave's two /// simdgroups instead of a threadgroup barrier, since the other waves need not be at /// the same exchange (see ). - private bool IsBridgedMeshWave64 => _stage == Gen5MslStage.Mesh && _meshInfo is { Amplified: true, Wave64: true }; + private bool IsBridgedMeshWave64 => _stage == Gen5MslStage.Mesh && _meshInfo is { GuestWaveLanes: 64 }; /// Where the two halves of a bridged wave64 meet: a threadgroup barrier when /// the wave is the whole threadgroup (compute), else the wave's own pair barrier. The @@ -551,6 +552,7 @@ public bool TryCompile(out Gen5MslShader shader, out string error) } } + _staticScratchSlots = CanUseStaticScratchSlots(); AssignLaneSlotLds(blocks); if (_stage is Gen5MslStage.Vertex or Gen5MslStage.Mesh) @@ -1322,6 +1324,9 @@ private Gen5MslPreludes ModulePreludes() preludes |= Gen5MslPreludes.MipStatistics; } + if (Enumerable.Range(0, _request.Resources.Info.Buffers.Count).Any(IsScalarConstantBuffer)) + preludes |= Gen5MslPreludes.ScalarConstants; + return preludes; } @@ -1346,6 +1351,9 @@ internal static void AppendTrailingPreludes(StringBuilder source, Gen5MslPrelude ("ballot_return", (preludes & Gen5MslPreludes.RealWaveLanes) != 0 ? "(uint)(uint64_t)simd_ballot(value)" : "value ? 1u : 0u"))); + if ((preludes & Gen5MslPreludes.ScalarConstants) != 0) + source.Append(MslTemplates.Render("scalar_constant_prelude")); + if ((preludes & Gen5MslPreludes.BufferOffset) != 0) { source.Append(MslTemplates.Render("buffer_offset_prelude")); @@ -2680,6 +2688,9 @@ private void EmitDecodedBufferFormatLoad( uint componentCount, FormatD16 d16) { + var packedRun = _options.PackedFormatLoads && !d16.Enabled && byteAddress.Contiguous && + memory.LoadWords is not null && TryParseUIntLiteral(descriptorWord3, out var packedWord3) && + BeginLiteralPackedFormatLoad(memory, byteAddress.Base, packedWord3, vectorData, componentCount); // Format and destination swizzle come from descriptor word 3, a register // or a specialization constant; the prelude table decodes the unified format. var word3 = Temp("uint", descriptorWord3); @@ -2769,7 +2780,7 @@ private void EmitDecodedBufferFormatLoad( } StoreFormatLoadComponents(vectorData, values, d16, numberFormat); - if (dwordRun) + if (dwordRun || packedRun) { _indent--; Line("}"); @@ -3130,6 +3141,14 @@ private bool TryEmitDataShare( } } + // Any intervening LDS writer may alias the cached record. Even a masked + // write invalidates every lane's copy, not just the writing lane's copy. + if (UseMeshLdsForwarding && !instruction.Opcode.StartsWith("DsRead", StringComparison.Ordinal) && + instruction.Opcode is not ("DsSwizzleB32" or "DsBpermuteB32" or "DsNop")) + Line("sharpemu_lds_forward_valid = false;"); + + if (TryEmitMeshHotLds(instruction, control)) return true; + void StoreLds(string index, string value) { // Exec-guarded like every other lane-visible write. @@ -3251,6 +3270,7 @@ void StoreLds(string index, string value) LdsIndex(address, offset + (uint)(dword * sizeof(uint))), RawSource(instruction, 1 + dword)); } + RememberMeshLdsWrite(instruction, address, offset); return true; } @@ -3321,6 +3341,7 @@ void StoreLds(string index, string value) var address = Temp("uint", RawSource(instruction, 0)); var offset = control.SingleOffsetBytes; + if (TryForwardMeshLdsRead(instruction, address, offset)) return true; for (var dword = 0; dword < dwordCount; dword++) { StoreVector( @@ -4420,7 +4441,7 @@ private void StoreVector(uint register, string expression, bool guardWithExec = return; } - if (guardWithExec && !_execKnownFull) + if (guardWithExec && !_execKnownFull && !_unobservedAluWrites.Contains(register)) { Line($"if (exec) {{ v[{register}] = {expression}; }}"); } diff --git a/src/SharpEmu.ShaderCompiler.Metal/Templates/scalar_constant_prelude.msl b/src/SharpEmu.ShaderCompiler.Metal/Templates/scalar_constant_prelude.msl new file mode 100644 index 000000000..b3d7fdd93 --- /dev/null +++ b/src/SharpEmu.ShaderCompiler.Metal/Templates/scalar_constant_prelude.msl @@ -0,0 +1,39 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +// Scalar-only read-only bindings use the uniform memory path, with identical +// dword truncation, wrapping and partial-run bounds semantics. +// A raw dword load: the address truncates to the dword it lands in, which is what the +// hardware does for S_BUFFER_LOAD_DWORD and BUFFER_LOAD_DWORD and what the SPIR-V +// backend emits. Out of range reads zero. +static inline uint sharpemu_load_dword(constant uint* b, uint bytes, uint addr) +{ + uint index = addr >> 2; + return index < (bytes >> 2) ? b[index] : 0u; +} + +// Up to four consecutive raw dwords of an S_BUFFER_LOAD/BUFFER_LOAD_DWORDXn, each what +// sharpemu_load_dword reads at addr + 4 * i. A run that lies inside the binding is one +// bounds check and adjacent loads; one that crosses the end reads dword by dword. +template +static inline uint4 sharpemu_load_dwords(constant uint* b, uint bytes, uint addr) +{ + uint4 result = uint4(0u); + uint index = addr >> 2; + uint words = bytes >> 2; + if (index < words && (uint)N <= words - index) + { + for (int i = 0; i < N; i++) + { + result[i] = b[index + (uint)i]; + } + } + else + { + for (int i = 0; i < N; i++) + { + result[i] = sharpemu_load_dword(b, bytes, addr + (uint)(4 * i)); + } + } + return result; +} diff --git a/src/SharpEmu.ShaderCompiler/Ir/Gen5ExecFullAnalysis.cs b/src/SharpEmu.ShaderCompiler/Ir/Gen5ExecFullAnalysis.cs index 7130d87b1..c7b1505bd 100644 --- a/src/SharpEmu.ShaderCompiler/Ir/Gen5ExecFullAnalysis.cs +++ b/src/SharpEmu.ShaderCompiler/Ir/Gen5ExecFullAnalysis.cs @@ -202,11 +202,11 @@ private static State Transfer(Gen5ShaderInstruction instruction, State input, bo nextExecFull = execFull; break; case "SMovB64" or "SMovB32" when sources.Count == 1: - nextExecFull = IsFullCopy(sources[0], copies, execFull, wide); + nextExecFull = IsFullCopy(sources[0], copies, execFull, wide, wave32); break; case "SOrB64" or "SOrB32" when sources.Count == 2: - nextExecFull = IsFullCopy(sources[0], copies, execFull, wide) || - IsFullCopy(sources[1], copies, execFull, wide); + nextExecFull = IsFullCopy(sources[0], copies, execFull, wide, wave32) || + IsFullCopy(sources[1], copies, execFull, wide, wave32); break; } } @@ -240,8 +240,20 @@ private static bool IsExec(Gen5Operand operand) => operand.Kind == Gen5OperandKind.ScalarRegister && operand.Value == ExecLow; // True when the operand is a full EXEC or a saved copy of one. - private static bool IsFullCopy(Gen5Operand operand, UInt128 copies, bool execFull, bool wide) + private static bool IsFullCopy(Gen5Operand operand, UInt128 copies, bool execFull, bool wide, bool wave32) { + // Inline -1 sign-extends for a 64-bit scalar operation. A literal dword + // zero-extends instead, so 0xffffffff covers only a 32-lane wave; + // it must not prove a wave64 EXEC restore full. + if (operand is { Kind: Gen5OperandKind.EncodedConstant, Value: 193 }) + { + return true; + } + if ((!wide || wave32) && operand is { Kind: Gen5OperandKind.LiteralConstant, Value: uint.MaxValue }) + { + return true; + } + if (operand.Kind != Gen5OperandKind.ScalarRegister) { return false; diff --git a/src/SharpEmu.ShaderCompiler/Ir/Gen5VectorObservationAnalysis.cs b/src/SharpEmu.ShaderCompiler/Ir/Gen5VectorObservationAnalysis.cs index 37eba4e5d..03d304a21 100644 --- a/src/SharpEmu.ShaderCompiler/Ir/Gen5VectorObservationAnalysis.cs +++ b/src/SharpEmu.ShaderCompiler/Ir/Gen5VectorObservationAnalysis.cs @@ -172,7 +172,7 @@ private static bool TryDescribe(Gen5ShaderInstruction instruction, out Effect ef { effect = default; var opcode = instruction.Opcode; - if (opcode.StartsWith("VMovrel", StringComparison.Ordinal) || + if (opcode.StartsWith("VMovrel", StringComparison.Ordinal) || opcode == "VSwaprelB32" || opcode.Contains("Raw", StringComparison.Ordinal) || opcode is "SSetpcB64" or "SSwappcB64" or "SRfeB64" or "SCbranchJoin" or "SCbranchGFork" or "SCbranchIFork") { @@ -279,7 +279,10 @@ private static void DescribeVector(Gen5ShaderInstruction instruction, bool wideN writes = writes.With(destination + 1); } + // DPP row/bank masks and inactive-source rules can retain the old + // destination even under a full EXEC. Such a write cannot kill its value. var partial = + instruction.Control is Gen5DppControl or Gen5Dpp8Control || opcode.Contains("Mac", StringComparison.Ordinal) || opcode.StartsWith("VDot", StringComparison.Ordinal) || opcode is "VWritelaneB32" || diff --git a/src/SharpEmu.ShaderCompiler/Resources/ResourceMaterializationCache.Validation.cs b/src/SharpEmu.ShaderCompiler/Resources/ResourceMaterializationCache.Validation.cs index 0f270cb08..c79d230fc 100644 --- a/src/SharpEmu.ShaderCompiler/Resources/ResourceMaterializationCache.Validation.cs +++ b/src/SharpEmu.ShaderCompiler/Resources/ResourceMaterializationCache.Validation.cs @@ -22,6 +22,29 @@ private sealed class JoinedReads } private const ulong JoinedGapBytes = 64; + + // Reused only during this cache's serialized validation. A range may move independently + // of its neighbors; never move a joined span or retain these addresses on the entry. + private ulong[] _validationMovedAddresses = []; + + private bool CompareValidationRanges(Entry entry, IReadOnlyList userData, AddressRebase rebase, + ResidentGuestBytesComparer comparer) + { + ReadOnlySpan addresses = entry.RangeAddresses; + if (rebase.Active.Length != 0) + { + var count = entry.RangeAddresses.Length; + if (_validationMovedAddresses.Length < count) + _validationMovedAddresses = new ulong[Math.Max(count, _validationMovedAddresses.Length * 2)]; + for (var index = 0; index < count; index++) + _validationMovedAddresses[index] = MoveValidationRead(entry, userData, rebase, + entry.RangeAddresses[index], entry.RangeLengths[index]); + addresses = _validationMovedAddresses.AsSpan(0, count); + } + using var profile = ResourceMaterializationProfile.MeasureSampled(ResourceMaterializationProfile.Phase.InPlaceComparison); + return comparer(addresses, entry.RangeLengths, entry.RangeOffsets, entry.Bytes); + } + private const int PageShift = 12; private ulong[] _joinedAddresses = []; @@ -119,8 +142,13 @@ static T[] Shared(Span words, T[]? hint) where T : IEquatable => private bool ValidateJoined(Entry entry, ResidentGuestBytesReader residentReader, ResidentGuestBytesComparer? comparer, out bool unreadable, out bool descriptorChanged, out RejectionSample rejection) { - if (comparer is null || !InPlaceValidateEnabled || - !comparer(entry.RangeAddresses, entry.RangeLengths, entry.RangeOffsets, entry.Bytes)) + var equal = false; + if (comparer is not null && InPlaceValidateEnabled) + { + using var profile = ResourceMaterializationProfile.MeasureSampled(ResourceMaterializationProfile.Phase.InPlaceComparison); + equal = comparer(entry.RangeAddresses, entry.RangeLengths, entry.RangeOffsets, entry.Bytes); + } + if (!equal) return ValidateJoinedByReading(entry, residentReader, out unreadable, out descriptorChanged, out rejection); unreadable = false; descriptorChanged = false; diff --git a/src/SharpEmu.ShaderCompiler/Resources/ResourceMaterializationCache.cs b/src/SharpEmu.ShaderCompiler/Resources/ResourceMaterializationCache.cs index 1de30ad55..c3f721e95 100644 --- a/src/SharpEmu.ShaderCompiler/Resources/ResourceMaterializationCache.cs +++ b/src/SharpEmu.ShaderCompiler/Resources/ResourceMaterializationCache.cs @@ -1471,7 +1471,8 @@ private bool MaterializeCore( for (var variant = cached; variant is not null; previous = variant, variant = variant.Next) { CountDiagnostic((Diagnostic)((int)Diagnostic.Variant1 + Math.Min(variantIndex++, MaxVariants - 1))); - if (!variant.Matches(plan, inputs, rebase.Active) || !rebase.AppliesTo(variant)) + // TryFind already checked the head against this draw's unchanged inputs. + if ((!ReferenceEquals(variant, cached) && !variant.Matches(plan, inputs, rebase.Active)) || !rebase.AppliesTo(variant)) continue; stale ??= variant; if (rebase.Active.Length == 0 && !rebase.HasLearnableMemoryBases && rejection.Rejects(variant)) @@ -1493,6 +1494,8 @@ private bool MaterializeCore( Hits++; Interlocked.Increment(ref _totalHits); snapshot = HitSnapshot(variant, inputs.UserData, rebase); + ResourceMaterializationProfile.RecordCacheHit(recalled: false, rebase.Active.Length != 0, + variant.RangeAddresses.Length, variant.Bytes.Length); specialization = variant.Specialization; failure = default; return true; @@ -2016,7 +2019,8 @@ private bool TryRecalledHit(ulong key, ShaderResourcePlan plan, ResourceRuntimeI for (Entry? previous = null, variant = cached; variant is not null; previous = variant, variant = variant.Next) { - if (!variant.Matches(plan, inputs, rebase.Active) || !rebase.AppliesTo(variant) || + // Only the head was checked by TryFind; older variants still need the full check. + if ((!ReferenceEquals(variant, cached) && !variant.Matches(plan, inputs, rebase.Active)) || !rebase.AppliesTo(variant) || !Validate(variant, residentReader, rebase, inputs.UserData, false, out _, out _, out _, inputs.CompareResidentMemory)) continue; if (previous is not null) @@ -2029,6 +2033,8 @@ private bool TryRecalledHit(ulong key, ShaderResourcePlan plan, ResourceRuntimeI Hits++; Interlocked.Increment(ref _totalHits); snapshot = HitSnapshot(variant, inputs.UserData, rebase); + ResourceMaterializationProfile.RecordCacheHit(recalled: true, rebase.Active.Length != 0, + variant.RangeAddresses.Length, variant.Bytes.Length); specialization = variant.Specialization; return true; } @@ -2101,6 +2107,7 @@ private static bool SameUserData(uint[] words, IReadOnlyList userData) // encoded later still holds exactly the snapshot it was given. private static ResourceSnapshot HitSnapshot(Entry entry, IReadOnlyList userData, AddressRebase rebase) { + using var profile = ResourceMaterializationProfile.MeasureSampled(ResourceMaterializationProfile.Phase.HitSnapshot); var kept = entry.HitSnapshots; if (kept is not null) { @@ -2588,6 +2595,13 @@ public bool RefutesWord(ulong address, bool clean, uint expected) => Length != 0 && Clean == clean && Address + (ulong)WordOffset == address && Word != expected; } + private static ulong MoveValidationRead(Entry entry, IReadOnlyList userData, AddressRebase rebase, + ulong address, int length) + { + using var profile = ResourceMaterializationProfile.MeasureSampled(ResourceMaterializationProfile.Phase.AddressRebasing); + return rebase.MoveRead(entry, userData, address, length); + } + private bool Validate(Entry entry, ResidentGuestBytesReader residentReader, AddressRebase rebase, IReadOnlyList userData, bool joined, out bool unreadable) => Validate(entry, residentReader, rebase, userData, joined, out unreadable, out _, out _); @@ -2596,11 +2610,36 @@ private bool Validate(Entry entry, ResidentGuestBytesReader residentReader, Addr bool joined, out bool unreadable, out bool descriptorChanged, out RejectionSample rejection, ResidentGuestBytesComparer? comparer = null) { + using var profile = ResourceMaterializationProfile.MeasureSampled(ResourceMaterializationProfile.Phase.HitValidation); unreadable = false; descriptorChanged = false; rejection = default; + // With no recorded guest reads there are no bytes or base-word probes to validate. + if (entry.RangeAddresses.Length == 0) return true; if (joined && rebase.Active.Length == 0) return ValidateJoined(entry, residentReader, comparer, out unreadable, out descriptorChanged, out rejection); + if (comparer is not null && InPlaceValidateEnabled && + CompareValidationRanges(entry, userData, rebase, comparer)) + { + if (!InPlaceValidateVerify) return true; + // Verification must use the original scalar reads, not enter this fast path again. + var read = ValidateByReading(entry, residentReader, rebase, userData, + out unreadable, out descriptorChanged, out rejection); + if (!read) + Console.Error.WriteLine($"[PERF][INPLACE_VALIDATE_VERIFY] mismatch: exact-address comparison found all {entry.RangeAddresses.Length} ranges equal, " + + $"scalar reading found them {(unreadable ? "unreadable" : "changed")}."); + return read; + } + return ValidateByReading(entry, residentReader, rebase, userData, + out unreadable, out descriptorChanged, out rejection); + } + + private bool ValidateByReading(Entry entry, ResidentGuestBytesReader residentReader, AddressRebase rebase, + IReadOnlyList userData, out bool unreadable, out bool descriptorChanged, out RejectionSample rejection) + { + unreadable = false; + descriptorChanged = false; + rejection = default; // A memory-loaded base usually moved when anything did: its few bytes are checked before // ranges that can span kilobytes. The full comparison below still decides a match. if (entry.BaseWordBuffers is not null) @@ -2608,7 +2647,7 @@ private bool Validate(Entry entry, ResidentGuestBytesReader residentReader, Addr Span word = stackalloc byte[sizeof(uint)]; foreach (var (range, offset) in entry.BaseWordProbes ??= BaseWordProbesOf(entry)) { - var address = rebase.MoveRead(entry, userData, entry.RangeAddresses[range], entry.RangeLengths[range]) + (ulong)offset; + var address = MoveValidationRead(entry, userData, rebase, entry.RangeAddresses[range], entry.RangeLengths[range]) + (ulong)offset; if (residentReader(address, word, entry.RangeClean[range]) && !word.SequenceEqual(entry.Bytes.AsSpan(entry.RangeOffsets[range] + offset, sizeof(uint)))) return false; @@ -2621,7 +2660,7 @@ private bool Validate(Entry entry, ResidentGuestBytesReader residentReader, Addr if (_scratch.Length < length) _scratch = new byte[Math.Max(length, _scratch.Length * 2)]; var current = _scratch.AsSpan(0, length); - var address = rebase.MoveRead(entry, userData, entry.RangeAddresses[index], length); + var address = MoveValidationRead(entry, userData, rebase, entry.RangeAddresses[index], length); if (!ReadWithProfile(residentReader, address, current, entry.RangeClean[index], CacheDetail.ValidateRead)) { unreadable = true; diff --git a/src/SharpEmu.ShaderCompiler/Resources/ResourceMaterializationProfile.cs b/src/SharpEmu.ShaderCompiler/Resources/ResourceMaterializationProfile.cs index 70210e9af..4d7754e4f 100644 --- a/src/SharpEmu.ShaderCompiler/Resources/ResourceMaterializationProfile.cs +++ b/src/SharpEmu.ShaderCompiler/Resources/ResourceMaterializationProfile.cs @@ -48,6 +48,10 @@ public enum Phase ReadRecording, ConsumerLookup, ConsumerGraph, + AddressRebasing, + HitValidation, + InPlaceComparison, + HitSnapshot, Count, } @@ -78,6 +82,35 @@ internal static void CountRead(ReadCounter counter, int amount = 1) if (Enabled) Interlocked.Add(ref _readCounts[(int)counter], amount); } + // One tiny counter record per participating thread. Hit recording never walks ranges + // and uses no atomics; reporting reads a cumulative, approximate live snapshot. + private sealed class HitCounts + { + internal long Normal, Recall, RebasedNormal, RebasedRecall, Ranges, Bytes; + } + private static readonly System.Collections.Concurrent.ConcurrentBag _hitCounts = new(); + [ThreadStatic] private static HitCounts? _threadHitCounts; + + internal static void RecordCacheHit(bool recalled, bool rebased, int ranges, int bytes) + { + if (!Enabled) return; + var counts = _threadHitCounts; + if (counts is null) + { + counts = _threadHitCounts = new HitCounts(); + _hitCounts.Add(counts); + } + if (recalled) counts.Recall++; + else counts.Normal++; + if (rebased) + { + if (recalled) counts.RebasedRecall++; + else counts.RebasedNormal++; + } + counts.Ranges += ranges; + counts.Bytes += bytes; + } + // The frame trace prints a line per guest flow and per mutex handoff, which on a // submission-heavy title costs more than the materializations being measured, so these // counters also answer to a switch of their own. @@ -154,7 +187,18 @@ internal static void WriteReport(TextWriter output) var parts = new List(); for (var index = 0; index < (int)Phase.Count; index++) parts.Add(FormattableString.Invariant($"{(Phase)index}={Interlocked.Read(ref _elapsedTicks[index]) * 1000.0 / Stopwatch.Frequency:F3}ms/n{Interlocked.Read(ref _calls[index])}")); - output.WriteLine($"[PERF][RESOURCE_MATERIALIZATION] cumulative=1 inclusive=1 sample_stride={SampleStride} sampled_phases=CompiledOutputAllocation,CompiledBody,CompiledWordRead,CompiledGroupRead,ReadRecording,ConsumerLookup allocated_bytes={Interlocked.Read(ref _allocatedBytesByPhase[(int)Phase.Total])} wrap_bytes={Interlocked.Read(ref _allocatedBytesByPhase[(int)Phase.RecorderWrap])} build_bytes={Interlocked.Read(ref _allocatedBytesByPhase[(int)Phase.EntryBuild])} branch_evaluations={Interlocked.Read(ref _branchEvaluations)} descriptor_sources={Interlocked.Read(ref _descriptorSources)} inactive_sources={Interlocked.Read(ref _inactiveSources)} table_words={Interlocked.Read(ref _tableWords)} descriptor_words={Interlocked.Read(ref _descriptorWords)} compiled_walks={Interlocked.Read(ref _compiledEvaluators)} interpreted_walks={Interlocked.Read(ref _interpretedEvaluators)} {string.Join(" ", parts)}"); + output.WriteLine($"[PERF][RESOURCE_MATERIALIZATION] cumulative=1 inclusive=1 sample_stride={SampleStride} sampled_phases=CompiledOutputAllocation,CompiledBody,CompiledWordRead,CompiledGroupRead,ReadRecording,ConsumerLookup,AddressRebasing,HitValidation,InPlaceComparison,HitSnapshot allocated_bytes={Interlocked.Read(ref _allocatedBytesByPhase[(int)Phase.Total])} wrap_bytes={Interlocked.Read(ref _allocatedBytesByPhase[(int)Phase.RecorderWrap])} build_bytes={Interlocked.Read(ref _allocatedBytesByPhase[(int)Phase.EntryBuild])} branch_evaluations={Interlocked.Read(ref _branchEvaluations)} descriptor_sources={Interlocked.Read(ref _descriptorSources)} inactive_sources={Interlocked.Read(ref _inactiveSources)} table_words={Interlocked.Read(ref _tableWords)} descriptor_words={Interlocked.Read(ref _descriptorWords)} compiled_walks={Interlocked.Read(ref _compiledEvaluators)} interpreted_walks={Interlocked.Read(ref _interpretedEvaluators)} {string.Join(" ", parts)}"); + long normal = 0, recalled = 0, rebasedNormal = 0, rebasedRecall = 0, ranges = 0, bytes = 0; + foreach (var counts in _hitCounts) + { + normal += Volatile.Read(ref counts.Normal); + recalled += Volatile.Read(ref counts.Recall); + rebasedNormal += Volatile.Read(ref counts.RebasedNormal); + rebasedRecall += Volatile.Read(ref counts.RebasedRecall); + ranges += Volatile.Read(ref counts.Ranges); + bytes += Volatile.Read(ref counts.Bytes); + } + output.WriteLine($"[PERF][RESOURCE_HIT_COUNTS] cumulative=1 sampled=0 live_snapshot=1 normal={normal} recalled={recalled} normal_active_rebase={rebasedNormal} recalled_active_rebase={rebasedRecall} recorded_ranges={ranges} recorded_bytes={bytes}"); var reads = new List(); for (var index = 0; index < (int)ReadCounter.Count; index++) reads.Add($"{(ReadCounter)index}={Interlocked.Read(ref _readCounts[index])}"); diff --git a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalCommandStream.cs b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalCommandStream.cs index 0c6f5cb1e..f1a06a1f4 100644 --- a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalCommandStream.cs +++ b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalCommandStream.cs @@ -77,6 +77,7 @@ public void BeginSubmission(int queueId, ulong submissionId, object? geometrySna _translation!.BeginSubmission(queueId, submissionId, geometrySnapshots, CommandStream.GetInterpreter(queueId)); _commandThread!.Post(() => { + NativeOcclusionBeginQueue(queueId); CommitCoalescedIfOverdue(); _ = EnsureCommandBuffer(); }); @@ -84,6 +85,7 @@ public void BeginSubmission(int queueId, ulong submissionId, object? geometrySna } DrainDraws(); + NativeOcclusionBeginQueue(queueId); _submissionsStarted++; CommitCoalescedIfOverdue(); _ = EnsureCommandBuffer(); @@ -177,7 +179,10 @@ private void FlushStream() var commit = ShouldCommitStreamFlush( coalesce, streamIdle: false, _coalescedSubmissions, _encodersInBuffer, now - _commitWantedSince, MetalV2Diagnostics.CoalesceMaxSubmissions, MetalV2Diagnostics.CoalesceMaxEncoders, CoalesceMaxTicks) || - !(_commandStream?.HasUnblockedPending ?? false); + // With a producer, an empty packet queue can still have many worker items + // behind this flush. Its ordered idle callback commits after those items; + // testing the producer queue here would commit each parsed submission. + (!UsesCommandThread && !(_commandStream?.HasUnblockedPending ?? false)); if (commit) { FlushCommandBuffer(MetalFlushCause.Stream); @@ -789,13 +794,16 @@ private ulong EncodeOrderedCopies( string profileWhat, string? profileKind, ulong profileBytes, - bool sourceIsFinal = true) + bool sourceIsFinal = true, + bool deferStableGpuResult = false) { ulong serial; // A copy whose source later work in the same encoder may still change (the GDS a // pixel or compute shader appends to) cannot wait for the boundary: it would copy // what that later work left there. - if (sourceIsFinal && + // A GPU-produced snapshot may wait for its producer encoder to end, but must + // never be read on the CPU for label replay before that producer has executed. + if ((sourceIsFinal || deferStableGpuResult) && (LabelsAtCommit || (MetalV2Diagnostics.DeferLabelWrites && _renderEncoder != 0) || (DefersCopiesInCompute && _computeEncoder != 0))) @@ -918,13 +926,28 @@ private ulong EncodeOrderedCopies( /// copied by the GPU where the stream has reached, or - when the GPU cannot write them, /// or a label is still owed by a completion - published by a completion behind it. /// + internal bool QueueCommandProcessorWrites { get; set; } = + Environment.GetEnvironmentVariable("SHARPEMU_METAL_V2_QUEUE_CP_WRITES") == "1"; + public void RecordCommandProcessorWrite(ulong address, ReadOnlySpan source, string kind) { if (OnCommandThread) { - // Recorded before the stream goes on, so its own reads see the bytes (the overlay). + if (source.IsEmpty) return; + if (address == 0 || (ulong)source.Length > ulong.MaxValue - address) + throw Fatal($"A command-processor write has an invalid range: address=0x{address:X16} bytes={source.Length}."); var queued = source.ToArray(); - RunTagged(() => RecordCommandProcessorWrite(address, queued, kind)); + if (QueueCommandProcessorWrites) + { + // Reads overlapping this queued write wait until the worker installed + // its overlay. Unrelated packets can continue; prefetch's non-draw gate + // still orders future resource reads after the write. + _commandThread!.PostRange(() => RecordCommandProcessorWrite(address, queued, kind), address, (ulong)queued.Length); + } + else + { + RunTagged(() => RecordCommandProcessorWrite(address, queued, kind)); + } return; } @@ -1035,16 +1058,27 @@ private bool TryEncodeCommandProcessorWrite(ulong address, ReadOnlySpan so /// public void RecordCommandProcessorQwords(ulong address, int count, ulong strideBytes, ulong value, string kind) { - if (RunOnWorker(() => RecordCommandProcessorQwords(address, count, strideBytes, value, kind))) + if (count <= 0) return; + if (OnCommandThread) { + // Publish the whole conservative hull before the producer advances. Reads in a + // sparse gap may wait unnecessarily, but no following overlapping read can miss + // the worker installing these writes. Invalid/wrapping runs keep the synchronous + // path and its original store-by-store failure/partial-write behavior. + if (QueueCommandProcessorWrites && address != 0 && address <= ulong.MaxValue - sizeof(ulong) && + (count == 1 || strideBytes <= (ulong.MaxValue - address - sizeof(ulong)) / (ulong)(count - 1))) + { + var size = (ulong)(count - 1) * strideBytes + sizeof(ulong); + _commandThread!.PostRange(() => RecordCommandProcessorQwords(address, count, strideBytes, value, kind), address, size); + } + else + { + RunTagged(() => RecordCommandProcessorQwords(address, count, strideBytes, value, kind)); + } return; } DrainDraws(); - if (count <= 0) - { - return; - } if (count > 1 && strideBytes >= sizeof(ulong) && address != 0 && CommandProcessorWriteBatching.Enabled && !GpuClobberDiagnostics.Enabled && @@ -1314,6 +1348,29 @@ private readonly record struct DeferredLabelCopy( private readonly List _deferredLabelCopies = []; private bool _encodingDeferredLabels; + private static readonly bool DeferredLabelFilterEnabled = + Environment.GetEnvironmentVariable("SHARPEMU_METAL_V2_LABEL_PAGE_FILTER") != "0"; + private readonly MetalPageRangeFilter _deferredGuestPages = new(); + private readonly MetalPageRangeFilter _deferredBufferPages = new(); + private long _labelLookupQueries, _labelHullRejects, _labelPageRejects, _labelExactScans, _labelEntriesInspected; + private int _labelMaxHeldCopies; + + private void NoteLabelScan(int entries) + { + if (!SharpEmu.Libs.VideoOut.RenderPhaseProfile.Enabled) return; + _labelExactScans++; + _labelEntriesInspected += entries; + } + + private string TakeDeferredLabelLookupStats() + { + if (!SharpEmu.Libs.VideoOut.RenderPhaseProfile.Enabled) return "disabled"; + var result = $"queries:{_labelLookupQueries},hull_rejects:{_labelHullRejects},page_rejects:{_labelPageRejects}," + + $"scans:{_labelExactScans},entries:{_labelEntriesInspected},max_held:{_labelMaxHeldCopies}"; + _labelLookupQueries = _labelHullRejects = _labelPageRejects = _labelExactScans = _labelEntriesInspected = 0; + _labelMaxHeldCopies = _deferredLabelCopies.Count; + return result; + } // The span of guest bytes, and of destination bytes when they all go to one buffer // (_deferredLabelBuffer, -1 for several), the held copies land in: every storage binding @@ -1337,12 +1394,24 @@ private void HoldLabelCopies(ReadOnlySpan copies) _deferredLabelDestinationLow = Math.Min(_deferredLabelDestinationLow, copy.DestinationOffset); _deferredLabelDestinationHigh = Math.Max(_deferredLabelDestinationHigh, copy.DestinationOffset + (ulong)copy.Bytes); _deferredLabelCopies.Add(copy); + if (DeferredLabelFilterEnabled) + { + _deferredGuestPages.Add(copy.GuestAddress, (ulong)copy.Bytes); + _deferredBufferPages.Add(copy.DestinationOffset, (ulong)copy.Bytes, unchecked((ulong)copy.DestinationBuffer)); + } + if (SharpEmu.Libs.VideoOut.RenderPhaseProfile.Enabled) + _labelMaxHeldCopies = Math.Max(_labelMaxHeldCopies, _deferredLabelCopies.Count); } } private void ClearHeldLabelCopies() { _deferredLabelCopies.Clear(); + if (DeferredLabelFilterEnabled) + { + _deferredGuestPages.Clear(); + _deferredBufferPages.Clear(); + } _deferredLabelLow = ulong.MaxValue; _deferredLabelHigh = 0; _deferredLabelBuffer = 0; @@ -1526,45 +1595,66 @@ private void EncodeLabelCopies(nint commandBuffer, ReadOnlySpan= _deferredLabelHigh || address + size <= _deferredLabelLow) { + if (SharpEmu.Libs.VideoOut.RenderPhaseProfile.Enabled) _labelHullRejects++; + return false; + } + if (DeferredLabelFilterEnabled && !_deferredGuestPages.MayOverlap(address, size)) + { + if (SharpEmu.Libs.VideoOut.RenderPhaseProfile.Enabled) _labelPageRejects++; return false; } - foreach (ref readonly var copy in - System.Runtime.InteropServices.CollectionsMarshal.AsSpan(_deferredLabelCopies)) + var copies = System.Runtime.InteropServices.CollectionsMarshal.AsSpan(_deferredLabelCopies); + for (var index = 0; index < copies.Length; index++) { + ref readonly var copy = ref copies[index]; if (copy.GuestAddress < address + size && address < copy.GuestAddress + (ulong)copy.Bytes) { + NoteLabelScan(index + 1); return true; } } + NoteLabelScan(copies.Length); return false; } - // The same question for a binding already resolved to host memory: whether a copy waiting - // for the open encoder to end lands in the window bytes the range names. + // Buffer identity is part of the page key: aliases with different guest addresses + // must still land before a binding to the same native buffer bytes. private bool DeferredLabelCopiesTouch(in MetalGuestRange range) { + if (SharpEmu.Libs.VideoOut.RenderPhaseProfile.Enabled) _labelLookupQueries++; if (_deferredLabelBuffer != -1 && (range.Buffer != _deferredLabelBuffer || range.Offset >= _deferredLabelDestinationHigh || range.Offset + range.Size <= _deferredLabelDestinationLow)) { + if (SharpEmu.Libs.VideoOut.RenderPhaseProfile.Enabled) _labelHullRejects++; + return false; + } + if (DeferredLabelFilterEnabled && + !_deferredBufferPages.MayOverlap(range.Offset, range.Size, unchecked((ulong)range.Buffer))) + { + if (SharpEmu.Libs.VideoOut.RenderPhaseProfile.Enabled) _labelPageRejects++; return false; } - foreach (ref readonly var copy in - System.Runtime.InteropServices.CollectionsMarshal.AsSpan(_deferredLabelCopies)) + var copies = System.Runtime.InteropServices.CollectionsMarshal.AsSpan(_deferredLabelCopies); + for (var index = 0; index < copies.Length; index++) { + ref readonly var copy = ref copies[index]; if (copy.DestinationBuffer == range.Buffer && copy.DestinationOffset < range.Offset + range.Size && range.Offset < copy.DestinationOffset + (ulong)copy.Bytes) { + NoteLabelScan(index + 1); return true; } } + NoteLabelScan(copies.Length); return false; } @@ -1805,8 +1895,9 @@ private void PublishClockAtCompletion(in EndOfPipeWrite label) /// public bool DefersEndOfPipeLabels => !PublishLabelsWhenParsed; - // Metal has no occlusion counter: the dumps keep the synthetic always-visible pairs. - public bool PublishesSyntheticOcclusionDumps => true; + // Keep conservative pairs by default. Precise GPU snapshots are opt-in while + // native query ordering and runtime behavior are validated; the audit changes no dumps. + public bool PublishesSyntheticOcclusionDumps => !NativeOcclusionEnabled; // A deferred label is encoded into the command buffer being recorded, before it is // committed, so it has landed once that submission completes. diff --git a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalGraphicsBindingMemo.cs b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalGraphicsBindingMemo.cs new file mode 100644 index 000000000..32c38cf70 --- /dev/null +++ b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalGraphicsBindingMemo.cs @@ -0,0 +1,33 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +namespace SharpEmu.Gpu.Metal.V2; + +internal enum MetalGraphicsBindingStage { Vertex, Mesh, Fragment } + +// Only graphics argument/push slots. The host resets these decisions whenever it forgets +// encoder state, including when a new native encoder reuses an old encoder's pointer. +internal sealed class MetalGraphicsBindingMemo +{ + private struct Entry + { + internal bool Known; + internal nint Selector; + internal nint Buffer; + internal ulong Offset; + } + + private readonly Entry[] _entries = new Entry[6]; + + internal bool Matches(MetalGraphicsBindingStage stage, nint selector, int slot, nint buffer, ulong offset) + { + ref var entry = ref _entries[(int)stage * 2 + slot]; + return entry.Known && entry.Selector == selector && entry.Buffer == buffer && entry.Offset == offset; + } + + // Call only after the corresponding native binding call succeeds. + internal void Record(MetalGraphicsBindingStage stage, nint selector, int slot, nint buffer, ulong offset) => + _entries[(int)stage * 2 + slot] = new Entry { Known = true, Selector = selector, Buffer = buffer, Offset = offset }; + + internal void Reset() => Array.Clear(_entries); +} diff --git a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalGuestMemory.cs b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalGuestMemory.cs index dbcc2a744..4fe3392a9 100644 --- a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalGuestMemory.cs +++ b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalGuestMemory.cs @@ -1577,6 +1577,14 @@ public ulong PairLockTableGpuAddress { get { + // The table lives until quiescent disposal. Publish its address only after + // zeroing and residency registration; subsequent draws need no monitor. + var published = Volatile.Read(ref _pairLockTableGpuAddress); + if (published != 0) + { + return published; + } + lock (_pairLockTableLock) { if (_pairLockTable == 0) @@ -1590,8 +1598,8 @@ public ulong PairLockTableGpuAddress AddResident(buffer); CommitResidency(); - _pairLockTableGpuAddress = gpuAddress; _pairLockTable = buffer; + Volatile.Write(ref _pairLockTableGpuAddress, gpuAddress); } return _pairLockTableGpuAddress; diff --git a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalLibraryCompilePool.cs b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalLibraryCompilePool.cs index 5cccaf8e5..1e0df32ed 100644 --- a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalLibraryCompilePool.cs +++ b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalLibraryCompilePool.cs @@ -31,6 +31,7 @@ internal sealed class Job(Action work) { private const int Queued = 0; private const int Taken = 1; + private const int Cancelled = 2; private readonly TaskCompletionSource _done = new(TaskCreationOptions.RunContinuationsAsynchronously); private int _state; @@ -40,6 +41,16 @@ internal sealed class Job(Action work) // Whether no thread has taken it yet; a thread may take it the moment after. internal bool IsQueued => Volatile.Read(ref _state) == Queued; + // A deferred module that no pipeline requested must not start compiling at + // teardown. Racing a claimant is safe: only one transition leaves Queued. + internal bool CancelIfQueued() + { + if (Interlocked.CompareExchange(ref _state, Cancelled, Queued) != Queued) + return false; + _done.SetCanceled(); + return true; + } + // True when this call ran the work; false when another thread has it (or had it). internal bool TryRun() { diff --git a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalPageRangeFilter.cs b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalPageRangeFilter.cs new file mode 100644 index 000000000..c67e44382 --- /dev/null +++ b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalPageRangeFilter.cs @@ -0,0 +1,67 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +namespace SharpEmu.Gpu.Metal.V2; + +// A bounded negative filter, never an overlap proof. Hash collisions and large ranges +// fall back to the caller's exact range list. The owner serializes Add/MayOverlap/Clear. +internal sealed class MetalPageRangeFilter +{ + private const int PageShift = 12; + private const int MaxPages = 64; + private readonly ulong[] _bits = new ulong[64]; + private bool _unbounded; + + public void Add(ulong address, ulong size, ulong space = 0) + { + if (size == 0 || _unbounded) return; + if (!TryPages(address, size, out var first, out var last)) + { + _unbounded = true; + return; + } + + for (var page = first; page <= last; page++) + { + var bit = Bit(page, space); + _bits[bit >> 6] |= 1UL << (bit & 63); + } + } + + public bool MayOverlap(ulong address, ulong size, ulong space = 0) + { + if (size == 0) return false; + if (_unbounded || !TryPages(address, size, out var first, out var last)) return true; + for (var page = first; page <= last; page++) + { + var bit = Bit(page, space); + if ((_bits[bit >> 6] & (1UL << (bit & 63))) != 0) return true; + } + + return false; + } + + public void Clear() + { + Array.Clear(_bits); + _unbounded = false; + } + + private static bool TryPages(ulong address, ulong size, out ulong first, out ulong last) + { + first = address >> PageShift; + last = 0; + if (size - 1 > ulong.MaxValue - address) return false; + last = (address + size - 1) >> PageShift; + return last - first < MaxPages; + } + + private static int Bit(ulong page, ulong space) + { + var hash = unchecked(page ^ (space * 0x9E3779B97F4A7C15UL)); + hash ^= hash >> 33; + hash = unchecked(hash * 0xFF51AFD7ED558CCDUL); + hash ^= hash >> 33; + return (int)(hash & 4095); + } +} diff --git a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalPendingWriteOverlay.cs b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalPendingWriteOverlay.cs index 19e35ff04..7944ca990 100644 --- a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalPendingWriteOverlay.cs +++ b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalPendingWriteOverlay.cs @@ -65,13 +65,23 @@ private sealed class Entry // of up to MaxPooledBytes are kept, by size, once they have left the page lists and the // order queue, and reused by the next write of that size. Guarded by the gate. private const int MaxPooledBytes = 16; - private const int MaxPooledPerSize = 256; + // A frame can retire thousands of occlusion-dump writes together. A 256-entry pool + // discarded most of that burst, allocating entries and byte arrays again next frame. + // Bound the total across sizes, rather than multiplying the limit by every size bin. + internal const int MaxPooledEntries = 8192; private readonly Stack[] _spareEntries = [new(), new(), new(), new(), new()]; + private int _spareEntryCount; + + internal int SpareEntryCount + { + get { lock (_gate) return _spareEntryCount; } + } private Entry RentEntry(int length) { if (length <= MaxPooledBytes && (length & 3) == 0 && _spareEntries[length >> 2] is { Count: > 0 } spare) { + _spareEntryCount--; return spare.Pop(); } @@ -83,7 +93,7 @@ private void RecycleEntry(Entry entry) { var length = entry.Bytes.Length; if (length == 0 || length > MaxPooledBytes || (length & 3) != 0 || - _spareEntries[length >> 2].Count >= MaxPooledPerSize) + _spareEntryCount >= MaxPooledEntries) { return; } @@ -92,6 +102,7 @@ private void RecycleEntry(Entry entry) entry.Dead = null; entry.Removed = false; _spareEntries[length >> 2].Push(entry); + _spareEntryCount++; } /// Entries not yet dropped. Zero is the fast path every read checks first. diff --git a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalPresentationCache.cs b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalPresentationCache.cs new file mode 100644 index 000000000..21d270439 --- /dev/null +++ b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalPresentationCache.cs @@ -0,0 +1,24 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +namespace SharpEmu.Gpu.Metal.V2; + +// A texture pointer alone is not a content identity: guest and CPU frame buffers +// are reused. Include their sequence and every output transform that changes pixels. +internal readonly record struct MetalPresentationStamp( + long GuestSequence, long HostSequence, long MovieSequence, nint Texture, + double Width, double Height, bool LayerHdr, bool GuestHdr, double Headroom); + +internal sealed class MetalPresentationCache +{ + private MetalPresentationStamp? _presented; + + internal bool NeedsPresent(in MetalPresentationStamp stamp, bool animatedOverlay) => + animatedOverlay || _presented is not { } previous || previous != stamp; + + // Called only after a successful submission, never after a failed drawable + // acquisition. An unsuccessful attempt must retry the same content. + internal void RecordPresented(in MetalPresentationStamp stamp) => _presented = stamp; + + internal void Invalidate() => _presented = null; +} diff --git a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.CommandThread.cs b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.CommandThread.cs index 7bb2024e0..f3bff9488 100644 --- a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.CommandThread.cs +++ b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.CommandThread.cs @@ -25,6 +25,9 @@ internal sealed partial class MetalRenderHost internal static readonly bool CommandThreadEnabled = Environment.GetEnvironmentVariable("SHARPEMU_METAL_V2_COMMAND_THREAD") == "1"; + private static readonly bool CommandThreadQosEnabled = + Environment.GetEnvironmentVariable("SHARPEMU_METAL_V2_COMMAND_THREAD_QOS") == "1"; + // Items run in batches; completions and relay commands are taken between them. private const int CommandItemBatch = 256; @@ -48,13 +51,16 @@ private void RunCommandThreadBatch() { if (_commandThread is null) { - _commandThread = new DeferredCommandThread(CommandStream, _memory, RunDeferredWork, _translation!.Prefetch, WakeWorker) + _commandThread = new DeferredCommandThread(CommandStream, _memory, RunDeferredWork, _translation!.Prefetch, WakeWorker, + useHostQos: CommandThreadQosEnabled) { ProducerIdle = RequestIdleCommit, }; _betweenCommandItems = RunRelayBetweenCommandItems; _translation.Sink = _commandThread; _commandThread.Start(); + if (CommandThreadQosEnabled) + Console.Error.WriteLine($"[METAL-V2][QOS] command_stream requested=UserInteractive applied={_commandThread.HasHostQos}."); } var deadline = System.Diagnostics.Stopwatch.GetTimestamp() + System.Diagnostics.Stopwatch.Frequency / 100; diff --git a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.Descriptors.cs b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.Descriptors.cs index 689e2d694..345a94470 100644 --- a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.Descriptors.cs +++ b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.Descriptors.cs @@ -45,6 +45,7 @@ internal sealed class MetalStageBindings : IPreparedBindings public ulong[]?[] MipTextureIds = []; public nint[] Samplers = []; public ulong[] SamplerIds = []; + public byte[] SamplerViewClasses = []; public ResourceSlotIdentifier[] Images = []; public ImageRequest[] Requests = []; public bool[] IsStorage = []; @@ -93,6 +94,7 @@ public void EnsureCapacity(int images, int samplers, int buffers, int shaderDwor { Samplers = new nint[samplers]; SamplerIds = new ulong[samplers]; + SamplerViewClasses = new byte[samplers]; } if (Buffers.Length < buffers) @@ -408,10 +410,15 @@ public IPreparedBindings PrepareBindings(ShaderStageResources stage) ResolveImageBinding(prepared, info.Images[index], snapshot.Images[index], index); } + // All image requests, including movie replacements and texture memo hits, are current. + // Classify their sampler pairings once rather than scanning every pair for every sampler. + MetalSamplerViewClasses.Resolve(info.SampledPairs, prepared.Requests.AsSpan(0, prepared.ImageCount), + prepared.SamplerViewClasses.AsSpan(0, info.Samplers.Count)); for (var index = 0; index < info.Samplers.Count; index++) { prepared.Samplers[index] = ResolveSampler( - info, prepared, info.Samplers[index], snapshot.Samplers[index], index, out prepared.SamplerIds[index]); + prepared, info.Samplers[index], snapshot.Samplers[index], index, + prepared.SamplerViewClasses[index] == MetalSamplerViewClasses.AllInteger, out prepared.SamplerIds[index]); } var shaderData = prepared.ShaderData; @@ -489,11 +496,11 @@ private void ResolveImageBinding(MetalStageBindings prepared, ImageResource imag } private nint ResolveSampler( - ShaderResourceInfo info, MetalStageBindings prepared, SamplerResource sampler, uint[] words, int index, + bool integerViews, out ulong resourceId) { if (words.Length < 4) @@ -520,30 +527,7 @@ private nint ResolveSampler( } var descriptor = new SamplerDescriptorWords(native); - return GetSampler(in descriptor, SamplesIntegerViews(info, prepared, index), out resourceId); - } - - // A sampler takes the numeric class of the views it samples; integer only when - // every paired view is integer, since a float view needs filtering. - private static bool SamplesIntegerViews(ShaderResourceInfo info, MetalStageBindings prepared, int sampler) - { - var paired = false; - foreach (var pair in info.SampledPairs) - { - if (pair.Sampler != sampler || pair.Image >= prepared.ImageCount) - { - continue; - } - - if (!ViewFormatRules.IsIntegerFormat(prepared.Requests[pair.Image].View.Format)) - { - return false; - } - - paired = true; - } - - return paired; + return GetSampler(in descriptor, integerViews, out resourceId); } // The binding layout gives a dynamic-mip storage image one element per level of its @@ -1939,23 +1923,37 @@ private void BindStageBuffers(nint encoder, int stageIndex, nint setBufferSelect if (_stageArgumentValid[stageIndex]) { - MetalNative.SendSetBuffer( - encoder, - setBufferSelector, - _stageArgumentBuffers[stageIndex], - (nuint)_stageArgumentOffsets[stageIndex], - Gen5MslArgumentLayout.ResourcesBufferIndex); + BindStageBuffer(encoder, setBufferSelector, _stageArgumentBuffers[stageIndex], + _stageArgumentOffsets[stageIndex], Gen5MslArgumentLayout.ResourcesBufferIndex); } if (_hasPushData && (_preparedStageMask & (1 << stageIndex)) != 0) { - MetalNative.SendSetBuffer( - encoder, - setBufferSelector, - _pushBuffer, - (nuint)_pushOffset, - Gen5MslArgumentLayout.PushDataBufferIndex); + BindStageBuffer(encoder, setBufferSelector, _pushBuffer, + _pushOffset, Gen5MslArgumentLayout.PushDataBufferIndex); + } + } + + private void BindStageBuffer(nint encoder, nint selector, nint buffer, ulong offset, int slot) + { + // Compute utility kernels overwrite their slots independently of this path. + // Keep their binding calls unchanged, and distinguish physical vertex/mesh state. + var stage = selector == SelSetVertexBuffer ? MetalGraphicsBindingStage.Vertex + : selector == SelSetMeshBuffer ? MetalGraphicsBindingStage.Mesh + : MetalGraphicsBindingStage.Fragment; + if (selector != SelSetVertexBuffer && selector != SelSetMeshBuffer && selector != SelSetFragmentBuffer) + { + MetalNative.SendSetBuffer(encoder, selector, buffer, (nuint)offset, (nuint)slot); + return; + } + _graphicsBindingAttempts[(int)stage]++; + if (GraphicsBindingReuseEnabled && _graphicsBindings.Matches(stage, selector, slot, buffer, offset)) + { + _graphicsBindingSkipped[(int)stage]++; + return; } + MetalNative.SendSetBuffer(encoder, selector, buffer, (nuint)offset, (nuint)slot); + if (GraphicsBindingReuseEnabled) _graphicsBindings.Record(stage, selector, slot, buffer, offset); } // One MTLSamplerState per distinct guest descriptor and view numeric class. diff --git a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.Occlusion.cs b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.Occlusion.cs new file mode 100644 index 000000000..434ec02a7 --- /dev/null +++ b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.Occlusion.cs @@ -0,0 +1,310 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using SharpEmu.Libs.Gpu.GpuCommands; + +namespace SharpEmu.Gpu.Metal.V2; + +// Experimental precise queries. A dump closes a visibility interval, not the render +// encoder. At the encoder boundary a GPU prefix sum preserves each dump's snapshot; +// the existing ordered-copy stream publishes them alongside guest WRITE_DATA stores. +internal sealed partial class MetalRenderHost +{ + internal bool NativeOcclusionEnabled { get; set; } = + Environment.GetEnvironmentVariable("SHARPEMU_METAL_V2_NATIVE_OCCLUSION") == "1"; + + private const int OcclusionIntervalCapacity = 4096; + internal int NativeOcclusionBatchBudget { get; set; } = 256; + private const ulong OcclusionReady = 1UL << 63; + private static readonly nint SelOcclusionNewBuffer = MetalNative.Selector("newBufferWithLength:options:"); + private static readonly nint SelOcclusionContents = MetalNative.Selector("contents"); + private static readonly nint SelOcclusionVisibility = MetalNative.Selector("setVisibilityResultBuffer:"); + private static readonly nint SelOcclusionMode = MetalNative.Selector("setVisibilityResultMode:offset:"); + private static readonly nint SelOcclusionPipeline = MetalNative.Selector("setComputePipelineState:"); + private static readonly nint SelOcclusionBuffer = MetalNative.Selector("setBuffer:offset:atIndex:"); + private static readonly nint SelOcclusionBytes = MetalNative.Selector("setBytes:length:atIndex:"); + private static readonly nint SelOcclusionDispatch = MetalNative.Selector("dispatchThreadgroups:threadsPerThreadgroup:"); + + private sealed class OcclusionBatch(nint counts, nint results) + { + internal readonly nint Counts = counts, Results = results; + internal ulong Serial; + internal int Slot, Queue; + internal int VisibilityMode = -1, VisibilitySlot = -1; + internal bool CountedDraw, HasSnapshot; + } + + private readonly List _oqBatches = []; + private OcclusionBatch? _oqOpen; + private int _oqQueue; + private nint _oqBank, _oqPipeline; + private long _oqDumps, _oqReductions, _oqBudgetWaits, _oqUnusedPasses, _oqInitializedSlots; + internal (long Dumps, long Reductions, int Buffers, long BudgetWaits, long UnusedPasses, long InitializedSlots) NativeOcclusionTotals => + (_oqDumps, _oqReductions, _oqBatches.Count, _oqBudgetWaits, _oqUnusedPasses, _oqInitializedSlots); + + private const string OcclusionPrefixSource = """ + #include + using namespace metal; + kernel void sharpemu_occlusion_prefix(device const ulong* counts [[buffer(0)]], + device ulong* snapshots [[buffer(1)]], device ulong* carry [[buffer(2)]], + constant uint& length [[buffer(3)]]) { + const ulong ready = ulong(1) << 63; + ulong sum = carry[0] & ~ready; + for (uint i = 0; i < length; i++) { + sum = (sum + counts[i]) & ~ready; + snapshots[i] = ready | sum; + } + carry[0] = ready | sum; + } + """; + + private unsafe void InitializeNativeOcclusion() + { + if (_oqBank != 0) return; + if (OcclusionAuditEnabled) throw Fatal("Native occlusion and the visibility audit must be tested separately."); + _oqBank = NewOcclusionBuffer((CommandStreamQueue.QueueCount + 1) * sizeof(ulong)); + new Span((void*)MetalNative.Send(_oqBank, SelOcclusionContents), CommandStreamQueue.QueueCount + 1).Fill(OcclusionReady); + var options = MetalNative.Send(MetalNative.Send(MetalNative.Class("MTLCompileOptions"), MetalNative.Selector("alloc")), MetalNative.Selector("init")); + MetalNative.ApplyMslCompileOptions(options); + nint error = 0; + var library = MetalNative.Send(_device, MetalNative.Selector("newLibraryWithSource:options:error:"), + MetalNative.NsString(OcclusionPrefixSource), options, ref error); + MetalNative.SendVoid(options, SelRelease); + if (library == 0) throw Fatal($"The occlusion prefix kernel did not compile: {MetalNative.DescribeError(error)}."); + var function = MetalNative.Send(library, MetalNative.Selector("newFunctionWithName:"), MetalNative.NsString("sharpemu_occlusion_prefix")); + MetalNative.SendVoid(library, SelRelease); + if (function == 0) throw Fatal("The occlusion prefix function is missing."); + _oqPipeline = MetalNative.Send(_device, MetalNative.Selector("newComputePipelineStateWithFunction:error:"), function, ref error); + MetalNative.SendVoid(function, SelRelease); + if (_oqPipeline == 0) throw Fatal($"The occlusion prefix pipeline did not build: {MetalNative.DescribeError(error)}."); + } + + private nint NewOcclusionBuffer(int bytes) + { + var buffer = MetalNative.SendNewBuffer(_device, SelOcclusionNewBuffer, (nuint)bytes, Mtl.ResourceStorageModeShared); + if (buffer == 0) throw Fatal($"Could not allocate {bytes} bytes for native occlusion."); + return buffer; + } + + private void ValidateOcclusionQueue(int queue) + { + if ((uint)queue >= CommandStreamQueue.QueueCount) throw Fatal($"Invalid occlusion queue {queue}."); + } + + private void NativeOcclusionBeginQueue(int queue) + { + if (NativeOcclusionEnabled) ValidateOcclusionQueue(queue); + if (!NativeOcclusionEnabled || _oqQueue == queue) return; + EndRendering(); + _oqQueue = queue; + } + + // Called between passes, before choosing the next command buffer or encoding its + // indirect arguments. Never recycle an unfinished batch or grow without a bound. + private void NativeOcclusionMakeRoom() + { + if (!NativeOcclusionEnabled || _oqBatches.Count < NativeOcclusionBatchBudget) return; + var completed = CompletedSerial; + var oldest = ulong.MaxValue; + foreach (var batch in _oqBatches) + { + if (batch.Serial <= completed) return; + oldest = Math.Min(oldest, batch.Serial); + } + if (oldest == ulong.MaxValue) throw Fatal("The native occlusion batch budget must be positive."); + if (oldest <= _previousSerial) + WaitForCommittedSerial(oldest, MetalWaitReason.Other); + else + FlushAndWaitForGpu(); + _oqBudgetWaits++; + } + + private unsafe void NativeOcclusionAttach(nint descriptor) + { + if (!NativeOcclusionEnabled) return; + InitializeNativeOcclusion(); + var completed = CompletedSerial; + OcclusionBatch? batch = null; + foreach (var candidate in _oqBatches) + { + if (candidate.Serial > completed) continue; + batch = candidate; + break; + } + if (batch is null) + { + // MakeRoom must run before selecting this pass's command buffer. + if (_oqBatches.Count >= NativeOcclusionBatchBudget) throw Fatal("Native occlusion did not retire a batch before opening a pass."); + var counts = NewOcclusionBuffer(OcclusionIntervalCapacity * sizeof(ulong)); + nint results; + try { results = NewOcclusionBuffer(OcclusionIntervalCapacity * sizeof(ulong)); } + catch { MetalNative.SendVoid(counts, SelRelease); throw; } + batch = new OcclusionBatch(counts, results); + _oqBatches.Add(batch); + } + batch.Serial = SubmissionSerial; + batch.Queue = _oqQueue; + batch.Slot = 0; + batch.VisibilityMode = batch.VisibilitySlot = -1; + batch.CountedDraw = batch.HasSnapshot = false; + // Expose only initialized slots. This batch's previous GPU use has retired, + // and the command buffer for the new pass has not been committed yet. + InitializeOcclusionSlot(batch); + _oqOpen = batch; + MetalNative.SendVoid(descriptor, SelOcclusionVisibility, batch.Counts); + } + + private unsafe void InitializeOcclusionSlot(OcclusionBatch batch) + { + ((ulong*)MetalNative.Send(batch.Counts, SelOcclusionContents))[batch.Slot] = 0; + _oqInitializedSlots++; + } + + private void NativeOcclusionPrepareDraw() + { + if (!NativeOcclusionEnabled || _oqOpen is not { } batch) return; + var enabled = (_dynamicState.DepthCountControl & (1u | (1u << 8))) == (1u << 8); + batch.CountedDraw |= enabled; + var mode = enabled ? 2 : 0; + if (batch.VisibilityMode == mode && batch.VisibilitySlot == batch.Slot) return; + MetalV2Native.SendVisibilityResultMode(_renderEncoder, SelOcclusionMode, + (uint)mode, (nuint)(batch.Slot * sizeof(ulong))); + batch.VisibilityMode = mode; + batch.VisibilitySlot = batch.Slot; + } + + // Called after the render encoder ended, before any deferred label copies land. + private unsafe void NativeOcclusionFinish() + { + if (_oqOpen is not { } batch) return; + _oqOpen = null; + // No counter changed and no dump references this batch's result storage. + // Keep its conservative serial retirement, but emit no utility encoder. + if (!batch.CountedDraw && !batch.HasSnapshot) + { + _oqUnusedPasses++; + return; + } + var length = (uint)Math.Min(batch.Slot + 1, OcclusionIntervalCapacity); + var encoder = RetainEncoder(MetalNative.Send(EnsureCommandBuffer(), SelComputeEncoder)); + if (encoder == 0) throw Fatal("Could not encode the occlusion prefix sum."); + WaitForPreviousEncoder(encoder, render: false); + MetalNative.SendVoid(encoder, SelOcclusionPipeline, _oqPipeline); + MetalNative.SendSetBuffer(encoder, SelOcclusionBuffer, batch.Counts, 0, 0); + MetalNative.SendSetBuffer(encoder, SelOcclusionBuffer, batch.Results, 0, 1); + MetalNative.SendSetBuffer(encoder, SelOcclusionBuffer, _oqBank, (nuint)(batch.Queue * sizeof(ulong)), 2); + MetalNative.SendSetBytes(encoder, SelOcclusionBytes, (nint)(&length), sizeof(uint), 3); + MetalNative.SendDispatch(encoder, SelOcclusionDispatch, + new MtlSize { Width = 1, Height = 1, Depth = 1 }, new MtlSize { Width = 1, Height = 1, Depth = 1 }); + EndAndReleaseEncoder(encoder, render: false); + _encodersInBuffer++; + _oqReductions++; + NoteRecordedWork(); + } + + public bool DumpOcclusionCounter(int queueId, ulong address, uint instanceMask, uint strideBytes) + { + if (!NativeOcclusionEnabled) return false; + if (OnCommandThread) + { + // No CPU result is needed here. Publish the pending guest range before + // returning so later producer reads cannot pass the queued GPU dump. + var mask = instanceMask & 0xFF_FFFFu; + var size = mask == 0 ? 0 : checked((ulong)System.Numerics.BitOperations.Log2(mask) * strideBytes + sizeof(ulong)); + _commandThread!.PostRange(() => DumpOcclusionCounter(queueId, address, instanceMask, strideBytes), address, size); + return true; + } + DrainDraws(); + ValidateOcclusionQueue(queueId); + InitializeNativeOcclusion(); + NativeOcclusionBeginQueue(queueId); + if (instanceMask == 0) return true; + Span copies = stackalloc DeferredLabelCopy[24]; + var count = 0; + for (var block = 0; block < 24; block++) + { + if ((instanceMask & (1u << block)) == 0) continue; + var destinationAddress = checked(address + (ulong)block * strideBytes); + var destination = _guestMemory.ResolveWritable(destinationAddress, sizeof(ulong)); + if (!destination.IsValid) throw Fatal($"Native occlusion has no writable result window at 0x{destinationAddress:X16}."); + copies[count++] = new DeferredLabelCopy(0, 0, destination.Buffer, destination.Offset, sizeof(ulong), destinationAddress); + } + var batch = _oqOpen; + var source = batch?.Results ?? _oqBank; + var offset = batch is null ? (ulong)queueId * sizeof(ulong) : (ulong)batch.Slot * sizeof(ulong); + var index = 0; + for (var block = 0; block < 24; block++) + { + if ((instanceMask & (1u << block)) == 0) continue; + copies[index] = copies[index] with + { + StagingBuffer = block == 0 ? source : _oqBank, + StagingOffset = block == 0 ? offset : (ulong)CommandStreamQueue.QueueCount * sizeof(ulong), + }; + _cpOverlay.Forget(copies[index].GuestAddress, sizeof(ulong)); + _labelOverlay.Forget(copies[index].GuestAddress, sizeof(ulong)); + index++; + } + if (batch is not null) + { + batch.HasSnapshot |= (instanceMask & 1) != 0; + batch.Slot++; + // Consecutive empty dumps still expose a zero interval. At capacity + // the pass ends below; there is no next slot to initialize. + if (batch.Slot < OcclusionIntervalCapacity) InitializeOcclusionSlot(batch); + } + _ = EncodeOrderedCopies(copies[..count], address, null, 0, MetalGpuWriteOrigin.Copy, + "occlusion dump", null, (ulong)count * sizeof(ulong), sourceIsFinal: false, deferStableGpuResult: batch is not null); + if (batch is { Slot: OcclusionIntervalCapacity }) EndRendering(); + _oqDumps++; + return true; + } + + public bool ResetOcclusionCounter(int queueId) + { + if (!NativeOcclusionEnabled) return false; + if (PostToWorker(() => ResetOcclusionCounter(queueId))) return true; + DrainDraws(); + ValidateOcclusionQueue(queueId); + InitializeNativeOcclusion(); + EndEncoders(PassEndCause.Other); + var copy = new DeferredLabelCopy(_oqBank, (ulong)CommandStreamQueue.QueueCount * sizeof(ulong), + _oqBank, (ulong)queueId * sizeof(ulong), sizeof(ulong), 0); + EncodeLabelCopies(EnsureCommandBuffer(), new ReadOnlySpan(in copy)); + NoteRecordedWork(); + return true; + } + + public unsafe bool TryReadOcclusionCounter(int queueId, out ulong value) + { + value = 0; + if (!NativeOcclusionEnabled) return false; + if (OnCommandThread) + { + ulong result = 0; + RunTagged(() => TryReadOcclusionCounter(queueId, out result)); + value = result; + return true; + } + DrainDraws(); + ValidateOcclusionQueue(queueId); + InitializeNativeOcclusion(); + FlushAndWaitForGpu(); + value = ((ulong*)MetalNative.Send(_oqBank, SelOcclusionContents))[queueId] & ~OcclusionReady; + return true; + } + + private void DisposeNativeOcclusion() + { + foreach (var batch in _oqBatches) + { + MetalNative.SendVoid(batch.Counts, SelRelease); + MetalNative.SendVoid(batch.Results, SelRelease); + } + _oqBatches.Clear(); + if (_oqBank != 0) MetalNative.SendVoid(_oqBank, SelRelease); + if (_oqPipeline != 0) MetalNative.SendVoid(_oqPipeline, SelRelease); + _oqBank = _oqPipeline = 0; + _oqOpen = null; + } +} diff --git a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.OcclusionAudit.cs b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.OcclusionAudit.cs new file mode 100644 index 000000000..7500ac108 --- /dev/null +++ b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.OcclusionAudit.cs @@ -0,0 +1,140 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +namespace SharpEmu.Gpu.Metal.V2; + +// Diagnostic only: guest query dumps remain conservative. Each enabled draw group +// gets a unique visibility slot in its encoder. Never recycle attached storage until +// its command buffer has completed, including empty encoders and failed submissions. +internal sealed partial class MetalRenderHost +{ + internal bool OcclusionAuditEnabled { get; set; } = + Environment.GetEnvironmentVariable("SHARPEMU_METAL_V2_AUDIT_OCCLUSION") == "1"; + + private const int OcclusionAuditSlots = 2048; + private const int OcclusionAuditBuffers = 64; + private static readonly nint SelAuditVisibilityBuffer = MetalNative.Selector("setVisibilityResultBuffer:"); + private static readonly nint SelAuditVisibilityMode = MetalNative.Selector("setVisibilityResultMode:offset:"); + private static readonly nint SelAuditNewBuffer = MetalNative.Selector("newBufferWithLength:options:"); + private static readonly nint SelAuditContents = MetalNative.Selector("contents"); + + private sealed class OcclusionAuditBuffer(nint buffer) + { + internal readonly nint Buffer = buffer; + internal readonly bool[] Drawn = new bool[OcclusionAuditSlots]; + internal int Used; + } + + private readonly List _oaAllocated = []; + private readonly Stack _oaFree = []; + private readonly Dictionary> _oaInFlight = []; + private List? _oaCurrent; + private OcclusionAuditBuffer? _oaOpen; + private int _oaSlot = -1; + private long _oaGroups, _oaZeroGroups, _oaEnabledDraws, _oaSkipped, _oaFailedGroups; + private ulong _oaSamples; + private long _oaLastReport; + + internal (long Groups, long ZeroGroups, ulong Samples, long EnabledDraws, long Skipped) OcclusionAuditTotals => + (_oaGroups, _oaZeroGroups, _oaSamples, _oaEnabledDraws, _oaSkipped); + + private void OcclusionAuditAttach(nint descriptor) + { + _oaOpen = null; + _oaSlot = -1; + if (!OcclusionAuditEnabled) return; + if (!_oaFree.TryPop(out var entry)) + { + if (_oaAllocated.Count == OcclusionAuditBuffers) return; + var buffer = MetalNative.SendNewBuffer(_device, SelAuditNewBuffer, + OcclusionAuditSlots * 8, Mtl.ResourceStorageModeShared); + if (buffer == 0) return; + entry = new OcclusionAuditBuffer(buffer); + _oaAllocated.Add(entry); + } + + entry.Used = 0; + Array.Clear(entry.Drawn); + _oaOpen = entry; + (_oaCurrent ??= []).Add(entry); + MetalNative.SendVoid(descriptor, SelAuditVisibilityBuffer, entry.Buffer); + } + + private void OcclusionAuditPrepareDraw() + { + if (!OcclusionAuditEnabled) return; + _oaSlot = -1; + var enabled = (_dynamicState.DepthCountControl & (1u | (1u << 8))) == (1u << 8); + if (enabled) + { + _oaEnabledDraws++; + if (_oaOpen is not null && _oaOpen.Used < OcclusionAuditSlots) + _oaSlot = _oaOpen.Used++; + else + _oaSkipped++; + } + + MetalV2Native.SendVisibilityResultMode(_renderEncoder, SelAuditVisibilityMode, + _oaSlot >= 0 ? 2u : 0u, _oaSlot >= 0 ? (nuint)(_oaSlot * 8) : 0); + } + + private void OcclusionAuditCountDraw() + { + if (_oaSlot >= 0 && _oaOpen is not null) _oaOpen.Drawn[_oaSlot] = true; + } + + private void OcclusionAuditClose() + { + _oaOpen = null; + _oaSlot = -1; + } + + private void OcclusionAuditCommit(ulong serial) + { + if (_oaCurrent is null) return; + _oaInFlight.Add(serial, _oaCurrent); + _oaCurrent = null; + } + + private unsafe void OcclusionAuditResolve(ulong serial, bool completed) + { + if (!_oaInFlight.Remove(serial, out var buffers)) return; + foreach (var entry in buffers) + { + if (!completed) + { + // Quarantine until host disposal: this includes dropped telemetry records, + // whose GPU writes might still be in flight. The global budget stays bounded. + _oaFailedGroups += entry.Used; + continue; + } + + var values = (ulong*)MetalNative.Send(entry.Buffer, SelAuditContents); + for (var i = 0; i < entry.Used; i++) + { + if (!entry.Drawn[i]) continue; + _oaGroups++; + if (values[i] == 0) _oaZeroGroups++; + _oaSamples += values[i]; + } + _oaFree.Push(entry); + } + + var now = System.Diagnostics.Stopwatch.GetTimestamp(); + if (now - _oaLastReport < 2 * System.Diagnostics.Stopwatch.Frequency) return; + _oaLastReport = now; + Console.Error.WriteLine($"[METAL-V2][OCCLUSION-AUDIT] serial={serial} groups={_oaGroups} " + + $"zero_groups={_oaZeroGroups} samples={_oaSamples} enabled_draws={_oaEnabledDraws} " + + $"skipped={_oaSkipped} failed_groups={_oaFailedGroups} buffers={_oaAllocated.Count}"); + } + + private void DisposeOcclusionAudit() + { + foreach (var entry in _oaAllocated) MetalNative.SendVoid(entry.Buffer, SelRelease); + _oaAllocated.Clear(); + _oaFree.Clear(); + _oaInFlight.Clear(); + _oaCurrent = null; + OcclusionAuditClose(); + } +} diff --git a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.Pipelines.cs b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.Pipelines.cs index eb2e77ca8..4ec8511c5 100644 --- a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.Pipelines.cs +++ b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.Pipelines.cs @@ -105,6 +105,7 @@ private sealed class MetalShaderModule // done, which RequireModule waits for, or runs itself while no pool thread has it. public Task? Compiling; public MetalLibraryCompilePool.Job? Job; + public bool Deferred; // What the compile spent waiting for the prewarm's compile of the same source, and // whether prewarm compiles were in the driver when it started: the split of the @@ -151,6 +152,11 @@ private sealed class MetalShaderModule /// Runs where a module's library compile starts, on whichever thread runs it; tests hold it. internal Action? BeforeLibraryCompile { get; set; } + // Experimental: retain an untaken job until a pipeline or its preparation + // requests the module. Source/module creation alone need not invoke Metal. + internal bool DeferLibraryCompiles { get; set; } = + Environment.GetEnvironmentVariable("SHARPEMU_METAL_V2_DEFER_LIBRARIES") == "1"; + public ulong CreateShaderModule(IGuestCompiledShader shader, ShaderStage stage, ulong hash, ulong programId) { _ = programId; @@ -169,6 +175,7 @@ public ulong CreateShaderModule(IGuestCompiledShader shader, ShaderStage stage, Compiled = metal, Hash = hash, Stage = stage, + Deferred = DeferLibraryCompiles, LastUsedTicks = System.Diagnostics.Stopwatch.GetTimestamp(), }; @@ -179,6 +186,8 @@ public ulong CreateShaderModule(IGuestCompiledShader shader, ShaderStage stage, // worker, with the translation of the second in between. void build() { + if (module.Deferred) + MetalShaderPrewarm.GameCompileStarted(); try { BeforeLibraryCompile?.Invoke(); @@ -200,15 +209,21 @@ void build() } } - // From here until the module is built the background prewarm holds back. - MetalShaderPrewarm.GameCompileStarted(); - if (MetalLibraryCompilePool.Enabled) + // An unused deferred module must not hold back prewarm indefinitely. + if (module.Deferred) + { + module.Job = new MetalLibraryCompilePool.Job(build); + module.Compiling = module.Job.Task; + } + else if (MetalLibraryCompilePool.Enabled) { + MetalShaderPrewarm.GameCompileStarted(); module.Job = MetalLibraryCompilePool.Shared.Run(build); module.Compiling = module.Job.Task; } else { + MetalShaderPrewarm.GameCompileStarted(); build(); } @@ -2066,6 +2081,8 @@ private void DisposePipelines() { try { + if (module.Deferred) + module.Job?.CancelIfQueued(); module.Compiling?.Wait(); } catch (AggregateException) diff --git a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.Profile.cs b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.Profile.cs index d5719fae3..033f9a7a2 100644 --- a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.Profile.cs +++ b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.Profile.cs @@ -16,6 +16,13 @@ namespace SharpEmu.Gpu.Metal.V2; // profiling mode pays for it; nothing here runs when the switch is off. internal sealed partial class MetalRenderHost { + // Diagnostic only: stage-boundary counters cannot time individual dispatches + // inside one compute encoder. Split after each dispatch, without committing or + // waiting, to identify its cost. This changes scheduling and is not a frame-time + // benchmark; normal pass profiling leaves encoder boundaries untouched. + private static readonly bool SplitComputeProfile = + Environment.GetEnvironmentVariable("SHARPEMU_METAL_V2_PROFILE_SPLIT_COMPUTE") == "1"; + private static readonly nint SelCommandBufferStatus = MetalNative.Selector("status"); // The pipeline entry's profile label, resolved once per pipeline so a draw only @@ -48,9 +55,10 @@ private void ProfileWork(MetalPipelineEntry entry, string kind, WorkShape shape) // what names the shaders in a slow command buffer's SLOW-WAIT line. _lastWorkKind = kind; _lastWorkLabel = entry.Profile; + var split = PassProfileEnabled && SplitComputeProfile && kind.StartsWith("dispatch", StringComparison.Ordinal); if (PassProfileEnabled) { - PassProfileNoteWork(entry.Profile, kind); + PassProfileNoteWork(split ? $"{entry.Profile} {shape}" : entry.Profile, kind); } if (DumpTargetsEnabled && kind.StartsWith("draw", StringComparison.Ordinal)) { @@ -61,6 +69,12 @@ private void ProfileWork(MetalPipelineEntry entry, string kind, WorkShape shape) NoteDispatchState(entry, kind, shape); } + if (split) + { + PassProfileNoteEnd("profile-dispatch"); + EndComputeEncoder(); + } + ProfileCommandBuffer(kind, entry.Profile ?? "stage=?", shape); } diff --git a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.Rendering.cs b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.Rendering.cs index b462051b5..443d4fed7 100644 --- a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.Rendering.cs +++ b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.Rendering.cs @@ -424,6 +424,7 @@ private void NoteAttachmentTraffic(in RenderingState state) private void ForgetEncoderState() { + _graphicsBindings.Reset(); _boundViewportKnown = false; _boundGraphicsPipeline = null; _boundRenderState = 0; @@ -998,6 +999,7 @@ public void BeginRendering(in RenderingState state) EndRenderingBefore(in state); EndComputeEncoder(); + NativeOcclusionMakeRoom(); EncodeStagedStores(); EncodeDeferredLabelCopies(); MaterializePendingClears(); @@ -1134,6 +1136,8 @@ public void BeginRendering(in RenderingState state) PassProfileAttachRender(descriptor); } + NativeOcclusionAttach(descriptor); + OcclusionAuditAttach(descriptor); _renderEncoder = RetainEncoder(MetalNative.Send(commandBuffer, SelRenderEncoder, descriptor)); MetalMemoryTelemetry.RenderEncoders.Add(); LabelEncoder(_renderEncoder, $"render {state.Width}x{state.Height}"); @@ -1267,8 +1271,10 @@ private void FinishRendering(in RenderingState next, bool hasNext) ResolveStoreActions(in next, hasNext); } + OcclusionAuditClose(); EndAndReleaseEncoder(_renderEncoder, render: true); _renderEncoder = 0; + NativeOcclusionFinish(); PassProfileClose(); } @@ -1489,6 +1495,8 @@ private void ApplyGraphicsState() _boundDepthStencilState = depthStencilState; } + NativeOcclusionPrepareDraw(); + OcclusionAuditPrepareDraw(); ApplyViewportAndScissor(); if (_boundBlendRed != _dynamicState.BlendRed || _boundBlendGreen != _dynamicState.BlendGreen || _boundBlendBlue != _dynamicState.BlendBlue || _boundBlendAlpha != _dynamicState.BlendAlpha) @@ -2312,6 +2320,7 @@ private bool TryGetMeshlets(Gen5MslMeshInfo mesh, uint indexCount, uint firstInd private void CountDraw(long startedTicks) { NoteRecordedWork(); + OcclusionAuditCountDraw(); Interlocked.Increment(ref _drawCount); NotePassDraw(_boundGraphicsPipeline?.Profile); Interlocked.Add(ref _drawTicks, System.Diagnostics.Stopwatch.GetTimestamp() - startedTicks); diff --git a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.StagedStorage.cs b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.StagedStorage.cs index ec4836661..52490bdb7 100644 --- a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.StagedStorage.cs +++ b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.StagedStorage.cs @@ -22,15 +22,18 @@ internal sealed partial class MetalRenderHost /// internal MetalGuestRange ResolveStorageBuffer(ulong address, ulong size, bool written) { - using var phaseScope = SharpEmu.Libs.VideoOut.RenderPhaseProfile.MeasureDetail(SharpEmu.Libs.VideoOut.RenderPhaseProfile.Phase.StorageBufferResolve); + using var phaseScope = SharpEmu.Libs.VideoOut.RenderPhaseProfile.MeasureDetail(SharpEmu.Libs.VideoOut.RenderPhaseProfile.Phase.StorageDeferredLabels); OrderAfterDeferredCopies(address, size); + phaseScope.SwitchPhase(SharpEmu.Libs.VideoOut.RenderPhaseProfile.Phase.StorageSharedLookup); var staged = StagedStorage; if (staged.TryShare(address, size, written, out var shared)) { return shared; } + phaseScope.SwitchPhase(SharpEmu.Libs.VideoOut.RenderPhaseProfile.Phase.StorageStoreOrder); OrderAfterStagedStores(address, size); + phaseScope.SwitchPhase(SharpEmu.Libs.VideoOut.RenderPhaseProfile.Phase.StorageBufferResolve); if (_guestMemory.TryResolveInPlace(address, size, out var window)) { return window; diff --git a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.cs b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.cs index a155ba749..bec20e0e3 100644 --- a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.cs +++ b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.cs @@ -32,6 +32,34 @@ internal sealed partial class MetalRenderHost : { private const uint MaxFramebufferDimension = 16384; + private readonly MetalGraphicsBindingMemo _graphicsBindings = new(); + private readonly long[] _graphicsBindingAttempts = new long[3]; + private readonly long[] _graphicsBindingSkipped = new long[3]; + private bool _graphicsBindingReuseEnabled = + Environment.GetEnvironmentVariable("SHARPEMU_METAL_V2_GRAPHICS_BINDING_REUSE") == "1"; + + internal bool GraphicsBindingReuseEnabled + { + get => _graphicsBindingReuseEnabled; + set + { + if (_graphicsBindingReuseEnabled == value) return; + _graphicsBindingReuseEnabled = value; + _graphicsBindings.Reset(); + } + } + + internal long GraphicsBindingAttempts => _graphicsBindingAttempts.Sum(); + internal long GraphicsBindingSkipped => _graphicsBindingSkipped.Sum(); + + internal (long Attempted, long Skipped) GraphicsBindingCounters(MetalGraphicsBindingStage stage) => + (_graphicsBindingAttempts[(int)stage], _graphicsBindingSkipped[(int)stage]); + + private string FormatGraphicsBindingCounters() => + $"graphics_bind=[cumulative=1 enabled={GraphicsBindingReuseEnabled} attempted={GraphicsBindingAttempts} skipped={GraphicsBindingSkipped} " + + $"vertex={_graphicsBindingAttempts[0]}/{_graphicsBindingSkipped[0]} mesh={_graphicsBindingAttempts[1]}/{_graphicsBindingSkipped[1]} " + + $"fragment={_graphicsBindingAttempts[2]}/{_graphicsBindingSkipped[2]}]"; + private static readonly nint SelCommandBuffer = MetalNative.Selector("commandBuffer"); private static readonly nint SelCommit = MetalNative.Selector("commit"); private static readonly nint SelEndEncoding = MetalNative.Selector("endEncoding"); @@ -592,6 +620,7 @@ private ulong FlushCommandBuffer(PassEndCause cause, MetalFlushCause why) _slotTrap?.OnCommit(serial); PassProfileCommit(serial, cause == PassEndCause.Present); + OcclusionAuditCommit(serial); if (cause == PassEndCause.Present) { _imageCache.MarkFrame(); @@ -714,6 +743,7 @@ private void RetainInFlight(nint buffer, ulong serial) MetalNative.SendVoid(oldest, SelWaitUntilCompleted); } + OcclusionAuditResolve(_inFlightSerials[(_inFlightHead + MaxInFlightSubmissions - 1) % MaxInFlightSubmissions], false); PassProfileResolve( _inFlightSerials[(_inFlightHead + MaxInFlightSubmissions - 1) % MaxInFlightSubmissions], PassProfileEnabled, @@ -786,7 +816,7 @@ private void SampleSubmission(nint buffer, ulong serial) } var status = MetalV2Native.SendValue(buffer, MetalNative.Selector("status")); - if (PassProfileEnabled && status != CommandBufferStatusCompleted && status != CommandBufferStatusError) + if ((PassProfileEnabled || OcclusionAuditEnabled) && status != CommandBufferStatusCompleted && status != CommandBufferStatusError) { // Its serial is signalled from inside it, so it is all but finished; the pass // profile needs its timestamps, which are readable only once it has completed. @@ -804,6 +834,7 @@ private void SampleSubmission(nint buffer, ulong serial) Console.Error.WriteLine( $"[METAL-V2][WARN] A Metal submission failed and will never signal its serial: serial={serial} " + $"completed={CompletedSerial} {MetalNative.DescribeError(error)}{DescribeEncoderStatus(error)}"); + OcclusionAuditResolve(serial, false); PassProfileResolve(serial, false, 0); return; } @@ -811,6 +842,7 @@ private void SampleSubmission(nint buffer, ulong serial) if (status != CommandBufferStatusCompleted) { _unsampledSubmissions++; + OcclusionAuditResolve(serial, false); PassProfileResolve(serial, false, 0); if (DsTrace.Gpu && DsTrace.Armed) { @@ -831,6 +863,7 @@ private void SampleSubmission(nint buffer, ulong serial) _gpuBusySamples++; } + OcclusionAuditResolve(serial, true); PassProfileResolve(serial, true, Math.Max(gpuEnd - gpuStart, 0)); var kernelStart = MetalNative.SendDouble(buffer, SelKernelStartTime); @@ -2444,9 +2477,10 @@ public void ReportCounters() $"prefetch=[{FormatPrefetchCounters()}] " + // The command thread's synchronous calls and their wait, its wait for room in the queue, and its items. $"cthread=[{FormatCommandThreadCounters()}] " + + (NativeOcclusionEnabled ? $"occlusion=[dumps={_oqDumps} reductions={_oqReductions} buffers={_oqBatches.Count} budget_waits={_oqBudgetWaits} unused_passes={_oqUnusedPasses} initialized_slots={_oqInitializedSlots}] " : string.Empty) + // Sampled bindings the texture memo answered, those whose stamp failed between the preparation // and the bind, lookups made in full, results memoised, and entries held. - $"texture_memo=({TakeTextureMemoCounters()}) " + + $"texture_memo=({TakeTextureMemoCounters()}) {FormatGraphicsBindingCounters()} " + // Modules and pipeline states resident against made, and what the bound released. $"{DescribeShaderCache()} " + $"images={_imageCache.ImageCount} image_mb={_imageCache.TotalUsedMemory / (1024 * 1024)} " + @@ -2609,7 +2643,7 @@ public void ReportCounters() $"end_other={endOtherCount} end_indirect={TakeIndirectPassEnds()} {TakeExecutorPassEnds()} targetless={targetlessCount} " + $"clears=[{TakeClearArrivals()}] clear_passes={clearPasses} " + $"clear_depth={clearDepthPasses} clear_mpx={(clearPixels / 1e6).ToString("F1", culture)} " + - $"labels_deferred={TakeDeferredLabelWrites()} clears_folded={foldedClears} " + + $"labels_deferred={TakeDeferredLabelWrites()} label_lookup=[{TakeDeferredLabelLookupStats()}] clears_folded={foldedClears} " + $"clears_encoded={encodedClears} depth_joins={depthJoins} depth_join_sampled={depthJoinsSampled} depth_join_copies={depthJoinsCopies} " + $"subset_joins={subsetJoins} subset_join_sampled={subsetJoinsSampled}{TakeSubsetRefusals()} " + $"transfers=[{TakeTransferCauses()}]"); @@ -2760,6 +2794,8 @@ public void Dispose() } _completions.Dispose(); + DisposeNativeOcclusion(); + DisposeOcclusionAudit(); DisposeWriteFences(); DisposeHostMovie(); DisposeBindlessImages(); diff --git a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalSamplerViewClasses.cs b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalSamplerViewClasses.cs new file mode 100644 index 000000000..7bf3e52be --- /dev/null +++ b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalSamplerViewClasses.cs @@ -0,0 +1,31 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using SharpEmu.Libs.Gpu.Images; +using SharpEmu.ShaderCompiler.Resources; + +namespace SharpEmu.Gpu.Metal.V2; + +// Recomputed from the resolved views each preparation: guest descriptors and the image +// cache may select a different numeric format even when the shader program is unchanged. +internal static class MetalSamplerViewClasses +{ + internal const byte AllInteger = 1; + private const byte AnyNonInteger = 2; + + internal static void Resolve(List pairs, ReadOnlySpan images, Span classes) + { + classes.Clear(); + if (classes.IsEmpty) return; + foreach (var pair in pairs) + { + // Invalid image entries do not establish a pairing, matching the per-sampler walk. + // Pairs for samplers outside this draw's table never participate in that walk. + if (pair.Sampler >= classes.Length || pair.Image >= images.Length) continue; + ref var classification = ref classes[(int)pair.Sampler]; + if (classification == AnyNonInteger) continue; + classification = ViewFormatRules.IsIntegerFormat(images[(int)pair.Image].View.Format) + ? AllInteger : AnyNonInteger; + } + } +} diff --git a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalV2Native.cs b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalV2Native.cs index f1546cea0..cd45c1b29 100644 --- a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalV2Native.cs +++ b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalV2Native.cs @@ -525,6 +525,10 @@ public static partial void SendSetStoreActionAtIndex( nuint action, nuint index); + // -setVisibilityResultMode:offset: + [LibraryImport(ObjCLibrary, EntryPoint = "objc_msgSend")] + public static partial void SendVisibilityResultMode(nint receiver, nint selector, nuint mode, nuint offset); + // -setBufferOffset:atIndex: [LibraryImport(ObjCLibrary, EntryPoint = "objc_msgSend")] public static partial void SendSetBufferOffset(nint receiver, nint selector, nuint offset, nuint index); diff --git a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalV2Presenter.HostMovie.cs b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalV2Presenter.HostMovie.cs index 7d37061da..d8548b020 100644 --- a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalV2Presenter.HostMovie.cs +++ b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalV2Presenter.HostMovie.cs @@ -31,7 +31,7 @@ internal static partial class MetalV2Presenter /// private static bool SelectHostMovieFallback(ref nint texture, ref uint width, ref uint height) { - var guestFrames = Interlocked.Read(ref _presentedGuestFrames); + var guestFrames = Interlocked.Read(ref _adoptedGuestFrameSequence); if (MoviePresentationServices.TryGetFallbackPresentationFrame( out var pixels, out var frameWidth, out var frameHeight, out var serial)) { diff --git a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalV2Presenter.cs b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalV2Presenter.cs index 8bb610c18..698555a15 100644 --- a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalV2Presenter.cs +++ b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalV2Presenter.cs @@ -79,6 +79,12 @@ private readonly record struct PendingFrame( private static uint _hostFrameHeight; private static long _guestFrameSequence; private static long _presentedGuestFrames; + // Independent of the resettable statistics counters and the published flip queue. + private static long _adoptedGuestFrameSequence; + private static long _hostFrameSequence; + private static readonly bool ReusePresentedFrame = + Environment.GetEnvironmentVariable("SHARPEMU_METAL_V2_REUSE_PRESENTED_FRAME") == "1"; + private static readonly MetalPresentationCache PresentationCache = new(); private static int _firstGuestFrameNoted; private static long _publishedGuestFrames; private static long _adoptionsBlocked; @@ -646,35 +652,13 @@ private static void PresentFrame() if (window.ConsumeSurfaceRestore()) { _drawableWidth = 0; + PresentationCache.Invalidate(); } SyncDrawableSize(); AdoptPendingFrame(); UploadPendingHostFrame(); UpdateOutputColor(window); - var drawable = MetalNative.Send(_metalLayer, MetalNative.Selector("nextDrawable")); - if (drawable == 0) - { - return; - } - - var drawableTexture = MetalNative.Send(drawable, MetalNative.Selector("texture")); - var commandBuffer = MetalNative.Send(_presentQueue, MetalNative.Selector("commandBuffer")); - MetalMemoryTelemetry.CommandBuffers.Add(); - if (drawableTexture == 0 || commandBuffer == 0) - { - return; - } - - var pass = CreateClearPass(drawableTexture); - var encoder = MetalNative.Send( - commandBuffer, MetalNative.Selector("renderCommandEncoderWithDescriptor:"), pass); - MetalMemoryTelemetry.RenderEncoders.Add(); - if (encoder == 0) - { - return; - } - // The GPU worker promotes frames too, when it publishes one and when it asks // for a flip slot, and a promotion releases the frame it replaces. So the // presented frame is read under the gate and held until the encoder has bound @@ -683,12 +667,14 @@ private static void PresentFrame() nint presented; uint presentedWidth, presentedHeight; bool presentedHdr; + long presentedSequence; lock (_frameGate) { presented = _presentTexture; presentedWidth = _presentWidth; presentedHeight = _presentHeight; presentedHdr = _presentIsHdr; + presentedSequence = _adoptedGuestFrameSequence; if (presented != 0) { MetalNative.Send(presented, MetalNative.Selector("retain")); @@ -701,6 +687,44 @@ private static void PresentFrame() var width = presented != 0 ? presentedWidth : _hostFrameWidth; var height = presented != 0 ? presentedHeight : _hostFrameHeight; var movie = SelectHostMovieFallback(ref texture, ref width, ref height); + var stamp = new MetalPresentationStamp( + presentedSequence, _hostFrameSequence, movie ? _movieFallbackSerial : 0, + texture, _drawableWidth, _drawableHeight, _layerHdrActive, + !movie && presented != 0 && presentedHdr, _edrCurrentHeadroom); + // Keep polling movies, window state and guest flips. The flip pacer is + // independent, so reusing a drawable never delays guest flip completion. + // Before a guest/movie image exists, the prewarm overlay may animate. + if (ReusePresentedFrame && !PresentationCache.NeedsPresent(in stamp, presented == 0 && !movie)) + { + ReportPresentationStatistics(presented: false); + // nextDrawable normally paces this loop; skipping it must not spin. + Thread.Sleep(8); + return; + } + + var drawable = MetalNative.Send(_metalLayer, MetalNative.Selector("nextDrawable")); + if (drawable == 0) + { + return; + } + + var drawableTexture = MetalNative.Send(drawable, MetalNative.Selector("texture")); + var commandBuffer = MetalNative.Send(_presentQueue, MetalNative.Selector("commandBuffer")); + MetalMemoryTelemetry.CommandBuffers.Add(); + if (drawableTexture == 0 || commandBuffer == 0) + { + return; + } + + var pass = CreateClearPass(drawableTexture); + var encoder = MetalNative.Send( + commandBuffer, MetalNative.Selector("renderCommandEncoderWithDescriptor:"), pass); + MetalMemoryTelemetry.RenderEncoders.Add(); + if (encoder == 0) + { + return; + } + if (texture != 0 && width != 0 && height != 0) { EncodePresent(encoder, texture, width, height, !movie && presented != 0 && presentedHdr); @@ -710,6 +734,14 @@ private static void PresentFrame() { EncodePrewarmProgress(encoder); } + + MetalNative.SendVoid(encoder, MetalNative.Selector("endEncoding")); + MetalNative.SendVoid(commandBuffer, MetalNative.Selector("presentDrawable:"), drawable); + MetalMemoryTelemetry.Commits.Add(); + MetalNative.SendVoid(commandBuffer, MetalNative.Selector("commit")); + PresentationCache.RecordPresented(in stamp); + PerfOverlay.RecordPresent(); + ReportPresentationStatistics(); } finally { @@ -718,13 +750,6 @@ private static void PresentFrame() MetalNative.SendVoid(presented, MetalNative.Selector("release")); } } - - MetalNative.SendVoid(encoder, MetalNative.Selector("endEncoding")); - MetalNative.SendVoid(commandBuffer, MetalNative.Selector("presentDrawable:"), drawable); - MetalMemoryTelemetry.Commits.Add(); - MetalNative.SendVoid(commandBuffer, MetalNative.Selector("commit")); - PerfOverlay.RecordPresent(); - ReportPresentationStatistics(); } finally { @@ -906,6 +931,7 @@ private static bool TryPromotePendingLocked() _presentWidth = frame.Width; _presentHeight = frame.Height; _presentIsHdr = frame.IsHdr; + Interlocked.Increment(ref _adoptedGuestFrameSequence); if (Interlocked.Increment(ref _presentedGuestFrames) == 1 && Interlocked.Exchange(ref _firstGuestFrameNoted, 1) == 0) { MetalShaderPrewarm.NoteFirstGuestFrame(); @@ -983,6 +1009,7 @@ private static unsafe void UploadPendingHostFrame() (nint)pixels, width * 4); } + _hostFrameSequence++; } private static nint CreateClearPass(nint targetTexture) @@ -1177,9 +1204,9 @@ private static nint CreateLinearSampler(nint device) } } - private static void ReportPresentationStatistics() + private static void ReportPresentationStatistics(bool presented = true) { - Interlocked.Increment(ref _hostPresents); + if (presented) Interlocked.Increment(ref _hostPresents); var now = System.Diagnostics.Stopwatch.GetTimestamp(); if (_statsTimestamp == 0) { diff --git a/tests/SharpEmu.Gpu.Metal.Tests/MetalAsyncPipelineTests.cs b/tests/SharpEmu.Gpu.Metal.Tests/MetalAsyncPipelineTests.cs index 89ad81e8c..ce3c55479 100644 --- a/tests/SharpEmu.Gpu.Metal.Tests/MetalAsyncPipelineTests.cs +++ b/tests/SharpEmu.Gpu.Metal.Tests/MetalAsyncPipelineTests.cs @@ -215,12 +215,15 @@ public void ADrawIntoATargetNotRebuiltEveryFrameWaitsForItsPipeline() // With the pool busy, the compile is still queued when the draw needs it, and runs off the worker. [Theory] - [InlineData(false)] - [InlineData(true)] - public void AGuestStoreIsServedWhileADrawWaitsForItsLibraryCompile(bool poolBusy) + [InlineData(false, false)] + [InlineData(true, false)] + [InlineData(false, true)] + [InlineData(true, true)] + public void AGuestStoreIsServedWhileADrawWaitsForItsLibraryCompile(bool poolBusy, bool deferred) { using var rig = CreateRig(); if (rig is null) return; + rig.Host.DeferLibraryCompiles = deferred; using var poolFree = new ManualResetEventSlim(false); try { diff --git a/tests/SharpEmu.Gpu.Metal.Tests/MetalCompileSchedulerTests.cs b/tests/SharpEmu.Gpu.Metal.Tests/MetalCompileSchedulerTests.cs index f02b8d4c0..9139cb0f8 100644 --- a/tests/SharpEmu.Gpu.Metal.Tests/MetalCompileSchedulerTests.cs +++ b/tests/SharpEmu.Gpu.Metal.Tests/MetalCompileSchedulerTests.cs @@ -236,4 +236,33 @@ public void PrewarmWaitsAreCollectedOnTheThreadThatWaited() CancellationToken.None, TaskCreationOptions.LongRunning, TaskScheduler.Default); + + [Fact] + public void AnUnusedDeferredLibraryCanBeCancelledWithoutRunningIt() + { + var runs = 0; + var job = new MetalLibraryCompilePool.Job(() => runs++); + Assert.True(job.CancelIfQueued()); + Assert.False(job.TryRun()); + Assert.False(job.CancelIfQueued()); + Assert.True(job.Task.IsCanceled); + Assert.Equal(0, runs); + } + + [Fact] + public void AClaimedDeferredLibraryCannotBeCancelled() + { + using var entered = new ManualResetEventSlim(); + using var release = new ManualResetEventSlim(); + var job = new MetalLibraryCompilePool.Job(() => { entered.Set(); release.Wait(); }); + var running = Task.Run(() => job.TryRun()); + try + { + Assert.True(entered.Wait(Released)); + Assert.False(job.CancelIfQueued()); + } + finally { release.Set(); } + Assert.True(running.Wait(Released)); + Assert.True(job.Task.IsCompletedSuccessfully); + } } diff --git a/tests/SharpEmu.Gpu.Metal.Tests/MetalDepthOnlyJoinTests.cs b/tests/SharpEmu.Gpu.Metal.Tests/MetalDepthOnlyJoinTests.cs index 866b93fe4..93eda7665 100644 --- a/tests/SharpEmu.Gpu.Metal.Tests/MetalDepthOnlyJoinTests.cs +++ b/tests/SharpEmu.Gpu.Metal.Tests/MetalDepthOnlyJoinTests.cs @@ -28,6 +28,7 @@ namespace SharpEmu.Gpu.Metal.Tests; public sealed unsafe class MetalDepthOnlyJoinTests(ITestOutputHelper output) { private const ulong GuestBase = 0x2_0000_0000; + private const ulong GuestAliasBase = GuestBase + 0x1000_0000; private const ulong TargetAddress = GuestBase + 0x4000; private const ulong OtherAddress = GuestBase + 0x8000; private const ulong LabelAddress = GuestBase + 0x1000; @@ -58,6 +59,7 @@ private enum Scene PendingClear, // A deferred copy lands in a buffer the depth-only draw binds. BindsCopyDestination, + BindsAliasedCopyDestination, // A deferred copy waits, in bytes the draw does not bind. BindsOtherBytes, } @@ -96,11 +98,13 @@ public void ADepthOnlyDrawDoesNotJoinWhileAClearWaits() AssertRedLeftGreenRight(outcome.Pixels); } - [Fact] - public void ADepthOnlyDrawDoesNotJoinWhileACopyOwesBytesItBinds() + [Theory] + [InlineData(false)] + [InlineData(true)] + public void ADepthOnlyDrawDoesNotJoinWhileACopyOwesBytesItBinds(bool alias) { if (!RequireDevice()) return; - var outcome = Run(Scene.BindsCopyDestination); + var outcome = Run(alias ? Scene.BindsAliasedCopyDestination : Scene.BindsCopyDestination); Assert.Equal(0L, outcome.Joins); Assert.Equal(1L, outcome.JoinsWaitingOnCopies); Assert.Equal(3L, outcome.Passes); @@ -196,7 +200,7 @@ private Outcome Run(Scene scene) Assert.Equal(1, host.PendingClearCount); } - if (scene is Scene.BindsCopyDestination or Scene.BindsOtherBytes) + if (scene is Scene.BindsCopyDestination or Scene.BindsAliasedCopyDestination or Scene.BindsOtherBytes) { host.RecordCommandProcessorWrite(LabelAddress, BitConverter.GetBytes(0x55UL), "WRITE_DATA"); Assert.Equal(1, host.DeferredLabelCopyCount); @@ -207,6 +211,7 @@ private Outcome Run(Scene scene) { Scene.SamplesAttachment => Rig.Stage(textures: [(nint)colourView.Handle]), Scene.BindsCopyDestination => Rig.Stage(buffers: [host.GuestMemory.ResolveWritable(LabelAddress, 64)]), + Scene.BindsAliasedCopyDestination => Rig.Stage(buffers: [host.GuestMemory.ResolveWritable(GuestAliasBase + (LabelAddress - GuestBase), 64)]), Scene.BindsOtherBytes => Rig.Stage(buffers: [host.GuestMemory.ResolveWritable(LabelAddress + 0x400, 64)]), _ => null, }; @@ -279,6 +284,8 @@ private sealed class Backing : IGuestBackedSpace, ICpuMemory, IDisposable public bool TryResolveBackingAlias(ulong address, ulong size, out ulong alias) { + if (address >= GuestAliasBase && address - GuestAliasBase < Size) + address = GuestBase + (address - GuestAliasBase); alias = Contains(address, size) ? (ulong)_block + (address - GuestBase) : 0; return alias != 0; } diff --git a/tests/SharpEmu.Gpu.Metal.Tests/MetalEndOfPipeOrderTests.cs b/tests/SharpEmu.Gpu.Metal.Tests/MetalEndOfPipeOrderTests.cs index cf56a5660..dc1db1945 100644 --- a/tests/SharpEmu.Gpu.Metal.Tests/MetalEndOfPipeOrderTests.cs +++ b/tests/SharpEmu.Gpu.Metal.Tests/MetalEndOfPipeOrderTests.cs @@ -285,8 +285,11 @@ public void ADispatchBindingBytesACopyStillOwesEndsTheComputeEncoderFirst() // The held copies' span only spares the scan: a binding between two of them that touches // neither keeps the encoder open, and once they land the span starts over, so bytes only // an earlier copy covered no longer count. - [Fact] - public void ABindingBetweenTwoHeldCopiesTouchesNeitherAndTheSpanStartsOverWhenTheyLand() + [Theory] + [InlineData(0x1000UL, 0x800UL, 1)] + [InlineData(0x4000UL, 0x2000UL, 1)] + [InlineData(0x4000UL, 0x2000UL, 32)] + public void ABindingBetweenTwoHeldCopiesTouchesNeitherAndTheSpanStartsOverWhenTheyLand(ulong secondOffset, ulong gapOffset, int copiesPerCluster) { if (!RequireDevice()) return; using var rig = new Rig(Device.Value); @@ -294,20 +297,23 @@ public void ABindingBetweenTwoHeldCopiesTouchesNeitherAndTheSpanStartsOverWhenTh Assert.True(host.DefersCopiesInCompute); _ = host.EnsureComputeEncoder(); - host.RecordCommandProcessorWrite(LaterLabel, BitConverter.GetBytes(0x55UL), "WRITE_DATA"); - host.RecordCommandProcessorWrite(LaterLabel + 0x1000, BitConverter.GetBytes(0x66UL), "WRITE_DATA"); - Assert.Equal(2, host.DeferredLabelCopyCount); + for (var index = 0; index < copiesPerCluster; index++) + { + host.RecordCommandProcessorWrite(LaterLabel + (ulong)index * 8, BitConverter.GetBytes(0x55UL), "WRITE_DATA"); + host.RecordCommandProcessorWrite(LaterLabel + secondOffset + (ulong)index * 8, BitConverter.GetBytes(0x66UL), "WRITE_DATA"); + } + Assert.Equal(copiesPerCluster * 2, host.DeferredLabelCopyCount); - host.OrderAfterDeferredCopies(LaterLabel + 0x800, 0x40); + host.OrderAfterDeferredCopies(LaterLabel + gapOffset, 0x40); Assert.True(host.ComputeEncoderOpen); - Assert.Equal(2, host.DeferredLabelCopyCount); + Assert.Equal(copiesPerCluster * 2, host.DeferredLabelCopyCount); - host.OrderAfterDeferredCopies(LaterLabel + 0x1000, 8); + host.OrderAfterDeferredCopies(LaterLabel + secondOffset, 8); Assert.False(host.ComputeEncoderOpen); Assert.Equal(0, host.DeferredLabelCopyCount); _ = host.EnsureComputeEncoder(); - host.RecordCommandProcessorWrite(LaterLabel + 0x1000, BitConverter.GetBytes(0x77UL), "WRITE_DATA"); + host.RecordCommandProcessorWrite(LaterLabel + secondOffset, BitConverter.GetBytes(0x77UL), "WRITE_DATA"); host.OrderAfterDeferredCopies(LaterLabel, 8); Assert.True(host.ComputeEncoderOpen); Assert.Equal(1, host.DeferredLabelCopyCount); @@ -316,7 +322,7 @@ public void ABindingBetweenTwoHeldCopiesTouchesNeitherAndTheSpanStartsOverWhenTh WaitForGpu(host, host.LastWorkSerial); WaitForCompletions(host); Assert.Equal(0x55UL, rig.Backing.Read64(LaterLabel)); - Assert.Equal(0x77UL, rig.Backing.Read64(LaterLabel + 0x1000)); + Assert.Equal(0x77UL, rig.Backing.Read64(LaterLabel + secondOffset)); } [Fact] diff --git a/tests/SharpEmu.Gpu.Metal.Tests/MetalGraphicsBindingMemoTests.cs b/tests/SharpEmu.Gpu.Metal.Tests/MetalGraphicsBindingMemoTests.cs new file mode 100644 index 000000000..08d7ae06c --- /dev/null +++ b/tests/SharpEmu.Gpu.Metal.Tests/MetalGraphicsBindingMemoTests.cs @@ -0,0 +1,58 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using SharpEmu.Gpu.Metal.V2; +using Xunit; + +namespace SharpEmu.Gpu.Metal.Tests; + +public sealed class MetalGraphicsBindingMemoTests +{ + [Fact] + public void OnlyAnExactlyRecordedBindingMatches() + { + var memo = new MetalGraphicsBindingMemo(); + Assert.False(memo.Matches(MetalGraphicsBindingStage.Vertex, 11, 0, 7, 256)); + // An attempted native call is not a recorded success. + Assert.False(memo.Matches(MetalGraphicsBindingStage.Vertex, 11, 0, 7, 256)); + memo.Record(MetalGraphicsBindingStage.Vertex, 11, 0, 7, 256); + Assert.True(memo.Matches(MetalGraphicsBindingStage.Vertex, 11, 0, 7, 256)); + Assert.False(memo.Matches(MetalGraphicsBindingStage.Vertex, 11, 0, 7, 512)); + Assert.False(memo.Matches(MetalGraphicsBindingStage.Vertex, 11, 0, 8, 256)); + Assert.False(memo.Matches(MetalGraphicsBindingStage.Vertex, 12, 0, 7, 256)); + memo.Record(MetalGraphicsBindingStage.Vertex, 11, 0, 7, 512); + Assert.True(memo.Matches(MetalGraphicsBindingStage.Vertex, 11, 0, 7, 512)); + Assert.False(memo.Matches(MetalGraphicsBindingStage.Vertex, 11, 0, 7, 256)); + } + + [Fact] + public void PhysicalStagesAndBothSlotsKeepIndependentState() + { + var memo = new MetalGraphicsBindingMemo(); + memo.Record(MetalGraphicsBindingStage.Vertex, 11, 0, 7, 256); + Assert.False(memo.Matches(MetalGraphicsBindingStage.Vertex, 11, 1, 7, 256)); + Assert.False(memo.Matches(MetalGraphicsBindingStage.Mesh, 11, 0, 7, 256)); + Assert.False(memo.Matches(MetalGraphicsBindingStage.Fragment, 11, 0, 7, 256)); + foreach (var stage in Enum.GetValues()) + for (var slot = 0; slot < 2; slot++) + memo.Record(stage, 11 + (int)stage, slot, 7 + slot, (ulong)(256 + slot * 256)); + foreach (var stage in Enum.GetValues()) + for (var slot = 0; slot < 2; slot++) + Assert.True(memo.Matches(stage, 11 + (int)stage, slot, 7 + slot, (ulong)(256 + slot * 256))); + // Returning from mesh to vertex still uses vertex's distinct physical binding. + Assert.True(memo.Matches(MetalGraphicsBindingStage.Vertex, 11, 0, 7, 256)); + } + + [Fact] + public void ExplicitEncoderResetForgetsBindingsEvenWhenNativePointersRepeat() + { + var memo = new MetalGraphicsBindingMemo(); + foreach (var stage in Enum.GetValues()) + for (var slot = 0; slot < 2; slot++) memo.Record(stage, 11, slot, 7, 256); + memo.Reset(); + foreach (var stage in Enum.GetValues()) + for (var slot = 0; slot < 2; slot++) Assert.False(memo.Matches(stage, 11, slot, 7, 256)); + memo.Record(MetalGraphicsBindingStage.Fragment, 11, 0, 7, 256); + Assert.True(memo.Matches(MetalGraphicsBindingStage.Fragment, 11, 0, 7, 256)); + } +} diff --git a/tests/SharpEmu.Gpu.Metal.Tests/MetalPageRangeFilterTests.cs b/tests/SharpEmu.Gpu.Metal.Tests/MetalPageRangeFilterTests.cs new file mode 100644 index 000000000..6a7e65f64 --- /dev/null +++ b/tests/SharpEmu.Gpu.Metal.Tests/MetalPageRangeFilterTests.cs @@ -0,0 +1,78 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using SharpEmu.Gpu.Metal.V2; +using Xunit; + +namespace SharpEmu.Gpu.Metal.Tests; + +public sealed class MetalPageRangeFilterTests +{ + [Fact] + public void CrossPageWritesAndNativeAliasesRemainPossibleOverlaps() + { + var filter = new MetalPageRangeFilter(); + filter.Add(0x1FFC, 8, space: 0x1234); + Assert.True(filter.MayOverlap(0x1FFF, 1, space: 0x1234)); + Assert.True(filter.MayOverlap(0x2000, 4, space: 0x1234)); + Assert.True(filter.MayOverlap(0x1000, 0x1001, space: 0x1234)); + Assert.False(filter.MayOverlap(0x2000, 0, space: 0x1234)); + // Native aliases supply the same buffer/offset key regardless of guest address. + filter.Add(0x1FFC, 8, space: 0x5678); + Assert.True(filter.MayOverlap(0x2000, 4, space: 0x5678)); + } + + [Fact] + public void LargeOrWrappingInsertionsFallBackUntilClear() + { + var filter = new MetalPageRangeFilter(); + foreach (var range in new[] { (0UL, 65UL * 4096), (ulong.MaxValue - 2, 8UL) }) + { + filter.Add(range.Item1, range.Item2); + Assert.True(filter.MayOverlap(0x12345678, 8)); + filter.Clear(); + Assert.False(filter.MayOverlap(0x12345678, 8)); + } + filter.Add(ulong.MaxValue, 1); + Assert.True(filter.MayOverlap(ulong.MaxValue, 1)); + Assert.True(filter.MayOverlap(ulong.MaxValue - 2, 8)); + Assert.True(filter.MayOverlap(0, 65UL * 4096)); + } + + [Fact] + public void ClearingRemovesOldPagesAndRepopulationKeepsNewPages() + { + var filter = new MetalPageRangeFilter(); + filter.Add(0x2000, 0); // No write. + Assert.False(filter.MayOverlap(0x2000, 1)); + filter.Add(0x2000, 8); + Assert.True(filter.MayOverlap(0x2000, 1)); + filter.Clear(); + Assert.False(filter.MayOverlap(0x2000, 1)); + filter.Add(0x4000, 8); + Assert.True(filter.MayOverlap(0x4000, 1)); + Assert.Contains(Enumerable.Range(0, 100), page => !filter.MayOverlap((ulong)page * 4096, 8)); + } + + [Fact] + public void SeededRangesNeverRejectAnExactOverlapAcrossAddressSpaces() + { + var random = new Random(8123); + var filter = new MetalPageRangeFilter(); + for (var batch = 0; batch < 100; batch++) + { + filter.Clear(); + var ranges = Enumerable.Range(0, 128).Select(_ => ( + Address: (ulong)random.NextInt64(1L << 40), + Size: (ulong)random.Next(1, 9000), + Space: (ulong)random.Next(0, 8))).ToArray(); + foreach (var range in ranges) filter.Add(range.Address, range.Size, range.Space); + foreach (var range in ranges) + { + Assert.True(filter.MayOverlap(range.Address, 1, range.Space)); + Assert.True(filter.MayOverlap(range.Address + range.Size - 1, 1, range.Space)); + Assert.True(filter.MayOverlap(range.Address, range.Size, range.Space)); + } + } + } +} diff --git a/tests/SharpEmu.Gpu.Metal.Tests/MetalPairLockTableTests.cs b/tests/SharpEmu.Gpu.Metal.Tests/MetalPairLockTableTests.cs new file mode 100644 index 000000000..bad35704c --- /dev/null +++ b/tests/SharpEmu.Gpu.Metal.Tests/MetalPairLockTableTests.cs @@ -0,0 +1,130 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using System.Diagnostics; +using System.Reflection; +using System.Runtime.CompilerServices; +using System.Text.Json; +using SharpEmu.Gpu.Metal.Tests.Scenes; +using SharpEmu.Gpu.Metal.V2; +using SharpEmu.ShaderCompiler.Metal; +using Xunit; +using Xunit.Abstractions; + +namespace SharpEmu.Gpu.Metal.Tests; + +public sealed unsafe class MetalPairLockTableTests(ITestOutputHelper output) +{ + private static readonly Lazy Device = new(MetalNative.MTLCreateSystemDefaultDevice); + + [Fact] + public void ConcurrentFirstUsePublishesOneZeroedResidentTableAndPreservesItsWords() + { + if (!RequireDevice()) return; + using var backing = new SceneMemory(); + using var memory = new MetalGuestMemory(Device.Value, backing, backing); + var initialBuffers = memory.ResidentBuffers.ToArray(); + var initialResidencyCount = memory.ResidencySet == 0 ? 0 : + MetalNative.Send(memory.ResidencySet, MetalNative.Selector("allocationCount")); + var addresses = new ulong[8]; + using var start = new ManualResetEventSlim(); + var readers = Enumerable.Range(0, addresses.Length).Select(index => Task.Factory.StartNew(() => + { + Assert.True(start.Wait(TimeSpan.FromSeconds(10))); + addresses[index] = memory.PairLockTableGpuAddress; + }, CancellationToken.None, TaskCreationOptions.LongRunning, TaskScheduler.Default)).ToArray(); + start.Set(); + Assert.True(Task.WaitAll(readers, TimeSpan.FromSeconds(10))); + Assert.NotEqual(0UL, addresses[0]); + Assert.All(addresses, address => Assert.Equal(addresses[0], address)); + var buffer = (nint)typeof(MetalGuestMemory).GetField("_pairLockTable", + BindingFlags.Instance | BindingFlags.NonPublic)!.GetValue(memory)!; + Assert.NotEqual(0, buffer); + AssertResident(); + Assert.Equal(addresses[0], (ulong)MetalNative.Send(buffer, MetalNative.Selector("gpuAddress"))); + var words = new Span((void*)MetalNative.Send(buffer, MetalNative.Selector("contents")), + Gen5MslArgumentLayout.PairLockTableWords); + Assert.True(words.IndexOfAnyExcept(0u) < 0, "The complete pair-lock table must start unlocked."); + + // Sequence words belong to the shaders after publication. Retrieving the address, + // including at a new submission, must neither allocate another table nor reset them. + words[0] = 2; + words[^1] = 0x12345678; + memory.BeginSubmission(1, 0); + Parallel.For(0, addresses.Length, index => + { + for (var read = 0; read < 1000; read++) + Assert.Equal(addresses[index], memory.PairLockTableGpuAddress); + }); + Assert.Equal(2u, words[0]); + Assert.Equal(0x12345678u, words[^1]); + AssertResident(); + + void AssertResident() + { + if (memory.ResidencySet == 0) + Assert.Equal(buffer, Assert.Single(memory.ResidentBuffers.ToArray().Except(initialBuffers))); + else + { + Assert.True(MetalNative.SendBool(memory.ResidencySet, MetalNative.Selector("containsAllocation:"), buffer)); + Assert.Equal(initialResidencyCount + 1, MetalNative.Send(memory.ResidencySet, MetalNative.Selector("allocationCount"))); + } + } + } + + [Fact] + public void MeasurePublishedTableLookupWhenRequested() + { + var path = Environment.GetEnvironmentVariable("SHARPEMU_PAIR_LOCK_BENCH_OUTPUT"); + if (string.IsNullOrEmpty(path) || !RequireDevice()) return; + using var backing = new SceneMemory(); + using var memory = new MetalGuestMemory(Device.Value, backing, backing); + const int iterations = 1_000_000; + var address = memory.PairLockTableGpuAddress; + ReadBatch(memory, 20_000); + var batches = new List(); + for (var batch = 0; batch < 9; batch++) + { + var allocated = GC.GetAllocatedBytesForCurrentThread(); + var start = Stopwatch.GetTimestamp(); + var sum = ReadBatch(memory, iterations); + var elapsed = Stopwatch.GetTimestamp() - start; + allocated = GC.GetAllocatedBytesForCurrentThread() - allocated; + Assert.Equal(unchecked(address * iterations), sum); + batches.Add(new + { + batch, + nsPerLookup = elapsed * (1_000_000_000.0 / Stopwatch.Frequency) / iterations, + bytesPerLookup = (double)allocated / iterations, + }); + } + File.WriteAllText(path, JsonSerializer.Serialize(new + { + architecture = System.Runtime.InteropServices.RuntimeInformation.ProcessArchitecture.ToString(), + iterations, + batches, + }, new JsonSerializerOptions { WriteIndented = true }) + "\n"); + output.WriteLine(path); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong ReadBatch(MetalGuestMemory memory, int iterations) + { + ulong sum = 0; + for (var index = 0; index < iterations; index++) + sum = unchecked(sum + memory.PairLockTableGpuAddress); + return sum; + } + + private bool RequireDevice() + { + if (Environment.GetEnvironmentVariable("SHARPEMU_TEST_REQUIRE_DEVICE") != "1") + { + output.WriteLine("Pair-lock table tests need the exclusive device gate."); + return false; + } + Assert.True(OperatingSystem.IsMacOS()); + Assert.NotEqual(0, Device.Value); + return true; + } +} diff --git a/tests/SharpEmu.Gpu.Metal.Tests/MetalPendingWriteOverlayTests.cs b/tests/SharpEmu.Gpu.Metal.Tests/MetalPendingWriteOverlayTests.cs index bd0e32c1e..fdba7587c 100644 --- a/tests/SharpEmu.Gpu.Metal.Tests/MetalPendingWriteOverlayTests.cs +++ b/tests/SharpEmu.Gpu.Metal.Tests/MetalPendingWriteOverlayTests.cs @@ -189,6 +189,60 @@ void Round() Assert.Equal(0, overlay.Count); } + [Theory] + [InlineData(512)] + [InlineData(2048)] + [InlineData(8192)] + public void CompletedOcclusionWriteBurstsReuseEntries(int count) + { + var overlay = new MetalPendingWriteOverlay(); + ulong serial = 0; + void Round() + { + overlay.AddQwords(0x7100_0000, count, 16, ++serial, serial); + overlay.Retire(serial); + } + + for (var warm = 0; warm < 8; warm++) Round(); + var before = GC.GetAllocatedBytesForCurrentThread(); + for (var round = 0; round < 16; round++) Round(); + var allocated = GC.GetAllocatedBytesForCurrentThread() - before; + Assert.True(allocated <= 512, $"A warm {count}-write burst allocated {allocated} bytes over16rounds."); + Assert.Equal(0, overlay.Count); + + // A reused entry must carry the new value, and retirement must still remove it. + overlay.AddQwords(0x7100_0000, count, 16, 0xFEDCBA9876543210, ++serial); + Span last = stackalloc byte[8]; + overlay.Apply(0x7100_0000 + (ulong)(count - 1) * 16, last); + Assert.Equal(0xFEDCBA9876543210UL, System.Buffers.Binary.BinaryPrimitives.ReadUInt64LittleEndian(last)); + overlay.Retire(serial); + Assert.Equal(0, overlay.Count); + } + + [Fact] + public void SpareEntryBudgetIsSharedAcrossSizesAndClearReturnsOnlyRemovedEntries() + { + var overlay = new MetalPendingWriteOverlay(); + Span bytes = stackalloc byte[16]; + bytes.Fill(0x55); + for (var length = 4; length <= 16; length += 4) + { + for (var i = 0; i < MetalPendingWriteOverlay.MaxPooledEntries; i++) + overlay.Add(0x7200_0000 + (ulong)i * 16, bytes[..length], serial: 1); + overlay.Clear(); + Assert.InRange(overlay.SpareEntryCount, 0, MetalPendingWriteOverlay.MaxPooledEntries); + Assert.Equal(0, overlay.Count); + } + + Assert.Equal(MetalPendingWriteOverlay.MaxPooledEntries, overlay.SpareEntryCount); + overlay.Add(0x7200_0000, bytes[..4], serial: 2); + Assert.Equal(MetalPendingWriteOverlay.MaxPooledEntries - 1, overlay.SpareEntryCount); + overlay.Retire(1); + Assert.Equal(1, overlay.Count); + overlay.Retire(2); + Assert.Equal(MetalPendingWriteOverlay.MaxPooledEntries, overlay.SpareEntryCount); + } + [Fact] public void ALongRangeIsWalkedRatherThanCounted() { diff --git a/tests/SharpEmu.Gpu.Metal.Tests/MetalPresentationCacheTests.cs b/tests/SharpEmu.Gpu.Metal.Tests/MetalPresentationCacheTests.cs new file mode 100644 index 000000000..f584cfd2e --- /dev/null +++ b/tests/SharpEmu.Gpu.Metal.Tests/MetalPresentationCacheTests.cs @@ -0,0 +1,81 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using SharpEmu.Gpu.Metal.V2; +using Xunit; + +namespace SharpEmu.Gpu.Metal.Tests; + +public sealed class MetalPresentationCacheTests +{ + private static readonly MetalPresentationStamp Room = new(1, 0, 0, 123, 1920, 1080, false, false, 1); + + [Fact] + public void ReusedGuestTexturePresentsEveryNewFlipButNotEveryDisplayTick() + { + var cache = new MetalPresentationCache(); + for (var frame = 1; frame <= 8; frame++) + { + var stamp = Room with { GuestSequence = frame }; + Assert.True(cache.NeedsPresent(in stamp, false)); + cache.RecordPresented(in stamp); + for (var displayTick = 0; displayTick < 12; displayTick++) + Assert.False(cache.NeedsPresent(in stamp, false)); + } + } + + [Fact] + public void FailedDrawableAcquisitionRetriesWithoutLosingTheNewFrame() + { + var cache = new MetalPresentationCache(); + cache.RecordPresented(in Room); + var next = Room with { GuestSequence = 2 }; + Assert.True(cache.NeedsPresent(in next, false)); + Assert.True(cache.NeedsPresent(in next, false)); + cache.RecordPresented(in next); + Assert.False(cache.NeedsPresent(in next, false)); + } + + [Fact] + public void MovieAndCpuUpdatesDoNotRequireANewGuestFlip() + { + var cache = new MetalPresentationCache(); + cache.RecordPresented(in Room); + var cpu = Room with { HostSequence = 1 }; + Assert.True(cache.NeedsPresent(in cpu, false)); + cache.RecordPresented(in cpu); + var movie = cpu with { MovieSequence = 99, Texture = 456 }; + Assert.True(cache.NeedsPresent(in movie, false)); + cache.RecordPresented(in movie); + Assert.False(cache.NeedsPresent(in movie, false)); + var nextMovie = movie with { MovieSequence = 100 }; + Assert.True(cache.NeedsPresent(in nextMovie, false)); + Assert.True(cache.NeedsPresent(in cpu, false)); // Movie ended; restore the guest. + } + + [Fact] + public void DisplayChangesAndSurfaceRestorationRedrawAStationaryGuest() + { + var cache = new MetalPresentationCache(); + cache.RecordPresented(in Room); + MetalPresentationStamp[] changes = + [ + Room with { Width = 1280 }, Room with { Height = 720 }, + Room with { LayerHdr = true }, Room with { GuestHdr = true }, + Room with { Headroom = 2 }, Room with { Texture = 789 }, + ]; + foreach (var changed in changes) + Assert.True(cache.NeedsPresent(in changed, false)); + cache.Invalidate(); + Assert.True(cache.NeedsPresent(in Room, false)); + } + + [Fact] + public void LoadingOverlayCanAnimateWithoutAnImageUpdate() + { + var cache = new MetalPresentationCache(); + cache.RecordPresented(in Room); + Assert.True(cache.NeedsPresent(in Room, true)); + Assert.False(cache.NeedsPresent(in Room, false)); + } +} diff --git a/tests/SharpEmu.Gpu.Metal.Tests/MetalSamplerViewClassesTests.cs b/tests/SharpEmu.Gpu.Metal.Tests/MetalSamplerViewClassesTests.cs new file mode 100644 index 000000000..304e0baf6 --- /dev/null +++ b/tests/SharpEmu.Gpu.Metal.Tests/MetalSamplerViewClassesTests.cs @@ -0,0 +1,91 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using SharpEmu.Gpu.Metal.V2; +using SharpEmu.Libs.Gpu.Abstractions; +using SharpEmu.Libs.Gpu.Images; +using SharpEmu.ShaderCompiler.Resources; +using Xunit; + +namespace SharpEmu.Gpu.Metal.Tests; + +public sealed class MetalSamplerViewClassesTests +{ + [Fact] + public void ASharedSamplerRequiresEveryPairedViewToBeInteger() + { + ImageRequest[] images = [Image(GpuFormat.R32Uint), Image(GpuFormat.R32Sint), Image(GpuFormat.R32Sfloat)]; + List pairs = [Pair(0, 0), Pair(1, 0), Pair(0, 1), Pair(2, 1), Pair(2, 2), Pair(1, 2)]; + var classes = new byte[4]; + MetalSamplerViewClasses.Resolve(pairs, images, classes); + Assert.Equal(MetalSamplerViewClasses.AllInteger, classes[0]); + Assert.NotEqual(MetalSamplerViewClasses.AllInteger, classes[1]); + Assert.NotEqual(MetalSamplerViewClasses.AllInteger, classes[2]); + Assert.Equal(0, classes[3]); // No pairing does not force integer filtering. + } + + [Fact] + public void InvalidImagesDoNotEstablishAPairingAndUnusedSamplerPairsAreIgnored() + { + ImageRequest[] images = [Image(GpuFormat.R32Uint)]; + List pairs = [Pair(uint.MaxValue, 0), Pair(1, 0), Pair(0, uint.MaxValue), Pair(1, 1), Pair(0, 1)]; + var classes = new byte[2]; + MetalSamplerViewClasses.Resolve(pairs, images, classes); + Assert.Equal(0, classes[0]); + Assert.Equal(MetalSamplerViewClasses.AllInteger, classes[1]); + } + + [Fact] + public void ReusedScratchReflectsCurrentFormatsAndCurrentImageAndSamplerCounts() + { + ImageRequest[] images = [Image(GpuFormat.R32Uint), Image(GpuFormat.R32Sint)]; + List pairs = [Pair(0, 0), Pair(1, 1)]; + var classes = new byte[3]; + MetalSamplerViewClasses.Resolve(pairs, images, classes); + Assert.Equal(MetalSamplerViewClasses.AllInteger, classes[0]); + Assert.Equal(MetalSamplerViewClasses.AllInteger, classes[1]); + + images[0] = Image(GpuFormat.R8G8B8A8Unorm); + classes[2] = 99; + MetalSamplerViewClasses.Resolve(pairs, images.AsSpan(0, 1), classes.AsSpan(0, 2)); + Assert.NotEqual(MetalSamplerViewClasses.AllInteger, classes[0]); + Assert.Equal(0, classes[1]); // Stale capacity is not a bound image. + Assert.Equal(99, classes[2]); // Stale sampler capacity is untouched. + + MetalSamplerViewClasses.Resolve([], images, classes); + Assert.Equal(new byte[3], classes); + MetalSamplerViewClasses.Resolve(pairs, images, Span.Empty); + } + + [Fact] + public void ClassificationMatchesThePreviousPerSamplerWalkForMixedPairGraphs() + { + var random = new Random(137); + for (var sample = 0; sample < 100; sample++) + { + var images = Enumerable.Range(0, random.Next(0, 12)) + .Select(_ => Image(random.Next(3) switch + { + 0 => GpuFormat.R32Uint, + 1 => GpuFormat.R32Sint, + _ => GpuFormat.R32Sfloat, + })).ToArray(); + var pairs = Enumerable.Range(0, random.Next(0, 40)) + .Select(_ => Pair((uint)random.Next(0, 15), (uint)random.Next(0, 12))).ToList(); + var classes = new byte[random.Next(0, 10)]; + MetalSamplerViewClasses.Resolve(pairs, images, classes); + for (var sampler = 0; sampler < classes.Length; sampler++) + { + var views = pairs.Where(pair => pair.Sampler == sampler && pair.Image < images.Length) + .Select(pair => images[(int)pair.Image].View.Format).ToArray(); + Assert.Equal(views.Length != 0 && views.All(ViewFormatRules.IsIntegerFormat), + classes[sampler] == MetalSamplerViewClasses.AllInteger); + } + } + } + + private static ImageRequest Image(GpuFormat format) => + new() { View = ImageViewDescription.Default with { Format = format } }; + + private static SampledImagePair Pair(uint image, uint sampler) => new() { Image = image, Sampler = sampler }; +} diff --git a/tests/SharpEmu.Gpu.Metal.Tests/MetalShaderSourceReleaseTests.cs b/tests/SharpEmu.Gpu.Metal.Tests/MetalShaderSourceReleaseTests.cs index c32b8c1b2..0fd25afc1 100644 --- a/tests/SharpEmu.Gpu.Metal.Tests/MetalShaderSourceReleaseTests.cs +++ b/tests/SharpEmu.Gpu.Metal.Tests/MetalShaderSourceReleaseTests.cs @@ -150,6 +150,37 @@ public void AModuleReleasedByTheShaderCacheBoundIsBuiltAgainFromItsRequest() Assert.Equal(expected, metal.PayloadHash); } + [Theory] + [InlineData(false)] + [InlineData(true)] + public void ADeferredModuleCompilesOnlyIfAPipelineRequestsIt(bool requested) + { + if (!RequireDevice()) return; + var compiles = 0; + using (var rig = new Rig(Device.Value)) + { + rig.Host.DeferLibraryCompiles = true; + rig.Host.BeforeLibraryCompile = () => Interlocked.Increment(ref compiles); + var request = Gen5ComputeFixtures.CreateComputeRequest(Gen5ComputeFixtures.ExecStore); + Assert.True(Gen5MslTranslator.TryCompileProgram(request, out var shader, out var error), error); + var metal = new MetalCompiledGuestShader(shader, request); + var program = new ShaderProgram(1, rig.Host.CreateShaderModule(metal, ShaderStage.Compute, 0x5EED, 1)); + Assert.Equal(0, Volatile.Read(ref compiles)); + if (requested) + { + rig.Host.CreateComputePipeline(new ComputePipelineDescription + { + Input = new ComputeInputInfo { ThreadsX = 32, ThreadsY = 1, ThreadsZ = 1 }, + Program = program, + Stage = new ShaderProgramInfo { Stage = ShaderStageKind.Compute, Hash = 0x5EED }, + }); + Assert.True(metal.SourceReleased); + Assert.Equal(1, Volatile.Read(ref compiles)); + } + } + Assert.Equal(requested ? 1 : 0, Volatile.Read(ref compiles)); + } + private static PipelineHandle Pipeline(MetalRenderHost host, MetalCompiledGuestShader shader, ulong id) { var program = new ShaderProgram(id, host.CreateShaderModule(shader, ShaderStage.Compute, 0x5EED + id, id)); diff --git a/tests/SharpEmu.Gpu.Metal.Tests/Scenes/BindingReuseSceneTests.cs b/tests/SharpEmu.Gpu.Metal.Tests/Scenes/BindingReuseSceneTests.cs index c9ae1f023..ee187fcea 100644 --- a/tests/SharpEmu.Gpu.Metal.Tests/Scenes/BindingReuseSceneTests.cs +++ b/tests/SharpEmu.Gpu.Metal.Tests/Scenes/BindingReuseSceneTests.cs @@ -2,6 +2,7 @@ // SPDX-License-Identifier: GPL-2.0-or-later using System.Runtime.InteropServices; +using SharpEmu.Gpu.Metal.V2; using Xunit; using Xunit.Abstractions; @@ -106,4 +107,66 @@ public void DrawsWithTheSameBindingsShareArgumentBuffersAndAChangedBindingIsSeen Assert.True(reuses > 0, "no draw reused an earlier draw's argument buffer"); Assert.True(writes > 0, "no argument buffer was written"); } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void FixedAttachmentDrawsFollowChangedArgumentOffsetsAndEncoderRestart(bool reuse) + { + using var rig = CreateRig(); + if (rig is null) return; + rig.Host.GraphicsBindingReuseEnabled = reuse; + const uint red = 0xFF0000FF; + const uint green = 0xFF00FF00; + const uint width = 16; + var redTexture = SolidTexture(rig, red); + var greenTexture = SolidTexture(rig, green); + var target = rig.ColorTarget(width, width, 16, 0xFF); + var state = new SceneState(width, width).ColorTarget(0, target, width, width, format: 14, numberType: 7); + SceneVertex[] quad = + [ + SceneVertex.At(-1, -1, 0.5f, 0, 1, 0, 1), SceneVertex.At(1, -1, 0.5f, 1, 1, 0, 1), + SceneVertex.At(-1, 1, 0.5f, 0, 0, 0, 1), SceneVertex.At(-1, 1, 0.5f, 0, 0, 0, 1), + SceneVertex.At(1, -1, 0.5f, 1, 1, 0, 1), SceneVertex.At(1, 1, 0.5f, 1, 0, 0, 1), + ]; + void Stripe(ScenePm4 stream, int left, int right, ulong texture) + { + state.ScreenScissor(left, 0, right, (int)width); + uint[] data = [.. Descriptors.Texture(texture, Rgba8, Size, Size), .. Descriptors.Sampler(), 0, 0, 0, 0]; + rig.DrawMesh(stream, state, quad, Sample, data); + } + float[] Pixel(int x) + { + var pixels = MemoryMarshal.Cast(rig.ReadImage(target, width, width, 16)); + return pixels.Slice((int)((8 * width + (uint)x) * 4), 4).ToArray(); + } + + var pm4 = new ScenePm4(); + Stripe(pm4, 0, 4, redTexture); + Stripe(pm4, 4, 8, redTexture); // Same fragment arguments: native state can survive. + Stripe(pm4, 8, 12, greenTexture); // Same ring buffer, different argument content/range. + Stripe(pm4, 12, 16, redTexture); // Return to the earlier argument range. + rig.Submit(pm4); + AssertColour("first red", Pixel(2), red); + AssertColour("repeated red", Pixel(6), red); + AssertColour("changed green", Pixel(10), green); + AssertColour("returned red", Pixel(14), red); + var first = rig.Host.GraphicsBindingCounters(MetalGraphicsBindingStage.Fragment); + Assert.True(first.Attempted > 0); + if (reuse) Assert.True(first.Skipped > 0, "No repeated fragment binding was skipped inside the fixed render pass."); + else Assert.Equal(0L, first.Skipped); + + // Submit drained and ended the old encoder. The first draw of the next one + // must bind every valid fragment slot even when its material was seen before. + var restart = new ScenePm4(); + Stripe(restart, 0, 16, redTexture); + rig.Submit(restart); + var after = rig.Host.GraphicsBindingCounters(MetalGraphicsBindingStage.Fragment); + Assert.True(after.Attempted > first.Attempted); + Assert.Equal(first.Skipped, after.Skipped); + AssertColour("restarted encoder", Pixel(8), red); + Assert.Empty(rig.Fatals); + output.WriteLine($"graphics binding reuse={reuse} attempted={rig.Host.GraphicsBindingAttempts} skipped={rig.Host.GraphicsBindingSkipped}"); + } + } diff --git a/tests/SharpEmu.Gpu.Metal.Tests/Scenes/GpuWordNoWaitSceneTests.cs b/tests/SharpEmu.Gpu.Metal.Tests/Scenes/GpuWordNoWaitSceneTests.cs index 938a99107..aad2fc97c 100644 --- a/tests/SharpEmu.Gpu.Metal.Tests/Scenes/GpuWordNoWaitSceneTests.cs +++ b/tests/SharpEmu.Gpu.Metal.Tests/Scenes/GpuWordNoWaitSceneTests.cs @@ -128,8 +128,7 @@ public void AReadAheadOfTheStreamGivesUpInsteadOfWaitingForTheGpu() { MetalV2Native.SendEventValue(host.CommandBuffer, SelEncodeWaitForEvent, gate, 1); rig.Enqueue(writer); - host.RunPendingCommands(); - host.RunCommandStreamSlices(64); + rig.RecordPending(); var waitsBefore = host.HostGpuWaits; Assert.Throws(() => @@ -183,8 +182,7 @@ public void APacketSizedCopyOfAWordTheGpuStillWritesIsCopiedOnTheGpu(bool queued var waitsBefore = host.HostGpuWaits; rig.Enqueue(stream); - host.RunPendingCommands(); - host.RunCommandStreamSlices(64); + rig.RecordPending(); Assert.False(host.CommandStream.HasUnblockedPending); // Parsing the copy waited for nothing. Assert.Equal(waitsBefore, host.HostGpuWaits); @@ -235,8 +233,7 @@ private static void RunBoth(SceneRig rig, uint[] first, uint[] second) { rig.Enqueue(first); rig.Enqueue(second); - rig.Host.RunPendingCommands(); - rig.Host.RunCommandStreamSlices(64); + rig.RecordPending(); Assert.False(rig.Host.CommandStream.HasPending); } diff --git a/tests/SharpEmu.Gpu.Metal.Tests/Scenes/GpuWordWaitSceneTests.cs b/tests/SharpEmu.Gpu.Metal.Tests/Scenes/GpuWordWaitSceneTests.cs index 1f5bddc9d..de2714c04 100644 --- a/tests/SharpEmu.Gpu.Metal.Tests/Scenes/GpuWordWaitSceneTests.cs +++ b/tests/SharpEmu.Gpu.Metal.Tests/Scenes/GpuWordWaitSceneTests.cs @@ -171,11 +171,9 @@ public void AWaitRegMemOnALabelAnotherQueueHasYetToWriteResolvesWithoutAHostWait var waitsBefore = host.HostGpuWaits; var suspendsBefore = SharpEmu.Libs.Gpu.GpuCommands.GpuCommandInterpreter.OperandSuspends; rig.Enqueue(producer); - host.RunPendingCommands(); - host.RunCommandStreamSlices(64); + rig.RecordPending(); rig.Enqueue(waiter, queue: 0x20); - host.RunPendingCommands(); - host.RunCommandStreamSlices(64); + rig.RecordPending(); Assert.Equal(0, Volatile.Read(ref released)); // The wait met the pending label: the compute queue ran to its end without a host wait or a park. @@ -212,8 +210,7 @@ private static void RunBoth(SceneRig rig, uint[] first, uint[] second) { rig.Enqueue(first); rig.Enqueue(second); - rig.Host.RunPendingCommands(); - rig.Host.RunCommandStreamSlices(64); + rig.RecordPending(); Assert.False(rig.Host.CommandStream.HasPending); } diff --git a/tests/SharpEmu.Gpu.Metal.Tests/Scenes/NativeOcclusionSceneTests.cs b/tests/SharpEmu.Gpu.Metal.Tests/Scenes/NativeOcclusionSceneTests.cs new file mode 100644 index 000000000..502f00f0e --- /dev/null +++ b/tests/SharpEmu.Gpu.Metal.Tests/Scenes/NativeOcclusionSceneTests.cs @@ -0,0 +1,289 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using Xunit; +using Xunit.Abstractions; + +namespace SharpEmu.Gpu.Metal.Tests.Scenes; + +public sealed class NativeOcclusionSceneTests(ITestOutputHelper output) : SceneTestBase(output) +{ + private const uint Size = 32; + private const ulong Ready = 1UL << 63; + private static ulong Query(SceneRig rig) => rig.Memory.Allocate(256, 64); + private static void Dump(ScenePm4 pm4, ulong address) => pm4.EventWithAddress(ScenePm4.EventPixelPipeStatDump, 1, address); + private static ulong Delta(SceneRig rig, ulong query) + { + var words = rig.Read(query, 32); + ulong samples = 0; + for (var block = 0; block < 16; block++) + { + Assert.Equal(Ready, words[block * 2] & Ready); + Assert.Equal(Ready, words[block * 2 + 1] & Ready); + samples += (words[block * 2 + 1] & ~Ready) - (words[block * 2] & ~Ready); + } + return samples; + } + + [Fact] + public void DisabledCountingPassesDoNotReduceOrChangeTheCounter() + { + using var rig = CreateRig(); + if (rig is null) return; + rig.Host.NativeOcclusionEnabled = true; + var targets = new[] { rig.ColorTarget(Size, Size), rig.ColorTarget(Size, Size), rig.ColorTarget(Size, Size) }; + var seed = new ScenePm4(); + var counted = new SceneState(Size, Size).ColorTarget(0, targets[0], Size, Size); + counted.Context[Cx.DbCountControl] = 1u << 8; + rig.DrawMesh(seed, counted, SceneDraws.Quad(0.1f, 1, 0, 0)); + rig.Submit(seed); + var before = rig.Host.NativeOcclusionTotals; + var sparse = rig.Memory.Allocate(128, 64); + const ulong Sentinel = 0x123456789ABCDEF0; + for (var i = 0; i < 16; i++) rig.Memory.Write64(sparse + (ulong)i * 8, Sentinel); + var pm4 = new ScenePm4(); + foreach (var target in targets) + { + var state = new SceneState(Size, Size).ColorTarget(0, target, Size, Size); + state.Context[Cx.DbCountControl] = 0; + rig.DrawMesh(pm4, state, SceneDraws.Quad(0.1f, 1, 0, 0)); + // Only hardware block three is selected; it publishes the ready-only + // word and must not make this disabled pass produce a prefix snapshot. + pm4.EventWithAddress(ScenePm4.EventPixelPipeStatControl, 1, (3UL << 9) | (8UL << 11)); + Dump(pm4, sparse); + } + rig.Submit(pm4); + Assert.True(rig.Host.TryReadOcclusionCounter(0, out var count)); + Assert.Equal((ulong)Size * Size, count); + var totals = rig.Host.NativeOcclusionTotals; + Assert.Equal(before.Reductions, totals.Reductions); + Assert.True(totals.UnusedPasses - before.UnusedPasses >= 3, $"Expected three disabled passes: {totals}."); + Assert.True(totals.InitializedSlots - before.InitializedSlots < 32); + var words = rig.Read(sparse, 16); + for (var i = 0; i < words.Length; i++) Assert.Equal(i == 12 ? Ready : Sentinel, words[i]); + foreach (var target in targets) + Assert.Equal(0xFF0000FFu, SceneDraws.Pixel(rig.ReadImage(target, Size, Size), Size, Size / 2, Size / 2)); + } + + [Fact] + public void CountingModeTransitionsAndEmptySnapshotsPreserveEarlierResults() + { + using var rig = CreateRig(); + if (rig is null) return; + rig.Host.NativeOcclusionEnabled = true; + var target = rig.ColorTarget(Size, Size); + var state = new SceneState(Size, Size).ColorTarget(0, target, Size, Size); + var first = Query(rig); var second = Query(rig); + var pm4 = new ScenePm4(); + Dump(pm4, first); + state.Context[Cx.DbCountControl] = 1u << 8; + rig.DrawMesh(pm4, state, SceneDraws.Quad(0.1f, 1, 0, 0)); + Dump(pm4, first + 8); + Dump(pm4, second); + state.Context[Cx.DbCountControl] = 0; + rig.DrawMesh(pm4, state, SceneDraws.Quad(0.1f, 0, 1, 0)); + state.Context[Cx.DbCountControl] = (1u << 8) | 1u; + rig.DrawMesh(pm4, state, SceneDraws.Quad(0.1f, 0, 1, 0)); + state.Context[Cx.DbCountControl] = 1u << 8; + rig.DrawMesh(pm4, state, SceneDraws.Quad(0.1f, 0, 0, 1)); + Dump(pm4, second + 8); + rig.Submit(pm4); + Assert.Equal((ulong)Size * Size, Delta(rig, first)); + Assert.Equal((ulong)Size * Size, Delta(rig, second)); + Assert.True(rig.Host.TryReadOcclusionCounter(0, out var count)); + Assert.Equal(2UL * Size * Size, count); + Assert.True(rig.Host.NativeOcclusionTotals.InitializedSlots < 32); + } + + [Fact] + public void HiddenVisibleAndEmptyQueriesKeepDistinctSnapshotsWithoutSplittingEveryDump() + { + using var rig = CreateRig(); + if (rig is null) return; + rig.Host.NativeOcclusionEnabled = true; + var d = new DepthRig(rig, Size, Size, 3); + var setup = d.State().DepthControl(true, true, CompareFunc.Always); + setup.Context[Cx.DbCountControl] = 0; + d.Draw(setup, d.Full(0.25f, 1, 0, 0)); + var tested = d.State().DepthControl(true, false, CompareFunc.Less); + tested.Context[Cx.DbCountControl] = 1u << 8; + var hidden = Query(rig); var visible = Query(rig); var empty = Query(rig); + Dump(d.Pm4, hidden); + d.Draw(tested, d.Full(0.75f, 0, 1, 0)); + Dump(d.Pm4, hidden + 8); + Dump(d.Pm4, visible); + d.Draw(tested, d.Full(0.1f, 0, 1, 0)); + d.Draw(tested, d.Full(0.1f, 0, 1, 0)); + Dump(d.Pm4, visible + 8); + Dump(d.Pm4, empty); Dump(d.Pm4, empty + 8); + rig.Submit(d.Pm4); + Assert.Equal(0UL, Delta(rig, hidden)); + Assert.Equal(2UL * Size * Size, Delta(rig, visible)); + Assert.Equal(0UL, Delta(rig, empty)); + var totals = rig.Host.NativeOcclusionTotals; + Assert.Equal(6, totals.Dumps); + Assert.True(totals.Reductions < totals.Dumps, $"Each dump split a pass: {totals}."); + Assert.True(rig.Host.TryReadOcclusionCounter(0, out var count)); + Assert.Equal(2UL * Size * Size, count); + } + + [Fact] + public void ResetAndSparseDumpPreserveOtherBytesAndQueues() + { + using var rig = CreateRig(); + if (rig is null) return; + rig.Host.NativeOcclusionEnabled = true; + var target = rig.ColorTarget(Size, Size); + var state = new SceneState(Size, Size).ColorTarget(0, target, Size, Size); + state.Context[Cx.DbCountControl] = 1u << 8; + var pm4 = new ScenePm4(); + rig.DrawMesh(pm4, state, SceneDraws.Quad(0.1f, 1, 0, 0)); + rig.Submit(pm4); + Assert.True(rig.Host.TryReadOcclusionCounter(0, out var drawn)); + Assert.Equal((ulong)Size * Size, drawn); + var result = rig.Memory.Allocate(128, 64); + for (var i = 0; i < 16; i++) rig.Memory.Write64(result + (ulong)i * 8, 0x123456789ABCDEF0); + Assert.True(rig.Host.DumpOcclusionCounter(0, result, 9, 32)); + Assert.True(rig.Host.ResetOcclusionCounter(1)); + Assert.True(rig.Host.TryReadOcclusionCounter(0, out var unchanged)); + Assert.Equal(drawn, unchanged); + var words = rig.Read(result, 16); + for (var i = 0; i < words.Length; i++) + Assert.Equal(i == 0 ? Ready | drawn : i == 12 ? Ready : 0x123456789ABCDEF0UL, words[i]); + Assert.True(rig.Host.ResetOcclusionCounter(0)); + Assert.True(rig.Host.TryReadOcclusionCounter(0, out var reset)); + Assert.Equal(0UL, reset); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void QueryPairsRemainCorrectAcrossCommandBuffersAndReusedGuestMemory(bool visible) + { + using var rig = CreateRig(); + if (rig is null) return; + rig.Host.NativeOcclusionEnabled = true; + var target = rig.ColorTarget(Size, Size); + var state = new SceneState(Size, Size).ColorTarget(0, target, Size, Size); + state.Context[Cx.DbCountControl] = 1u << 8; + var query = Query(rig); + for (var round = 0; round < 4; round++) + { + // A previous ready pair must not become the result of a new empty query. + var begin = new ScenePm4(); Dump(begin, query); rig.Submit(begin); + var end = new ScenePm4(); + if (visible && (round & 1) == 0) rig.DrawMesh(end, state, SceneDraws.Quad(0.1f, 1, 0, 0)); + Dump(end, query + 8); rig.Submit(end); + Assert.Equal(visible && (round & 1) == 0 ? (ulong)Size * Size : 0UL, Delta(rig, query)); + } + } + [Theory] + [InlineData(false, false)] + [InlineData(true, false)] + [InlineData(false, true)] + [InlineData(true, true)] + public void ClearedQueryMemoryAndPendingPredicationRespectVisibility(bool visible, bool noWait) + { + using var rig = CreateRig(); + if (rig is null) return; + rig.Host.NativeOcclusionEnabled = true; + var query = Query(rig); + // Seed a stale visible pair, then replace it through guest WRITE_DATA. + for (var block = 0; block < 16; block++) + { + rig.Memory.Write64(query + (ulong)block * 16, Ready); + rig.Memory.Write64(query + (ulong)block * 16 + 8, Ready | 123); + } + var target = rig.ColorTarget(Size, Size); + var state = new SceneState(Size, Size).ColorTarget(0, target, Size, Size); + state.Context[Cx.DbCountControl] = 1u << 8; + var pm4 = new ScenePm4(); + var clear = new uint[67]; + clear[0] = 1u << 8; clear[1] = (uint)query; clear[2] = (uint)(query >> 32); + pm4.Packet(ScenePm4.OpWriteData, clear); + Dump(pm4, query); + if (visible) rig.DrawMesh(pm4, state, SceneDraws.Quad(0.1f, 1, 0, 0, right: 0f)); + Dump(pm4, query + 8); + pm4.SetPredication(1, true, noWait, false, query); + rig.DrawMesh(pm4, state, SceneDraws.Quad(0.1f, 0, 1, 0, left: 0f), predicated: true); + rig.Submit(pm4); + var pixel = SceneDraws.Pixel(rig.ReadImage(target, Size, Size), Size, Size * 3 / 4, Size / 2); + Assert.Equal(visible || noWait ? 0xFF00FF00u : 0u, pixel); + Assert.Equal(visible ? (ulong)Size * Size / 2 : 0UL, Delta(rig, query)); + } + + [Fact] + public void PacketResetAndComputeQueueLeaveGraphicsCounterIndependent() + { + using var rig = CreateRig(); + if (rig is null) return; + rig.Host.NativeOcclusionEnabled = true; + var query = Query(rig); + var target = rig.ColorTarget(Size, Size); + var state = new SceneState(Size, Size).ColorTarget(0, target, Size, Size); + state.Context[Cx.DbCountControl] = 1u << 8; + var graphics = new ScenePm4(); + rig.DrawMesh(graphics, state, SceneDraws.Quad(0.1f, 1, 0, 0)); + rig.Submit(graphics); + var compute = new ScenePm4().Event(ScenePm4.EventPixelPipeStatReset); + Dump(compute, query); Dump(compute, query + 8); + rig.Submit(compute.ToArray(), queue: 0x20); + Assert.Equal(0UL, Delta(rig, query)); + Assert.True(rig.Host.TryReadOcclusionCounter(0, out var graphicsCount)); + Assert.Equal((ulong)Size * Size, graphicsCount); + rig.Submit(new ScenePm4().Event(ScenePm4.EventPixelPipeStatReset)); + Assert.True(rig.Host.TryReadOcclusionCounter(0, out var reset)); + Assert.Equal(0UL, reset); + } + + [Fact] + public void MoreThanOneBatchOfDumpsPreservesEverySnapshot() + { + using var rig = CreateRig(); + if (rig is null) return; + rig.Host.NativeOcclusionEnabled = true; + const int count = 4100; + var results = rig.Memory.Allocate(count * 256, 64); + var target = rig.ColorTarget(Size, Size); + var state = new SceneState(Size, Size).ColorTarget(0, target, Size, Size); + state.Context[Cx.DbCountControl] = 1u << 8; + var pm4 = new ScenePm4(); + for (var i = 0; i < count; i++) + { + if (i == 0 || i == 4096) rig.DrawMesh(pm4, state, SceneDraws.Quad(0.1f, 1, 0, 0)); + Dump(pm4, results + (ulong)i * 256); + } + rig.Submit(pm4); + var words = rig.Read(results, count * 32); + for (var i = 0; i < count; i++) + Assert.Equal(Ready | ((ulong)Size * Size * (i < 4096 ? 1UL : 2UL)), words[i * 32]); + } + + [Fact] + public void BatchBudgetWaitsForCompletedStorageWithoutLosingDrawsOrDumps() + { + using var rig = CreateRig(); + if (rig is null) return; + rig.Host.NativeOcclusionEnabled = true; + rig.Host.NativeOcclusionBatchBudget = 2; + var targets = new[] { rig.ColorTarget(Size, Size), rig.ColorTarget(Size, Size) }; + var query = Query(rig); + var pm4 = new ScenePm4(); + Dump(pm4, query); + for (var i = 0; i < 12; i++) + { + var state = new SceneState(Size, Size).ColorTarget(0, targets[i % 2], Size, Size); + state.Context[Cx.DbCountControl] = 1u << 8; + rig.DrawMesh(pm4, state, SceneDraws.Quad(0.1f, 1, 0, 0)); + } + Dump(pm4, query + 8); + rig.Submit(pm4); + Assert.Equal(12UL * Size * Size, Delta(rig, query)); + var totals = rig.Host.NativeOcclusionTotals; + Assert.Equal(2, totals.Buffers); + Assert.True(totals.BudgetWaits > 0, $"The tiny budget did not exercise backpressure: {totals}."); + foreach (var target in targets) + Assert.Equal(0xFF0000FFu, SceneDraws.Pixel(rig.ReadImage(target, Size, Size), Size, Size / 2, Size / 2)); + } + +} diff --git a/tests/SharpEmu.Gpu.Metal.Tests/Scenes/OcclusionSceneTests.cs b/tests/SharpEmu.Gpu.Metal.Tests/Scenes/OcclusionSceneTests.cs index 0b4728a81..8e01de4ad 100644 --- a/tests/SharpEmu.Gpu.Metal.Tests/Scenes/OcclusionSceneTests.cs +++ b/tests/SharpEmu.Gpu.Metal.Tests/Scenes/OcclusionSceneTests.cs @@ -21,6 +21,43 @@ public sealed class OcclusionSceneTests(ITestOutputHelper output) : SceneTestBas private const uint Red = 0xFF0000FF; private const uint Green = 0xFF00FF00; + [Fact] + public void HardwareAuditDistinguishesHiddenDrawsWithoutChangingGuestQueries() + { + using var rig = CreateRig(); + if (rig is null) return; + rig.Host.OcclusionAuditEnabled = true; + var depth = new DepthRig(rig, Size, Size, 3); + var pm4 = depth.Pm4; + var occluder = depth.State().DepthControl(true, true, CompareFunc.Always); + occluder.Context[Cx.DbCountControl] = 0; + depth.Draw(occluder, depth.Full(0.25f, 1, 0, 0)); + var query = Query(rig); + Begin(pm4, query); + var tested = depth.State().DepthControl(true, false, CompareFunc.Less); + tested.Context[Cx.DbCountControl] = 1u << 8; + depth.Draw(tested, depth.Full(0.75f, 0, 1, 0)); + End(pm4, query); + depth.Draw(tested, depth.Full(0.1f, 0, 1, 0)); + rig.Submit(pm4); + var totals = rig.Host.OcclusionAuditTotals; + Assert.Equal(2, totals.Groups); + Assert.Equal(1, totals.ZeroGroups); + Assert.Equal((ulong)(Size * Size), totals.Samples); + Assert.Equal(0, totals.Skipped); + var words = rig.Read(query, DepthBlocks * 2); + Assert.True((words[1] & ~Ready) > (words[0] & ~Ready)); + + // Recycled buffers and later submissions cannot expose stale visible counts. + var hidden = new ScenePm4(); + rig.DrawMesh(hidden, tested, depth.Full(0.75f, 1, 0, 0)); + rig.Submit(hidden); + totals = rig.Host.OcclusionAuditTotals; + Assert.Equal(3, totals.Groups); + Assert.Equal(2, totals.ZeroGroups); + Assert.Equal((ulong)(Size * Size), totals.Samples); + } + private sealed class Scene { public required SceneRig Rig { get; init; } diff --git a/tests/SharpEmu.Gpu.Metal.Tests/Scenes/QueuedCommandWriteSceneTests.cs b/tests/SharpEmu.Gpu.Metal.Tests/Scenes/QueuedCommandWriteSceneTests.cs new file mode 100644 index 000000000..d590c7538 --- /dev/null +++ b/tests/SharpEmu.Gpu.Metal.Tests/Scenes/QueuedCommandWriteSceneTests.cs @@ -0,0 +1,62 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using Xunit; +using Xunit.Abstractions; + +namespace SharpEmu.Gpu.Metal.Tests.Scenes; + +public sealed class QueuedCommandWriteSceneTests(ITestOutputHelper output) : SceneTestBase(output) +{ + private static void Write(ScenePm4 stream, ulong address, params uint[] words) => + stream.Packet(ScenePm4.OpWriteData, [1u << 8, (uint)address, (uint)(address >> 32), .. words]); + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void RewrittenIndirectPacketsAreReadAfterTheQueuedWrite(bool queued) + { + using var rig = CreateRig(); + if (rig is null) return; + rig.Host.QueueCommandProcessorWrites = queued; + var outputWord = rig.Memory.Allocate(16); + var stale = new ScenePm4(); Write(stale, outputWord, 0xBAD); + var child = rig.Memory.Upload(stale.ToArray()); + var parent = new ScenePm4(); + // Rewrite the packet's payload, then follow it without a GPU completion wait. + // The parser must see the worker's pending CP overlay rather than stale RAM. + Write(parent, child + 16, 0xC0FFEE); + parent.Packet(0x3F, (uint)child, (uint)(child >> 32), (uint)stale.Count); + Write(parent, outputWord + 4, 0x1234); + rig.Submit(parent); + Assert.Equal(new uint[] { 0xC0FFEE, 0x1234 }, rig.Read(outputWord, 2)); + Assert.Empty(rig.Fatals); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void OverlappingQueuedWritesPreserveBytesAndPredicationOrder(bool queued) + { + using var rig = CreateRig(); + if (rig is null) return; + rig.Host.QueueCommandProcessorWrites = queued; + var query = rig.Memory.Allocate(16); + var outputWord = rig.Memory.Allocate(16); + var stream = new ScenePm4(); + Write(stream, query, 0, 0, 0xAA, 0xBB); + Write(stream, query + 4, 1); + // BOOL64 is nonzero only because the second write changes the high word. + stream.SetPredication(3, true, false, false, query); + stream.Packet(ScenePm4.OpWriteData, true, 1u << 8, (uint)outputWord, (uint)(outputWord >> 32), 0x55); + stream.SetPredication(0, false, false, false, 0); + Write(stream, query, 0, 0); + stream.SetPredication(3, true, false, false, query); + stream.Packet(ScenePm4.OpWriteData, true, 1u << 8, (uint)outputWord, (uint)(outputWord >> 32), 0xBAD); + stream.SetPredication(0, false, false, false, 0); + rig.Submit(stream); + Assert.Equal(0x55u, rig.Read(outputWord, 1)[0]); + Assert.Equal(new uint[] { 0, 0, 0xAA, 0xBB }, rig.Read(query, 4)); + Assert.Empty(rig.Fatals); + } +} diff --git a/tests/SharpEmu.Gpu.Metal.Tests/Scenes/QueuedQwordWriteSceneTests.cs b/tests/SharpEmu.Gpu.Metal.Tests/Scenes/QueuedQwordWriteSceneTests.cs new file mode 100644 index 000000000..bafaa662f --- /dev/null +++ b/tests/SharpEmu.Gpu.Metal.Tests/Scenes/QueuedQwordWriteSceneTests.cs @@ -0,0 +1,212 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using System.Reflection; +using SharpEmu.Gpu.Metal.V2; +using SharpEmu.Libs.Gpu.GpuCommands; +using Xunit; +using Xunit.Abstractions; + +namespace SharpEmu.Gpu.Metal.Tests.Scenes; + +public sealed class QueuedQwordWriteSceneTests(ITestOutputHelper output) : SceneTestBase(output) +{ + private static void Write(ScenePm4 stream, ulong address, params uint[] words) => + stream.Packet(ScenePm4.OpWriteData, [1u << 8, (uint)address, (uint)(address >> 32), .. words]); + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void ProducerQwordRewriteIsVisibleToTheFollowingIndirectPacket(bool queued) + { + using var rig = CreateRig(); + if (rig is null) return; + rig.Host.QueueCommandProcessorWrites = queued; + var destination = rig.Memory.Allocate(16); + var childStream = new ScenePm4(); + Write(childStream, destination, 0xBAD, 0xBAD); + var child = rig.Memory.Upload(childStream.ToArray()); + using var producer = new ProducerHarness(rig); + using var returned = new ManualResetEventSlim(); + producer.Enqueue(() => + { + Assert.True(producer.Thread.IsProducerThread); + rig.Host.RecordCommandProcessorQwords(child + 16, 1, ulong.MaxValue, 0x123400C0FFEEUL, "test-indirect"); + returned.Set(); + }); + var parent = new ScenePm4(); + parent.Packet(0x3F, (uint)child, (uint)(child >> 32), (uint)childStream.Count); + rig.Enqueue(parent.ToArray()); + producer.Start(); + if (queued) + { + // No worker has run: returning here proves the actual producer queued the run. + Assert.True(returned.Wait(TimeSpan.FromSeconds(5))); + var stale = new byte[4]; + Assert.True(rig.Memory.TryRead(child + 16, stale)); + Assert.Equal(0xBADu, BitConverter.ToUInt32(stale)); + } + else + { + Assert.True(SpinWait.SpinUntil(() => producer.Thread.TakeCounters().SyncRuns != 0, TimeSpan.FromSeconds(5))); + Assert.False(returned.IsSet); // Disabled mode waits for the worker as before. + } + producer.Drain(); + Assert.True(returned.IsSet); + Assert.Equal(new uint[] { 0xC0FFEE, 0x1234 }, rig.Read(destination, 2)); + Assert.Empty(rig.Fatals); + } + + [Theory] + [InlineData(false, 16UL)] + [InlineData(true, 16UL)] + [InlineData(false, 0UL)] + [InlineData(true, 0UL)] + public void ByteAndQwordWritesKeepPredicationOrderAndSparseGaps(bool queued, ulong stride) + { + using var rig = CreateRig(); + if (rig is null) return; + rig.Host.QueueCommandProcessorWrites = queued; + var query = rig.Memory.Upload(new ulong[] { 0, 0xAAAAAAAAAAAAAAAA, 0, 0xBBBBBBBBBBBBBBBB }); + var destination = rig.Memory.Allocate(16); + using var producer = new ProducerHarness(rig); + producer.Enqueue(() => + { + rig.Host.RecordCommandProcessorWrite(query, BitConverter.GetBytes(0UL), "test-before"); + rig.Host.RecordCommandProcessorQwords(query, 2, stride, 1, "test-query"); + rig.Host.RecordCommandProcessorWrite(query + 4, BitConverter.GetBytes(2u), "test-after"); + }); + var stream = new ScenePm4(); + stream.SetPredication(3, true, false, false, query); + stream.Packet(ScenePm4.OpWriteData, true, 1u << 8, (uint)destination, (uint)(destination >> 32), 0x55); + stream.SetPredication(0, false, false, false, 0); + rig.Enqueue(stream.ToArray()); + producer.Start(); + producer.Drain(); + Assert.Equal(0x55u, rig.Read(destination, 1)[0]); + Assert.Equal(new ulong[] { 0x200000001, 0xAAAAAAAAAAAAAAAA, stride == 0 ? 0UL : 1UL, 0xBBBBBBBBBBBBBBBB }, + rig.Read(query, 4)); + Assert.Empty(rig.Fatals); + } + + [Fact] + public void AReadInsideASparseGapWaitsForTheConservativePendingHull() + { + using var rig = CreateRig(); + if (rig is null) return; + rig.Host.QueueCommandProcessorWrites = true; + var address = rig.Memory.Upload(new ulong[] { 0, 0x1234, 0 }); + using var producer = new ProducerHarness(rig); + using var queued = new ManualResetEventSlim(); + using var readReturned = new ManualResetEventSlim(); + ulong gap = 0; + producer.Enqueue(() => + { + rig.Host.RecordCommandProcessorQwords(address, 2, 16, 7, "test-sparse"); + queued.Set(); + var bytes = new byte[8]; + Assert.True(rig.Host.TryReadGuest(address + 8, bytes)); + gap = BitConverter.ToUInt64(bytes); + readReturned.Set(); + }); + producer.Start(); + Assert.True(queued.Wait(TimeSpan.FromSeconds(5))); + // Pending-range synchronization is installed before the producer advances. + Assert.True(SpinWait.SpinUntil(() => producer.Thread.TakeCounters().SyncRuns != 0, TimeSpan.FromSeconds(5))); + Assert.False(readReturned.IsSet); + producer.Drain(); + Assert.Equal(0x1234UL, gap); + Assert.Equal(new ulong[] { 7, 0x1234, 7 }, rig.Read(address, 3)); + Assert.Empty(rig.Fatals); + } + + [Fact] + public void WrappingRunsRetainSynchronousStoreByStoreBehavior() + { + using var rig = CreateRig(); + if (rig is null) return; + rig.Host.QueueCommandProcessorWrites = true; + var address = rig.Memory.Allocate(32) + 8; + using var producer = new ProducerHarness(rig); + producer.Enqueue(() => rig.Host.RecordCommandProcessorQwords(address, 2, ulong.MaxValue, + 0x0807060504030201, "test-wrap")); + producer.Start(); + Assert.True(SpinWait.SpinUntil(() => producer.Thread.TakeCounters().SyncRuns != 0, TimeSpan.FromSeconds(5))); + producer.Drain(); + Assert.Equal(new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 8 }, rig.Read(address - 1, 9)); + Assert.Empty(rig.Fatals); + } + + [Theory] + [InlineData(0)] + [InlineData(-1)] + public void NonpositiveRunsDoNotQueueOrSynchronize(int count) + { + using var rig = CreateRig(); + if (rig is null) return; + rig.Host.QueueCommandProcessorWrites = true; + using var producer = new ProducerHarness(rig); + using var returned = new ManualResetEventSlim(); + producer.Enqueue(() => + { + rig.Host.RecordCommandProcessorQwords(ulong.MaxValue, count, ulong.MaxValue, 1, "test-empty"); + returned.Set(); + }); + producer.Start(); + Assert.True(returned.Wait(TimeSpan.FromSeconds(5))); + Assert.Equal(0L, producer.Thread.TakeCounters().Items); + producer.Drain(); + Assert.Empty(rig.Fatals); + } + + // Install a real producer on the host without changing process-wide environment flags. + // These scenes issue no draws, so no translation sink or prefetch worker is required. + private sealed class ProducerHarness : IDisposable + { + private static readonly FieldInfo CommandThread = typeof(MetalRenderHost).GetField("_commandThread", + BindingFlags.Instance | BindingFlags.NonPublic)!; + private readonly SceneRig _rig; + private bool _started; + internal DeferredCommandThread Thread { get; } + + internal ProducerHarness(SceneRig rig) + { + _rig = rig; + Assert.Null(CommandThread.GetValue(rig.Host)); + Thread = new DeferredCommandThread(rig.Host.CommandStream, rig.Memory, + _ => throw new InvalidOperationException("This test must not enqueue draws."), null, static () => { }); + CommandThread.SetValue(rig.Host, Thread); + } + + internal void Enqueue(Action action) => + Assert.True(_rig.Host.CommandStream.TryEnqueueControlBarrier(action, static () => { })); + + internal void Start() { Thread.Start(); _started = true; } + + internal void Drain() + { + using var consumed = new ManualResetEventSlim(); + Enqueue(consumed.Set); + var deadline = DateTime.UtcNow.AddSeconds(30); + while (!consumed.IsSet || Thread.HasWork) + { + Thread.ThrowIfFailed(); + if (DateTime.UtcNow > deadline) throw new TimeoutException("The queued qword scene did not drain."); + Thread.RunBatch(long.MaxValue, 64, static () => { }); + System.Threading.Thread.Yield(); + } + _rig.Host.CommitCoalescedFlush(); + _rig.Host.FlushAndWaitForGpu(); + _rig.Host.PollGpuCompletions(); + } + + public void Dispose() + { + try + { + if (_started) Thread.Stop(() => Thread.RunBatch(long.MaxValue, 64, static () => { })); + } + finally { CommandThread.SetValue(_rig.Host, null); } + } + } +} diff --git a/tests/SharpEmu.Gpu.Metal.Tests/Scenes/SceneRig.cs b/tests/SharpEmu.Gpu.Metal.Tests/Scenes/SceneRig.cs index b66a08d45..d1cc13486 100644 --- a/tests/SharpEmu.Gpu.Metal.Tests/Scenes/SceneRig.cs +++ b/tests/SharpEmu.Gpu.Metal.Tests/Scenes/SceneRig.cs @@ -152,15 +152,42 @@ public void Enqueue(uint[] words, int queue = 0) /// The relay the rig's host posts GPU-queue work through, for driving the worker's own pass. internal MetalGpuWorkerRelay Relay => _relay; + /// Records all queued packets without waiting for their GPU work to finish. + public void RecordPending() + { + var recorded = 0; + if (!Host.CommandStream.TryEnqueueControlBarrier( + () => Volatile.Write(ref recorded, 1), () => Volatile.Write(ref recorded, -1))) + throw new InvalidOperationException("The scene could not enqueue its recording barrier."); + var deadline = DateTime.UtcNow.AddSeconds(30); + do + { + if (DateTime.UtcNow > deadline || Volatile.Read(ref recorded) < 0) + throw new TimeoutException("The scene's recording barrier did not finish."); + Host.RunPendingCommands(); + Host.RunCommandStreamSlices(64); + Thread.Yield(); + } while (Volatile.Read(ref recorded) != 1 || Host.CommandThreadHasWork || _relay.HasPendingCommands); + } + /// Runs the worker loop until no submission is runnable, then waits for the GPU. public void Drain() { + // HasPending excludes a slice the producer is currently interpreting. A + // trailing control barrier proves it handed every earlier item to the worker. + var producerDrained = Host.UsesCommandThread ? 0 : 1; + if (Host.UsesCommandThread && !Host.CommandStream.TryEnqueueControlBarrier( + () => Volatile.Write(ref producerDrained, 1), + () => Volatile.Write(ref producerDrained, -1))) + throw new InvalidOperationException("The scene could not enqueue its producer barrier."); var deadline = DateTime.UtcNow.AddSeconds(30); while (true) { + if (DateTime.UtcNow > deadline || Volatile.Read(ref producerDrained) < 0) + throw new TimeoutException("The scene's command stream did not finish its producer barrier."); Host.RunPendingCommands(); Host.RunCommandStreamSlices(64); - if (Host.CommandStream.HasUnblockedPending || _relay.HasPendingCommands) + if (Host.CommandStream.HasUnblockedPending || Host.CommandThreadHasWork || _relay.HasPendingCommands) { continue; } @@ -168,7 +195,10 @@ public void Drain() Host.CommitCoalescedFlush(); Host.FlushAndWaitForGpu(); Host.PollGpuCompletions(); - if (!Host.CommandStream.HasPending) + // A producer may have consumed the final packet while its worker items + // still own the draws and result copies. Stream-empty alone is not done. + if (Volatile.Read(ref producerDrained) == 1 && !Host.CommandStream.HasPending && + !Host.CommandThreadHasWork && !_relay.HasPendingCommands) { return; } diff --git a/tests/SharpEmu.Libs.Tests/Agc/AgcGeometrySnapshotValidationTests.cs b/tests/SharpEmu.Libs.Tests/Agc/AgcGeometrySnapshotValidationTests.cs index 643a654b8..70ea26822 100644 --- a/tests/SharpEmu.Libs.Tests/Agc/AgcGeometrySnapshotValidationTests.cs +++ b/tests/SharpEmu.Libs.Tests/Agc/AgcGeometrySnapshotValidationTests.cs @@ -4,6 +4,7 @@ using System.Buffers.Binary; using SharpEmu.HLE; using SharpEmu.Libs.Agc; +using SharpEmu.Libs.Gpu.GpuCommands; using Xunit; namespace SharpEmu.Libs.Tests.Agc; @@ -79,6 +80,65 @@ private static object CapturedAsThePrepassWould(ulong drawPacketAddress, ulong i instanceCount: 1, [(UserDataRegister, UserDataValue)]); + private static void SubmitDriver(FakeCpuMemory memory, uint dwords) + { + var context = new CpuContext(memory, Generation.Gen5); + const ulong submissionAddress = BaseAddress + 0x40; + context[CpuRegister.Rdi] = submissionAddress; + Assert.True(context.TryWriteUInt64(submissionAddress, CommandAddress)); + Span length = stackalloc byte[4]; + BinaryPrimitives.WriteUInt32LittleEndian(length, dwords); + Assert.True(memory.TryWrite(submissionAddress + 8, length)); + Assert.Equal(0, AgcExports.DriverSubmitDcb(context)); + } + + [Fact] + public void ExecutedShaderRegistersKeepTheInterpreterAliasAcrossClearAndReset() + { + var (memory, dwords, _) = WriteStream(SetShaderRegister(UserDataRegister, 0)); + SubmitDriver(memory, dwords); + var interpreter = AgcExports.GetHeadlessCommandStreamForTests(memory).Queue.GetInterpreter(0); + var alias = AgcExports.GetExecutedShaderRegistersForTests(memory); + Assert.Same(interpreter.Registers.Shader, alias); + Assert.True(alias.TryGetValue(UserDataRegister, out var zero)); + Assert.Equal(0u, zero); + alias.Clear(); + Assert.Empty(interpreter.Registers.Shader); + interpreter.Registers.Shader[UserDataRegister] = 9; + Assert.Equal(9u, alias[UserDataRegister]); + interpreter.Registers.Reset(); + Assert.Same(alias, interpreter.Registers.Shader); + Assert.Empty(alias); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void CaptureRetainsOverflowShaderKeysEvenWhenExecutionSkipsTheWrite(bool indirect) + { + const ulong table = PredicateAddress + 0x100; + uint[] write = indirect + ? [Pm4Header(5, PacketOpcode.SetShaderRegisterIndirect), unchecked((uint)table), (uint)(table >> 32), 0, 1] + : SetShaderRegister(uint.MaxValue, 0); + var (memory, dwords, _) = WriteStream( + ConditionalExecute(PredicateAddress, (uint)write.Length), write, + IndexType(0), DrawIndex2(IndexAddress)); + Span zero = stackalloc byte[4]; + zero.Clear(); + Assert.True(memory.TryWrite(PredicateAddress, zero)); + Span entry = stackalloc byte[8]; + BinaryPrimitives.WriteUInt32LittleEndian(entry, uint.MaxValue); + BinaryPrimitives.WriteUInt32LittleEndian(entry[4..], 0); + Assert.True(memory.TryWrite(table, entry)); + SubmitDriver(memory, dwords); + var captured = AgcExports.GetGeometryCaptureRegistersForTests(memory); + var key = indirect ? uint.MaxValue & ~0x7000_0000u : uint.MaxValue; + Assert.True(captured.TryGetValue(key, out var value)); + Assert.Equal(0u, value); + var executed = AgcExports.GetExecutedShaderRegistersForTests(memory); + Assert.False(executed.ContainsKey(key)); + } + [Fact] public void MatchingState_UsesTheSnapshot() { diff --git a/tests/SharpEmu.Libs.Tests/Ampr/AprSessionLifetimeTests.cs b/tests/SharpEmu.Libs.Tests/Ampr/AprSessionLifetimeTests.cs new file mode 100644 index 000000000..5003b861d --- /dev/null +++ b/tests/SharpEmu.Libs.Tests/Ampr/AprSessionLifetimeTests.cs @@ -0,0 +1,336 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using SharpEmu.HLE; +using SharpEmu.HLE.FileSystem; +using SharpEmu.Libs.Ampr; +using SharpEmu.Libs.Kernel; +using SharpEmu.Libs.Tests.FileSystem; +using Xunit; + +namespace SharpEmu.Libs.Tests.Ampr; + +[CollectionDefinition("APR session lifetime", DisableParallelization = true)] +public sealed class AprSessionLifetimeCollection +{ + public const string Name = "APR session lifetime"; +} + +[Collection(AprSessionLifetimeCollection.Name)] +public sealed class AprSessionLifetimeTests +{ + private const ulong Base = 0x3_0000_0000; + private const ulong Gate = Base + 0x800; + private const ulong Done = Base + 0x810; + private const ulong Destination = Base + 0x1000; + + [Fact] + public void StopJoinsAnActiveReadBeforeArchiveAndMemoryCanBeReleased() + { + var directory = Directory.CreateTempSubdirectory("sharpemu-apr-lifetime-").FullName; + var archivePath = Path.Combine(directory, "game.zar"); + var root = Path.Combine(directory, "cache"); + var bytes = Enumerable.Range(0, 4096).Select(i => (byte)i).ToArray(); + File.WriteAllBytes(archivePath, ZArchiveTestWriter.Build([("asset.bin", bytes)])); + App0Archive.Mount(ZArchive.Open(archivePath)!, root); + var memory = new ObservedMemory { BlockFill = true }; + var session = AmprExports.BeginRuntimeSession(memory); + Task? stopping = null; + try + { + var ctx = new CpuContext(memory, Generation.Gen5); + var first = BuildBuffer(ctx, 0x100); + var fileId = AmprFileRegistry.RegisterAprResolvedPath("/app0/asset.bin", Path.Combine(root, "asset.bin")); + ctx[CpuRegister.Rsp] = Base + 0x900; + Assert.True(ctx.TryWriteUInt64(Base + 0x908, 0)); + ctx[CpuRegister.Rdi] = first; + ctx[CpuRegister.Rcx] = fileId; + ctx[CpuRegister.R8] = Destination; + ctx[CpuRegister.R9] = (ulong)bytes.Length; + Assert.Equal(0, AmprExports.AprCommandBufferReadFile(ctx)); + Assert.Equal(0, Submit(ctx, first)); + Assert.True(memory.Entered.Wait(TimeSpan.FromSeconds(5))); + + var queued = BuildBuffer(ctx, 0x400); + AppendWrite(ctx, queued, Done, 1); + Assert.Equal(0, Submit(ctx, queued)); + stopping = Task.Run(session.Dispose); + // Rejection proves that stop has acquired the admission gate. The blocked + // callback still owns the archive and destination until shutdown joins it. + Assert.True(SpinWait.SpinUntil(() => Submit(ctx, queued) != 0, TimeSpan.FromSeconds(5))); + Assert.False(stopping.IsCompleted); + Assert.True(App0Archive.IsMounted); + memory.Release.Set(); + Assert.True(stopping.Wait(TimeSpan.FromSeconds(5))); + Assert.Equal(0UL, Read(ctx, Done)); + var actual = new byte[bytes.Length]; + Assert.True(memory.TryRead(Destination, actual)); + Assert.Equal(bytes, actual); + App0Archive.Unmount(); + memory.Released = true; + // A late submission is rejected without capturing or writing guest memory. + Assert.NotEqual(0, Submit(ctx, queued)); + Assert.Equal(0, memory.AccessAfterRelease); + session.Dispose(); + } + finally + { + memory.Release.Set(); + session.Dispose(); + stopping?.Wait(TimeSpan.FromSeconds(5)); + App0Archive.Unmount(); + Directory.Delete(directory, recursive: true); + } + } + + [Fact] + public void StopCancelsAnUnsatisfiedAddressWaitAndANewSessionCanReuseTheMemory() + { + var memory = new ObservedMemory(); + var ctx = new CpuContext(memory, Generation.Gen5); + using (var session = AmprExports.BeginRuntimeSession(memory)) + { + var buffer = BuildBuffer(ctx, 0x100); + var manager = new ModuleManager(); + manager.RegisterExports(SharpEmu.Generated.SysAbiExportRegistry.CreateExports(Generation.Gen5)); + ctx[CpuRegister.Rdi] = buffer; + ctx[CpuRegister.Rsi] = Gate; + ctx[CpuRegister.Rdx] = 7; + ctx[CpuRegister.Rcx] = 0; + ctx[CpuRegister.R8] = 0; + Assert.Equal(OrbisGen2Result.ORBIS_GEN2_OK, manager.Dispatch("V7GQTEeUfhw", ctx)); + AppendWrite(ctx, buffer, Done, 1); + Assert.Equal(0, Submit(ctx, buffer)); + Assert.True(memory.Entered.Wait(TimeSpan.FromSeconds(5))); + var stop = Task.Run(session.Dispose); + Assert.True(stop.Wait(TimeSpan.FromSeconds(5))); + Assert.Equal(0UL, Read(ctx, Done)); + Assert.NotEqual(0, Submit(ctx, buffer)); + } + using (AmprExports.BeginRuntimeSession(memory)) + { + var buffer = BuildBuffer(ctx, 0x400); + AppendWrite(ctx, buffer, Done, 2); + Assert.Equal(0, Submit(ctx, buffer, returnId: true)); + Assert.True(ctx.TryReadUInt32(Base + 0x40, out var id)); + ctx[CpuRegister.Rdi] = id; + Assert.Equal(0, KernelAprCompatExports.KernelAprWaitCommandBuffer(ctx)); + Assert.Equal(2UL, Read(ctx, Done)); + } + } + + [Fact] + public void CacheEvictionDoesNotCloseAFileWithAnActiveReadLease() + { + var directory = Directory.CreateTempSubdirectory("sharpemu-apr-lease-").FullName; + var path = Path.Combine(directory, "asset.bin"); + var bytes = new byte[] { 4, 5, 6, 7 }; + File.WriteAllBytes(path, bytes); + var memory = new ObservedMemory { BlockFill = true }; + var ctx = new CpuContext(memory, Generation.Gen5); + using var session = AmprExports.BeginRuntimeSession(memory); + try + { + var buffer = BuildReadBuffer(ctx, path, bytes.Length); + Assert.Equal(0, Submit(ctx, buffer, returnId: true)); + Assert.True(ctx.TryReadUInt32(Base + 0x40, out var id)); + Assert.True(memory.Entered.Wait(TimeSpan.FromSeconds(5))); + // Exercise the same eviction used for EMFILE recovery and session teardown. + typeof(AmprExports).GetMethod("EvictAllCachedHostFiles", + System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Static)!.Invoke(null, null); + memory.Release.Set(); + ctx[CpuRegister.Rdi] = id; + Assert.Equal(0, KernelAprCompatExports.KernelAprWaitCommandBuffer(ctx)); + Assert.True(ctx.TryReadUInt64(Base + 0x60, out var result)); + Assert.Equal(0UL, result); + var actual = new byte[bytes.Length]; + Assert.True(memory.TryRead(Destination, actual)); + Assert.Equal(bytes, actual); + } + finally + { + memory.Release.Set(); + session.Dispose(); + Directory.Delete(directory, recursive: true); + } + } + + [Fact] + public void RestartWithTheSameArchivePathUsesTheNewMount() + { + var directory = Directory.CreateTempSubdirectory("sharpemu-apr-remount-").FullName; + var path = Path.Combine(directory, "game.zar"); + var root = Path.Combine(directory, "cache"); + var memory = new ObservedMemory(); + var ctx = new CpuContext(memory, Generation.Gen5); + try + { + foreach (var value in new byte[] { 1, 2 }) + { + File.WriteAllBytes(path, ZArchiveTestWriter.Build([("asset.bin", new byte[] { value })])); + App0Archive.Mount(ZArchive.Open(path)!, root); + using (AmprExports.BeginRuntimeSession(memory)) + { + var buffer = BuildReadBuffer(ctx, Path.Combine(root, "asset.bin"), 1); + Assert.Equal(0, Submit(ctx, buffer, returnId: true)); + Assert.True(ctx.TryReadUInt32(Base + 0x40, out var id)); + ctx[CpuRegister.Rdi] = id; + Assert.Equal(0, KernelAprCompatExports.KernelAprWaitCommandBuffer(ctx)); + var actual = new byte[1]; + Assert.True(memory.TryRead(Destination, actual)); + Assert.Equal(value, actual[0]); + } + App0Archive.Unmount(); + } + } + finally + { + App0Archive.Unmount(); + Directory.Delete(directory, recursive: true); + } + } + + [Fact] + public void StopWaitsForAnAdmittedIdWriteWithoutHoldingTheGlobalGate() + { + var memory = new ObservedMemory(); + var ctx = new CpuContext(memory, Generation.Gen5); + var session = AmprExports.BeginRuntimeSession(memory); + var buffer = BuildBuffer(ctx, 0x100); + AppendWrite(ctx, buffer, Done, 1); + memory.BlockIdWrite = true; + Task? producer = null; + Task? stop = null; + try + { + producer = Task.Run(() => Submit(ctx, buffer, returnId: true)); + Assert.True(memory.Entered.Wait(TimeSpan.FromSeconds(5))); + stop = Task.Run(session.Dispose); + var stopping = session.GetType().GetField("Stopping", + System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Instance)!; + // Captured command state can be cached. Park the actual id write so + // admission is deterministically inside guest memory, outside the gate. + Assert.True(SpinWait.SpinUntil(() => (bool)stopping.GetValue(session)!, TimeSpan.FromSeconds(5))); + Assert.False(stop.IsCompleted); + + // A newly allocated tracker must resolve the stopped root owner too. + var lateContext = new CpuContext(new SharpEmu.Core.Cpu.TrackedCpuMemory(memory), Generation.Gen5); + var late = Task.Run(() => Submit(lateContext, buffer)); + Assert.True(late.Wait(TimeSpan.FromSeconds(5))); + Assert.NotEqual(0, late.Result); + memory.Release.Set(); + Assert.True(producer.Wait(TimeSpan.FromSeconds(5))); + Assert.NotEqual(0, producer.Result); + Assert.True(stop.Wait(TimeSpan.FromSeconds(5))); + memory.BlockIdWrite = false; + Assert.Equal(0UL, Read(ctx, Done)); + memory.Released = true; + var afterStop = new CpuContext(new SharpEmu.Core.Cpu.TrackedCpuMemory(memory), Generation.Gen5); + Assert.NotEqual(0, Submit(afterStop, buffer)); + Assert.Equal(0, memory.AccessAfterRelease); + } + finally + { + memory.Release.Set(); + session.Dispose(); + producer?.Wait(TimeSpan.FromSeconds(5)); + stop?.Wait(TimeSpan.FromSeconds(5)); + } + } + + private static ulong BuildReadBuffer(CpuContext ctx, string path, int length) + { + var buffer = BuildBuffer(ctx, 0x100); + var id = AmprFileRegistry.RegisterAprResolvedPath("/app0/" + Guid.NewGuid() + "/asset.bin", path); + ctx[CpuRegister.Rsp] = Base + 0x900; + Assert.True(ctx.TryWriteUInt64(Base + 0x908, 0)); + ctx[CpuRegister.Rdi] = buffer; + ctx[CpuRegister.Rcx] = id; + ctx[CpuRegister.R8] = Destination; + ctx[CpuRegister.R9] = (ulong)length; + Assert.Equal(0, AmprExports.AprCommandBufferReadFile(ctx)); + return buffer; + } + + private static ulong BuildBuffer(CpuContext ctx, ulong offset) + { + var buffer = Base + offset; + ctx[CpuRegister.Rdi] = buffer; + ctx[CpuRegister.Rsi] = buffer + 0x40; + ctx[CpuRegister.Rdx] = 0x100; + Assert.Equal(0, AmprExports.CommandBufferConstructor(ctx)); + Assert.Equal(0, AmprExports.CommandBufferSetBuffer(ctx)); + return buffer; + } + + private static void AppendWrite(CpuContext ctx, ulong buffer, ulong address, ulong value) + { + var manager = new ModuleManager(); + manager.RegisterExports(SharpEmu.Generated.SysAbiExportRegistry.CreateExports(Generation.Gen5)); + ctx[CpuRegister.Rdi] = buffer; + ctx[CpuRegister.Rsi] = address; + ctx[CpuRegister.Rdx] = value; + Assert.Equal(OrbisGen2Result.ORBIS_GEN2_OK, manager.Dispatch("j0+3uJMxYJY", ctx)); + } + + private static int Submit(CpuContext ctx, ulong buffer, bool returnId = false) + { + return AmprExports.SubmitCommandBuffer(ctx, buffer, priority: 0, + resultAddress: Base + 0x60, outIdAddress: returnId ? Base + 0x40 : 0, + dsReturnAddress: 0, out _); + } + + private static ulong Read(CpuContext ctx, ulong address) + { + Assert.True(ctx.TryReadUInt64(address, out var value)); + return value; + } + + private sealed class ObservedMemory : ICpuMemory + { + private readonly FakeCpuMemory _inner = new(Base, 0x4000); + public readonly ManualResetEventSlim Entered = new(); + public readonly ManualResetEventSlim Release = new(); + public bool BlockFill; + public volatile bool BlockIdWrite; + public volatile bool Released; + public int AccessAfterRelease; + + private void CheckAlive() + { + if (Released) Interlocked.Increment(ref AccessAfterRelease); + } + + public bool TryRead(ulong address, Span destination) + { + CheckAlive(); + if (address == Gate) Entered.Set(); + return _inner.TryRead(address, destination); + } + + public bool TryWrite(ulong address, ReadOnlySpan source) + { + CheckAlive(); + if (BlockIdWrite && address == Base + 0x40) + { + Entered.Set(); + Release.Wait(); + } + return _inner.TryWrite(address, source); + } + + public bool TryFillInPlace(ulong address, int length, TState state, + GuestSpanFill fill, out int filled) + { + CheckAlive(); + if (BlockFill) + { + Entered.Set(); + Release.Wait(); + } + var bytes = new byte[length]; + filled = fill(state, bytes); + return _inner.TryWrite(address, bytes.AsSpan(0, filled)); + } + } +} diff --git a/tests/SharpEmu.Libs.Tests/Audio/AudioOut2QueueDispatchTests.cs b/tests/SharpEmu.Libs.Tests/Audio/AudioOut2QueueDispatchTests.cs new file mode 100644 index 000000000..4b297421d --- /dev/null +++ b/tests/SharpEmu.Libs.Tests/Audio/AudioOut2QueueDispatchTests.cs @@ -0,0 +1,117 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later +using System.Diagnostics; +using System.Reflection; +using System.Runtime.InteropServices; +using SharpEmu.Core.Cpu.Native; +using SharpEmu.HLE; +using Xunit; + +namespace SharpEmu.Libs.Tests.Audio; + +public sealed partial class AudioOut2QueueTests +{ + [NativeX64Theory] + [InlineData(false, "empty")] + [InlineData(true, "empty")] + [InlineData(false, "full")] + [InlineData(true, "full")] + [InlineData(false, "available-only")] + [InlineData(true, "available-only")] + [InlineData(false, "aliased")] + [InlineData(true, "aliased")] + [InlineData(false, "invalid-level")] + [InlineData(true, "invalid-level")] + [InlineData(false, "invalid-available")] + [InlineData(true, "invalid-available")] + [InlineData(false, "missing-context")] + [InlineData(true, "missing-context")] + public void QueueQueryDispatcherMatchesFullGateway(bool lowLatency, string scenario) + { + using var gateway = new AudioQueueGateway(_ctx, lowLatency); + var context = CreateContext(4, LongGrain); + if (scenario == "full") for (var i = 0; i < 4; i++) Assert.Equal(0, Push(context, NonBlocking)); + var level = scenario == "available-only" ? 0UL : scenario == "invalid-level" ? MemoryBase + MemorySize : LevelAddress; + var available = scenario == "aliased" ? LevelAddress : scenario == "invalid-available" ? MemoryBase + MemorySize : AvailableAddress; + gateway.Arguments(scenario == "missing-context" ? 0 : context, level, available); + var expected = gateway.Dispatch(1); // Same export, forced through the full production gateway. + var expectedLevel = ReadUInt32(LevelAddress); + var expectedAvailable = ReadUInt32(AvailableAddress); + _memory.TryWrite(LevelAddress, new byte[8]); + _ctx[CpuRegister.Rax] = 0xBAD; + Assert.Equal(expected, gateway.Dispatch(0)); + Assert.Equal(expected, _ctx[CpuRegister.Rax]); + Assert.Equal(expectedLevel, ReadUInt32(LevelAddress)); + Assert.Equal(expectedAvailable, ReadUInt32(AvailableAddress)); + Assert.Equal(0x800010000L, Marshal.ReadInt64(gateway.Pack, 96)); + } + + [NativeX64Fact] + public void QueueQueryDispatchMeasurement() + { + // Explicit opt-in: ordinary correctness runs do not collect noisy timing data. + var outputPath = Environment.GetEnvironmentVariable("SHARPEMU_AUDIO_QUEUE_BENCH_OUTPUT"); + if (string.IsNullOrEmpty(outputPath)) return; + using var gateway = new AudioQueueGateway(_ctx, false); + gateway.Arguments(CreateContext(4, LongGrain), 0, AvailableAddress); + for (var i = 0; i < 20000; i++) { gateway.Dispatch(0); gateway.Dispatch(1); } + const int count = 200000; + var rows = new List(); + foreach (var index in new[] { 1, 0, 0, 1, 0, 1, 1, 0 }) + { + var allocated = GC.GetAllocatedBytesForCurrentThread(); + var start = Stopwatch.GetTimestamp(); + ulong result = 0; + for (var i = 0; i < count; i++) result |= gateway.Dispatch(index); + var elapsed = Stopwatch.GetElapsedTime(start); + var bytes = GC.GetAllocatedBytesForCurrentThread() - allocated; + Assert.Equal(0UL, result); + Assert.Equal(4U, ReadUInt32(AvailableAddress)); + rows.Add(new { path = index == 0 ? "classified" : "full", nsPerCall = elapsed.TotalNanoseconds / count, bytesPerCall = (double)bytes / count, count }); + } + File.WriteAllText(outputPath, System.Text.Json.JsonSerializer.Serialize(new { enabled = Environment.GetEnvironmentVariable("SHARPEMU_AUDIO_OUT2_QUEUE_LEAF"), rows }, new System.Text.Json.JsonSerializerOptions { WriteIndented = true })); + } + + private sealed class AudioQueueGateway : IDisposable + { + private const BindingFlags Instance = BindingFlags.Instance | BindingFlags.NonPublic; + private readonly DirectExecutionBackend _backend; + private readonly nint _allocation = Marshal.AllocHGlobal(512); + private readonly Func _dispatch; + internal nint Pack => _allocation + 192; + + internal AudioQueueGateway(CpuContext context, bool lowLatency) + { + var modules = new ModuleManager(); modules.Freeze(); + _backend = new DirectExecutionBackend(modules); + var type = typeof(DirectExecutionBackend); + type.GetField("_cpuContext", Instance)!.SetValue(_backend, context); + Marshal.Copy(new byte[512], 0, _allocation, 512); + Marshal.WriteInt64(Pack, 96, 0x800010000L); + var nid = lowLatency ? "Q8DZkKQ-SYc" : "R7d0F1g2qsU"; + var entryType = type.GetNestedType("ImportStubEntry", BindingFlags.NonPublic)!; + var entries = Array.CreateInstance(entryType, 2); + for (var index = 0; index < 2; index++) + { + var dispatchNid = index == 0 ? nid : "test-full-audio-queue-query"; + var export = new ExportedFunction("libSceAudioOut2", dispatchNid, "queue query", Generation.Gen5, + lowLatency ? SharpEmu.Libs.Audio.AudioOut2Exports.AudioOut2LoContextGetQueueLevel : SharpEmu.Libs.Audio.AudioOut2Exports.AudioOut2ContextGetQueueLevel); + entries.SetValue(Activator.CreateInstance(entryType, [0x200000UL, dispatchNid, export, false, false, false, false, 0UL]), index); + } + type.GetField("_importEntries", Instance)!.SetValue(_backend, entries); + Assert.Equal(Environment.GetEnvironmentVariable("SHARPEMU_AUDIO_OUT2_QUEUE_LEAF") == "1", DirectExecutionBackend.IsTrivialLeafImport(nid)); + Assert.False(DirectExecutionBackend.IsTrivialLeafImport("test-full-audio-queue-query")); + Assert.False((bool)type.GetField("_disableTrivialLeafDispatch", BindingFlags.Static | BindingFlags.NonPublic)!.GetValue(null)!); + _dispatch = type.GetMethod("DispatchImportCore", Instance)!.CreateDelegate>(_backend); + } + + internal void Arguments(ulong context, ulong level, ulong available) + { + Marshal.WriteInt64(Pack, 0, unchecked((long)context)); + Marshal.WriteInt64(Pack, 8, unchecked((long)level)); + Marshal.WriteInt64(Pack, 16, unchecked((long)available)); + } + internal ulong Dispatch(int index) => _dispatch(index, Pack); + public void Dispose() { _backend.Dispose(); Marshal.FreeHGlobal(_allocation); } + } +} diff --git a/tests/SharpEmu.Libs.Tests/Audio/AudioOut2QueueTests.cs b/tests/SharpEmu.Libs.Tests/Audio/AudioOut2QueueTests.cs index a171146eb..910664e1d 100644 --- a/tests/SharpEmu.Libs.Tests/Audio/AudioOut2QueueTests.cs +++ b/tests/SharpEmu.Libs.Tests/Audio/AudioOut2QueueTests.cs @@ -13,7 +13,7 @@ namespace SharpEmu.Libs.Tests.Audio; // The context queue the way Wwise drives it: Advance then a non-blocking Push for each grain the // queue level says is available. [Collection(AudioOutStateCollection.Name)] -public sealed class AudioOut2QueueTests : IDisposable +public sealed partial class AudioOut2QueueTests : IDisposable { private const ulong MemoryBase = 0x1_0000_0000; private const ulong ContextParamAddress = MemoryBase + 0x100; diff --git a/tests/SharpEmu.Libs.Tests/Cpu/HostShutdownLeafDispatchTests.cs b/tests/SharpEmu.Libs.Tests/Cpu/HostShutdownLeafDispatchTests.cs new file mode 100644 index 000000000..ed024c92b --- /dev/null +++ b/tests/SharpEmu.Libs.Tests/Cpu/HostShutdownLeafDispatchTests.cs @@ -0,0 +1,146 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later +using System.Buffers.Binary; +using System.Reflection; +using System.Runtime.InteropServices; +using SharpEmu.Core.Cpu.Native; +using SharpEmu.HLE; +using Xunit; + +namespace SharpEmu.Libs.Tests.Cpu; + +public sealed class HostShutdownLeafDispatchTests +{ + [NativeX64Theory] + [InlineData("trivial", false, true)] + [InlineData("ordinary", false, true)] + [InlineData("hot-memory", false, true)] + [InlineData("builder", false, true)] + [InlineData("trivial", true, true)] + [InlineData("ordinary", true, true)] + [InlineData("hot-memory", true, true)] + [InlineData("builder", true, true)] + [InlineData("trivial", false, false)] + [InlineData("ordinary", false, false)] + [InlineData("hot-memory", false, false)] + [InlineData("builder", false, false)] + [InlineData("trivial", true, false)] + [InlineData("ordinary", true, false)] + [InlineData("hot-memory", true, false)] + [InlineData("builder", true, false)] + public void StopBeforeOrDuringLeafRedirectsTheCurrentSlice(string path, bool duringImport, bool mainContext) + { + var modules = new ModuleManager(); modules.Freeze(); + using var backend = new DirectExecutionBackend(modules); + var type = typeof(DirectExecutionBackend); + const BindingFlags statics = BindingFlags.Static | BindingFlags.NonPublic; + const BindingFlags instances = BindingFlags.Instance | BindingFlags.NonPublic; + // These tests exercise enabled managed leaf dispatch, not environment-forced full dispatch. + if (path == "trivial" && (bool)type.GetField("_disableTrivialLeafDispatch", statics)!.GetValue(null)!) return; + if (path == "builder" && !(bool)type.GetField("_builderLeafEnabled", statics)!.GetValue(null)!) return; + var memory = new SlotMemory(); + var context = new CpuContext(memory, Generation.Gen5); + var bodyCalls = 0; + void Body() + { + bodyCalls++; + if (duringImport) backend.RequestHostShutdown("leaf-race-test"); + } + memory.Copy = () => { Body(); return true; }; + var nid = path switch + { + "trivial" => "V++UgBtQhn0", + "hot-memory" => "Q3VBxCXhUHs", + "builder" => "q88lQ+GP5Yk", + _ => "shutdown-test-ordinary-leaf", + }; + var export = new ExportedFunction("test", nid, "shutdown test", Generation.Gen5, c => { Body(); c[CpuRegister.Rax] = 9; return 0; }); + var entryType = type.GetNestedType("ImportStubEntry", BindingFlags.NonPublic)!; + var constructor = entryType.GetConstructors(BindingFlags.Instance | BindingFlags.Public).Single(); + var entry = constructor.Invoke([0x200000UL, nid, export, true, true, false, false, 0UL]); + var entries = Array.CreateInstance(entryType, 1); entries.SetValue(entry, 0); + type.GetField("_importEntries", instances)!.SetValue(backend, entries); + var names = new[] { "_activeExecutionBackend", "_activeCpuContext", "_activeEntryReturnSentinelRip", "_activeGuestReturnSlotAddress", "_activeForcedGuestExit", "_activeGuestThreadYieldRequested" }; + var fields = names.Select(n => type.GetField(n, statics)!).ToArray(); + var saved = fields.Select(f => f.GetValue(null)).ToArray(); + var allocation = Marshal.AllocHGlobal(512); + var pack = allocation + 192; // vector-save space and stack arguments are valid too. + var sentinel = mainContext ? 0x123456UL : 0x654321UL; + try + { + Marshal.Copy(new byte[512], 0, allocation, 512); + Marshal.WriteInt64(pack, 0, 0x2000); Marshal.WriteInt64(pack, 8, 0x3000); + Marshal.WriteInt64(pack, 16, 4); Marshal.WriteInt64(pack, 96, 0xABCDEF); + fields[0].SetValue(null, backend); fields[1].SetValue(null, context); + fields[2].SetValue(null, sentinel); fields[3].SetValue(null, SlotMemory.Address); + fields[4].SetValue(null, false); fields[5].SetValue(null, false); + type.GetField("_cpuContext", instances)!.SetValue(backend, mainContext ? context : new CpuContext(memory, Generation.Gen5)); + type.GetField("_entryReturnSentinelRip", instances)!.SetValue(backend, 0x777777UL); + type.GetField("_hostShutdownEntryInFlight", instances)!.SetValue(backend, 1); + if (!duringImport) backend.RequestHostShutdown("leaf-pre-entry-test"); + var result = (ulong)type.GetMethod("DispatchImportCore", instances)!.Invoke(backend, [0, pack])!; + Assert.Equal(duringImport ? 1 : 0, bodyCalls); + Assert.Equal(1UL, result); + Assert.Equal(1UL, context[CpuRegister.Rax]); + Assert.Equal((long)sentinel, Marshal.ReadInt64(pack, 96)); + Assert.Equal(sentinel, memory.Value); + Assert.Equal(mainContext ? 1 : 0, (int)type.GetField("_hostShutdownEntryRedirected", instances)!.GetValue(backend)!); + } + finally + { + for (var i = 0; i < fields.Length; i++) fields[i].SetValue(null, saved[i]); + Marshal.FreeHGlobal(allocation); + } + } + + [NativeX64Fact] + public void LateLeafCheckPreservesAPendingEntryExit() + { + var modules = new ModuleManager(); modules.Freeze(); + using var backend = new DirectExecutionBackend(modules); + var memory = new SlotMemory(); + var context = new CpuContext(memory, Generation.Gen5); + var state = GuestThreadExecution.State; + var pending = state.PendingEntryExit; + var type = typeof(DirectExecutionBackend); + const BindingFlags flags = BindingFlags.Static | BindingFlags.NonPublic; + var names = new[] { "_activeExecutionBackend", "_activeCpuContext", "_activeEntryReturnSentinelRip", "_activeGuestReturnSlotAddress", "_activeForcedGuestExit", "_activeGuestThreadYieldRequested" }; + var fields = names.Select(n => type.GetField(n, flags)!).ToArray(); + var saved = fields.Select(f => f.GetValue(null)).ToArray(); + var pack = Marshal.AllocHGlobal(128); + try + { + fields[0].SetValue(null, backend); fields[1].SetValue(null, context); + fields[2].SetValue(null, 0x123456UL); fields[3].SetValue(null, SlotMemory.Address); + fields[4].SetValue(null, false); fields[5].SetValue(null, false); + state.PendingEntryExit = true; // a must-not-return outcome belongs to its existing dispatcher. + backend.RequestHostShutdown("pending-entry-exit-test"); + Marshal.WriteInt64(pack, 96, 0xABCDEF); + var result = (ulong)typeof(DirectExecutionBackend).GetMethod("CompleteLeafImportShutdown", BindingFlags.Instance | BindingFlags.NonPublic)! + .Invoke(backend, [context, pack, 9UL])!; + Assert.Equal(9UL, result); + Assert.Equal(0xABCDEFL, Marshal.ReadInt64(pack, 96)); + Assert.True(state.PendingEntryExit); + } + finally + { + state.PendingEntryExit = pending; + for (var i = 0; i < fields.Length; i++) fields[i].SetValue(null, saved[i]); + Marshal.FreeHGlobal(pack); + } + } + + private sealed class SlotMemory : ICpuMemory + { + internal const ulong Address = 0x100000; + internal ulong Value; + internal Func? Copy; + public bool TryRead(ulong address, Span bytes) => false; + public bool TryWrite(ulong address, ReadOnlySpan bytes) + { + if (address != Address || bytes.Length != 8) return false; + Value = BinaryPrimitives.ReadUInt64LittleEndian(bytes); return true; + } + public bool TryCopy(ulong destinationAddress, ulong sourceAddress, ulong length) => Copy?.Invoke() == true; + } +} diff --git a/tests/SharpEmu.Libs.Tests/Gpu/GpuCommands/CommandStreamQueueTests.cs b/tests/SharpEmu.Libs.Tests/Gpu/GpuCommands/CommandStreamQueueTests.cs index bcf89ff20..b53f94da8 100644 --- a/tests/SharpEmu.Libs.Tests/Gpu/GpuCommands/CommandStreamQueueTests.cs +++ b/tests/SharpEmu.Libs.Tests/Gpu/GpuCommands/CommandStreamQueueTests.cs @@ -33,6 +33,50 @@ private static (RecordingCommandStreamHost Host, CommandStreamQueue Queue) NewQu return (host, new CommandStreamQueue(host)); } + [Theory] + [InlineData(0UL)] + [InlineData(4UL)] + public void APartialLabelReplacementCannotAnswerWithTheOldQword(ulong offset) + { + var (host, queue) = NewQueue(); + host.PendingLabelTick = 5; + var interpreter = queue.GetInterpreter(0); + interpreter.PendingLabelRecorded!(Label, 8, 0x22222222_11111111UL); + interpreter.PendingLabelRecorded!(Label + offset, 4, 0x33333333); + Assert.False(interpreter.PendingLabelLookup!(Label, true, out _)); + Assert.True(interpreter.PendingLabelLookup!(Label + offset, false, out var replacement)); + Assert.Equal(0x33333333UL, replacement); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void AQwordReplacementRemovesAnOlderUpperDword(bool batch) + { + var (host, queue) = NewQueue(); + host.PendingLabelTick = 5; + var interpreter = queue.GetInterpreter(0); + interpreter.PendingLabelRecorded!(Label + 4, 4, 0xBAD); + if (batch) interpreter.PendingQwordsRecorded!(Label, 2, 16, 0x22222222_11111111UL); + else interpreter.PendingLabelRecorded!(Label, 8, 0x22222222_11111111UL); + Assert.True(interpreter.PendingLabelLookup!(Label + 4, false, out var high)); + Assert.Equal(0x22222222UL, high); + queue.NotifyCompletedGpuTick(5); + Assert.False(interpreter.PendingLabelLookup!(Label, true, out _)); + } + + [Fact] + public void AnUnalignedReplacementRefusesStaleCachedAnswers() + { + var (host, queue) = NewQueue(); + host.PendingLabelTick = 5; + var interpreter = queue.GetInterpreter(0); + interpreter.PendingLabelRecorded!(Label, 8, 0x22222222_11111111UL); + interpreter.PendingLabelRecorded!(Label + 2, 4, 0x33333333); + Assert.False(interpreter.PendingLabelLookup!(Label, true, out _)); + Assert.False(interpreter.PendingLabelLookup!(Label + 2, false, out _)); + } + private static void Enqueue(RecordingCommandStreamHost host, CommandStreamQueue queue, ulong address, ulong submissionId, params uint[][] packets) { var words = StreamRunner.Concat(packets); diff --git a/tests/SharpEmu.Libs.Tests/Gpu/GpuCommands/DeferredCommandItemReuseTests.cs b/tests/SharpEmu.Libs.Tests/Gpu/GpuCommands/DeferredCommandItemReuseTests.cs new file mode 100644 index 000000000..89bf8dedf --- /dev/null +++ b/tests/SharpEmu.Libs.Tests/Gpu/GpuCommands/DeferredCommandItemReuseTests.cs @@ -0,0 +1,265 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using System.Buffers.Binary; +using System.Diagnostics; +using System.Reflection; +using System.Runtime.InteropServices; +using System.Text.Json; +using SharpEmu.Libs.Agc; +using SharpEmu.Libs.Gpu.GpuCommands; +using Xunit; + +namespace SharpEmu.Libs.Tests.Gpu.GpuCommands; + +[Collection(CommandStreamQueueStateCollection.Name)] +public sealed class DeferredCommandItemReuseTests +{ + private sealed record Draw(int Index) : AgcExports.DeferredWork; + private static readonly Action Nothing = static () => { }; + + private static DeferredCommandThread NewThread(RecordingCommandStreamHost host, bool reuse, + Action? draw = null) => + new(new CommandStreamQueue(host), host.Memory, draw ?? (_ => { }), null, Nothing, + recycleAsyncItems: reuse); + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void RepeatedMixedKindsKeepOrderingAndPendingWriteOverlay(bool reuse) + { + var host = new RecordingCommandStreamHost(); + var seen = new List(); + var thread = NewThread(host, reuse, work => seen.Add(((Draw)work).Index)); + const ulong address = RecordingCommandStreamHost.MemoryBase; + Span actual = stackalloc byte[4]; + for (var iteration = 0; iteration < 512; iteration++) + { + var start = iteration * 4; + thread.EnqueueDeferred(new Draw(start), isDraw: true); + thread.Post(() => seen.Add(start + 1)); + thread.EnqueueDeferred(new Draw(start + 2), isDraw: false); + var bytes = BitConverter.GetBytes(iteration + 1); + thread.PostWrite(address, bytes); + // The queued write is visible before it executes and owns a copy of the payload. + bytes.AsSpan().Clear(); + Assert.True(thread.ReadOverlay(address, actual)); + Assert.Equal(iteration + 1, BinaryPrimitives.ReadInt32LittleEndian(actual)); + thread.PostRange(() => seen.Add(start + 3), address, 4); + Assert.Equal(5, thread.RunBatch(long.MaxValue, 64, Nothing)); + Assert.False(thread.HasWork); + // A stale pending range would enqueue a synchronous item and block this read. + thread.WaitForPendingRange(address, 4); + Assert.True(thread.ReadOverlay(address, actual)); + Assert.Equal(iteration + 1, BinaryPrimitives.ReadInt32LittleEndian(actual)); + } + Assert.Equal(Enumerable.Range(0, 2048), seen); + Assert.Equal(512 * 4, thread.Gate); + AssertCacheContainsNoPayloads(thread, reuse); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void TwoThreadsWrapTheReturnRingAndPreserveSynchronousFailureResults(bool reuse) + { + var host = new RecordingCommandStreamHost(); + var queue = new CommandStreamQueue(host); + using var wake = new AutoResetEvent(false); + using var completed = new ManualResetEventSlim(); + var observed = new List(); + var thread = new DeferredCommandThread(queue, host.Memory, + work => observed.Add(((Draw)work).Index), null, () => wake.Set(), recycleAsyncItems: reuse); + Exception? failure = null; + var syncFailures = 0; + const int count = 20000; + Assert.True(queue.TryEnqueueControlBarrier(() => + { + try + { + for (var i = 0; i < count; i++) + { + if ((i & 1) == 0) thread.EnqueueDeferred(new Draw(i), isDraw: true); + else + { + var value = i; + thread.Post(() => observed.Add(value)); + } + if (i % 127 == 0) + { + var expected = new InvalidOperationException($"sync-{i}"); + var caught = Assert.Throws(() => thread.Run(() => throw expected)); + Assert.Same(expected, caught.InnerException); + syncFailures++; + } + } + // Completion of a synchronous callback publishes all preceding output. + thread.Run(Nothing); + } + catch (Exception exception) { failure = exception; } + finally { completed.Set(); } + }, Nothing)); + thread.Start(); + try + { + var deadline = Stopwatch.GetTimestamp() + 30 * Stopwatch.Frequency; + while (!completed.IsSet && Stopwatch.GetTimestamp() < deadline) + { + thread.ConsumerIdle = true; + if (thread.RunBatch(long.MaxValue, 17, Nothing) == 0) wake.WaitOne(1); + thread.ConsumerIdle = false; + } + Assert.True(completed.IsSet); + Assert.Null(failure); + thread.ThrowIfFailed(); + Assert.Equal(158, syncFailures); + Assert.Equal(Enumerable.Range(0, count), observed); + } + finally + { + thread.Stop(() => thread.RunBatch(long.MaxValue, int.MaxValue, Nothing)); + } + Assert.False(thread.IsAlive); + AssertCacheContainsNoPayloads(thread, reuse); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void FailedAsynchronousItemsAreNotReturnedOrReexecuted(bool reuse) + { + var host = new RecordingCommandStreamHost(); + var thread = NewThread(host, reuse); + var failure = new InvalidOperationException("asynchronous failure"); + thread.Post(() => throw failure); + Assert.Same(failure, Assert.Throws( + () => thread.RunBatch(long.MaxValue, 64, Nothing))); + Assert.Same(failure, Assert.Throws(thread.ThrowIfFailed).InnerException); + if (reuse) + foreach (var item in CachedItems(thread)!) Assert.Null(item); + } + + [Fact] + public void CacheStaysBoundedAcrossChangingBatchSizes() + { + var thread = NewThread(new RecordingCommandStreamHost(), true); + for (var batch = 0; batch < 300; batch++) + { + var count = batch % 64 + 1; + for (var i = 0; i < count; i++) thread.Post(Nothing); + while (thread.HasWork) thread.RunBatch(long.MaxValue, 7, Nothing); + } + Assert.Equal(64, CachedItems(thread)!.Length); + AssertCacheContainsNoPayloads(thread, true); + } + + private static Array? CachedItems(DeferredCommandThread thread) => + (Array?)typeof(DeferredCommandThread).GetField("_recycledItems", BindingFlags.Instance | BindingFlags.NonPublic)!.GetValue(thread); + + private static void AssertCacheContainsNoPayloads(DeferredCommandThread thread, bool enabled) + { + var cache = CachedItems(thread); + if (!enabled) { Assert.Null(cache); return; } + Assert.NotNull(cache); + Assert.Equal(64, cache.Length); + foreach (var item in cache) + { + if (item is null) continue; + foreach (var field in new[] { "Work", "Action", "Data", "Failure" }) + Assert.Null(item.GetType().GetField(field)!.GetValue(item)); + foreach (var field in new[] { "IsDraw", "Sync", "Done" }) + Assert.Equal(false, item.GetType().GetField(field)!.GetValue(item)); + Assert.Equal(0UL, item.GetType().GetField("RangeSize")!.GetValue(item)); + } + } + + [Fact] + public void MeasureQueueItemReuseWhenRequested() + { + var path = Environment.GetEnvironmentVariable("SHARPEMU_COMMAND_ITEM_BENCH_OUTPUT"); + if (string.IsNullOrEmpty(path)) return; + var rows = new List(); + foreach (var batch in new[] { 1, 16, 64 }) + { + var baseline = NewThread(new RecordingCommandStreamHost(), false); + var candidate = NewThread(new RecordingCommandStreamHost(), true); + foreach (var thread in new[] { baseline, candidate }) Exercise(thread, batch, 10000); + for (var trial = 0; trial < 8; trial++) + foreach (var enabled in (trial & 1) == 0 ? new[] { false, true } : new[] { true, false }) + { + var thread = enabled ? candidate : baseline; + const int count = 200000; + var bytes = GC.GetAllocatedBytesForCurrentThread(); + var start = Stopwatch.GetTimestamp(); + var ran = Exercise(thread, batch, count); + var elapsed = Stopwatch.GetElapsedTime(start); + bytes = GC.GetAllocatedBytesForCurrentThread() - bytes; + Assert.Equal(count, ran); + rows.Add(new { mode = "same-thread", batch, enabled, trial, + nsPerItem = elapsed.TotalNanoseconds / count, bytesPerItem = (double)bytes / count }); + } + } + for (var trial = 0; trial < 6; trial++) + foreach (var enabled in (trial & 1) == 0 ? new[] { false, true } : new[] { true, false }) + rows.Add(MeasureConcurrent(enabled, trial)); + File.WriteAllText(path, JsonSerializer.Serialize(new { architecture = RuntimeInformation.ProcessArchitecture.ToString(), rows }, + new JsonSerializerOptions { WriteIndented = true })); + } + + private static object MeasureConcurrent(bool reuse, int trial) + { + var host = new RecordingCommandStreamHost(); + var queue = new CommandStreamQueue(host); + using var wake = new AutoResetEvent(false); + using var done = new ManualResetEventSlim(); + var thread = new DeferredCommandThread(queue, host.Memory, _ => { }, null, () => wake.Set(), + recycleAsyncItems: reuse); + Exception? failure = null; + const int count = 100000; + double ns = 0, allocated = 0; + Assert.True(queue.TryEnqueueControlBarrier(() => + { + try + { + for (var i = 0; i < 5000; i++) thread.Post(Nothing); + thread.Run(Nothing); + var bytes = GC.GetAllocatedBytesForCurrentThread(); + var start = Stopwatch.GetTimestamp(); + for (var i = 0; i < count; i++) thread.Post(Nothing); + thread.Run(Nothing); + ns = Stopwatch.GetElapsedTime(start).TotalNanoseconds / count; + allocated = (double)(GC.GetAllocatedBytesForCurrentThread() - bytes) / count; + } + catch (Exception exception) { failure = exception; } + finally { done.Set(); } + }, Nothing)); + thread.Start(); + try + { + var deadline = Stopwatch.GetTimestamp() + 30 * Stopwatch.Frequency; + while (!done.IsSet && Stopwatch.GetTimestamp() < deadline) + { + thread.ConsumerIdle = true; + if (thread.RunBatch(long.MaxValue, 64, Nothing) == 0) wake.WaitOne(1); + thread.ConsumerIdle = false; + } + Assert.True(done.IsSet); + Assert.Null(failure); + } + finally { thread.Stop(() => thread.RunBatch(long.MaxValue, int.MaxValue, Nothing)); } + return new { mode = "two-thread", batch = 64, enabled = reuse, trial, nsPerItem = ns, bytesPerItem = allocated }; + } + + private static int Exercise(DeferredCommandThread thread, int batch, int count) + { + var ran = 0; + for (var i = 0; i < count;) + { + var length = Math.Min(batch, count - i); + for (var j = 0; j < length; j++) thread.Post(Nothing); + ran += thread.RunBatch(long.MaxValue, int.MaxValue, Nothing); + i += length; + } + return ran; + } +} diff --git a/tests/SharpEmu.Libs.Tests/Gpu/GpuCommands/DeferredCommandThreadTests.cs b/tests/SharpEmu.Libs.Tests/Gpu/GpuCommands/DeferredCommandThreadTests.cs new file mode 100644 index 000000000..2679c5eab --- /dev/null +++ b/tests/SharpEmu.Libs.Tests/Gpu/GpuCommands/DeferredCommandThreadTests.cs @@ -0,0 +1,78 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using SharpEmu.Libs.Gpu.GpuCommands; +using Xunit; + +namespace SharpEmu.Libs.Tests.Gpu.GpuCommands; + +public sealed class DeferredCommandThreadTests +{ + [Theory] + [InlineData(false)] + [InlineData(true)] + public void ProducerIdentityAndOrderedConsumerWorkSurviveThreadSelection(bool useHostQos) + { + var host = new RecordingCommandStreamHost(); + var queue = new CommandStreamQueue(host); + var producer = new DeferredCommandThread(queue, host.Memory, _ => { }, null, () => { }, useHostQos); + using var entered = new ManualResetEventSlim(); + using var completed = new ManualResetEventSlim(); + bool producerIdentity = false, consumerIdentity = true; + var values = new List(); + Assert.True(queue.TryEnqueueControlBarrier(() => + { + producerIdentity = producer.IsProducerThread; + producer.Post(() => values.Add(1)); + entered.Set(); + producer.Run(() => { consumerIdentity = producer.IsProducerThread; values.Add(2); }); + completed.Set(); + }, () => { })); + producer.Start(); + try + { + Assert.False(producer.IsProducerThread); + Assert.True(entered.Wait(TimeSpan.FromSeconds(10))); + Assert.True(producer.IsAlive); + Assert.Throws(producer.Start); + Assert.True(SpinWait.SpinUntil(() => + { + producer.RunBatch(long.MaxValue, 64, () => { }); + return completed.IsSet; + }, TimeSpan.FromSeconds(10))); + producer.ThrowIfFailed(); + Assert.True(producerIdentity); + Assert.False(consumerIdentity); + Assert.Equal(new[] { 1, 2 }, values); + } + finally + { + producer.Stop(() => producer.RunBatch(long.MaxValue, 64, () => { })); + } + Assert.False(producer.IsAlive); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void ProducerFailureIsReportedAndThreadCanBeStopped(bool useHostQos) + { + var host = new RecordingCommandStreamHost(); + var queue = new CommandStreamQueue(host); + using var failed = new ManualResetEventSlim(); + var expected = new InvalidOperationException("producer failure"); + var producer = new DeferredCommandThread(queue, host.Memory, _ => { }, null, failed.Set, useHostQos); + Assert.True(queue.TryEnqueueControlBarrier(() => throw expected, () => { })); + producer.Start(); + try + { + Assert.True(failed.Wait(TimeSpan.FromSeconds(10))); + Assert.Same(expected, Assert.Throws(producer.ThrowIfFailed).InnerException); + } + finally + { + producer.Stop(() => { }); + } + Assert.False(producer.IsAlive); + } +} diff --git a/tests/SharpEmu.Libs.Tests/Gpu/GpuCommands/DenseShaderRegisterMapTests.cs b/tests/SharpEmu.Libs.Tests/Gpu/GpuCommands/DenseShaderRegisterMapTests.cs new file mode 100644 index 000000000..9693af3d6 --- /dev/null +++ b/tests/SharpEmu.Libs.Tests/Gpu/GpuCommands/DenseShaderRegisterMapTests.cs @@ -0,0 +1,148 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using SharpEmu.Libs.Gpu.GpuCommands; +using Xunit; + +namespace SharpEmu.Libs.Tests.Gpu.GpuCommands; + +public sealed class DenseShaderRegisterMapTests +{ + [Fact] + public void PresenceCountAndOverflowDistinguishUnwrittenFromZero() + { + var map = new DenseShaderRegisterMap(); + foreach (var key in new[] { 0u, 63u, 64u, 767u, 768u, uint.MaxValue }) + { + Assert.False(map.TryGetValue(key, out var missing)); + Assert.Equal(0u, missing); + Assert.Throws(() => map[key]); + map[key] = 0; + Assert.True(map.ContainsKey(key)); + Assert.True(map.TryGetValue(key, out var zero)); + Assert.Equal(0u, zero); + map[key] = key; + Assert.Equal(key, map[key]); + } + Assert.Equal(6, map.Count); + Assert.Throws(() => map.Add(0, 1)); + Assert.Equal(6, map.Count); + } + + [Fact] + public void InsertionOrderAndLiveViewsSurviveClearThroughAnAlias() + { + var map = new DenseShaderRegisterMap(); + IDictionary alias = map; + IReadOnlyDictionary readOnly = map; + var keys = alias.Keys; + var values = alias.Values; + alias[900] = 9; + alias[2] = 2; + alias[1] = 1; + alias[2] = 22; + Assert.Equal(new uint[] { 900, 2, 1 }, keys); + Assert.Equal(new uint[] { 9, 22, 1 }, values); + Assert.Equal(22u, readOnly[2]); + Assert.True(alias.Remove(2)); + Assert.False(alias.Remove(2)); + alias.Add(2, 23); + Assert.Equal(new uint[] { 900, 1, 2 }, readOnly.Keys); + Assert.Equal(3, new Dictionary(map).Count); + alias.Clear(); + Assert.Empty(map); + Assert.Empty(keys); + Assert.Empty(values); + Assert.Equal(0, readOnly.Count); + Assert.False(map.ContainsKey(900)); + Assert.False(map.ContainsKey(1)); + alias[1] = 0; + Assert.Single(map); + Assert.Equal(new uint[] { 1 }, keys); + Assert.Equal(new uint[] { 0 }, values); + Assert.Throws(() => keys.Add(3)); + Assert.Throws(() => values.Clear()); + } + + [Fact] + public void PairAndCopyContractsPreserveExactMembership() + { + ICollection> map = new DenseShaderRegisterMap(); + map.Add(new(12, 34)); + Assert.False(map.Remove(new(12, 35))); + Assert.True(map.Contains(new(12, 34))); + var copied = new KeyValuePair[3]; + map.CopyTo(copied, 1); + Assert.Equal(new KeyValuePair(12, 34), copied[1]); + Assert.Throws(() => map.CopyTo(null!, 0)); + Assert.Throws(() => map.CopyTo(copied, -1)); + Assert.Throws(() => map.CopyTo(copied, 3)); + Assert.True(map.Remove(new(12, 34))); + Assert.Empty(map); + } + + [Fact] + public void EnumerationDetectsInsertionButAllowsOverwriteRemovalAndClear() + { + var map = new DenseShaderRegisterMap { [1] = 1, [2] = 2 }; + var overwritten = map.GetEnumerator(); + map[1] = 7; + Assert.True(overwritten.MoveNext()); + Assert.Equal(7u, overwritten.Current.Value); + map.Remove(2); + Assert.False(overwritten.MoveNext()); + var cleared = map.GetEnumerator(); + map.Clear(); + Assert.False(cleared.MoveNext()); + var inserted = map.GetEnumerator(); + map[3] = 3; + Assert.Throws(() => inserted.MoveNext()); + var keys = map.Keys.GetEnumerator(); + map[4] = 4; + Assert.Throws(() => keys.MoveNext()); + } + + [Fact] + public void ReinsertCompactionPreservesLiveInsertionOrder() + { + var map = new DenseShaderRegisterMap { [1] = 1, [900] = 9, [2] = 0 }; + for (var i = 0; i < 2000; i++) + { + Assert.True(map.Remove(2)); + map[2] = (uint)i; + } + Assert.Equal(new uint[] { 1, 900, 2 }, map.Keys); + Assert.Equal(3, map.Count); + Assert.Equal(1999u, map[2]); + } + + [Fact] + public void SeededMixedOperationsMatchDictionaryValuesAndCounts() + { + var random = new System.Random(7721); + var map = new DenseShaderRegisterMap(); + var reference = new Dictionary(); + for (var i = 0; i < 10000; i++) + { + var key = i % 17 == 0 ? uint.MaxValue : (uint)random.Next(0, 900); + switch (random.Next(0, 12)) + { + case 0: + Assert.Equal(reference.Remove(key), map.Remove(key)); + break; + case 1: + map.Clear(); reference.Clear(); + break; + default: + var value = (uint)random.Next(); + map[key] = value; reference[key] = value; + break; + } + Assert.Equal(reference.Count, map.Count); + Assert.Equal(reference.TryGetValue(key, out var expected), map.TryGetValue(key, out var actual)); + Assert.Equal(expected, actual); + if (i % 100 == 0) + Assert.Equal(reference.OrderBy(x => x.Key), map.OrderBy(x => x.Key)); + } + } +} diff --git a/tests/SharpEmu.Libs.Tests/Gpu/GpuCommands/GpuCommandInterpreterPacketSemanticsTests.cs b/tests/SharpEmu.Libs.Tests/Gpu/GpuCommands/GpuCommandInterpreterPacketSemanticsTests.cs index 962b12d0b..bd355d487 100644 --- a/tests/SharpEmu.Libs.Tests/Gpu/GpuCommands/GpuCommandInterpreterPacketSemanticsTests.cs +++ b/tests/SharpEmu.Libs.Tests/Gpu/GpuCommands/GpuCommandInterpreterPacketSemanticsTests.cs @@ -240,6 +240,18 @@ public void RegisterSpaceWait_ComparesTheRegisterThisQueueWrote() Assert.Contains("not modelled", runner.RunExpectingFatal(RegisterWait(3, 0xA100, 0, 0)).Message); } + [Fact] + public void ShaderRegisterWaitDistinguishesWrittenZeroFromClearedPresence() + { + var runner = new StreamRunner(); + var writeZero = StreamRunner.Packet(PacketOpcode.SetShaderRegister, 0x0C, 0); + Assert.Equal(SubmissionProgress.Complete, runner.Run(writeZero, RegisterWait(3, 0x2C0C, 0, uint.MaxValue))); + var alias = runner.Interpreter.Registers.Shader; + runner.Interpreter.Registers.Reset(); + Assert.Same(alias, runner.Interpreter.Registers.Shader); + Assert.Contains("never wrote", runner.RunExpectingFatal(RegisterWait(3, 0x2C0C, 0, uint.MaxValue)).Message); + } + [Fact] public void AcquireMemory_BarriersOnlyForCacheActions() { diff --git a/tests/SharpEmu.Libs.Tests/Gpu/GpuCommands/GpuCommandInterpreterRegisterTests.cs b/tests/SharpEmu.Libs.Tests/Gpu/GpuCommands/GpuCommandInterpreterRegisterTests.cs index 8a910c516..f811bc3c6 100644 --- a/tests/SharpEmu.Libs.Tests/Gpu/GpuCommands/GpuCommandInterpreterRegisterTests.cs +++ b/tests/SharpEmu.Libs.Tests/Gpu/GpuCommands/GpuCommandInterpreterRegisterTests.cs @@ -120,6 +120,21 @@ public void RegistersOutsideTheBank_AreFatal() Assert.Contains("range leaves the bank", runner.RunExpectingFatal(StreamRunner.Packet(PacketOpcode.SetContextRegister, 0x3FF, 1, 2)).Message); } + [Fact] + public void DirectShaderRangeWithInvalidTailPreservesOnlyItsRawPrefix() + { + var runner = new StreamRunner(); + var before = runner.Interpreter.TypedRegisters.Snapshot(); + var last = RegisterBankLayout.ShaderRegisterCount - 1; + var error = runner.RunExpectingFatal(StreamRunner.Packet(PacketOpcode.SetShaderRegister, last, 0x1234, 0x5678)); + Assert.Contains("shader register range leaves the bank", error.Message); + Assert.Equal(0x1234u, runner.Interpreter.Registers.Shader[last]); + Assert.False(runner.Interpreter.Registers.Shader.ContainsKey(last + 1)); + Assert.Same(before.Shader.Pixel, runner.Interpreter.TypedRegisters.Shader.Pixel); + Assert.Same(before.Shader.Vertex, runner.Interpreter.TypedRegisters.Shader.Vertex); + Assert.Same(before.Shader.Compute, runner.Interpreter.TypedRegisters.Shader.Compute); + } + [Fact] public void DirectDepthWrites_ClearTheCompositeExtent() { diff --git a/tests/SharpEmu.Libs.Tests/Gpu/GpuCommands/GpuResourceOriginTests.cs b/tests/SharpEmu.Libs.Tests/Gpu/GpuCommands/GpuResourceOriginTests.cs new file mode 100644 index 000000000..84c7fed9c --- /dev/null +++ b/tests/SharpEmu.Libs.Tests/Gpu/GpuCommands/GpuResourceOriginTests.cs @@ -0,0 +1,73 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using SharpEmu.Libs.Gpu.GpuCommands; +using SharpEmu.Libs.Gpu.GpuCommands.Registers; +using Xunit; + +namespace SharpEmu.Libs.Tests.Gpu.GpuCommands; + +public sealed class GpuResourceOriginTests +{ + [Theory] + [InlineData(0)] + [InlineData(-1)] + public void RingRequiresPositiveCapacity(int capacity) => + Assert.Throws(() => new GpuResourceOriginRing(capacity)); + + [Fact] + public void RingKeepsOnlyTheLast128WritesInSequenceOrder() + { + var ring = new GpuResourceOriginRing(128); + Assert.Empty(ring.Snapshot()); + for (uint value = 0; value < 300; value++) + ring.Add(new GpuResourceOriginWrite { Value = value }); + var writes = ring.Snapshot(); + Assert.Equal(128, writes.Length); + Assert.Equal(Enumerable.Range(172, 128).Select(value => (uint)value), writes.Select(write => write.Value)); + Assert.Equal(Enumerable.Range(173, 128).Select(value => (long)value), writes.Select(write => write.Sequence)); + ring.Add(new GpuResourceOriginWrite { Value = 300 }); + Assert.Equal(172u, writes[0].Value); // A failure-report snapshot cannot be overwritten by producers. + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void CaptureRetainsPacketSourceAndResultingPixelState(bool table) + { + var banks = new RegisterBanks(message => new InvalidOperationException(message)); + banks.Shader.Pixel.Address = 0x808B440000; + banks.Shader.Pixel.UserScalars.Set(28, 0xE90AFF18, UserScalarKind.Unknown); + banks.Shader.Pixel.UserScalars.Set(29, 0x80, UserScalarKind.Unknown); + var packet = new PacketContext(0xC0047600, 0x8123400000, 3, 16, 32); + var write = GpuResourceOrigin.Capture(2, 1234, packet, table, 0x8123400010, + GpuResourceOrigin.PointerLow, 0xABCD0028, 0xE90AFF18, 2, 6, banks); + banks.Shader.Pixel.UserScalars.Set(28, 0, UserScalarKind.Unknown); + banks.Shader.Pixel.Address = 0; + + Assert.Equal(2, write.Queue); + Assert.Equal(1234UL, write.Submit); + Assert.Equal(packet.PacketAddress, write.PacketAddress); + Assert.Equal(packet.Header, write.Header); + Assert.Equal(table, write.Table); + Assert.Equal(0x8123400010UL, write.SourceAddress); + Assert.Equal(GpuResourceOrigin.PointerLow, write.Offset); + Assert.Equal(0xABCD0028u, write.RawOffset); + Assert.Equal(2u, write.Entry); + Assert.Equal(6u, write.Count); + Assert.Equal(0xE90AFF18u, write.Value); + Assert.Equal(0x808B440000UL, write.ShaderAddress); + Assert.Equal(0xE90AFF18u, write.PointerLow); + Assert.Equal(0x80u, write.PointerHigh); + } + + [Fact] + public void ConcurrentProducersStillProduceABoundedOrderedSnapshot() + { + var ring = new GpuResourceOriginRing(128); + Parallel.For(0, 1000, value => ring.Add(new GpuResourceOriginWrite { Value = (uint)value })); + var writes = ring.Snapshot(); + Assert.Equal(128, writes.Length); + Assert.Equal(Enumerable.Range(873, 128).Select(value => (long)value), writes.Select(write => write.Sequence)); + } +} diff --git a/tests/SharpEmu.Libs.Tests/Gpu/GpuCommands/Registers/IdenticalScalarWriteTests.cs b/tests/SharpEmu.Libs.Tests/Gpu/GpuCommands/Registers/IdenticalScalarWriteTests.cs new file mode 100644 index 000000000..3fd61f619 --- /dev/null +++ b/tests/SharpEmu.Libs.Tests/Gpu/GpuCommands/Registers/IdenticalScalarWriteTests.cs @@ -0,0 +1,193 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using System.Diagnostics; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Text.Json; +using SharpEmu.Libs.Gpu.GpuCommands; +using SharpEmu.Libs.Gpu.GpuCommands.Registers; +using Xunit; +using static SharpEmu.Libs.Gpu.GpuCommands.Registers.ShaderRegisterOffset; + +namespace SharpEmu.Libs.Tests.Gpu.GpuCommands.Registers; + +public sealed class IdenticalScalarWriteTests +{ + private static bool Enabled => Environment.GetEnvironmentVariable("SHARPEMU_REGISTER_IDENTICAL_SCALARS") == "1"; + private static RegisterBanks Banks() => new(static message => new InvalidOperationException(message)); + private static uint Write(RegisterBanks banks, uint offset, ReadOnlySpan values) + { + var packet = new PacketContext(PacketHeader.Make((uint)values.Length + 2, PacketOpcode.SetShaderRegister), 0x10000, 0, 100, 100); + return RegisterWriteTable.WriteShaderPacket(banks, in packet, offset, values); + } + + private static UserScalarRegisters Scalars(RegisterBanks banks, uint first) => first switch + { + SpiShaderUserDataPs0 => banks.Shader.Pixel.UserScalars, + SpiShaderUserDataGs0 => banks.Shader.Vertex.GeometryUserScalars, + SpiShaderUserDataHs0 => banks.Shader.Vertex.HullUserScalars, + ComputeUserData0 => banks.Shader.Compute.UserScalars, + SpiShaderUserDataVs0 => banks.Shader.Vertex.LegacyVertexUserScalars, + SpiShaderUserDataEs0 => banks.Shader.Vertex.ExportUserScalars, + _ => throw new ArgumentOutOfRangeException(nameof(first)), + }; + + [Theory] + [InlineData(SpiShaderUserDataPs0, 32)] + [InlineData(SpiShaderUserDataGs0, 32)] + [InlineData(SpiShaderUserDataHs0, 32)] + [InlineData(ComputeUserData0, 16)] + public void RepeatedPacketPreservesSnapshotUntilValueKindOrCountChanges(uint first, int capacity) + { + var banks = Banks(); + var offset = first + (uint)capacity - 2; + banks.UserDataMarker = UserScalarKind.Region; + Write(banks, offset, [17, 19]); + var original = banks.Snapshot(); + var oldScalars = Scalars(original, first); + banks.UserDataMarker = UserScalarKind.Region; + Assert.Equal(2u, Write(banks, offset, [17, 19])); + Assert.Equal(UserScalarKind.Unknown, banks.UserDataMarker); + if (Enabled) + { + Assert.Same(oldScalars, Scalars(banks, first)); + if (Environment.GetEnvironmentVariable("SHARPEMU_REGISTER_SNAPSHOT_REUSE") != "0") + Assert.Same(original, banks.Snapshot()); + } + + // Equal values with a different marker still require a copy. + Write(banks, offset, [17, 19]); + Assert.NotSame(oldScalars, Scalars(banks, first)); + Assert.Equal(UserScalarKind.Unknown, Scalars(banks, first).Kinds[capacity - 1]); + Assert.Equal(UserScalarKind.Region, oldScalars.Kinds[capacity - 1]); + var changedKind = banks.Snapshot(); + Write(banks, offset + 1, [23]); + Assert.Equal(19u, Scalars(changedKind, first).Values[capacity - 1]); + Assert.Equal(23u, Scalars(banks, first).Values[capacity - 1]); + + // A zero write extends Count even if the uninitialized storage already contains zero. + banks = Banks(); + var empty = banks.Snapshot(); + Write(banks, offset, [0, 0]); + Assert.Equal((uint)capacity, Scalars(banks, first).Count); + Assert.Equal(0u, Scalars(empty, first).Count); + } + + [Theory] + [InlineData(SpiShaderUserDataPs0, 32)] + [InlineData(SpiShaderUserDataGs0, 32)] + [InlineData(SpiShaderUserDataHs0, 32)] + [InlineData(ComputeUserData0, 16)] + public void InvalidPacketsStillFailWithoutChangingScalarContents(uint first, int capacity) + { + var banks = Banks(); + Write(banks, first + (uint)capacity - 1, [19]); + var original = banks.Snapshot(); + banks.UserDataMarker = UserScalarKind.BufferResource; + Assert.Contains("outside the stage window", Assert.Throws( + () => Write(banks, first + (uint)capacity - 1, [19, 0])).Message); + Assert.Equal(UserScalarKind.BufferResource, banks.UserDataMarker); + Assert.Equal(19u, Scalars(banks, first).Values[capacity - 1]); + Assert.Equal(19u, Scalars(original, first).Values[capacity - 1]); + Assert.Contains("consumed no values", Assert.Throws( + () => Write(banks, first, [])).Message); + // The original empty direct writer clears its marker before reporting zero consumption. + Assert.Equal(UserScalarKind.Unknown, banks.UserDataMarker); + } + + [Theory] + [InlineData(SpiShaderUserDataVs0)] + [InlineData(SpiShaderUserDataEs0)] + public void LegacyIndirectPacketsStillApplyMarkerOnlyToFirstEntry(uint first) + { + var banks = Banks(); + Write(banks, first, [7, 9]); + var original = banks.Snapshot(); + banks.UserDataMarker = UserScalarKind.Region; + Write(banks, first, [7, 9]); + Assert.Equal(UserScalarKind.Region, Scalars(banks, first).Kinds[0]); + Assert.Equal(UserScalarKind.Unknown, Scalars(banks, first).Kinds[1]); + Assert.Equal(UserScalarKind.Unknown, Scalars(original, first).Kinds[0]); + Assert.Equal(UserScalarKind.Unknown, banks.UserDataMarker); + } + + [Theory] + [InlineData(SpiShaderUserDataPs0, 32)] + [InlineData(SpiShaderUserDataGs0, 32)] + [InlineData(SpiShaderUserDataHs0, 32)] + [InlineData(ComputeUserData0, 16)] + public void MixedPartialWritesAgreeWithIndependentStateAndPreserveAllEarlierSnapshots(uint first, int capacity) + { + var banks = Banks(); + var random = new System.Random(4729); + var words = new uint[capacity]; + var kinds = new UserScalarKind[capacity]; + uint count = 0; + var history = new List<(RegisterBanks Banks, uint[] Words, UserScalarKind[] Kinds, uint Count)>(); + for (var step = 0; step < 200; step++) + { + var slot = random.Next(capacity); + var length = random.Next(1, capacity - slot + 1); + var marker = (UserScalarKind)random.Next(3); + var values = words.AsSpan(slot, length).ToArray(); + if (step % 3 == 0) values[^1] = (uint)random.Next(); + banks.UserDataMarker = marker; + Assert.Equal((uint)length, Write(banks, first + (uint)slot, values)); + values.CopyTo(words, slot); + Array.Fill(kinds, marker, slot, length); + count = Math.Max(count, (uint)(slot + length)); + history.Add((banks.Snapshot(), (uint[])words.Clone(), (UserScalarKind[])kinds.Clone(), count)); + } + foreach (var saved in history) + { + var scalars = Scalars(saved.Banks, first); + Assert.Equal(saved.Count, scalars.Count); + Assert.Equal(saved.Words, ((ReadOnlySpan)scalars.Values)[..capacity].ToArray()); + Assert.Equal(saved.Kinds, ((ReadOnlySpan)scalars.Kinds)[..capacity].ToArray()); + Assert.Equal(UserScalarKind.Unknown, saved.Banks.UserDataMarker); + } + } + + [Fact] + public void MeasureScalarSnapshotWritesWhenRequested() + { + var output = Environment.GetEnvironmentVariable("SHARPEMU_SCALAR_REUSE_BENCH_OUTPUT"); + if (string.IsNullOrEmpty(output)) return; + const int count = 100000; + var rows = new List(); + foreach (var length in new[] { 2, 16, 32 }) + foreach (var pattern in new[] { "identical", "alternating", "mostly-changing" }) + { + var banks = Banks(); + var values = Enumerable.Range(0, length).Select(index => (uint)index).ToArray(); + Exercise(banks, values, pattern, 10000); + for (var trial = 0; trial < 7; trial++) + { + var bytes = GC.GetAllocatedBytesForCurrentThread(); + var start = Stopwatch.GetTimestamp(); + var checksum = Exercise(banks, values, pattern, count); + var duration = Stopwatch.GetElapsedTime(start); + rows.Add(new { length, pattern, trial, checksum, nsPerWrite = duration.TotalNanoseconds / count, + bytesPerWrite = (double)(GC.GetAllocatedBytesForCurrentThread() - bytes) / count }); + } + } + File.WriteAllText(output, JsonSerializer.Serialize(new { enabled = Enabled, + architecture = RuntimeInformation.ProcessArchitecture.ToString(), rows }, new JsonSerializerOptions { WriteIndented = true })); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong Exercise(RegisterBanks banks, uint[] values, string pattern, int count) + { + ulong checksum = 0; + for (var i = 0; i < count; i++) + { + if (pattern == "alternating") values[^1] = (uint)i; + else if (pattern == "mostly-changing" && i % 8 != 0) values[^1] = (uint)i; + Write(banks, SpiShaderUserDataPs0, values); + var snapshot = banks.Snapshot(); + checksum += snapshot.Shader.Pixel.UserScalars.Values[values.Length - 1]; + } + return checksum; + } +} diff --git a/tests/SharpEmu.Libs.Tests/Gpu/GpuCommands/Registers/RegisterWriteTableTests.cs b/tests/SharpEmu.Libs.Tests/Gpu/GpuCommands/Registers/RegisterWriteTableTests.cs index 8d9b00923..f6c3b8a23 100644 --- a/tests/SharpEmu.Libs.Tests/Gpu/GpuCommands/Registers/RegisterWriteTableTests.cs +++ b/tests/SharpEmu.Libs.Tests/Gpu/GpuCommands/Registers/RegisterWriteTableTests.cs @@ -16,6 +16,54 @@ public sealed class RegisterWriteTableTests { private const ulong PacketAddress = 0x1_0000_1000; + [Theory] + [InlineData(SpiShaderUserDataPs0, 32u)] + [InlineData(SpiShaderUserDataGs0, 32u)] + [InlineData(SpiShaderUserDataHs0, 32u)] + [InlineData(ComputeUserData0, 16u)] + public void DirectScalarBatchPreservesKindsCountAndSnapshotsAtTheStageBoundary(uint first, uint capacity) + { + static UserScalarRegisters Scalars(RegisterBanks banks, uint first) => first switch + { + SpiShaderUserDataPs0 => banks.Shader.Pixel.UserScalars, + SpiShaderUserDataGs0 => banks.Shader.Vertex.GeometryUserScalars, + SpiShaderUserDataHs0 => banks.Shader.Vertex.HullUserScalars, + ComputeUserData0 => banks.Shader.Compute.UserScalars, + _ => throw new InvalidOperationException(), + }; + + var banks = NewBanks(); + banks.UserDataMarker = UserScalarKind.Region; + Assert.Equal(2u, WriteShader(banks, first + capacity - 2, 17, 19)); + var before = banks.Snapshot(); + var original = Scalars(before, first); + Assert.Equal(capacity, original.Count); + Assert.Equal(17u, original.Values[(int)capacity - 2]); + Assert.Equal(19u, original.Values[(int)capacity - 1]); + Assert.Equal(UserScalarKind.Region, original.Kinds[(int)capacity - 2]); + Assert.Equal(UserScalarKind.Region, original.Kinds[(int)capacity - 1]); + Assert.Equal(0u, original.Values[(int)capacity - 3]); + Assert.Equal(UserScalarKind.Unknown, banks.UserDataMarker); + + WriteShader(banks, first + capacity - 2, 23); + var current = Scalars(banks, first); + Assert.NotSame(original, current); + Assert.Equal(capacity, current.Count); + Assert.Equal(23u, current.Values[(int)capacity - 2]); + Assert.Equal(UserScalarKind.Unknown, current.Kinds[(int)capacity - 2]); + Assert.Equal(19u, current.Values[(int)capacity - 1]); + Assert.Equal(UserScalarKind.Region, current.Kinds[(int)capacity - 1]); + Assert.Equal(17u, original.Values[(int)capacity - 2]); + Assert.Equal(UserScalarKind.Region, original.Kinds[(int)capacity - 2]); + + banks.UserDataMarker = UserScalarKind.Region; + var error = Assert.Throws(() => WriteShader(banks, first + capacity - 1, 29, 31)); + Assert.Contains("outside the stage window", error.Message); + Assert.Equal(19u, Scalars(banks, first).Values[(int)capacity - 1]); + Assert.Equal(capacity, Scalars(banks, first).Count); + Assert.Equal(UserScalarKind.Region, banks.UserDataMarker); + } + [Fact] public void GeometryUserDataPointerPreservesBothWordsAndSubmissionCopy() { diff --git a/tests/SharpEmu.Libs.Tests/Gpu/Images/ColorTargetMemoTests.cs b/tests/SharpEmu.Libs.Tests/Gpu/Images/ColorTargetMemoTests.cs new file mode 100644 index 000000000..c62ed654f --- /dev/null +++ b/tests/SharpEmu.Libs.Tests/Gpu/Images/ColorTargetMemoTests.cs @@ -0,0 +1,80 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using SharpEmu.Libs.Gpu.Images; +using SharpEmu.Libs.Tests.Gpu.Scheduling; +using Xunit; + +namespace SharpEmu.Libs.Tests.Gpu.Images; + +[Collection(SchedulingStateCollection.Name)] +public sealed class ColorTargetMemoTests +{ + [Fact] + public void EquivalentMasksAndCollidingTargetsReuseBuiltLayouts() + { + // These addresses collide in the original eight-slot address hash. + var targets = Enumerable.Range(0, 4).Select(i => + RegisterWords.Color(0x31000000UL + (ulong)i * 0x800000, 64, 64)).ToArray(); + foreach (var words in targets) + { + Assert.NotNull(ImageRequestBuilders.ColorTarget(words, 1, 0, false)); + } + + ulong sum = 0; + var before = GC.GetAllocatedBytesForCurrentThread(); + for (var repeat = 0; repeat < 128; repeat++) + { + foreach (var words in targets) + { + var result = ImageRequestBuilders.ColorTarget(words, 0xFFFF, 0, true)!.Value; + sum += result.Request.Description.MipLayout[0].Size; + } + } + var allocated = GC.GetAllocatedBytesForCurrentThread() - before; + Assert.True(sum > 0); + Assert.True(allocated <= 512, $"Warm equivalent targets allocated {allocated} bytes."); + } + + [Fact] + public void NativeMetadataModeIsPartOfTheMemoKey() + { + var previous = ImageRequestBuilders.NativeColorClearMetadata; + var words = RegisterWords.Color(0x32000000, 64, 64, GuestTileMode.RenderTarget); + words = words with { Info = words.Info | (1u << 13), CmaskAddress = 0x34000000, ClearWord0 = 0xff000000 }; + try + { + ImageRequestBuilders.NativeColorClearMetadata = true; + var enabled = ImageRequestBuilders.ColorTarget(words, 15, 0, false)!.Value; + Assert.True(enabled.Request.Description.Metadata.NativeColorClear); + ImageRequestBuilders.NativeColorClearMetadata = false; + var disabled = ImageRequestBuilders.ColorTarget(words, 15, 0, false)!.Value; + Assert.False(disabled.Request.Description.Metadata.NativeColorClear); + Assert.Equal(MetadataKind.None, disabled.Request.Description.Metadata.Kind); + ImageRequestBuilders.NativeColorClearMetadata = true; + Assert.True(ImageRequestBuilders.ColorTarget(words, 15, 0, false)!.Value.Request.Description.Metadata.NativeColorClear); + } + finally + { + ImageRequestBuilders.NativeColorClearMetadata = previous; + } + } + + [Fact] + public void MaskedTargetAndChangedRegistersDoNotReuseAnActiveRequest() + { + var words = RegisterWords.Color(0x35000000, 64, 64); + var active = ImageRequestBuilders.ColorTarget(words, 1, 0, false)!.Value; + Assert.Null(ImageRequestBuilders.ColorTarget(words, 0xF0, 0, false)); + Assert.NotNull(ImageRequestBuilders.ColorTarget(words, 0, 0, true)); + Assert.Null(ImageRequestBuilders.ColorTarget(words with { BaseAddress = 0 }, 0, 0, true)); + var changed = RegisterWords.Color(words.BaseAddress, 128, 32); + var resolved = ImageRequestBuilders.ColorTarget(changed, 2, 0, false)!.Value; + Assert.Equal(128u, resolved.Extent.Width); + Assert.Equal(32u, resolved.Extent.Height); + Assert.Equal(64u, active.Extent.Width); + var copied = active.Request.Description; + copied.MipLayout[0] = default; + Assert.NotEqual(0UL, ImageRequestBuilders.ColorTarget(words, 15, 0, false)!.Value.Request.Description.MipLayout[0].Size); + } +} diff --git a/tests/SharpEmu.Libs.Tests/Gpu/Images/DepthTargetMemoTests.cs b/tests/SharpEmu.Libs.Tests/Gpu/Images/DepthTargetMemoTests.cs new file mode 100644 index 000000000..d6795b6a9 --- /dev/null +++ b/tests/SharpEmu.Libs.Tests/Gpu/Images/DepthTargetMemoTests.cs @@ -0,0 +1,110 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using System.Diagnostics; +using SharpEmu.Libs.Gpu.Abstractions; +using SharpEmu.Libs.Gpu.Buffers; +using SharpEmu.Libs.Gpu.Images; +using SharpEmu.ShaderCompiler; +using Xunit; +using Xunit.Abstractions; + +namespace SharpEmu.Libs.Tests.Gpu.Images; + +public sealed class DepthTargetMemoTests(ITestOutputHelper output) +{ + private sealed class FormatSupport : IImageFormatSupport + { + public int Queries; + public bool TryGetImageFormatProperties(GpuFormat format, GpuImageType type, GpuImageTiling tiling, + GpuImageUsage usage, GpuImageCreateFlags flags, out GpuImageFormatProperties properties) + { + Queries++; + properties = new GpuImageFormatProperties { SampleCounts = GpuSampleCount.Count1Bit }; + return true; + } + } + + private static DepthTargetWords[] Targets() => Enumerable.Range(0, 4) + .Select(i => RegisterWords.Depth(0x61000000UL + (ulong)i * 0x800000, 64, 64)).ToArray(); + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void AlternatingEqualReadWriteBasesRetainsAllFourTargets(bool sameAddress) + { + using var reuse = WorkerScratch.Override(true); + var device = new FormatSupport(); + var targets = sameAddress + ? Enumerable.Range(1, 4).Select(i => RegisterWords.Depth(0x61000000, (uint)i * 64, 64)).ToArray() + : Targets(); + foreach (var words in targets) Assert.NotNull(ImageRequestBuilders.DepthTarget(words, device)); + var queries = device.Queries; + var before = GC.GetAllocatedBytesForCurrentThread(); + ulong sum = 0; + for (var repeat = 0; repeat < 128; repeat++) + foreach (var words in targets) + sum += ImageRequestBuilders.DepthTarget(words, device)!.Value.DepthSize; + var bytes = GC.GetAllocatedBytesForCurrentThread() - before; + Assert.True(sum > 0); + Assert.Equal(queries, device.Queries); + Assert.True(bytes <= 512, $"Warm alternating targets allocated {bytes} bytes."); + } + + [Fact] + public void DeviceStateAndCopiedLayoutsRemainIndependent() + { + using var reuse = WorkerScratch.Override(true); + var first = new FormatSupport(); + var second = new FormatSupport(); + var words = Targets()[0]; + var original = ImageRequestBuilders.DepthTarget(words, first)!.Value; + Assert.NotNull(ImageRequestBuilders.DepthTarget(words, second)); + Assert.True(first.Queries > 0 && second.Queries > 0); + var changed = ImageRequestBuilders.DepthTarget(RegisterWords.Depth(words.ZReadBase, 128, 32, depthClear: true), first)!.Value; + Assert.Equal(128u, changed.Width); + Assert.Equal(32u, changed.Height); + Assert.True(changed.DepthClearEnabled); + var copied = original.Request.Description; + copied.MipLayout[0] = default; + var again = ImageRequestBuilders.DepthTarget(words, first)!.Value; + Assert.Equal(64u, again.Width); + Assert.False(again.DepthClearEnabled); + Assert.NotEqual(0UL, again.Request.Description.MipLayout[0].Size); + Assert.Null(ImageRequestBuilders.DepthTarget(RegisterWords.Depth(words.ZReadBase, 64, 64, depthTest: false, depthWrite: false), first)); + } + + [Fact] + public void AlternatingDepthTargetBenchmark() + { + if (Environment.GetEnvironmentVariable("SHARPEMU_DEPTH_MEMO_BENCH") != "1") return; + var architecture = System.Runtime.InteropServices.RuntimeInformation.ProcessArchitecture; + // The native clock agrees with elapsed CPU time on arm64. Rosetta's result was not + // calibrated, so do not label its raw values as nanoseconds in benchmark reports. + var cpuTimingSupported = OperatingSystem.IsMacOS() && + architecture == System.Runtime.InteropServices.Architecture.Arm64; + output.WriteLine($"process_arch={architecture} cpu_clock_supported={cpuTimingSupported}"); + using var reuse = WorkerScratch.Override(true); + var device = new FormatSupport(); + foreach (var count in new[] { 1, 4 }) + { + var targets = Targets().Take(count).ToArray(); + ulong sum = 0; + for (var i = 0; i < 20000; i++) sum += ImageRequestBuilders.DepthTarget(targets[i % count], device)!.Value.DepthSize; + const int iterations = 100000; + for (var batch = 0; batch < 7; batch++) + { + var queries = device.Queries; + var bytes = GC.GetAllocatedBytesForCurrentThread(); + var cpuStart = cpuTimingSupported ? SharpEmu.HLE.DsTrace.DsTrace.SelfCpuNs() : 0; + var start = Stopwatch.GetTimestamp(); + for (var i = 0; i < iterations; i++) sum += ImageRequestBuilders.DepthTarget(targets[i % count], device)!.Value.DepthSize; + var elapsed = Stopwatch.GetTimestamp() - start; + var cpu = cpuTimingSupported ? SharpEmu.HLE.DsTrace.DsTrace.SelfCpuNs() - cpuStart : 0; + bytes = GC.GetAllocatedBytesForCurrentThread() - bytes; + output.WriteLine($"depth_memo count={count} batch={batch} ns={elapsed * 1e9 / Stopwatch.Frequency / iterations:F2} bytes={bytes / (double)iterations:F2} format_queries={device.Queries - queries} cpu_ns={cpu / (double)iterations:F2}"); + } + Assert.True(sum > 0); + } + } +} diff --git a/tests/SharpEmu.Libs.Tests/Gpu/Images/MipLevelLayoutsTests.cs b/tests/SharpEmu.Libs.Tests/Gpu/Images/MipLevelLayoutsTests.cs new file mode 100644 index 000000000..49ef389a6 --- /dev/null +++ b/tests/SharpEmu.Libs.Tests/Gpu/Images/MipLevelLayoutsTests.cs @@ -0,0 +1,76 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using SharpEmu.Libs.Gpu.Images; +using Xunit; + +namespace SharpEmu.Libs.Tests.Gpu.Images; + +public sealed class MipLevelLayoutsTests +{ + [Theory] + [InlineData(1)] + [InlineData(7)] + [InlineData(16)] + public void BulkLayoutMatchesIndividualWritesAndOwnsItsInput(int count) + { + var values = Enumerable.Range(0, count).Select(i => new MipLevelLayout + { + Offset = (ulong)i * 4096, Size = (ulong)(16 - i) * 256, + Pitch = (uint)(128 >> Math.Min(i, 7)), Height = (uint)(64 >> Math.Min(i, 6)), + }).ToArray(); + var individual = new MipLevelLayouts(); + for (var i = 0; i < count; i++) individual[i] = values[i]; + var bulk = new MipLevelLayouts(); + bulk.SetPrefix(values); + Array.Clear(values); + for (var i = 0; i < ImageDescription.MaxLevels; i++) Assert.Equal(individual[i], bulk[i]); + + var copy = bulk; + bulk.SetPrefix([new MipLevelLayout { Size = 99 }]); + Assert.Equal(individual[0], copy[0]); + Assert.Equal(99UL, bulk[0].Size); + for (var i = 1; i < ImageDescription.MaxLevels; i++) Assert.Equal(copy[i], bulk[i]); + copy[count - 1] = default; + Assert.NotEqual(0UL, bulk[count - 1].Size); + } + + [Fact] + public void EmptyOrInvalidPrefixDoesNotChangeExistingValues() + { + var layouts = new MipLevelLayouts(); + layouts[15] = new MipLevelLayout { Size = 17 }; + layouts.SetPrefix([]); + Assert.Throws(() => layouts.SetPrefix(new MipLevelLayout[17])); + Assert.Equal(17UL, layouts[15].Size); + Assert.Equal(default, layouts[-1]); + Assert.Equal(default, layouts[16]); + } + + [Fact] + public void FullChainAllocatesOneArrayInsteadOfSixteen() + { + var values = new MipLevelLayout[16]; + values[15].Size = 1; + var warm = new MipLevelLayouts(); + warm.SetPrefix(values); + warm[15] = values[15]; + var before = GC.GetAllocatedBytesForCurrentThread(); + for (var repeat = 0; repeat < 128; repeat++) + { + var chain = new MipLevelLayouts(); + for (var i = 0; i < values.Length; i++) chain[i] = values[i]; + GC.KeepAlive(chain); + } + var individual = GC.GetAllocatedBytesForCurrentThread() - before; + before = GC.GetAllocatedBytesForCurrentThread(); + for (var repeat = 0; repeat < 128; repeat++) + { + var chain = new MipLevelLayouts(); + chain.SetPrefix(values); + GC.KeepAlive(chain); + } + var bulk = GC.GetAllocatedBytesForCurrentThread() - before; + Assert.True(bulk * 6 < individual, $"Bulk={bulk}, individual={individual} bytes."); + } +} diff --git a/tests/SharpEmu.Libs.Tests/Gpu/Images/NativeColorClearRequestTests.cs b/tests/SharpEmu.Libs.Tests/Gpu/Images/NativeColorClearRequestTests.cs index e8b1bc46a..da852a698 100644 --- a/tests/SharpEmu.Libs.Tests/Gpu/Images/NativeColorClearRequestTests.cs +++ b/tests/SharpEmu.Libs.Tests/Gpu/Images/NativeColorClearRequestTests.cs @@ -2,13 +2,26 @@ // SPDX-License-Identifier: GPL-2.0-or-later using SharpEmu.Libs.Gpu.Images; +using SharpEmu.Libs.Tests.Gpu.Scheduling; using SharpEmu.ShaderCompiler; using Xunit; namespace SharpEmu.Libs.Tests.Gpu.Images; -public sealed class NativeColorClearRequestTests +[Collection(SchedulingStateCollection.Name)] +public sealed class NativeColorClearRequestTests : IDisposable { + private readonly bool _previousNativeMetadata = ImageRequestBuilders.NativeColorClearMetadata; + + public NativeColorClearRequestTests() + { + // FakeCpuMemory registers the Metal backend in other tests, disabling this + // process-wide mode. These tests exercise the native metadata builder explicitly. + ImageRequestBuilders.NativeColorClearMetadata = true; + } + + public void Dispose() => ImageRequestBuilders.NativeColorClearMetadata = _previousNativeMetadata; + [Fact] public void CmaskTargetUsesItsOwnMetadataSizeAndPackedClear() { diff --git a/tests/SharpEmu.Libs.Tests/Gpu/Pipelines/ShaderPipelineLookupCacheTests.cs b/tests/SharpEmu.Libs.Tests/Gpu/Pipelines/ShaderPipelineLookupCacheTests.cs index 48b80658d..96dc3f1b4 100644 --- a/tests/SharpEmu.Libs.Tests/Gpu/Pipelines/ShaderPipelineLookupCacheTests.cs +++ b/tests/SharpEmu.Libs.Tests/Gpu/Pipelines/ShaderPipelineLookupCacheTests.cs @@ -227,6 +227,39 @@ public void ARewrittenAttributeTablePointerIsSeenOnTheNextDraw() Assert.Equal(B + 0x22030, AddressOf(second, 2)); } + [Theory] + [InlineData(true, false)] + [InlineData(false, true)] + [InlineData(true, true)] + public void RelocatedIdenticalVertexTablesReuseLayoutsButRewrittenBytesDoNot(bool moveAttributes, bool moveBuffers) + { + var first = Draw(); + var attributes = moveAttributes ? B + 0x9600 : AttributeTable; + var buffers = moveBuffers ? B + 0x9800 : BufferTable; + Span attributeBytes = stackalloc byte[3 * 4]; + Span bufferBytes = stackalloc byte[3 * 16]; + Assert.True(_memory.TryRead(AttributeTable, attributeBytes)); + Assert.True(_memory.TryRead(BufferTable, bufferBytes)); + Assert.True(_memory.TryWrite(attributes, attributeBytes)); + Assert.True(_memory.TryWrite(buffers, bufferBytes)); + SetAttributeTable(attributes); + var user = _banks.Shader.Vertex.GeometryUserScalars; + user.Set(10, unchecked((uint)buffers), UserScalarKind.Unknown); + user.Set(11, (uint)(buffers >> 32), UserScalarKind.Unknown); + + var moved = Draw(); + Assert.Same(first.VertexInput.Attributes, moved.VertexInput.Attributes); + Assert.Same(first.VertexInput.Buffers, moved.VertexInput.Buffers); + + WriteWord(buffers + 2 * 16, unchecked((uint)(B + 0x30000))); + var changed = Draw(); + Assert.NotSame(first.VertexInput.Buffers, changed.VertexInput.Buffers); + Assert.Equal(B + 0x30000, AddressOf(changed, 2)); + Assert.Equal(B + 0x22000, AddressOf(first, 2)); + WriteAttribute(attributes, 1, 1, offset: 32); + Assert.Equal(B + 0x21020, AddressOf(Draw(), 1)); + } + [Fact] public void ARewrittenBufferDescriptorIsSeenOnTheNextDraw() { diff --git a/tests/SharpEmu.Libs.Tests/Logging/MemoryTelemetryTests.cs b/tests/SharpEmu.Libs.Tests/Logging/MemoryTelemetryTests.cs new file mode 100644 index 000000000..2c1b78183 --- /dev/null +++ b/tests/SharpEmu.Libs.Tests/Logging/MemoryTelemetryTests.cs @@ -0,0 +1,58 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using System.ComponentModel; +using System.Text; +using SharpEmu.Logging; +using Xunit; + +namespace SharpEmu.Libs.Tests.Logging; + +public sealed class MemoryTelemetryTests +{ + [Fact] + public void FailedProcessInspectionDoesNotStopOtherSectionsOrTheNextSample() + { + using var output = new StringWriter(); + var calls = 0; + (string, Func)[] providers = + [ + ("broken", () => throw new InvalidOperationException("provider unavailable")), + ("healthy", () => $"sample={++calls}"), + ]; + MemoryTelemetry.ReportSections(output, + () => throw new Win32Exception(12, "process inspection unavailable"), providers); + MemoryTelemetry.ReportSections(output, () => "threads=8", providers); + + Assert.Equal(2, calls); + var lines = output.ToString().Split(Environment.NewLine, StringSplitOptions.RemoveEmptyEntries); + Assert.Equal(6, lines.Length); + Assert.Contains("dotnet error=Win32Exception", lines[0]); + Assert.Contains("broken error=InvalidOperationException", lines[1]); + Assert.Equal("[MEM] healthy sample=1", lines[2]); + Assert.Equal("[MEM] dotnet threads=8", lines[3]); + Assert.Equal("[MEM] healthy sample=2", lines[5]); + } + + [Fact] + public void AFailedDiagnosticSinkCannotTerminateReporting() + { + using var output = new RecoveringWriter(); + MemoryTelemetry.ReportSections(output, () => "threads=8", + [("healthy", () => "continued")]); + Assert.Equal(3, output.Attempts); + Assert.Equal("[MEM] healthy continued" + Environment.NewLine, output.Saved.ToString()); + } + + private sealed class RecoveringWriter : TextWriter + { + public override Encoding Encoding => Encoding.UTF8; + public int Attempts { get; private set; } + public StringBuilder Saved { get; } = new(); + public override void WriteLine(string? value) + { + if (++Attempts <= 2) throw new IOException("diagnostic output unavailable"); + Saved.AppendLine(value); + } + } +} diff --git a/tests/SharpEmu.Libs.Tests/Memory/GuestMemory/GuestViewTableTests.cs b/tests/SharpEmu.Libs.Tests/Memory/GuestMemory/GuestViewTableTests.cs index dd634104c..67868a477 100644 --- a/tests/SharpEmu.Libs.Tests/Memory/GuestMemory/GuestViewTableTests.cs +++ b/tests/SharpEmu.Libs.Tests/Memory/GuestMemory/GuestViewTableTests.cs @@ -48,6 +48,29 @@ private static GuestViewLogEntry[] Log(GuestViewTable table, ulong from = 0) return records.ToArray(); } + [Fact] + public void TranslatorDiagnosticsReadPublishedChangesWithoutChangingTranslations() + { + using var table = NewTable(logCapacity: 4); + var translator = new GuestMirrorTranslator(table.Region.Address, table.Region.Size, MirrorBase, MirrorBytes); + table.Map(Base, Block, 0, HostPageProtection.ReadWrite); + Assert.True(translator.TryTranslate(Base, 4, out var before)); + for (ulong i = 1; i < 6; i++) table.Map(Base + i * 2 * Block, Block, i * Block, HostPageProtection.ReadWrite); + var generation = translator.MappingGeneration; + Assert.Equal(table.LogSequence, translator.ChangeSequence); + Assert.False(translator.TryReadChange(0, out _)); // overwritten ring slot + Assert.False(translator.TryReadChange(table.LogSequence, out _)); // unpublished future slot + for (ulong i = 2; i < table.LogSequence; i++) + { + Assert.True(table.TryReadLog(i, out var expected)); + Assert.True(translator.TryReadChange(i, out var actual)); + Assert.Equal(expected, actual); + } + Assert.Equal(generation, translator.MappingGeneration); + Assert.True(translator.TryTranslate(Base, 4, out var after)); + Assert.Equal(before, after); + } + [Fact] public void TheLogRecordsEveryChangeInTheOrderItWasMade() { diff --git a/tests/SharpEmu.Libs.Tests/Pad/HostWindowInputTests.cs b/tests/SharpEmu.Libs.Tests/Pad/HostWindowInputTests.cs index 938404c3d..438aaa488 100644 --- a/tests/SharpEmu.Libs.Tests/Pad/HostWindowInputTests.cs +++ b/tests/SharpEmu.Libs.Tests/Pad/HostWindowInputTests.cs @@ -169,6 +169,38 @@ public void CommonHostInputUsesPublishedWindowSource() } } + [Fact] + public void ScriptedOnlyInputIgnoresPhysicalKeysControllersAndOutputs() + { + var output = new RecordingGamepadOutput(); + var input = new WindowHostInput(scriptedOnly: true); + HostWindowInput.Connect(output); + try + { + HostWindowInput.SetKey(0x57, true); + HostWindowInput.SetGamepad("Physical pad", new HostGamepadState( + true, HostGamepadButtons.Cross, 0, 255, 0, 255, 255, 255)); + Span states = stackalloc HostGamepadState[1]; + + Assert.False(input.IsKeyDown(0x57)); + Assert.False(input.IsHostWindowFocused()); + Assert.Equal(0, input.GetGamepadStates(states)); + Assert.Null(input.DescribeConnectedGamepad()); + input.SetRumble(10, 20); + input.SetTriggerRumble(30, 40); + input.SetLightbar(50, 60, 70); + input.ResetLightbar(); + Assert.Equal(((byte)0, (byte)0), output.Rumble); + Assert.Equal(((byte?)null, (byte?)null), output.TriggerRumble); + Assert.Equal(((byte)0, (byte)0, (byte)0), output.Lightbar); + Assert.False(output.LightbarReset); + } + finally + { + HostWindowInput.Disconnect(); + } + } + private sealed class RecordingGamepadOutput : IHostGamepadOutput { public (byte Large, byte Small) Rumble { get; private set; } diff --git a/tests/SharpEmu.ShaderCompiler.Metal.Tests/Gen5NggFixtures.cs b/tests/SharpEmu.ShaderCompiler.Metal.Tests/Gen5NggFixtures.cs index c75804561..f35f5add8 100644 --- a/tests/SharpEmu.ShaderCompiler.Metal.Tests/Gen5NggFixtures.cs +++ b/tests/SharpEmu.ShaderCompiler.Metal.Tests/Gen5NggFixtures.cs @@ -145,6 +145,83 @@ void Add(string opcode, Gen5ShaderEncoding encoding, Gen5Operand[] sources, private static uint Branch(uint opcodeWord, uint fromPc, uint toPc) => opcodeWord | (ushort)(short)(((int)toPc - (int)(fromPc + 4)) / 4); + // The two waves take opposite exits after their LDS exchange. One keeps the + // original parameter; the other overwrites it. Both must reach the mesh epilogue. + public static Gen5ShaderProgram ConditionalMeshExit(bool firstWaveExitsEarly) + { + var instructions = CullingTriangle().Instructions.SkipLast(1).ToList(); + var pc = instructions[^1].Pc + 4; + instructions.Add(new(pc, Gen5ShaderEncoding.Sop2, "SAndB32", [0u], + [Scalar(Gen5NggExports.MergedWaveInfoRegister), Literal(1u << 24)], [Scalar(30)], null)); + pc += 4; + instructions.Add(new(pc, Gen5ShaderEncoding.Sopc, "SCmpEqU32", [0u], + [Scalar(30), Literal(firstWaveExitsEarly ? 0u : 1u << 24)], [], null)); + pc += 4; + instructions.Add(new(pc, Gen5ShaderEncoding.Sopp, "SCbranchScc1", + [Branch(BranchScc1Word, pc, pc + 16)], [], [], null)); + pc += 4; + instructions.Add(new(pc, Gen5ShaderEncoding.Vop1, "VMovB32", [0u], + [Literal(Bits(0.25f))], [Vector(15)], null)); + pc += 4; + instructions.Add(new(pc, Gen5ShaderEncoding.Exp, "Exp", [0u], + [Vector(15), Vector(15), Vector(15), Vector(15)], [], + new Gen5ExportControl(Gen5NggExports.FirstParameterTarget, 0xF, false, false, false))); + pc += 4; + instructions.Add(new(pc, Gen5ShaderEncoding.Sopp, "SEndpgm", [EndProgramWord], [], [], null)); + pc += 4; + instructions.Add(new(pc, Gen5ShaderEncoding.Sopp, "SEndpgm", [EndProgramWord], [], [], null)); + return new Gen5ShaderProgram(0, instructions); + } + + public static Gen5ShaderProgram MaskedMeshAlu() + { + var instructions = MeshExitLoop(3).Instructions.SkipLast(1).ToList(); + var pc = instructions[^1].Pc + 4; + void Add(string opcode, Gen5ShaderEncoding encoding, Gen5Operand[] sources, + Gen5Operand[] destinations, Gen5InstructionControl? control = null) + { + instructions.Add(new(pc, encoding, opcode, [0u], sources, destinations, control)); + pc += 4; + } + Add("VMovB32", Gen5ShaderEncoding.Vop1, [Literal(Bits(0.125f))], [Vector(11)]); + Add("SMovB32", Gen5ShaderEncoding.Sop1, [Literal(5)], [Scalar(126)]); + Add("VMovB32", Gen5ShaderEncoding.Vop1, [Literal(Bits(0.75f))], [Vector(12)]); + Add("VAddF32", Gen5ShaderEncoding.Vop2, [Vector(12), Vector(12)], [Vector(11)]); + Add("SMovB64", Gen5ShaderEncoding.Sop1, [new Gen5Operand(Gen5OperandKind.EncodedConstant, 193)], [Scalar(126)]); + Add("Exp", Gen5ShaderEncoding.Exp, [Vector(11), Vector(11), Vector(11), Vector(11)], [], + new Gen5ExportControl(Gen5NggExports.FirstParameterTarget, 0xF, false, false, false)); + instructions.Add(new(pc, Gen5ShaderEncoding.Sopp, "SEndpgm", [EndProgramWord], [], [], null)); + return new Gen5ShaderProgram(0, instructions); + } + + public static Gen5ShaderProgram MeshExitLoop(uint iterations) + { + // One wave: read back its own LDS rather than the absent second wave. + var instructions = CullingTriangle().Instructions.SkipLast(1).Select(instruction => + instruction.Opcode == "VXorB32" + ? instruction with { Sources = [Inline(0), Vector(Gen5NggExports.EsVertexIndexRegister)] } + : instruction).ToList(); + var pc = instructions[^1].Pc + 4; + void Add(string opcode, Gen5ShaderEncoding encoding, Gen5Operand[] sources, + Gen5Operand[] destinations, Gen5InstructionControl? control = null, uint word = 0) + { + instructions.Add(new(pc, encoding, opcode, [word], sources, destinations, control)); + pc += 4; + } + Add("SMovB32", Gen5ShaderEncoding.Sop1, [Literal(iterations)], [Scalar(31)]); + var loop = pc; + Add("SSubU32", Gen5ShaderEncoding.Sop2, [Scalar(31), Inline(1)], [Scalar(31)]); + Add("SCmpEqU32", Gen5ShaderEncoding.Sopc, [Scalar(31), Inline(0)], []); + Add("SCbranchScc0", Gen5ShaderEncoding.Sopp, [], [], word: Branch(BranchScc0Word, pc, loop)); + // Only a normal loop exit executes this export. A capped loop preserves the + // preceding parameter, making an incorrect fall-through visible in the image. + Add("VMovB32", Gen5ShaderEncoding.Vop1, [Literal(Bits(0.25f))], [Vector(15)]); + Add("Exp", Gen5ShaderEncoding.Exp, [Vector(15), Vector(15), Vector(15), Vector(15)], [], + new Gen5ExportControl(Gen5NggExports.FirstParameterTarget, 0xF, false, false, false)); + Add("SEndpgm", Gen5ShaderEncoding.Sopp, [], [], word: EndProgramWord); + return new Gen5ShaderProgram(0, instructions); + } + private static void AppendTail(List instructions, ref uint pc) { var at = pc; @@ -249,6 +326,129 @@ public static Gen5ShaderProgram BarrierAfterLoopWithTwoExits() return new Gen5ShaderProgram(0, instructions); } + /// Every visible vertex samples the same LDS record, so its red output has + /// an exact CPU oracle. Mutations cover masked writers, aliasing stores/atomics, + /// another region, non-affine writers and wrapped/unaligned read addresses. + public static Gen5ShaderProgram MeshLdsForwardingProbe(uint readLane, int mutation, uint baseAddress = 0, bool allComponents = false) + { + var instructions = new List(); + uint pc = 0; + void Add(string op, Gen5ShaderEncoding encoding, Gen5Operand[] sources, + Gen5Operand[] destinations, Gen5InstructionControl? control = null) + { + instructions.Add(new Gen5ShaderInstruction(pc, encoding, op, [0], sources, destinations, control)); + pc += 4; + } + static Gen5DataShareControl Offset(uint value) => new(value & 255u, value >> 8, false); + var all = new Gen5Operand(Gen5OperandKind.EncodedConstant, 193); + void Restore() => Add("SMovB64", Gen5ShaderEncoding.Sop1, [all], [Scalar(126), Scalar(127)]); + Restore(); + Add("VMbcntLoU32B32", Gen5ShaderEncoding.Vop3, [all, Inline(0)], [Vector(40)]); + Add("VMovB32", Gen5ShaderEncoding.Vop1, [Vector(40)], [Vector(44)]); + if (mutation == 3) + Add("VXorB32", Gen5ShaderEncoding.Vop2, [Inline(1), Vector(44)], [Vector(44)]); + Add("VLshlrevB32", Gen5ShaderEncoding.Vop2, [Inline(4), Vector(44)], [Vector(44)]); + if (baseAddress != 0) + Add("VAddI32", Gen5ShaderEncoding.Vop2, [Literal(baseAddress), Vector(44)], [Vector(44)]); + for (uint i = 1; i < 4; i++) Add("VMovB32", Gen5ShaderEncoding.Vop1, [Inline(i)], [Vector(40 + i)]); + Add("DsWriteB128", Gen5ShaderEncoding.Ds, [Vector(44), Vector(40), Vector(41), Vector(42), Vector(43)], [], Offset(128)); + if (mutation == 1) + { + Add("SMovB64", Gen5ShaderEncoding.Sop1, [Literal(255)], [Scalar(126), Scalar(127)]); + Add("VAddI32", Gen5ShaderEncoding.Vop2, [Literal(100), Vector(40)], [Vector(40)]); + Add("DsWriteB128", Gen5ShaderEncoding.Ds, [Vector(44), Vector(40), Vector(41), Vector(42), Vector(43)], [], Offset(128)); + Restore(); + } + if (mutation == 4) + Add("DsWriteB128", Gen5ShaderEncoding.Ds, [Vector(44), Vector(40), Vector(41), Vector(42), Vector(43)], [], Offset(640)); + if (mutation is 2 or 5 or 6 or 10) + { + Add("SMovB64", Gen5ShaderEncoding.Sop1, [Inline(2)], [Scalar(126), Scalar(127)]); + Add("VMovB32", Gen5ShaderEncoding.Vop1, [Literal(baseAddress + readLane * 16 + (mutation == 10 ? 128u : 0u))], [Vector(45)]); + Add("VMovB32", Gen5ShaderEncoding.Vop1, [Literal(mutation is 2 or 10 ? 999u : mutation == 5 ? 127u : 1u)], [Vector(46)]); + Add(mutation is 2 or 10 ? "DsWriteB32" : mutation == 5 ? "DsWriteB8" : "DsAddU32", + Gen5ShaderEncoding.Ds, [Vector(45), Vector(46)], [], Offset(mutation == 10 ? 0u : 128u)); + Restore(); + } + Add("SBarrier", Gen5ShaderEncoding.Sopp, [], []); + var readAddress = baseAddress + readLane * 16 + (mutation == 7 ? 1024u : mutation == 8 ? 1u : 0u); + Add("VMovB32", Gen5ShaderEncoding.Vop1, [Literal(readAddress)], [Vector(45)]); + Add("DsReadB128", Gen5ShaderEncoding.Ds, [Vector(45)], [Vector(50), Vector(51), Vector(52), Vector(53)], Offset(128)); + if (allComponents) + { + for (uint i = 1; i < 4; i++) + { + Add("VLshlrevB32", Gen5ShaderEncoding.Vop2, [Inline(i * 5), Vector(50 + i)], [Vector(55)]); + Add("VAddI32", Gen5ShaderEncoding.Vop2, [Vector(55), Vector(50)], [Vector(50)]); + } + } + Add("VCvtF32U32", Gen5ShaderEncoding.Vop1, [Vector(50)], [Vector(54)]); + foreach (var instruction in GeometryQuad(false, 0).Instructions) + { + var next = instruction with { Pc = pc }; + if (next.Control is Gen5ExportControl { Target: Gen5NggExports.FirstParameterTarget }) + next = next with { Sources = new[] { Vector(54), Vector(20), Vector(20), Vector(21) } }; + instructions.Add(next); + pc += 4; + } + return new Gen5ShaderProgram(0, instructions); + } + + /// A fused geometry program emits a quad from one input triangle. With a + /// nonzero first vertex its output lives in a later guest wave. The lane exchange + /// and LDS round trip exercise both halves of wave64 and the workgroup barrier. + public static Gen5ShaderProgram GeometryQuad(bool wave64, uint firstVertex) + { + var instructions = new List(); + uint pc = 0; + void Add(string opcode, Gen5ShaderEncoding encoding, Gen5Operand[] sources, + Gen5Operand[] destinations, Gen5InstructionControl? control = null, uint word = 0) + { + instructions.Add(new Gen5ShaderInstruction(pc, encoding, opcode, [word], sources, destinations, control)); + pc += sizeof(uint); + } + var minusOne = new Gen5Operand(Gen5OperandKind.EncodedConstant, 193); + Add("VMbcntLoU32B32", Gen5ShaderEncoding.Vop3, [minusOne, Inline(0)], [Vector(10)]); + if (wave64) + Add("VMbcntHiU32B32", Gen5ShaderEncoding.Vop3, [minusOne, Vector(10)], [Vector(10)]); + Add("SBfeU32", Gen5ShaderEncoding.Sop2, [Scalar(3), Literal(0x00040018)], [Scalar(20)]); + Add("VLshlAddU32", Gen5ShaderEncoding.Vop3, [Scalar(20), Inline(wave64 ? 6u : 5u), Vector(10)], [Vector(11)]); + Add("VLshlrevB32", Gen5ShaderEncoding.Vop2, [Inline(2), Vector(11)], [Vector(12)]); + Add("DsWriteB32", Gen5ShaderEncoding.Ds, [Vector(12), Vector(11)], [], new Gen5DataShareControl(0, 0, false)); + Add("SBarrier", Gen5ShaderEncoding.Sopp, [], []); + Add("DsReadB32", Gen5ShaderEncoding.Ds, [Vector(12)], [Vector(11)], new Gen5DataShareControl(0, 0, false)); + Add("VReadlaneB32", Gen5ShaderEncoding.Vop3, [Vector(10), Inline(wave64 ? 63u : 31u)], [Scalar(21)]); + Add("SAddI32", Gen5ShaderEncoding.Sop2, [Scalar(21), Literal(unchecked(0u - (wave64 ? 63u : 31u)))], [Scalar(21)]); + Add("VAddI32", Gen5ShaderEncoding.Vop2, [Literal(unchecked(0u - firstVertex)), Vector(11)], [Vector(14)]); + Add("VAddI32", Gen5ShaderEncoding.Vop2, [Scalar(21), Vector(14)], [Vector(14)]); + Add("VAndB32", Gen5ShaderEncoding.Vop2, [Inline(1), Vector(14)], [Vector(15)]); + Add("VCvtF32U32", Gen5ShaderEncoding.Vop1, [Vector(15)], [Vector(16)]); + Add("VMulF32", Gen5ShaderEncoding.Vop2, [Literal(Bits(1.6f)), Vector(16)], [Vector(16)]); + Add("VAddF32", Gen5ShaderEncoding.Vop2, [Literal(Bits(-0.8f)), Vector(16)], [Vector(16)]); + Add("VLshrrevB32", Gen5ShaderEncoding.Vop2, [Inline(1), Vector(14)], [Vector(17)]); + Add("VCvtF32U32", Gen5ShaderEncoding.Vop1, [Vector(17)], [Vector(18)]); + Add("VMulF32", Gen5ShaderEncoding.Vop2, [Literal(Bits(1.6f)), Vector(18)], [Vector(19)]); + Add("VAddF32", Gen5ShaderEncoding.Vop2, [Literal(Bits(-0.8f)), Vector(19)], [Vector(19)]); + Add("VMovB32", Gen5ShaderEncoding.Vop1, [Inline(0)], [Vector(20)]); + Add("VMovB32", Gen5ShaderEncoding.Vop1, [Literal(Bits(1.0f))], [Vector(21)]); + Add("SMovB32", Gen5ShaderEncoding.Sop1, + [Literal((firstVertex + 4) | (2u << Gen5NggExports.AllocationPrimitiveCountShift))], [Scalar(124)]); + Add("SSendmsg", Gen5ShaderEncoding.Sopp, [], [], word: SendMessageWord); + Add("Exp", Gen5ShaderEncoding.Exp, [Vector(16), Vector(19), Vector(20), Vector(21)], [], + new Gen5ExportControl(Gen5NggExports.FirstPositionTarget, 0xF, false, false, false)); + Add("Exp", Gen5ShaderEncoding.Exp, [Vector(20), Vector(20), Vector(20), Vector(21)], [], + new Gen5ExportControl(Gen5NggExports.FirstParameterTarget, 0xF, false, false, false)); + // Primitive 0 joins vertices 0,1,2; primitive 1 joins 1,3,2. + var a = firstVertex | ((firstVertex + 1) << 10) | ((firstVertex + 2) << 20); + var b = (firstVertex + 1) | ((firstVertex + 3) << 10) | ((firstVertex + 2) << 20); + Add("VCmpEqU32", Gen5ShaderEncoding.Vop3, [Inline(1), Vector(11)], [Scalar(30)]); + Add("VCndmaskB32", Gen5ShaderEncoding.Vop3, [Literal(a), Literal(b), Scalar(30)], [Vector(22)]); + Add("Exp", Gen5ShaderEncoding.Exp, [Vector(22), Vector(22), Vector(22), Vector(22)], [], + new Gen5ExportControl(Gen5NggExports.PrimitiveTarget, 0x1, false, true, false)); + Add("SEndpgm", Gen5ShaderEncoding.Sopp, [], [], word: EndProgramWord); + return new Gen5ShaderProgram(0, instructions); + } + /// The stage state of a wave64 primitive shader with no GS half. public static Gen5NggStageInfo Wave64StageInfo { get; } = StageInfo with { diff --git a/tests/SharpEmu.ShaderCompiler.Metal.Tests/Goldens/exec-store-layout.msl b/tests/SharpEmu.ShaderCompiler.Metal.Tests/Goldens/exec-store-layout.msl index 2bb658f3f..5d71a4c1e 100644 --- a/tests/SharpEmu.ShaderCompiler.Metal.Tests/Goldens/exec-store-layout.msl +++ b/tests/SharpEmu.ShaderCompiler.Metal.Tests/Goldens/exec-store-layout.msl @@ -659,9 +659,6 @@ static inline void sharpemu_note_step_cap(device atomic_uint* report) s126 = t15; exec = ((t15) >> sharpemu_lane & 1u) != 0u; uint t16 = (0x10u + 0u + 0u + (0u * 0u)); - if (exec) - { - sharpemu_store_dword(b0, sharpemu_resources.buffer_bytes[0], t16 + 0u, v0); - } + sharpemu_store_dword(b0, sharpemu_resources.buffer_bytes[0], t16 + 0u, v0); } } diff --git a/tests/SharpEmu.ShaderCompiler.Metal.Tests/MetalNative.Mesh.cs b/tests/SharpEmu.ShaderCompiler.Metal.Tests/MetalNative.Mesh.cs index fa5232002..949440f77 100644 --- a/tests/SharpEmu.ShaderCompiler.Metal.Tests/MetalNative.Mesh.cs +++ b/tests/SharpEmu.ShaderCompiler.Metal.Tests/MetalNative.Mesh.cs @@ -56,7 +56,8 @@ public static bool TryRenderMesh( IReadOnlyList? extraResidentBuffers = null, uint layers = 1, bool volume = false, - MeshIndirectLaunch? indirect = null) + MeshIndirectLaunch? indirect = null, + IDictionary? readbackBuffers = null) { pixels = []; error = string.Empty; @@ -148,6 +149,7 @@ public static bool TryRenderMesh( SendVoid(encoder, Selector("setRenderPipelineState:"), pipeline); var residents = new List(); + var bound = new List<(nint Buffer, uint Index, int Length)>(); foreach (var (bytes, index) in meshBuffers) { var buffer = SharedBuffer(bytes.Length == 0 ? new byte[4] : bytes); @@ -158,6 +160,7 @@ public static bool TryRenderMesh( } residents.Add(buffer); + bound.Add((buffer, index, bytes.Length)); SendSetBuffer(encoder, Selector("setMeshBuffer:offset:atIndex:"), buffer, 0, index); } @@ -202,6 +205,10 @@ public static bool TryRenderMesh( indirect.Written = ReadSharedBuffer(launchSlot, indirect.Slot.Length); } + if (readbackBuffers is not null) + foreach (var (buffer, index, length) in bound) + readbackBuffers[index] = ReadSharedBuffer(buffer, length); + var bytesOut = volume ? ReadFloatTexture(target, width, height, 1, layers) : ReadFloatTexture(target, width, height, layers, 1); diff --git a/tests/SharpEmu.ShaderCompiler.Metal.Tests/MslConstantExecRestoreTests.cs b/tests/SharpEmu.ShaderCompiler.Metal.Tests/MslConstantExecRestoreTests.cs new file mode 100644 index 000000000..7929673b5 --- /dev/null +++ b/tests/SharpEmu.ShaderCompiler.Metal.Tests/MslConstantExecRestoreTests.cs @@ -0,0 +1,66 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using System.Buffers.Binary; +using Xunit; + +namespace SharpEmu.ShaderCompiler.Metal.Tests; + +public sealed class MslConstantExecRestoreTests +{ + [Theory] + [InlineData(32u, false, false)] + [InlineData(32u, false, true)] + [InlineData(32u, true, false)] + [InlineData(32u, true, true)] + [InlineData(64u, false, false)] + [InlineData(64u, false, true)] + [InlineData(64u, true, false)] + [InlineData(64u, true, true)] + public void ConstantRestore_OnlyWritesActiveLanes(uint lanes, bool wide, bool literal) + { + var restore = wide ? 0xBEFE0400u : 0xBEFE0300u; + uint[] source = literal ? [restore | 255u, uint.MaxValue] : [restore | 193u]; + var fixture = new Gen5ComputeFixture("constant-exec-restore", + [ + 0xD7650000, 0x000100C1, // v_mbcnt_lo_u32_b32 v0, -1, 0 + 0xD7660000, 0x000200C1, // v_mbcnt_hi_u32_b32 v0, -1, v0 + 0x34080082, // v_lshlrev_b32 v4, 2, v0 + 0x7E040287, // v_mov_b32 v2, 7 + 0xBEFE0480, // s_mov_b64 exec, 0 + 0x7E04028D, // v_mov_b32 v2, 13 (must stay masked) + .. source, + 0x7E04028B, // v_mov_b32 v2, 11 + 0xE0701000, 0x80020204, // buffer_store_dword v2, v4, s[8:11], 0 offen + 0xBF810000, + ], 8, checked((int)lanes * 4)); + var request = Gen5ComputeFixtures.CreateComputeRequest(fixture, lanes, lanes); + Assert.True(Gen5MslTranslator.TryCompileProgram(request, out var optimized, out var error), error); + Assert.True(Gen5MslTranslator.TryCompileProgram(request, out var guarded, out error, + new Gen5MslCodegenOptions(ExecGuardElision: false)), error); + if (Environment.GetEnvironmentVariable("SHARPEMU_TEST_REQUIRE_DEVICE") != "1") + { + return; + } + Assert.True(MetalNative.IsAvailable, "A Metal device is required."); + var expected = Execute(guarded); + var actual = Execute(optimized); + Assert.Equal(expected, actual); + for (var lane = 0; lane < lanes; lane++) + { + var active = lane < 32 || (wide && !literal); + Assert.Equal(active ? 11u : 0xCDCDCDCDu, + BinaryPrimitives.ReadUInt32LittleEndian(actual.AsSpan(lane * 4, 4))); + } + + byte[] Execute(Gen5MslShader shader) + { + Assert.True(MetalNative.TryCompileLibrary(shader.Source, out var library, out var compileError), compileError); + var bytes = Enumerable.Repeat((byte)0xCD, fixture.StoreBackingBytes).ToArray(); + Assert.True(MetalNative.TryExecuteWithArgumentBuffer(library, shader.EntryPoint, bytes, + new byte[Resources.PushData.ByteSize], shader.ArgumentLayout!, lanes, + out var result, out var runError), runError); + return result; + } + } +} diff --git a/tests/SharpEmu.ShaderCompiler.Metal.Tests/MslLiteralFormatLoadTests.cs b/tests/SharpEmu.ShaderCompiler.Metal.Tests/MslLiteralFormatLoadTests.cs index 3460eac49..d07f43b47 100644 --- a/tests/SharpEmu.ShaderCompiler.Metal.Tests/MslLiteralFormatLoadTests.cs +++ b/tests/SharpEmu.ShaderCompiler.Metal.Tests/MslLiteralFormatLoadTests.cs @@ -109,6 +109,46 @@ public void TheRunReadsWhatThePerComponentRulesRead(uint format, uint x, uint y, } } + [Theory] + [InlineData(14u)] // 8x2 UNORM: missing Z/W and a partial final word. + [InlineData(44u)] // 2:10:10:10 UNORM, all components in one word. + [InlineData(56u)] // 8x4 UNORM. + [InlineData(57u)] // 8x4 SNORM. + [InlineData(60u, false)] // 8x4 UINT. + [InlineData(60u, true)] // Coherent loads use the same packed span. + [InlineData(61u)] // 8x4 SINT. + [InlineData(65u)] // 16x4 UNORM, two words. + [InlineData(71u)] // 16x4 FLOAT, including arbitrary half bit patterns. + public void PackedRunsMatchTheDecoderAtAlignedUnalignedAndPartialElements(uint format, bool glc = false) + { + if (!RequireDevice()) return; + foreach (var offset in new[] { 16, 17, 61, 63 }) + foreach (var swizzle in new[] { Identity, Swizzle(7, 6, 5, 4), Swizzle(4, 1, 0, 4) }) + { + var initial = new byte[BufferBytes]; + for (var i = 0; i < initial.Length; i++) initial[i] = (byte)((i * 37) + 11); + byte[]? reference = null; + foreach (var fast in new[] { false, true }) + { + var shader = Compile(format, swizzle, offset, packed: fast, glc: glc); + if (fast) Assert.Contains(glc ? "sharpemu_load_words_coherent<" : "sharpemu_load_words<", shader.Source, StringComparison.Ordinal); + Assert.True(MetalNative.TryCompileLibrary(shader.Source, out var library, out var compileError), compileError); + Assert.True(MetalNative.TryExecuteWithArgumentBuffer(library, shader.EntryPoint, + initial, new byte[PushData.ByteSize], shader.ArgumentLayout!, 1, + out var result, out var runError), runError); + if (!fast) reference = result; + else Assert.Equal(reference, result); + // Independent oracle for the four unsigned-byte components. + if (format == 60 && swizzle == Identity) + { + for (var i = 0; i < 4; i++) + Assert.Equal(offset <= BufferBytes - 4 ? (uint)initial[offset + i] : 0u, + BinaryPrimitives.ReadUInt32LittleEndian(result.AsSpan(i * 4))); + } + } + } + } + private bool RequireDevice() { if (!MetalNative.IsAvailable) @@ -125,13 +165,13 @@ private bool RequireDevice() // v1 = the byte offset (OFFEN); buffer_load_format_xyzw v[4:7] from s[8:11]; the four // registers are stored at bytes 0, 4, 8 and 12 of the same buffer. - private static Gen5MslShader Compile(uint format, uint swizzle, int offset) + private static Gen5MslShader Compile(uint format, uint swizzle, int offset, bool packed = false, bool glc = false) { var fixture = new Gen5ComputeFixture( "format-load-literal", [ 0x7E0202FF, (uint)offset, // v_mov_b32 v1, offset - 0xE0000000u | (3u << 18) | (1u << 12), // buffer_load_format_xyzw v[4:7], v1, s[8:11], 0 offen + 0xE0000000u | (3u << 18) | (1u << 12) | (glc ? 1u << 14 : 0), // buffer_load_format_xyzw v[4:7], v1, s[8:11], 0 offen 0x80020401, 0xE0700000, 0x80020400, // buffer_store_dword v4 offset:0 0xE0700004, 0x80020500, // buffer_store_dword v5 offset:4 @@ -150,7 +190,8 @@ private static Gen5MslShader Compile(uint format, uint swizzle, int offset) Gen5ComputeFixtures.DecodeOrThrow(fixture), ShaderStage.Compute, runtimeInputs: new ResourceRuntimeInputs { UserData = userData }); - Assert.True(Gen5MslTranslator.TryCompileProgram(request, out var shader, out var error), error); + Assert.True(Gen5MslTranslator.TryCompileProgram(request, out var shader, out var error, + new Gen5MslCodegenOptions { PackedFormatLoads = packed }), error); return shader!; } } diff --git a/tests/SharpEmu.ShaderCompiler.Metal.Tests/MslStaticScratchSlotTests.cs b/tests/SharpEmu.ShaderCompiler.Metal.Tests/MslStaticScratchSlotTests.cs new file mode 100644 index 000000000..11de2e792 --- /dev/null +++ b/tests/SharpEmu.ShaderCompiler.Metal.Tests/MslStaticScratchSlotTests.cs @@ -0,0 +1,132 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using System.Buffers.Binary; +using SharpEmu.ShaderCompiler.Resources; +using Xunit; + +namespace SharpEmu.ShaderCompiler.Metal.Tests; + +public sealed class MslStaticScratchSlotTests +{ + [Theory] + [InlineData(7u, 32u, false)] + [InlineData(7u, 64u, false)] + [InlineData(46u, 32u, false)] + [InlineData(46u, 64u, false)] + [InlineData(46u, 1024u, false)] + [InlineData(46u, 1024u, true)] + public void RotatedSlots_PreserveAliasesAndMaskedStores(uint dwords, uint threads, bool relativeRead) + { + var request = Request(dwords, threads, relativeRead ? "relative-read" : ""); + var slow = Compile(request, false); + var fast = Compile(request, true); + Assert.Contains($"% {dwords}u]", slow.Source, StringComparison.Ordinal); + Assert.DoesNotContain($"% {dwords}u]", fast.Source, StringComparison.Ordinal); + Assert.Contains("sharpemu_scratch[0u]", fast.Source, StringComparison.Ordinal); + if (Environment.GetEnvironmentVariable("SHARPEMU_TEST_REQUIRE_DEVICE") != "1") return; + Assert.True(MetalNative.IsAvailable, "A Metal device is required."); + var expected = Execute(slow); + var actual = Execute(fast); + Assert.Equal(expected, actual); + for (var lane = 0; lane < threads; lane++) + { + Assert.Equal(lane % 2 == 0 ? 17u : 11u, + BinaryPrimitives.ReadUInt32LittleEndian(actual.AsSpan(lane * 8, 4))); + Assert.Equal(13u, BinaryPrimitives.ReadUInt32LittleEndian(actual.AsSpan(lane * 8 + 4, 4))); + } + + byte[] Execute(Gen5MslShader shader) + { + Assert.True(MetalNative.TryCompileLibrary(shader.Source, out var library, out var error), error); + Assert.True(MetalNative.TryExecuteWithArgumentBuffer(library, shader.EntryPoint, + new byte[threads * 8], new byte[PushData.ByteSize], shader.ArgumentLayout!, threads, + out var result, out error), error); + return result; + } + } + + [Theory] + [InlineData("changed-base")] + [InlineData("mixed-base")] + [InlineData("unaligned")] + [InlineData("negative")] + [InlineData("dynamic")] + [InlineData("return-to-access")] + [InlineData("relative-read-base")] + public void UnprovenAddressing_KeepsDynamicScratch(string variant) + { + var request = Request(46, 64, variant); + var shader = Compile(request, true); + Assert.Contains("% 46u]", shader.Source, StringComparison.Ordinal); + } + + private static Gen5MslShader Compile(ShaderCompileRequest request, bool enabled) + { + Assert.True(Gen5MslTranslator.TryCompileProgram(request, out var shader, out var error, + new Gen5MslCodegenOptions { StaticScratchSlots = enabled }), error); + return shader; + } + + private static ShaderCompileRequest Request(uint dwords, uint threads, string variant = "") + { + uint[] relative = variant is "relative-read" or "relative-read-base" + ? [0xBEFC0380, variant == "relative-read" ? 0x7E0C8700u : 0x7E008700u] + : []; + var fixture = new Gen5ComputeFixture("static-scratch", + [ + .. relative, + 0x34080083, // v_lshlrev_b32 v4, 3, v0 (local id * 8) + 0x7E04028B, // v_mov_b32 v2, 11 + 0xDC704000, 0x007D0200, // scratch_store_dword v0, v2, off + 0x7E04028D, // v_mov_b32 v2, 13 + 0xDC704004, 0x007D0200, // scratch_store_dword v0, v2, off offset:4 + 0xBEFE03FF, 0x55555555, // s_mov_b32 exec_lo, alternating lanes + 0x7E040291, // v_mov_b32 v2, 17 + 0xDC704000 | (dwords * 4), 0x007D0200, // wrap to the first slot + 0xBEFE04C1, // restore all lanes + 0xDC304000, 0x037D0000, // scratch_load_dword v3, v0, off + 0xDC304004, 0x057D0000, // scratch_load_dword v5, v0, off offset:4 + 0x7E000280, // v0 may change after the final scratch access + 0xE0701000, 0x80020304, // store v3 at local id * 8 + 0xE0701004, 0x80020504, // store v5 at local id * 8 + 4 + 0xBF810000, + ], 8, checked((int)threads * 8)); + var program = Gen5ComputeFixtures.DecodeOrThrow(fixture); + var instructions = program.Instructions.ToArray(); + var index = Array.FindIndex(instructions, i => i.Opcode.StartsWith("ScratchStore", StringComparison.Ordinal)); + var first = instructions[index]; + var control = (Gen5GlobalMemoryControl)first.Control!; + if (variant == "changed-base") + { + instructions[0] = new Gen5ShaderInstruction(instructions[0].Pc, Gen5ShaderEncoding.Vop1, + "VMovB32", [0u], [new Gen5Operand(Gen5OperandKind.EncodedConstant, 128)], [Gen5Operand.Vector(0)], null); + } + else if (variant == "return-to-access") + { + var branch = instructions[^1]; + var displacement = unchecked((ushort)(short)((int)(first.Pc - branch.Pc - 4) / 4)); + instructions[^1] = new Gen5ShaderInstruction(branch.Pc, Gen5ShaderEncoding.Sopp, + "SBranch", [0xBF820000u | displacement], [], [], null); + } + else + { + control = variant switch + { + "mixed-base" => control with { VectorAddress = 1 }, + "unaligned" => control with { OffsetBytes = 2 }, + "negative" => control with { OffsetBytes = -4 }, + "dynamic" => control with { DynamicOffsetRegister = 0 }, + _ => control, + }; + instructions[index] = first with { Control = control }; + } + program = new Gen5ShaderProgram(program.Address, instructions); + var data = new uint[16]; + data[8] = 0x200000; + data[10] = threads * 8; + data[11] = DescriptorConstants.IdentityDestinationSelect; + return Gen5ComputeFixtures.RequestOrThrow(program, ShaderStage.Compute, + localSizeX: threads, scratchDwords: dwords, runtimeInputs: new ResourceRuntimeInputs { UserData = data }); + } +} diff --git a/tests/SharpEmu.ShaderCompiler.Metal.Tests/MslUnobservedAluTests.cs b/tests/SharpEmu.ShaderCompiler.Metal.Tests/MslUnobservedAluTests.cs new file mode 100644 index 000000000..1d97dae8c --- /dev/null +++ b/tests/SharpEmu.ShaderCompiler.Metal.Tests/MslUnobservedAluTests.cs @@ -0,0 +1,70 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using System.Buffers.Binary; +using Xunit; + +namespace SharpEmu.ShaderCompiler.Metal.Tests; + +public sealed class MslUnobservedAluTests +{ + [Theory] + [InlineData(0u, false)] + [InlineData(0u, true)] + [InlineData(0x55555555u, false)] + [InlineData(0x55555555u, true)] + [InlineData(0xAAAAAAAAu, false)] + [InlineData(0xAAAAAAAAu, true)] + [InlineData(0x80000000u, false)] + [InlineData(0x80000000u, true)] + public void InactiveTemporariesCanBeOverwrittenButObservedValuesSurvive(uint mask, bool crossRead) + { + uint[] read = crossRead ? [0xD7600000, 0x00010301] : []; // v_readlane_b32 s0, v1, 1 + var fixture = new Gen5ComputeFixture("unobserved-alu", + [ + 0x34080083, // v4 = local id * 8 + 0xBE800380, // s0 = 0 + 0x7E020287, // v1 = 7 + 0x7E040285, // v2 = 5 + 0xBEFE03FF, mask, // mask EXEC_LO + 0x7E020289, // v1 = 9, only observed inside this mask unless readlane is present + .. read, + 0x4A040281, // v_add_nc_u32 v2, 1, v1 + 0xBEFE04C1, // restore EXEC + 0x7E060200, // v3 = s0 + 0xE0701000, 0x80020204, // store v2 + 0xE0701004, 0x80020304, // store v3 + 0xBF810000, + ], 8, 32 * 8); + var request = Gen5ComputeFixtures.CreateComputeRequest(fixture); + Assert.True(Gen5MslTranslator.TryCompileProgram(request, out var slow, out var error, + new Gen5MslCodegenOptions { UnobservedAluWrites = false }), error); + Assert.True(Gen5MslTranslator.TryCompileProgram(request, out var fast, out error, + new Gen5MslCodegenOptions { UnobservedAluWrites = true }), error); + if (crossRead) + Assert.Contains("if (exec) { v1 = ", fast.Source, StringComparison.Ordinal); + else + Assert.DoesNotContain("if (exec) { v1 = ", fast.Source, StringComparison.Ordinal); + Assert.Contains("if (exec) { v2 = ", fast.Source, StringComparison.Ordinal); + if (Environment.GetEnvironmentVariable("SHARPEMU_TEST_REQUIRE_DEVICE") != "1") return; + Assert.True(MetalNative.IsAvailable, "A Metal device is required."); + var reference = Execute(slow); + var actual = Execute(fast); + Assert.Equal(reference, actual); + for (var lane = 0; lane < 32; lane++) + { + Assert.Equal(((mask >> lane) & 1) != 0 ? 10u : 5u, + BinaryPrimitives.ReadUInt32LittleEndian(actual.AsSpan(lane * 8, 4))); + Assert.Equal(crossRead ? ((mask & 2) != 0 ? 9u : 7u) : 0u, + BinaryPrimitives.ReadUInt32LittleEndian(actual.AsSpan(lane * 8 + 4, 4))); + } + + byte[] Execute(Gen5MslShader shader) + { + Assert.True(MetalNative.TryCompileLibrary(shader.Source, out var library, out var compileError), compileError); + Assert.True(MetalNative.TryExecuteWithArgumentBuffer(library, shader.EntryPoint, new byte[fixture.StoreBackingBytes], + new byte[Resources.PushData.ByteSize], shader.ArgumentLayout!, 32, out var result, out var runError), runError); + return result; + } + } +} diff --git a/tests/SharpEmu.ShaderCompiler.Metal.Tests/NggMeshIndirectLaunchTests.cs b/tests/SharpEmu.ShaderCompiler.Metal.Tests/NggMeshIndirectLaunchTests.cs index b493bc9d4..1757fe8ac 100644 --- a/tests/SharpEmu.ShaderCompiler.Metal.Tests/NggMeshIndirectLaunchTests.cs +++ b/tests/SharpEmu.ShaderCompiler.Metal.Tests/NggMeshIndirectLaunchTests.cs @@ -280,6 +280,11 @@ private static (uint[] Grid, uint[]? Words) CpuLaunch(Gen5MslMeshInfo mesh, Laun ThreadsPerThreadgroup: 64, WavesPerThreadgroup: 2, MaxVertices: 64, MaxPrimitives: 64, VerticesPerPrimitive: 3, Topology: Gen5MslMeshTopology.Triangle, ThreadgroupMemoryBytes: 4096, DrawInfoBufferIndex: 2, IndexBufferIndex: 3, ReadsIndexBuffer: true, GeometryVerticesOut: 4)); + yield return ("geometry separate input and output capacities", new Gen5MslMeshInfo( + ThreadsPerThreadgroup: 128, WavesPerThreadgroup: 2, MaxVertices: 96, MaxPrimitives: 88, + VerticesPerPrimitive: 3, Topology: Gen5MslMeshTopology.Triangle, ThreadgroupMemoryBytes: 4096, + DrawInfoBufferIndex: 2, IndexBufferIndex: 3, ReadsIndexBuffer: true, GeometryVerticesOut: 24, + Wave64: true, GeometryInputVertices: 24, GeometryInputPrimitives: 4)); yield return ("points", new Gen5MslMeshInfo( ThreadsPerThreadgroup: 32, WavesPerThreadgroup: 1, MaxVertices: 32, MaxPrimitives: 32, VerticesPerPrimitive: 1, Topology: Gen5MslMeshTopology.Point, ThreadgroupMemoryBytes: 4096, diff --git a/tests/SharpEmu.ShaderCompiler.Metal.Tests/NggMeshNarrowWave64Tests.cs b/tests/SharpEmu.ShaderCompiler.Metal.Tests/NggMeshNarrowWave64Tests.cs index a57c8aed1..8333a71b1 100644 --- a/tests/SharpEmu.ShaderCompiler.Metal.Tests/NggMeshNarrowWave64Tests.cs +++ b/tests/SharpEmu.ShaderCompiler.Metal.Tests/NggMeshNarrowWave64Tests.cs @@ -9,12 +9,13 @@ namespace SharpEmu.ShaderCompiler.Metal.Tests; public sealed class NggMeshNarrowWave64Tests { private static Gen5MslShader Compile( - Gen5ShaderProgram program, Gen5NggStageInfo? stage, bool structured) + Gen5ShaderProgram program, Gen5NggStageInfo? stage, bool structured, bool meshExitJoin = true) { var request = Gen5NggFixtures.Request(program, stage); Assert.True( Gen5MslTranslator.TryCompileProgram( - request, out var shader, out var error, new Gen5MslCodegenOptions(StructuredControlFlow: structured)), + request, out var shader, out var error, + new Gen5MslCodegenOptions(StructuredControlFlow: structured) { MeshExitJoin = meshExitJoin }), error); return shader; } @@ -66,12 +67,18 @@ public void AControlFlowTheStructuredFormCannotNestTakesTheLinearForm() PrimitivesPerSubgroup = 1, MaxOutputPerSubgroup = 32, }; - var shader = Compile(Gen5NggFixtures.BarrierLoopWithTwoExits(), stage, structured: true); + var shader = Compile(Gen5NggFixtures.BarrierLoopWithTwoExits(), stage, structured: true, meshExitJoin: false); Assert.DoesNotContain("while (active && !sharpemu_mesh_wait)", shader.Source, StringComparison.Ordinal); Assert.Contains("while (active && pc >= 1u && pc <= 3u)", shader.Source, StringComparison.Ordinal); Assert.Contains("if (active && pc == 2u)", shader.Source, StringComparison.Ordinal); Assert.Contains("threadgroup_barrier(mem_flags::mem_threadgroup);", shader.Source, StringComparison.Ordinal); + // Joining the exits now lets this one-wave loop use structured control flow; + // the kill switch above still exercises the linear fallback itself. + var joined = Compile(Gen5NggFixtures.BarrierLoopWithTwoExits(), stage, structured: true); + Assert.DoesNotContain("pc ==", joined.Source, StringComparison.Ordinal); + Assert.Contains("while (true)", joined.Source, StringComparison.Ordinal); + // Several waves around a barrier inside a loop: back to the dispatcher. var wide = Compile(Gen5NggFixtures.BarrierLoopWithTwoExits(), null, structured: true); Assert.Contains("while (active && !sharpemu_mesh_wait)", wide.Source, StringComparison.Ordinal); @@ -82,7 +89,7 @@ public void ALinearBarrierOfSeveralWavesStandsOutsideItsPcGuard() { // A wave that runs out of steps clears active; a barrier under that guard would be // skipped by it while the other waves wait. So it follows the guarded block. - var shader = Compile(Gen5NggFixtures.BarrierAfterLoopWithTwoExits(), null, structured: true); + var shader = Compile(Gen5NggFixtures.BarrierAfterLoopWithTwoExits(), null, structured: true, meshExitJoin: false); Assert.Equal(2u, Assert.IsType(shader.MeshInfo).WavesPerThreadgroup); Assert.Contains("while (active && pc >= ", shader.Source, StringComparison.Ordinal); Assert.DoesNotContain("while (active && !sharpemu_mesh_wait)", shader.Source, StringComparison.Ordinal); @@ -102,7 +109,7 @@ public void ALinearBarrierOfSeveralWavesStandsOutsideItsPcGuard() PrimitivesPerSubgroup = 1, MaxOutputPerSubgroup = 32, }; - var oneWave = Compile(Gen5NggFixtures.BarrierAfterLoopWithTwoExits(), stage, structured: true); + var oneWave = Compile(Gen5NggFixtures.BarrierAfterLoopWithTwoExits(), stage, structured: true, meshExitJoin: false); var oneLines = oneWave.Source.Split('\n'); var oneBody = Array.FindIndex(oneLines, static line => line.Contains("if (active && pc == 0u)", StringComparison.Ordinal)); var oneBarrier = Array.FindIndex(oneLines, oneBody, static line => line.Trim() == "threadgroup_barrier(mem_flags::mem_threadgroup);"); diff --git a/tests/SharpEmu.ShaderCompiler.Metal.Tests/NggMeshRuntimeTests.cs b/tests/SharpEmu.ShaderCompiler.Metal.Tests/NggMeshRuntimeTests.cs index 390dc936e..53e00804f 100644 --- a/tests/SharpEmu.ShaderCompiler.Metal.Tests/NggMeshRuntimeTests.cs +++ b/tests/SharpEmu.ShaderCompiler.Metal.Tests/NggMeshRuntimeTests.cs @@ -37,6 +37,381 @@ private static string FindCheckoutRoot() return AppContext.BaseDirectory; } + [Theory] + [InlineData(1u)] + [InlineData(2u)] + [InlineData(4u)] + [InlineData(8u)] + [InlineData(16u)] + public void ScalarConstantBuffersPreserveRawBoundsAndWideLoads(uint count) + { + if (SkipWithoutMetalDevice("scalar constant mesh buffers")) return; + var data = new byte[64]; + for (var i = 0; i < 16; i++) BitConverter.GetBytes((uint)(i + 1)).CopyTo(data, i * 4); + foreach (var offset in new[] { 0u, 1u, 56u, 60u, 64u, 0xFFFFFFFCu }) + { + float[]? baseline = null; + foreach (var enabled in new[] { false, true }) + { + var request = ScalarConstantRequest(count, offset); + Assert.True(Gen5MslTranslator.TryCompileProgram(request, out var shader, out var error, + new Gen5MslCodegenOptions { ScalarConstantBuffers = enabled }), error); + Assert.Equal(enabled, shader.Source.Contains("constant uint* b0 =", StringComparison.Ordinal)); + Assert.True(MetalNative.TryCompileLibrary(shader.Source + FragmentSource, out var library, out var compileError), compileError); + var mesh = shader.MeshInfo!; + Assert.True(MetalNative.TryRenderMeshWithArgumentBuffer(library, shader.EntryPoint, "gen5_mesh_test_fs", + data, new byte[PushData.ByteSize], BuildDrawInfo(mesh), shader.ArgumentLayout!, + mesh.DrawInfoBufferIndex, mesh.IndexBufferIndex, 1, mesh.ThreadsPerThreadgroup, + TargetSize, TargetSize, out var pixels, out var renderError), renderError); + uint expected = 0; + for (uint i = 0; i < count; i++) + { + var index = unchecked(offset + i * 4) >> 2; + if (index < 16) expected += index + 1; + } + Assert.Equal((float)expected, Red(pixels, 16, 16), 2); + Assert.Equal(1f, Green(pixels, 16, 16), 3); + if (!enabled) baseline = pixels; + else Assert.Equal(baseline, pixels); + } + } + } + + [Theory] + [InlineData(true, false, false)] + [InlineData(false, true, false)] + [InlineData(false, false, true)] + public void ScalarConstantBuffersRefuseCoherentVectorOrDeviceWriteAccess(bool coherent, bool vectorAccess, bool deviceWrite) + { + var request = ScalarConstantRequest(4, 0, coherent, vectorAccess, deviceWrite: deviceWrite); + Assert.True(Gen5MslTranslator.TryCompileProgram(request, out var shader, out var error, + new Gen5MslCodegenOptions { ScalarConstantBuffers = true }), error); + Assert.DoesNotContain("constant uint* b0 =", shader.Source, StringComparison.Ordinal); + } + + [Fact] + public void ScalarConstantBuffersRequireOptInAndRawDwordSemantics() + { + foreach (var options in new[] { new Gen5MslCodegenOptions(), + new Gen5MslCodegenOptions(RawDwordAccess: false) { ScalarConstantBuffers = true } }) + { + Assert.True(Gen5MslTranslator.TryCompileProgram(ScalarConstantRequest(4, 0), + out var shader, out var error, options), error); + Assert.DoesNotContain("constant uint* b0 =", shader.Source, StringComparison.Ordinal); + } + } + + [Fact] + public void ScalarConstantBuffersPreserveMixedPointerArgumentLayout() + { + if (SkipWithoutMetalDevice("mixed scalar constant mesh buffers")) return; + float[]? baseline = null; + foreach (var enabled in new[] { false, true }) + { + var request = ScalarConstantRequest(1, 12, vectorAccess: true, separateScalarBinding: true); + Assert.True(Gen5MslTranslator.TryCompileProgram(request, out var shader, out var error, + new Gen5MslCodegenOptions { ScalarConstantBuffers = enabled }), error); + Assert.Equal(2u, shader.ArgumentLayout!.Find(DescriptorBindingKind.Buffers, MslArgumentFieldKind.BufferPointers)!.Count); + if (enabled) + { + Assert.Contains("device uint* buffer_0 [[id(0)]]", shader.Source, StringComparison.Ordinal); + Assert.Contains("constant uint* buffer_1 [[id(1)]]", shader.Source, StringComparison.Ordinal); + Assert.Contains("buffer_bytes [[id(2)]]", shader.Source, StringComparison.Ordinal); + } + Assert.True(MetalNative.TryCompileLibrary(shader.Source + FragmentSource, out var library, out var compileError), compileError); + var mesh = shader.MeshInfo!; + Assert.True(MetalNative.TryRenderMeshWithArgumentBuffer(library, shader.EntryPoint, "gen5_mesh_test_fs", + PositionBuffer(), new byte[PushData.ByteSize], BuildDrawInfo(mesh), shader.ArgumentLayout!, + mesh.DrawInfoBufferIndex, mesh.IndexBufferIndex, 1, mesh.ThreadsPerThreadgroup, + TargetSize, TargetSize, out var pixels, out var renderError), renderError); + Assert.Equal(1f, Green(pixels, 16, 16), 3); + Assert.InRange(Red(pixels, 16, 16), 1065353216f - 128f, 1065353216f + 128f); + if (!enabled) baseline = pixels; + else Assert.Equal(baseline, pixels); + } + } + + private static ShaderCompileRequest ScalarConstantRequest(uint count, uint offset, bool coherent = false, bool vectorAccess = false, bool separateScalarBinding = false, bool deviceWrite = false) + { + var instructions = new List(); + foreach (var instruction in Gen5NggFixtures.CullingTriangle(bufferFetchPosition: vectorAccess).Instructions) + { + if (instruction.Control is Gen5ExportControl { Target: Gen5NggExports.FirstParameterTarget }) + { + instructions.Add(new(0, Gen5ShaderEncoding.Smem, count == 1 ? "SBufferLoadDword" : $"SBufferLoadDwordx{count}", + [0u, 0u], [Gen5Operand.Scalar(separateScalarBinding ? 12u : 8u), Gen5Operand.Source(125)], + [.. Enumerable.Range(32, (int)count).Select(i => Gen5Operand.Scalar((uint)i))], + new Gen5ScalarMemoryControl(count, unchecked((int)offset), null, Glc: coherent))); + for (uint i = 1; i < count; i++) + instructions.Add(new(0, Gen5ShaderEncoding.Sop2, "SAddU32", [0u], + [Gen5Operand.Scalar(32), Gen5Operand.Scalar(32 + i)], [Gen5Operand.Scalar(32)], null)); + instructions.Add(new(0, Gen5ShaderEncoding.Vop1, "VCvtF32U32", [0u], + [Gen5Operand.Scalar(32)], [Gen5Operand.Vector(15)], null)); + if (deviceWrite) + instructions.Add(new(0, Gen5ShaderEncoding.Mubuf, "BufferStoreDword", [0u, 0u], + [Gen5Operand.Vector(15), Gen5Operand.Scalar(12), Gen5Operand.Source(128)], [], + new Gen5BufferMemoryControl(1, 0, 15, 12, 0, false, false, false, false))); + } + instructions.Add(instruction); + } + var program = new Gen5ShaderProgram(0, instructions.Select((i, n) => i with { Pc = (uint)n * 8 }).ToArray()); + var userData = Gen5NggFixtures.PositionUserData(); + if (separateScalarBinding || deviceWrite) + { + Array.Copy(userData, 0, userData, 4, 4); + userData[4] += 0x10000; // Distinct guest binding; the device harness populates both slots. + } + return Gen5NggFixtures.Request(program, runtimeInputs: new ResourceRuntimeInputs { UserData = userData }); + } + + [Theory] + [InlineData(0)] + [InlineData(1)] + [InlineData(2)] + [InlineData(3)] + [InlineData(4)] + [InlineData(8)] + [InlineData(10)] + public void HotLdsBankPreservesColdFallbackAliasingAndWrap(int mutation) + { + if (SkipWithoutMetalDevice("compact hot LDS bank")) return; + foreach (var baseAddress in new[] { 0u, 500u, 1024u, 4096u, 8048u, 0xFFFFFF80u }) + foreach (var readLane in new[] { 0u, 17u, 31u }) + { + float[]? baseline = null; + foreach (var enabled in new[] { false, true }) + { + var stage = Gen5NggFixtures.Wave64StageInfo with { LocalDataShareBytes = 8192u }; + Assert.True(Gen5MslTranslator.TryCompileProgram( + Gen5NggFixtures.Request(Gen5NggFixtures.MeshLdsForwardingProbe(readLane, mutation, baseAddress, allComponents: true), stage), + out var shader, out var error, new Gen5MslCodegenOptions + { MeshHotLds = enabled, MeshDeviceLdsThresholdBytes = 0 }), error); + var mesh = shader.MeshInfo!; + Assert.Equal(enabled, shader.Source.Contains("threadgroup uint sharpemu_hot_lds[", StringComparison.Ordinal)); + Assert.Equal(enabled, mesh.ThreadgroupMemoryBytes != 0); + Assert.True(mesh.DeviceLds); + Assert.True(MetalNative.TryCompileLibrary(shader.Source + FragmentSource, out var library, out var compileError), compileError); + Assert.True(MetalNative.TryRenderMesh(library, shader.EntryPoint, "gen5_mesh_test_fs", + [(BuildArgumentBuffer(shader), 0u), (new byte[64], 1u), (BuildDrawInfo(mesh), mesh.DrawInfoBufferIndex), + (new byte[16], mesh.IndexBufferIndex), + (new byte[(Gen5MslMeshInfo.LdsPoolHeaderDwords + Gen5MslMeshInfo.LdsPoolSlotDwords) * 4], Gen5MslMeshInfo.LdsPoolBufferIndex)], + 1, mesh.ThreadsPerThreadgroup, TargetSize, TargetSize, out var pixels, out var renderError), renderError); + var expected = mutation switch + { + 1 when readLane < 8 => readLane + 100, + 2 or 10 => 999u, 3 => readLane ^ 1, _ => readLane, + }; + expected += 100384; // 1<<5 + 2<<10 + 3<<15: check every loaded component. + Assert.Equal((float)expected, Red(pixels, 16, 16), 2); + Assert.Equal(1f, Green(pixels, 16, 16), 3); + if (!enabled) baseline = pixels; + else Assert.Equal(baseline, pixels); + } + } + } + + [Fact] + public void HotLdsBankRefusesMultipleWavesAndOtherStorageModes() + { + foreach (var (stage, threshold, forwarding) in new[] + { + (Gen5NggFixtures.StageInfo, 0u, false), + (Gen5NggFixtures.Wave64StageInfo, 16384u, false), + (Gen5NggFixtures.Wave64StageInfo, 0u, true), + }) + { + Assert.True(Gen5MslTranslator.TryCompileProgram( + Gen5NggFixtures.Request(Gen5NggFixtures.MeshLdsForwardingProbe(3, 0), stage), + out var shader, out var error, new Gen5MslCodegenOptions + { MeshHotLds = true, MeshDeviceLdsThresholdBytes = threshold, MeshLdsForwarding = forwarding }), error); + Assert.DoesNotContain("sharpemu_hot_lds", shader.Source, StringComparison.Ordinal); + } + } + + [Theory] + [InlineData(5)] + [InlineData(6)] + public void HotLdsBankRefusesUnsupportedMemoryOperations(int mutation) + { + Assert.True(Gen5MslTranslator.TryCompileProgram( + Gen5NggFixtures.Request(Gen5NggFixtures.MeshLdsForwardingProbe(3, mutation), Gen5NggFixtures.Wave64StageInfo), + out var shader, out var error, new Gen5MslCodegenOptions { MeshHotLds = true, MeshDeviceLdsThresholdBytes = 0 }), error); + Assert.DoesNotContain("sharpemu_hot_lds", shader.Source, StringComparison.Ordinal); + } + + [Fact] + public void LdsForwardingRefusesAGroupWithMoreThanOneHardwareWave() + { + Assert.True(Gen5MslTranslator.TryCompileProgram( + Gen5NggFixtures.Request(Gen5NggFixtures.MeshLdsForwardingProbe(3, 0)), + out var shader, out var error, new Gen5MslCodegenOptions { MeshLdsForwarding = true }), error); + Assert.Equal(64u, shader.MeshInfo!.ThreadsPerThreadgroup); + Assert.DoesNotContain("sharpemu_lds_forward_", shader.Source, StringComparison.Ordinal); + } + + [Theory] + [InlineData(0)] + [InlineData(1)] + [InlineData(2)] + [InlineData(3)] + [InlineData(4)] + [InlineData(5)] + [InlineData(6)] + [InlineData(7)] + [InlineData(8)] + public void SingleWaveLdsForwardingPreservesNeighborReadsAndAliasing(int mutation) + { + if (SkipWithoutMetalDevice("guarded mesh LDS forwarding")) return; + foreach (var threshold in new[] { 0u, 12288u }) + foreach (var readLane in new[] { 0u, 3u, 17u, 31u }) + { + float[]? reference = null; + foreach (var forwarding in new[] { false, true }) + { + var stage = Gen5NggFixtures.Wave64StageInfo with { LocalDataShareBytes = mutation == 4 ? 2048u : 1024u }; + Assert.True(Gen5MslTranslator.TryCompileProgram( + Gen5NggFixtures.Request(Gen5NggFixtures.MeshLdsForwardingProbe(readLane, mutation), stage), + out var shader, out var error, new Gen5MslCodegenOptions + { MeshLdsForwarding = forwarding, MeshDeviceLdsThresholdBytes = threshold }), error); + var mesh = shader.MeshInfo!; + Assert.Equal(32u, mesh.ThreadsPerThreadgroup); + Assert.Equal(forwarding, shader.Source.Contains("simd_shuffle(sharpemu_lds_forward_values", StringComparison.Ordinal)); + Assert.True(MetalNative.TryCompileLibrary(shader.Source + FragmentSource, out var library, out var compileError), compileError); + var buffers = new List<(byte[], uint)> + { + (BuildArgumentBuffer(shader), 0u), (new byte[64], 1u), + (BuildDrawInfo(mesh), mesh.DrawInfoBufferIndex), (new byte[16], mesh.IndexBufferIndex), + }; + if (mesh.DeviceLds) buffers.Add((new byte[(Gen5MslMeshInfo.LdsPoolHeaderDwords + Gen5MslMeshInfo.LdsPoolSlotDwords) * 4], Gen5MslMeshInfo.LdsPoolBufferIndex)); + Assert.True(MetalNative.TryRenderMesh(library, shader.EntryPoint, "gen5_mesh_test_fs", buffers, + 1, mesh.ThreadsPerThreadgroup, TargetSize, TargetSize, out var pixels, out var renderError), renderError); + var expected = mutation switch + { + 1 when readLane < 8 => readLane + 100, + 2 => 999u, 3 => readLane ^ 1, 5 => 127u, 6 => readLane + 1, + _ => readLane, + }; + Assert.Equal((float)expected, Red(pixels, 16, 16), 2); + Assert.Equal(1f, Green(pixels, 16, 16), 3); + if (!forwarding) reference = pixels; + else Assert.Equal(reference, pixels); + } + } + } + + [Theory] + [InlineData(1024u)] + [InlineData(5120u)] + [InlineData(9216u)] + [InlineData(15872u)] + public void DeviceLdsSlotCannotOverlapAnOccupiedSlotOfALargerShader(uint ldsBytes) + { + if (SkipWithoutMetalDevice("mixed-size device LDS slot isolation")) return; + Assert.True(Gen5MslTranslator.TryCompileProgram( + Gen5NggFixtures.Request(Gen5NggFixtures.CullingTriangle(), + Gen5NggFixtures.StageInfo with { LocalDataShareBytes = ldsBytes }), + out var shader, out var error, new Gen5MslCodegenOptions { MeshDeviceLdsThresholdBytes = 0 }), error); + var mesh = shader.MeshInfo!; + var pool = new byte[Gen5MslMeshInfo.LdsPoolByteSize]; + var words = System.Runtime.InteropServices.MemoryMarshal.Cast(pool.AsSpan()); + words[0] = 1; // The next acquisition takes slot 1. + words[1] = 1; // Slot 0 belongs to a still-live shader using the full 16 KiB. + var held = words.Slice((int)Gen5MslMeshInfo.LdsPoolHeaderDwords, (int)Gen5MslMeshInfo.LdsPoolSlotDwords); + held.Fill(0xA5B6C7D8u); + var expectedHeld = held.ToArray(); + Assert.True(MetalNative.TryCompileLibrary(shader.Source + FragmentSource, out var library, out var compileError), compileError); + var readback = new Dictionary(); + Assert.True(MetalNative.TryRenderMesh(library, shader.EntryPoint, "gen5_mesh_test_fs", + [(BuildArgumentBuffer(shader), 0u), (new byte[64], 1u), (BuildDrawInfo(mesh), mesh.DrawInfoBufferIndex), + (new byte[16], mesh.IndexBufferIndex), (pool, Gen5MslMeshInfo.LdsPoolBufferIndex)], + 1, mesh.ThreadsPerThreadgroup, TargetSize, TargetSize, out var pixels, out var renderError, + readbackBuffers: readback), renderError); + Assert.Equal(1f, Green(pixels, TargetSize / 2, TargetSize / 2), 3); + var actual = System.Runtime.InteropServices.MemoryMarshal.Cast(readback[Gen5MslMeshInfo.LdsPoolBufferIndex]); + Assert.Equal(1u, actual[1]); // The other owner's reservation survives. + Assert.Equal(0u, actual[2]); // This invocation released slot 1. + Assert.Equal(expectedHeld, actual.Slice((int)Gen5MslMeshInfo.LdsPoolHeaderDwords, + (int)Gen5MslMeshInfo.LdsPoolSlotDwords).ToArray()); + } + + [Theory] + [InlineData(Gen5NggExports.OutputPrimitiveTriangleStrip, 88u)] + [InlineData(Gen5NggExports.OutputPrimitiveLineStrip, 92u)] + [InlineData(Gen5NggExports.OutputPrimitivePointList, 96u)] + public void GeometryInputAndOutputBudgetsRemainIndependent(uint topology, uint outputPrimitives) + { + var stage = Gen5NggFixtures.StageInfo with + { + VerticesPerSubgroup = 24, PrimitivesPerSubgroup = 4, MaxOutputPerSubgroup = 96, + GeometryVerticesOut = 24, OutputPrimitiveType = topology, IsWave64 = true, + }; + Assert.True(Gen5MslTranslator.TryCompileProgram( + Gen5NggFixtures.Request(Gen5NggFixtures.GeometryQuad(true, 0), stage), + out var shader, out var error, new Gen5MslCodegenOptions { GeometryOutputCapacity = true }), error); + var mesh = shader.MeshInfo!; + Assert.Equal(24u, mesh.GeometryInputVertices); + Assert.Equal(4u, mesh.GeometryInputPrimitives); + Assert.Equal(96u, mesh.MaxVertices); + Assert.Equal(outputPrimitives, mesh.MaxPrimitives); + Assert.Equal(128u, mesh.ThreadsPerThreadgroup); + Assert.Equal(2u, mesh.WavesPerThreadgroup); + foreach (var input in Enum.GetValues()) + Assert.Equal(4u, mesh.InputPrimitivesPerSubgroup(input)); + Assert.Equal(2u, Gen5MslMeshLaunch.SubgroupCount(mesh, Gen5MslMeshInputTopology.TriangleList, 15)); + Assert.Equal(2u, Gen5MslMeshLaunch.SubgroupCount(mesh, Gen5MslMeshInputTopology.TriangleStrip, 7)); + Assert.Equal(2u, Gen5MslMeshLaunch.SubgroupCount(mesh, Gen5MslMeshInputTopology.PointList, 5)); + } + + [Theory] + [InlineData(false, 0u, 12288u)] + [InlineData(true, 0u, 12288u)] + [InlineData(false, 64u, 12288u)] + [InlineData(true, 64u, 12288u)] + [InlineData(true, 64u, 0u)] + public void GeometryOutputCapacityPreservesAmplifiedVerticesAndPrimitives(bool wave64, uint firstVertex, uint ldsThreshold) + { + if (SkipWithoutMetalDevice("geometry output capacity")) return; + var stage = Gen5NggFixtures.StageInfo with + { + VerticesPerSubgroup = 3, PrimitivesPerSubgroup = 1, + MaxOutputPerSubgroup = firstVertex + 4, GeometryVerticesOut = firstVertex + 4, + IsWave64 = wave64, + }; + foreach (var capacity in new[] { false, true }) + { + Assert.True(Gen5MslTranslator.TryCompileProgram( + Gen5NggFixtures.Request(Gen5NggFixtures.GeometryQuad(wave64, firstVertex), stage), + out var shader, out var error, new Gen5MslCodegenOptions + { GeometryOutputCapacity = capacity, MeshDeviceLdsThresholdBytes = ldsThreshold }), error); + var mesh = shader.MeshInfo!; + Assert.Equal(capacity ? firstVertex + 4 : 3u, mesh.MaxVertices); + Assert.Equal(capacity ? firstVertex + 2 : 1u, mesh.MaxPrimitives); + if (capacity) + { + Assert.Equal(wave64 ? 64u : 32u, mesh.GuestWaveLanes); + Assert.Equal(mesh.ThreadsPerThreadgroup / mesh.GuestWaveLanes, mesh.WavesPerThreadgroup); + } + Assert.True(MetalNative.TryCompileLibrary(shader.Source + FragmentSource, out var library, out var compileError), compileError); + var words = new uint[Gen5MslMeshInfo.DrawInfoDwordCount]; + Gen5MslMeshLaunch.FillDrawInfo(mesh, Gen5MslMeshInputTopology.TriangleList, words, 3, 1, false, 0, 0, 0, 0); + var drawInfo = new byte[words.Length * 4]; + Buffer.BlockCopy(words, 0, drawInfo, 0, drawInfo.Length); + var bindings = new List<(byte[], uint)> + { + (BuildArgumentBuffer(shader), 0u), (new byte[64], 1u), + (drawInfo, mesh.DrawInfoBufferIndex), (new byte[16], mesh.IndexBufferIndex), + }; + if (mesh.DeviceLds) bindings.Add((new byte[Gen5MslMeshInfo.LdsPoolByteSize], Gen5MslMeshInfo.LdsPoolBufferIndex)); + Assert.True(MetalNative.TryRenderMesh(library, shader.EntryPoint, "gen5_mesh_test_fs", bindings, + 1, mesh.ThreadsPerThreadgroup, TargetSize, TargetSize, out var pixels, out var renderError), renderError); + // The upper-right half belongs only to the second triangle. The old input + // bounds clip its fourth vertex and second primitive; the fixed bounds retain it. + Assert.Equal(capacity ? 1f : 0f, Green(pixels, 24, 8), 3); + if (capacity) Assert.Equal(1f, Green(pixels, 8, 24), 3); + Assert.Equal(0f, Green(pixels, 1, 1), 3); + } + } + [Fact] public void SyntheticPrimitiveShadersCompileWithTheRuntimeMetalCompiler() { @@ -176,20 +551,25 @@ public void HandedVertexValuesReachTheFragmentInPrimitiveOrder(bool passthrough) } [Theory] - [InlineData(false)] - [InlineData(true)] - public void DeviceLdsPrimitiveShaderExchangesThroughTheSlotAndBarrier(bool nullPrimitive) + [InlineData(false, 0u)] + [InlineData(true, 0u)] + [InlineData(false, 12288u)] + [InlineData(true, 12288u)] + [InlineData(false, 16384u)] + [InlineData(true, 16384u)] + public void DeviceLdsPrimitiveShaderExchangesThroughTheSlotAndBarrier(bool nullPrimitive, uint threshold) { if (SkipWithoutMetalDevice("the device LDS mesh execution test")) return; - // The same program as the threadgroup-memory case, with its LDS in the device pool: - // the exchange across both waves only survives if the slot is acquired before the - // first access and every barrier orders device memory. - var shader = Gen5NggFixtures.CompileOrThrow( + // Exercise the same cross-wave exchange with both LDS placements. Device-pool + // execution also requires acquisition before the first access and device fences. + var request = Gen5NggFixtures.Request( Gen5NggFixtures.CullingTriangle(nullPrimitive), Gen5NggFixtures.StageInfo with { LocalDataShareBytes = 15 * 1024 }); + Assert.True(Gen5MslTranslator.TryCompileProgram(request, out var shader, out var error, + new Gen5MslCodegenOptions { MeshDeviceLdsThresholdBytes = threshold }), error); var mesh = shader.MeshInfo!; - Assert.True(mesh.DeviceLds); + Assert.Equal(threshold < 15 * 1024, mesh.DeviceLds); var source = shader.Source + FragmentSource; if (!MetalNative.TryCompileLibrary(source, out var library, out var compileError)) { @@ -642,9 +1022,100 @@ instruction.Control is Gen5BufferMemoryControl && Gen5NggFixtures.StageInfo with { GeometryVerticesOut = 3 })); } - private float[] RenderOrThrow(Gen5ShaderProgram program) + [Theory] + [InlineData(false)] + [InlineData(true)] + public void ConditionalMeshExitsReconvergeAndKeepEachWavesOutput(bool firstWaveExitsEarly) + { + if (SkipWithoutMetalDevice("conditional mesh exits")) return; + var request = Gen5NggFixtures.Request(Gen5NggFixtures.ConditionalMeshExit(firstWaveExitsEarly)); + float[]? reference = null; + foreach (var options in new[] + { + new Gen5MslCodegenOptions(StructuredControlFlow: false), + new Gen5MslCodegenOptions { MeshExitJoin = false }, + new Gen5MslCodegenOptions { MeshExitJoin = true }, + }) + { + Assert.True(Gen5MslTranslator.TryCompileProgram(request, out var shader, out var error, options), error); + if (options.StructuredControlFlow && options.MeshExitJoin) + { + Assert.DoesNotContain("pc ==", shader.Source, StringComparison.Ordinal); + Assert.DoesNotContain("while (active", shader.Source, StringComparison.Ordinal); + } + + var pixels = RenderOrThrow(shader); + Assert.Equal(1.0f, Green(pixels, TargetSize / 2, TargetSize / 2), 3); + Assert.Equal(firstWaveExitsEarly ? 1.0f : 0.25f, Red(pixels, TargetSize / 2, TargetSize / 2), 1); + Assert.Equal(0.0f, Green(pixels, 1, 1), 3); + if (reference is not null) Assert.Equal(reference, pixels); + reference = pixels; + } + } + + [Theory] + [InlineData(3u, false)] + [InlineData(3u, true)] + [InlineData(100001u, false)] + [InlineData(100001u, true)] + public void OneWaveLoopExitPublishesOnceIncludingAtTheStepLimit(uint iterations, bool deviceLds) + { + if (SkipWithoutMetalDevice("mesh loop exit")) return; + var stage = Gen5NggFixtures.StageInfo with + { + VerticesPerSubgroup = 32, MaxOutputPerSubgroup = 32, + LocalDataShareBytes = deviceLds ? 15u * 1024 : 1024u, + }; + var request = Gen5NggFixtures.Request(Gen5NggFixtures.MeshExitLoop(iterations), stage); + float[]? reference = null; + foreach (var structured in new[] { false, true }) + { + Assert.True(Gen5MslTranslator.TryCompileProgram(request, out var shader, out var error, + new Gen5MslCodegenOptions(StructuredControlFlow: structured) { MeshExitJoin = true }), error); + Assert.Equal(1u, shader.MeshInfo!.WavesPerThreadgroup); + if (structured) Assert.DoesNotContain("pc ==", shader.Source, StringComparison.Ordinal); + var pixels = RenderOrThrow(shader); + Assert.Equal(1.0f, Green(pixels, TargetSize / 2, TargetSize / 2), 3); + Assert.Equal(iterations > 100000 ? 1.0f : 0.25f, Red(pixels, TargetSize / 2, TargetSize / 2), 1); + if (reference is not null) Assert.Equal(reference, pixels); + reference = pixels; + } + } + + [Theory] + [InlineData(false, false)] + [InlineData(false, true)] + [InlineData(true, false)] + [InlineData(true, true)] + public void UnobservedMeshAluPreservesInactiveExportValues(bool deviceLds, bool structured) + { + if (SkipWithoutMetalDevice("masked mesh ALU")) return; + var stage = Gen5NggFixtures.StageInfo with + { + VerticesPerSubgroup = 32, MaxOutputPerSubgroup = 32, + LocalDataShareBytes = deviceLds ? 15u * 1024 : 1024u, + }; + var request = Gen5NggFixtures.Request(Gen5NggFixtures.MaskedMeshAlu(), stage); + float[]? reference = null; + foreach (var enabled in new[] { false, true }) + { + Assert.True(Gen5MslTranslator.TryCompileProgram(request, out var shader, out var error, + new Gen5MslCodegenOptions(StructuredControlFlow: structured) + { UnobservedAluWrites = enabled, MeshExitJoin = true }), error); + Assert.Equal(32u, shader.MeshInfo!.ThreadsPerThreadgroup); + var pixels = RenderOrThrow(shader); + Assert.Equal(1.0f, Green(pixels, TargetSize / 2, TargetSize / 2), 3); + Assert.InRange(Red(pixels, TargetSize / 2, TargetSize / 2), 0.75f, 0.9f); + if (reference is not null) Assert.Equal(reference, pixels); + reference = pixels; + } + } + + private float[] RenderOrThrow(Gen5ShaderProgram program) => + RenderOrThrow(Gen5NggFixtures.CompileOrThrow(program)); + + private float[] RenderOrThrow(Gen5MslShader shader) { - var shader = Gen5NggFixtures.CompileOrThrow(program); var mesh = shader.MeshInfo!; var source = shader.Source + FragmentSource; if (!MetalNative.TryCompileLibrary(source, out var library, out var compileError)) @@ -652,16 +1123,21 @@ private float[] RenderOrThrow(Gen5ShaderProgram program) throw new InvalidOperationException($"Metal rejected the mesh MSL: {compileError}\n{source}"); } + var bindings = new List<(byte[], uint)> + { + (BuildArgumentBuffer(shader), 0u), + (new byte[64], 1u), + (BuildDrawInfo(mesh), mesh.DrawInfoBufferIndex), + (new byte[16], mesh.IndexBufferIndex), + }; + if (mesh.DeviceLds) + bindings.Add((new byte[Gen5MslMeshInfo.LdsPoolByteSize], Gen5MslMeshInfo.LdsPoolBufferIndex)); + if (!MetalNative.TryRenderMesh( library, shader.EntryPoint, "gen5_mesh_test_fs", - [ - (BuildArgumentBuffer(shader), 0u), - (new byte[64], 1u), - (BuildDrawInfo(mesh), mesh.DrawInfoBufferIndex), - (new byte[16], mesh.IndexBufferIndex), - ], + bindings, threadgroups: 1, mesh.ThreadsPerThreadgroup, TargetSize, diff --git a/tests/SharpEmu.ShaderCompiler.Tests/Gen5ExecFullAnalysisTests.cs b/tests/SharpEmu.ShaderCompiler.Tests/Gen5ExecFullAnalysisTests.cs index 5b6f0fd0a..936c373a1 100644 --- a/tests/SharpEmu.ShaderCompiler.Tests/Gen5ExecFullAnalysisTests.cs +++ b/tests/SharpEmu.ShaderCompiler.Tests/Gen5ExecFullAnalysisTests.cs @@ -44,6 +44,71 @@ private void PointBranch(uint branchPc, uint targetPc) private IReadOnlySet Analyze() => Gen5ExecFullAnalysis.Analyze(new Gen5ShaderProgram(0x1000, _program), wave32: false); + [Theory] + [InlineData(false, false, false, false)] + [InlineData(false, false, true, false)] + [InlineData(false, true, false, true)] + [InlineData(false, true, true, false)] + [InlineData(true, false, false, true)] + [InlineData(true, false, true, true)] + [InlineData(true, true, false, true)] + [InlineData(true, true, true, true)] + public void ConstantExecRestore_RespectsWidthAndLiteralExtension(bool wave32, bool wide, bool literal, bool expected) + { + Kill(); + var mask = literal ? new Gen5Operand(Gen5OperandKind.LiteralConstant, uint.MaxValue) + : new Gen5Operand(Gen5OperandKind.EncodedConstant, 193); + Scalar(wide ? "SMovB64" : "SMovB32", Exec, mask); + var after = VectorAdd(); + var full = Gen5ExecFullAnalysis.Analyze(new Gen5ShaderProgram(0x1000, _program), wave32); + Assert.Equal(expected, full.Contains(after)); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void OrWithInlineMinusOne_RestoresExec(bool constantFirst) + { + Kill(); + var mask = new Gen5Operand(Gen5OperandKind.EncodedConstant, 193); + Scalar("SOrB64", Exec, constantFirst ? mask : Gen5Operand.Scalar(Exec), + constantFirst ? Gen5Operand.Scalar(Exec) : mask); + Assert.Contains(VectorAdd(), Analyze()); + } + + [Theory] + [InlineData(Gen5OperandKind.EncodedConstant, 192u)] + [InlineData(Gen5OperandKind.EncodedConstant, 194u)] + [InlineData(Gen5OperandKind.LiteralConstant, 0x7fffffffu)] + [InlineData(Gen5OperandKind.LiteralConstant, 0xfffffffeu)] + public void PartialConstantExecRestore_RemainsGuarded(Gen5OperandKind kind, uint value) + { + Kill(); + Scalar("SMovB64", Exec, new Gen5Operand(kind, value)); + Assert.DoesNotContain(VectorAdd(), Analyze()); + } + + [Fact] + public void ConstantRestoreOnOnlyOneBranch_DoesNotProveJoinFull() + { + Kill(); + var branch = Branch("SCbranchScc0"); + Scalar("SMovB64", Exec, new Gen5Operand(Gen5OperandKind.EncodedConstant, 193)); + var join = VectorAdd(); + PointBranch(branch, join); + Assert.DoesNotContain(join, Analyze()); + } + + [Fact] + public void LiteralSavedHighWord_DoesNotProveWave32ExecFull() + { + Scalar("SMovB64", 40, new Gen5Operand(Gen5OperandKind.LiteralConstant, uint.MaxValue)); + Kill(); + Scalar("SMovB32", Exec, Gen5Operand.Scalar(41)); + var after = VectorAdd(); + Assert.DoesNotContain(after, Gen5ExecFullAnalysis.Analyze(new Gen5ShaderProgram(0x1000, _program), true)); + } + [Fact] public void StraightLineCode_RunsWithAFullExec() { diff --git a/tests/SharpEmu.ShaderCompiler.Tests/Gen5VectorObservationAnalysisTests.cs b/tests/SharpEmu.ShaderCompiler.Tests/Gen5VectorObservationAnalysisTests.cs index e49dc6e92..08453d0c7 100644 --- a/tests/SharpEmu.ShaderCompiler.Tests/Gen5VectorObservationAnalysisTests.cs +++ b/tests/SharpEmu.ShaderCompiler.Tests/Gen5VectorObservationAnalysisTests.cs @@ -87,6 +87,28 @@ public void AnAccumulatingWriteReadsItsDestination() Assert.False(Analyze().UnobservedWrites(write).Contains(3)); } + [Fact] + public void AMaskedDppWriteUnderFullExecDoesNotKillTheOldDestination() + { + SaveExec(); + Narrow(); + var write = Add(Vop2(0, "VAddU32", 3, Operand(1), Gen5Operand.Vector(2))); + RestoreExec(); + Add(new Gen5ShaderInstruction(0, Gen5ShaderEncoding.Vop1, "VMovB32", [0u, 0u], + [Gen5Operand.Vector(4)], [Gen5Operand.Vector(3)], + new Gen5DppControl(0x111, false, false, 0, 0, 0x1, 0xF))); + Add(Vop2(0, "VAddU32", 5, Operand(1), Gen5Operand.Vector(3))); + Assert.False(Analyze().UnobservedWrites(write).Contains(3)); + } + + [Fact] + public void ARelativeSwapDisablesThePlan() + { + Add(Vop1(0, "VSwaprelB32", 3, Gen5Operand.Vector(4))); + Add(EndProgram(0)); + Assert.Null(Gen5VectorObservationAnalysis.Analyze(Program([.. _instructions]), wave32: false)); + } + [Fact] public void ARelativeMoveDisablesThePlan() { diff --git a/tests/SharpEmu.ShaderCompiler.Tests/Resources/ResourceMaterializationExactComparisonTests.cs b/tests/SharpEmu.ShaderCompiler.Tests/Resources/ResourceMaterializationExactComparisonTests.cs new file mode 100644 index 000000000..cc86829fa --- /dev/null +++ b/tests/SharpEmu.ShaderCompiler.Tests/Resources/ResourceMaterializationExactComparisonTests.cs @@ -0,0 +1,237 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using SharpEmu.ShaderCompiler.Resources; +using Xunit; +using static SharpEmu.ShaderCompiler.Tests.Resources.ResourceTestProgram; + +namespace SharpEmu.ShaderCompiler.Tests.Resources; + +public sealed class ResourceMaterializationExactComparisonTests +{ + private sealed class Draws + { + private readonly ShaderResourcePlan _plan; + private readonly ResourceRuntimeInputs _inputs; + private readonly ResidentGuestBytesReader _resident; + private ResourceSnapshot _snapshot = new(); + private ResourceSpecialization _specialization = new(); + public readonly TestWordMemory Memory = new() { Base = 0x1000, Words = new uint[0x20000] }; + public readonly ResourceMaterializationCache Cache = new(); + public ResourceSnapshot Snapshot => _snapshot; + public int Compares, ResidentReads, ProbeReads; + public bool RefuseComparison, ForceEqual, ObserveOrder, SawReversedOrder; + + public Draws(ShaderResourcePlan plan) + { + _plan = plan; + _resident = Resident; + _inputs = new ResourceRuntimeInputs + { + ReadMemory = Memory.Read, ReadCleanMemory = Memory.Read, CompareResidentMemory = Compare, + }; + } + + public bool Run(uint[] userData) + { + _inputs.UserData = userData; + return Cache.Materialize(_plan, _inputs, _resident, ref _snapshot, ref _specialization, out _); + } + + public void AssertFullWalk(uint[] userData) + { + var expected = new ResourceSnapshot(); + var specialization = new ResourceSpecialization(); + Assert.True(ResourceMaterializer.Materialize(_plan, Inputs(userData, Memory.Read, Memory.Read), + ref expected, ref specialization)); + Assert.Equal(expected.Buffers.Select(x => x.ToArray()), _snapshot.Buffers.Select(x => x.ToArray())); + Assert.Equal(expected.FlattenedResourceTable, _snapshot.FlattenedResourceTable); + Assert.Equal(specialization, _specialization); + } + + private bool Compare(ReadOnlySpan addresses, ReadOnlySpan lengths, ReadOnlySpan offsets, + ReadOnlySpan bytes) + { + Compares++; + if (RefuseComparison) return false; + if (ObserveOrder) + for (var index = 1; index < addresses.Length; index++) + SawReversedOrder |= addresses[index] < addresses[index - 1]; + if (ForceEqual) return true; + for (var range = 0; range < addresses.Length; range++) + for (var offset = 0; offset < lengths[range]; offset += 4) + if (!Memory.Read(addresses[range] + (ulong)offset, out var word) || + word != BitConverter.ToUInt32(bytes.Slice(offsets[range] + offset, 4))) return false; + return true; + } + + private bool Resident(ulong address, Span destination, bool clean) + { + ResidentReads++; + if (destination.Length == sizeof(uint)) ProbeReads++; + for (var offset = 0; offset < destination.Length; offset += 4) + { + if (!Memory.Read(address + (ulong)offset, out var word)) return false; + BitConverter.TryWriteBytes(destination[offset..], word); + } + return true; + } + + public void Write(ulong address, params uint[] values) + { + for (var index = 0; index < values.Length; index++) Memory.At(address + (ulong)index * 4) = values[index]; + } + } + + private static ShaderResourcePlan TablePlan() + { + var program = new List(); + for (uint index = 0; index < 6; index++) + program.Add(ScalarLoad(index * 8, 0, 8 + index * 4, 4, (int)index * 16)); + for (uint index = 0; index < 6; index++) program.Add(BufferLoad(48 + index * 8, 8 + index * 4)); + program.Add(EndProgram(96)); + return Extract(Program([.. program]), userDataCount: 2); + } + + [Fact] + public void ReturningTablePointersCompareWithoutCopiesAndRefusalsStillRead() + { + if (!ResourceMaterializationCache.InPlaceValidateEnabled) return; + var draws = new Draws(TablePlan()); + uint[][] pointers = Enumerable.Range(0, 8).Select(i => new uint[] { 0x10000u + (uint)i * 0x300, 0 }).ToArray(); + foreach (var pointer in pointers) + for (uint slot = 0; slot < 6; slot++) draws.Write(pointer[0] + slot * 16, 0x50000 + slot * 0x1000, 0, 256, 0x2_0000); + for (var draw = 0; draw < 40; draw++) Assert.True(draws.Run(pointers[draw % pointers.Length])); + var hits = draws.Cache.Hits; + var compares = draws.Compares; + var reads = draws.ResidentReads; + Assert.True(draws.Run(pointers[5])); + Assert.Equal(hits + 1, draws.Cache.Hits); + Assert.True(draws.Compares > compares); + if (!ResourceMaterializationCache.InPlaceValidateVerify) Assert.Equal(reads, draws.ResidentReads); + draws.AssertFullWalk(pointers[5]); + + draws.RefuseComparison = true; + reads = draws.ResidentReads; + hits = draws.Cache.Hits; + Assert.True(draws.Run(pointers[5])); + Assert.Equal(hits + 1, draws.Cache.Hits); + Assert.True(draws.ResidentReads > reads); + draws.AssertFullWalk(pointers[5]); + + draws.RefuseComparison = false; + draws.Memory.At(pointers[5][0] + 8) = 512; + Assert.True(draws.Run(pointers[5])); + draws.AssertFullWalk(pointers[5]); + Assert.Equal(512u, draws.Snapshot.Buffers[0][2]); + draws.Memory.FailAddress = pointers[5][0] + 4; + hits = draws.Cache.Hits; + Assert.False(draws.Run(pointers[5])); + Assert.Equal(hits, draws.Cache.Hits); + } + + [Fact] + public void IndependentlyMovedPointersCompareExactRangesEvenWhenTheirOrderReverses() + { + if (!ResourceMaterializationCache.InPlaceValidateEnabled) return; + var plan = Extract(Program(ScalarLoad(0, 0, 8, 4), ScalarLoad(8, 2, 12, 4), ScalarLoad(16, 4, 16, 4), + BufferLoad(24, 8), BufferLoad(32, 12), BufferLoad(40, 16), EndProgram(48)), userDataCount: 6); + var draws = new Draws(plan) { ObserveOrder = true }; + draws.Write(0x7000, 0x80000, 0, 128, 0x2_0000); + for (uint draw = 0; draw < 32; draw++) + { + uint[] pointers = [0x10000 + draw * 0x1000, 0, 0x30800 - draw * 0x1000, 0, 0x7000, 0]; + draws.Write(pointers[0], 0x60000, 0, 256, 0x2_0000); + draws.Write(pointers[2], 0x70000, 0, 512, 0x2_0000); + Assert.True(draws.Run(pointers)); + draws.AssertFullWalk(pointers); + } + Assert.True(draws.Cache.Hits > 0); + Assert.True(draws.Compares > 0); + Assert.True(draws.SawReversedOrder); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void MemoryLoadedBaseChangesStillReachTheOriginalProbeAndRefreshPath(bool movingTable) + { + var plan = Extract(Program(ScalarLoad(0, 0, 8, 4), ScalarBufferLoad(4, 8, 12, 4), + Vop1(8, "VMovB32", 0, Gen5Operand.Scalar(12)), BufferLoad(12, 8), EndProgram(16)), userDataCount: 2); + var draws = new Draws(plan); + uint[] pointer = [0x1000, 0]; + for (uint draw = 0; draw < 20; draw++) + { + // First confirm the raw table pointer independently, then change the base + // loaded from that table. Entries share immutable input arrays like ShaderSource. + var valueDraw = movingTable && draw < 8 ? 0 : draw; + var address = 0x10000 + valueDraw * 0x100; + pointer = [movingTable ? 0x2000 + draw * 0x100 : 0x1000, 0]; + draws.Write(pointer[0], address, 0, 256, 0x2_0000); + draws.Write(address, valueDraw * 10, valueDraw * 10 + 1, valueDraw * 10 + 2, valueDraw * 10 + 3); + Assert.True(draws.Run(pointer), $"draw={draw} movingTable={movingTable} table=0x{pointer[0]:X} base=0x{address:X}"); + draws.AssertFullWalk(pointer); + if (movingTable && draw == 7) + Assert.True(draws.Cache.Hits > 0, + $"Raw table pointer did not warm: hits={draws.Cache.Hits} misses={draws.Cache.Misses} compares={draws.Compares} probes={draws.ProbeReads}"); + } + draws.Write(pointer[0] + 8, 512); + Assert.True(draws.Run(pointer), $"Resized final table failed: movingTable={movingTable} table=0x{pointer[0]:X}"); + draws.AssertFullWalk(pointer); + Assert.Equal(512u, draws.Snapshot.Buffers[0][2]); + if (movingTable && ResourceMaterializationCache.InPlaceValidateEnabled) + Assert.True(draws.ProbeReads > 0, + $"Expected a scalar base-word probe after learned raw-pointer movement: hits={draws.Cache.Hits} misses={draws.Cache.Misses} compares={draws.Compares} residentReads={draws.ResidentReads} probes={draws.ProbeReads}"); + } + + [Fact] + public void WarmRebasedComparisonDoesNotAllocateAndEmptyRangesDoNotCallTheComparer() + { + if (!ResourceMaterializationCache.InPlaceValidateEnabled) return; + var plan = Extract(Program(ScalarLoad(0, 0, 8, 4), BufferLoad(8, 8), EndProgram(16)), userDataCount: 2); + var draws = new Draws(plan); + uint[] final = []; + for (uint draw = 0; draw < 20; draw++) + { + final = [0x10000 + draw * 0x100, 0]; + draws.Write(final[0], 0x60000, 0, 256, 0x2_0000); + Assert.True(draws.Run(final)); + } + for (var warmup = 0; warmup < 20; warmup++) Assert.True(draws.Run(final)); + Assert.True(draws.Compares > 0); + var before = GC.GetAllocatedBytesForCurrentThread(); + var result = draws.Run(final); + var allocated = GC.GetAllocatedBytesForCurrentThread() - before; + Assert.True(result); + Assert.Equal(0, allocated); + + var empty = new Draws(Extract(Program(BufferLoad(0, 4), EndProgram(8)), userDataCount: 8)); + for (uint draw = 0; draw < 20; draw++) + Assert.True(empty.Run([0, 0, 0, 0, 0x10000 + draw * 0x100, 0, 256, 0x2_0000])); + Assert.True(empty.Cache.Hits > 0); + Assert.Equal(0, empty.Compares); + Assert.Equal(0, empty.ResidentReads); + } + + [Fact] + public void VerificationReadsInsteadOfTrustingAnIncorrectSuccessfulComparison() + { + if (!ResourceMaterializationCache.InPlaceValidateEnabled || !ResourceMaterializationCache.InPlaceValidateVerify) return; + var plan = Extract(Program(ScalarLoad(0, 0, 8, 4), BufferLoad(8, 8), EndProgram(16)), userDataCount: 2); + var draws = new Draws(plan); + for (uint draw = 0; draw < 12; draw++) + { + uint[] pointer = [0x10000 + draw * 0x100, 0]; + draws.Write(pointer[0], 0x60000, 0, 256, 0x2_0000); + Assert.True(draws.Run(pointer)); + } + draws.ForceEqual = true; + uint[] changed = [0x10B00, 0]; + draws.Memory.At(changed[0] + 8) = 512; + var reads = draws.ResidentReads; + Assert.True(draws.Run(changed)); + Assert.True(draws.ResidentReads > reads); + draws.AssertFullWalk(changed); + Assert.Equal(512u, draws.Snapshot.Buffers[0][2]); + } +} diff --git a/tests/SharpEmu.ShaderCompiler.Tests/Resources/ResourceMaterializationProfileTests.cs b/tests/SharpEmu.ShaderCompiler.Tests/Resources/ResourceMaterializationProfileTests.cs index bc66df769..c0673cb4d 100644 --- a/tests/SharpEmu.ShaderCompiler.Tests/Resources/ResourceMaterializationProfileTests.cs +++ b/tests/SharpEmu.ShaderCompiler.Tests/Resources/ResourceMaterializationProfileTests.cs @@ -14,6 +14,23 @@ public sealed class ResourceMaterializationProfileCollection; [Collection("ResourceMaterializationProfile")] public sealed class ResourceMaterializationProfileTests { + [Fact] + public void CacheHitCountsDistinguishRecallAndRebaseAcrossThreads() + { + var before = ReadReport(); + Parallel.For(0, 100, index => + ResourceMaterializationProfile.RecordCacheHit(recalled: (index & 1) != 0, + rebased: index % 4 < 2, ranges: 3, bytes: 128)); + var after = ReadReport(); + if (!ResourceMaterializationProfile.Enabled) { Assert.Empty(after); return; } + Assert.Equal(ReadCounter(before, "normal") + 50, ReadCounter(after, "normal")); + Assert.Equal(ReadCounter(before, "recalled") + 50, ReadCounter(after, "recalled")); + Assert.Equal(ReadCounter(before, "normal_active_rebase") + 25, ReadCounter(after, "normal_active_rebase")); + Assert.Equal(ReadCounter(before, "recalled_active_rebase") + 25, ReadCounter(after, "recalled_active_rebase")); + Assert.Equal(ReadCounter(before, "recorded_ranges") + 300, ReadCounter(after, "recorded_ranges")); + Assert.Equal(ReadCounter(before, "recorded_bytes") + 12800, ReadCounter(after, "recorded_bytes")); + } + [Fact] public void ReadCountersAreExactAndGated() { diff --git a/tools/SharpEmu.Tools.MaterializationBench/Benches.Materialization.cs b/tools/SharpEmu.Tools.MaterializationBench/Benches.Materialization.cs index 782e96ab8..1c6ddb433 100644 --- a/tools/SharpEmu.Tools.MaterializationBench/Benches.Materialization.cs +++ b/tools/SharpEmu.Tools.MaterializationBench/Benches.Materialization.cs @@ -70,8 +70,35 @@ bool ReadWord(ulong address, out uint word) } bool ReadResident(ulong address, Span destination, bool clean) => memory.TryRead(address, destination); + // Reuse the capturing delegate as the renderer does. Re-converting the local + // function at every call measures a harness allocation on each cache hit. + ResidentGuestBytesReader residentReader = ReadResident; - var inputs = new ResourceRuntimeInputs { ReadMemory = ReadWord, ReadCleanMemory = ReadWord, ReadResidentMemory = ReadResident }; + bool CompareResident(ReadOnlySpan addresses, ReadOnlySpan lengths, + ReadOnlySpan offsets, ReadOnlySpan recorded) + { + if (addresses.Length != lengths.Length || addresses.Length != offsets.Length) return false; + for (var index = 0; index < addresses.Length; index++) + { + var address = addresses[index]; + var length = lengths[index]; + var offset = offsets[index]; + if (length < 0 || offset < 0 || offset > recorded.Length - length || + address < NativeGuestMemory.Base || address - NativeGuestMemory.Base > memory.Size || + (ulong)length > memory.Size - (address - NativeGuestMemory.Base)) return false; + if (!new ReadOnlySpan(memory.Raw(address), length).SequenceEqual(recorded.Slice(offset, length))) + return false; + } + return true; + } + + var compareResident = Environment.GetEnvironmentVariable("PA_COMPARE_RESIDENT") == "1"; + Console.WriteLine($"resident_comparer={compareResident} (synthetic fixed native allocation; no Metal ownership checks)"); + var inputs = new ResourceRuntimeInputs + { + ReadMemory = ReadWord, ReadCleanMemory = ReadWord, ReadResidentMemory = residentReader, + CompareResidentMemory = compareResident ? CompareResident : null, + }; var snapshot = new ResourceSnapshot(); var specialization = new ResourceSpecialization(); var table0 = TableRegion; @@ -90,13 +117,13 @@ bool ReadWord(ulong address, out uint word) // 3b: warm cache hit, same descriptors every draw. var cache = new ResourceMaterializationCache(); - if (!cache.Materialize(plan, inputs, ReadResident, ref snapshot, ref specialization, out var warmFailure)) throw new InvalidOperationException(warmFailure.ToString()); + if (!cache.Materialize(plan, inputs, residentReader, ref snapshot, ref specialization, out var warmFailure)) throw new InvalidOperationException(warmFailure.ToString()); var hitsBefore = cache.Hits; var warm = Harness.Measure("3b", "WARM cache hit, same descriptors every call (validation re-read)", Batch * 5, 13825, _ => { for (var i = 0; i < Batch * 5; i++) { - if (!cache.Materialize(plan, inputs, ReadResident, ref snapshot, ref specialization, out var failure)) throw new InvalidOperationException(failure.ToString()); + if (!cache.Materialize(plan, inputs, residentReader, ref snapshot, ref specialization, out var failure)) throw new InvalidOperationException(failure.ToString()); } }, unit: "cache-hit call", note: "validation re-read"); Console.WriteLine($" -> cache hits {cache.Hits - hitsBefore}, misses {cache.Misses}, refreshes {cache.TableRefreshes}"); @@ -110,7 +137,7 @@ bool ReadWord(ulong address, out uint word) { var table = TableRegion + (ulong)(cursor++ % Pool) * TableStride; inputs.UserData = new uint[] { (uint)table, (uint)(table >> 32) }; - if (!cold.Materialize(plan, inputs, ReadResident, ref snapshot, ref specialization, out var failure)) throw new InvalidOperationException(failure.ToString()); + if (!cold.Materialize(plan, inputs, residentReader, ref snapshot, ref specialization, out var failure)) throw new InvalidOperationException(failure.ToString()); } }, unit: "miss", note: "cold: includes allocating the user-data array and the entry build"); Console.WriteLine($" -> hits {cold.Hits} misses {cold.Misses} refreshes {cold.TableRefreshes} uncacheable {cold.Uncacheable} cold_bypasses {ColdBypasses(cold)}"); @@ -118,7 +145,7 @@ bool ReadWord(ulong address, out uint word) // 3d: same key, the table's descriptors change between calls (stale same-key): tries refresh, falls back to the walk. var stale = new ResourceMaterializationCache(); inputs.UserData = new uint[] { (uint)table0, (uint)(table0 >> 32) }; - stale.Materialize(plan, inputs, ReadResident, ref snapshot, ref specialization, out _); + stale.Materialize(plan, inputs, residentReader, ref snapshot, ref specialization, out _); var salt = 0; var refreshesBefore = stale.TableRefreshes; Harness.Measure("3d", "STALE same key: descriptor words change every call (refresh or re-walk)", Batch, 1404, _ => @@ -126,7 +153,7 @@ bool ReadWord(ulong address, out uint word) for (var i = 0; i < Batch; i++) { WriteTable(memory, table0, 1 + (salt++ % (Pool - 1))); - if (!stale.Materialize(plan, inputs, ReadResident, ref snapshot, ref specialization, out var failure)) throw new InvalidOperationException(failure.ToString()); + if (!stale.Materialize(plan, inputs, residentReader, ref snapshot, ref specialization, out var failure)) throw new InvalidOperationException(failure.ToString()); } }, unit: "stale", note: "includes WriteTable (14 small writes) in the timed loop"); Console.WriteLine($" -> hits {stale.Hits} misses {stale.Misses} refreshes {stale.TableRefreshes - refreshesBefore} (slot refreshes {stale.SlotRefreshes}, cold_bypasses {ColdBypasses(stale)})"); @@ -148,7 +175,7 @@ bool ReadWord(ulong address, out uint word) for (var i = 0; i < Draws; i++) { inputs.UserData = userData[i % Tables]; - if (!alternating.Materialize(plan, inputs, ReadResident, ref snapshot, ref specialization, out var failure)) throw new InvalidOperationException(failure.ToString()); + if (!alternating.Materialize(plan, inputs, residentReader, ref snapshot, ref specialization, out var failure)) throw new InvalidOperationException(failure.ToString()); } Console.WriteLine($" 3f first {Draws} draws (cold cache): hits {alternating.Hits} misses {alternating.Misses} refreshes {alternating.TableRefreshes} -> hit rate {100.0 * alternating.Hits / Draws:F1}%"); var hits0 = alternating.Hits; var misses0 = alternating.Misses; var refreshes0 = alternating.TableRefreshes; @@ -158,7 +185,7 @@ bool ReadWord(ulong address, out uint word) for (var i = 0; i < Draws; i++) { inputs.UserData = userData[draw++ % Tables]; - if (!alternating.Materialize(plan, inputs, ReadResident, ref snapshot, ref specialization, out var failure)) throw new InvalidOperationException(failure.ToString()); + if (!alternating.Materialize(plan, inputs, residentReader, ref snapshot, ref specialization, out var failure)) throw new InvalidOperationException(failure.ToString()); } }, unit: "draw"); var calls = (alternating.Hits - hits0) + (alternating.Misses - misses0) + (alternating.TableRefreshes - refreshes0); @@ -183,7 +210,7 @@ void Frame() var table = TableRegion + (ulong)(RingFirst + start + m) * TableStride; WriteTable(memory, table, 100 + m); inputs.UserData = new uint[] { (uint)table, (uint)(table >> 32) }; - if (!ring.Materialize(plan, inputs, ReadResident, ref snapshot, ref specialization, out var failure)) throw new InvalidOperationException(failure.ToString()); + if (!ring.Materialize(plan, inputs, residentReader, ref snapshot, ref specialization, out var failure)) throw new InvalidOperationException(failure.ToString()); } } diff --git a/tools/SharpEmu.Tools.MaterializationBench/README.md b/tools/SharpEmu.Tools.MaterializationBench/README.md index 6db122bf7..929f29d7e 100644 --- a/tools/SharpEmu.Tools.MaterializationBench/README.md +++ b/tools/SharpEmu.Tools.MaterializationBench/README.md @@ -36,3 +36,19 @@ load drift. Cache counters must distinguish actual hits from rebuilds; correctne and invalidation remain the responsibility of the regression suites. These are synthetic per-operation timings, not game frame-time projections. + +Set `PA_COMPARE_RESIDENT=1` to offer an exact in-place comparer over the benchmark's +native allocation. Use identical settings in both source/binary arms when comparing +warm or moving-pointer validation. The default retains read-and-copy validation. +This synthetic comparer checks bounds and every recorded byte, but has no GPU +ownership, pending-write, alias-lifetime or released-page costs; its times are not +Metal-backend or game-FPS measurements. `SHARPEMU_RESOURCE_INPLACE_VALIDATE=0` +disables use of the offered comparer, and +`SHARPEMU_RESOURCE_INPLACE_VALIDATE_VERIFY=1` also checks accepted comparisons +through the original reader. + +The resident-reader delegate is now reused across calls. Earlier versions converted +a capturing local function into a delegate on every cache call; their per-operation +allocation figures therefore include harness overhead. Keep the benchmark binary +fixed across compiler comparisons, and do not attribute changes from this harness +correction to the emulator. diff --git a/tools/SharpEmu.Tools.PerfAudit/LabelFilterBenches.cs b/tools/SharpEmu.Tools.PerfAudit/LabelFilterBenches.cs new file mode 100644 index 000000000..304c15992 --- /dev/null +++ b/tools/SharpEmu.Tools.PerfAudit/LabelFilterBenches.cs @@ -0,0 +1,60 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using System.Diagnostics; +using System.Runtime.CompilerServices; +using SharpEmu.Gpu.Metal.V2; + +namespace SharpEmu.PerfAudit; + +internal static class LabelFilterBenches +{ + public static void Run() + { + Console.WriteLine($"arch={System.Runtime.InteropServices.RuntimeInformation.ProcessArchitecture} runtime={Environment.Version}"); + foreach (var count in new[] { 16, 256, 4096 }) + { + var copies = Enumerable.Range(0, count).Select(i => new Copy(1, 0, 2, (ulong)i * 16, 8, + 0x100000UL + (i < count / 2 ? 0UL : 0x1000000UL) + (ulong)(i % (count / 2)) * 16)).ToArray(); + var filter = new MetalPageRangeFilter(); + foreach (var c in copies) filter.Add(c.Address, (ulong)c.Bytes); + foreach (var (name, address) in new[] { ("outside-hull", 0x2000000UL), ("interpage-gap", 0x900000UL), ("first-hit", copies[0].Address), ("last-hit", copies[^1].Address), ("same-page-gap", copies[0].Address + 8) }) + { + var baseline = Touch(copies, filter, address, false); + if (baseline != Touch(copies, filter, address, true)) throw new Exception("mismatch"); + for (var i = 0; i < 20000; i++) { _ = Touch(copies, filter, address, false); _ = Touch(copies, filter, address, true); } + for (var trial = 0; trial < 4; trial++) + foreach (var enabled in (trial & 1) == 0 ? new[] { false, true } : new[] { true, false }) + { + const int iterations = 100000; + var alloc = GC.GetAllocatedBytesForCurrentThread(); var start = Stopwatch.GetTimestamp(); var hits = 0; + for (var i = 0; i < iterations; i++) if (Touch(copies, filter, address, enabled)) hits++; + var ns = Stopwatch.GetElapsedTime(start).TotalNanoseconds / iterations; + var bytes = GC.GetAllocatedBytesForCurrentThread() - alloc; + if (hits != (baseline ? iterations : 0)) throw new Exception("result mismatch"); + Console.WriteLine($"count={count} case={name} filter={enabled} trial={trial} ns_op={ns:F2} B_op={bytes/(double)iterations:F4}"); + } + } + var before = GC.GetAllocatedBytesForCurrentThread(); var watch = Stopwatch.GetTimestamp(); + for (var batch = 0; batch < 10000; batch++) + { + filter.Clear(); + foreach (var c in copies) filter.Add(c.Address, (ulong)c.Bytes); + } + Console.WriteLine($"count={count} add_clear_ns_per_copy={Stopwatch.GetElapsedTime(watch).TotalNanoseconds/(10000*count):F2} bytes={GC.GetAllocatedBytesForCurrentThread()-before}"); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + static bool Touch(Copy[] copies, MetalPageRangeFilter filter, ulong address, bool enabled) + { + const ulong size = 4; + if (address >= copies[^1].Address + 8 || address + size <= copies[0].Address) return false; + if (enabled && !filter.MayOverlap(address, size)) return false; + foreach (ref readonly var c in copies.AsSpan()) + if (c.Address < address + size && address < c.Address + (ulong)c.Bytes) return true; + return false; + } + } + + private readonly record struct Copy(nint Source, ulong SourceOffset, nint Destination, ulong DestinationOffset, int Bytes, ulong Address); +} diff --git a/tools/SharpEmu.Tools.PerfAudit/Program.cs b/tools/SharpEmu.Tools.PerfAudit/Program.cs index 51e9c0c24..b87e32b8d 100644 --- a/tools/SharpEmu.Tools.PerfAudit/Program.cs +++ b/tools/SharpEmu.Tools.PerfAudit/Program.cs @@ -29,6 +29,8 @@ if (filter.Split(',').Contains("xlate")) TranslatorBenches.Run(); if (filter.Split(',').Contains("gword")) MetalRawBenches.GpuWordWait(); if (filter.Split(',').Contains("wranges")) WriteRangeBenches.Run(); +if (filter.Split(',').Contains("labels")) LabelFilterBenches.Run(); +if (filter.Split(',').Contains("shmap")) ShaderRegisterMapBenches.Run(); Console.WriteLine(); Console.WriteLine(Harness.Table()); diff --git a/tools/SharpEmu.Tools.PerfAudit/README.md b/tools/SharpEmu.Tools.PerfAudit/README.md index 11d995918..06c01141e 100644 --- a/tools/SharpEmu.Tools.PerfAudit/README.md +++ b/tools/SharpEmu.Tools.PerfAudit/README.md @@ -31,6 +31,8 @@ The executable has its own assembly name and cannot overwrite the Metal test DLL | `w8` | Staged range/cache behavior | | `xlate` | Single, fragmented and strided guest view layouts | | `wranges` | GPU-written range tracking | +| `labels` | Held-label overlap lookup, exact scan versus bounded page filter | +| `shmap` | Dictionary versus standalone dense shader-register map candidate; shared interface dispatch | | `metal` | Raw Metal binding/encoding and submission timings | | `e2e` | Generated textured draws through the interpreter and Metal backend | | `gword` | Synthetic GPU-word wait path | @@ -55,3 +57,13 @@ For the final cold-admission A/B experiment, use the dedicated sampling and counters match the published final result. Historical measurements were made with the corresponding historical source, not this newly packaged binary. See [benchmark results and scope](../../docs/performance/benchmark-index.md). + +The `labels` selector uses fixed parameters: 20,000 warmup pairs followed by four +alternating trials of 100,000 lookups for each list size and overlap shape. It +prints nanoseconds and allocations per lookup plus filter insertion/reset cost; +`--quick` does not alter this selector. To match the native ARM64 lookup experiment, +publish this tool for `osx-arm64` with `--self-contained true` and +`-p:PublishReadyToRun=false`, then run the resulting executable with +`DOTNET_TieredCompilation=0` and selector `labels`. The original measurement used +a standalone build of the same benchmark body linked to `MetalPageRangeFilter.cs`; +packaged-tool validation is recorded separately. diff --git a/tools/SharpEmu.Tools.PerfAudit/ShaderRegisterMapBenches.cs b/tools/SharpEmu.Tools.PerfAudit/ShaderRegisterMapBenches.cs new file mode 100644 index 000000000..6aed40e74 --- /dev/null +++ b/tools/SharpEmu.Tools.PerfAudit/ShaderRegisterMapBenches.cs @@ -0,0 +1,71 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using System.Diagnostics; +using System.Runtime.CompilerServices; +using SharpEmu.Libs.Gpu.GpuCommands; + +namespace SharpEmu.PerfAudit; + +// Both arms use the same interface-dispatched operations. Production integration is separate. +internal static class ShaderRegisterMapBenches +{ + public static void Run() + { + const int rounds = 2000; + Console.WriteLine("shader-map candidate only; synthetic CPU operations, not frame-time evidence"); + foreach (var scenario in new[] { "direct-runs", "repeated", "mixed-reads", "overflow" }) + { + var expected = Exercise(new Dictionary(), scenario, 20); + if (Exercise(new DenseShaderRegisterMap(), scenario, 20) != expected) + throw new InvalidOperationException($"Shader-map checksum mismatch: {scenario}."); + for (var warmup = 0; warmup < 4; warmup++) + { + _ = Exercise(new Dictionary(), scenario, 200); + _ = Exercise(new DenseShaderRegisterMap(), scenario, 200); + } + ulong? reference = null; + for (var trial = 0; trial < 4; trial++) + foreach (var dense in (trial & 1) == 0 ? new[] { false, true } : new[] { true, false }) + { + var allocated = GC.GetAllocatedBytesForCurrentThread(); + var started = Stopwatch.GetTimestamp(); + IDictionary map = dense ? new DenseShaderRegisterMap() : new Dictionary(); + var checksum = Exercise(map, scenario, rounds); + var elapsed = Stopwatch.GetElapsedTime(started); + var bytes = GC.GetAllocatedBytesForCurrentThread() - allocated; + reference ??= checksum; + if (checksum != reference.Value) throw new InvalidOperationException($"Trial checksum mismatch: {scenario}."); + Console.WriteLine($"case={scenario} dense={dense} trial={trial} rounds={rounds} " + + $"ns_round={elapsed.TotalNanoseconds / rounds:F2} B_round={bytes / (double)rounds:F3} checksum={checksum:X16}"); + } + } + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong Exercise(IDictionary map, string scenario, int rounds) + { + ulong checksum = 0; + for (var round = 0; round < rounds; round++) + { + if (scenario == "direct-runs") map.Clear(); + for (uint run = 0; run < 8; run++) + { + var first = run * 64; + for (uint word = 0; word < 16; word++) + { + var key = first + word; + if (scenario == "overflow" && word == 15) key = uint.MaxValue - run; + map[key] = (uint)round ^ key; + } + } + if (scenario == "mixed-reads") + for (uint key = 0; key < 768; key += 3) + checksum += map.TryGetValue(key, out var value) ? value + 1UL : 0UL; + else + checksum += map[64] + (ulong)map.Count; + } + foreach (var item in map) checksum += item.Key + (ulong)item.Value; + return checksum; + } +} diff --git a/tools/SharpEmu.Tools.ShaderCompileBench/Observation.cs b/tools/SharpEmu.Tools.ShaderCompileBench/Observation.cs new file mode 100644 index 000000000..ea69bf429 --- /dev/null +++ b/tools/SharpEmu.Tools.ShaderCompileBench/Observation.cs @@ -0,0 +1,76 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using System.Globalization; +using System.Text.Json; +using SharpEmu.Libs.Gpu.Pipelines; +using SharpEmu.ShaderCompiler; +using SharpEmu.ShaderCompiler.Ir; +using SharpEmu.ShaderCompiler.Metal; + +internal static class Observation +{ + // Reports analysis opportunities; emit= optionally writes translated source without executing it. + public static int Analyze(string directory, Func arg) + { + var selected = arg("hash", "").Split(',', StringSplitOptions.RemoveEmptyEntries) + .Select(value => ulong.Parse(value.Replace("0x", "", StringComparison.OrdinalIgnoreCase), NumberStyles.HexNumber)) + .ToHashSet(); + using var list = GraphicsPrewarmList.Open(directory) ?? throw new InvalidOperationException("no graphics list"); + var seen = new HashSet(); + var failures = 0; + foreach (var item in list.Loaded()) + { + var record = GraphicsPrewarmList.Decode(item); + if ((selected.Count != 0 && !selected.Contains(record.Hash)) || !seen.Add(record.Hash)) continue; + if (!ShaderCorpus.TryBuildRequest(record, out var request, out var error) || request is null) + { + Console.WriteLine(JsonSerializer.Serialize(new { hash = record.Hash.ToString("X16"), error })); + failures++; + continue; + } + if (arg("emit", "") is { Length: > 0 } emit) + { + if (!Gen5MslTranslator.TryCompileProgram(request, out var translated, out var translationError)) + { + Console.WriteLine(JsonSerializer.Serialize(new { hash = record.Hash.ToString("X16"), translationError })); + failures++; + continue; + } + Directory.CreateDirectory(emit); + File.WriteAllText(Path.Combine(emit, $"{record.Hash:X16}.metal"), translated.Source); + Console.WriteLine(JsonSerializer.Serialize(new + { + hash = record.Hash.ToString("X16"), translated.MeshInfo, + hotLds = translated.Source.Contains("sharpemu_hot_lds", StringComparison.Ordinal), + sourceBytes = System.Text.Encoding.UTF8.GetByteCount(translated.Source), + })); + } + var full = Gen5ExecFullAnalysis.Analyze(request.Program, request.WaveSize == 32); + var plan = Gen5VectorObservationAnalysis.Analyze(request.Program, request.WaveSize == 32); + var instructions = request.Program.Instructions; + var candidates = instructions.Where(i => !full.Contains(i.Pc)) + .Sum(i => plan?.UnobservedWrites(i.Pc).Count ?? 0); + var pureCandidates = instructions.Where(i => !full.Contains(i.Pc) && + i.Encoding is (Gen5ShaderEncoding.Vop1 or Gen5ShaderEncoding.Vop2 or Gen5ShaderEncoding.Vop3 or Gen5ShaderEncoding.Vop3p) && + i.Control is not (Gen5DppControl or Gen5Dpp8Control)) + .Sum(i => plan?.UnobservedWrites(i.Pc).Count ?? 0); + Console.WriteLine(JsonSerializer.Serialize(new + { + hash = request.Hash.ToString("X16"), stage = request.Stage.ToString(), wave = request.WaveSize, + mesh = Gen5MslTranslator.RequiresMeshEmission(request), instructions = instructions.Count, + primitiveShader = request.PrimitiveShader, + fullExecInstructions = full.Count, planAvailable = plan is not null, candidates, pureCandidates, + relative = instructions.Where(i => i.Opcode.Contains("rel", StringComparison.Ordinal)) + .Select(i => i.Opcode).Distinct().ToArray(), + dppInstructions = instructions.Count(i => i.Control is Gen5DppControl or Gen5Dpp8Control), + })); + } + foreach (var missing in selected.Except(seen)) + { + Console.WriteLine(JsonSerializer.Serialize(new { hash = missing.ToString("X16"), error = "requested shader is absent from the corpus" })); + failures++; + } + return failures == 0 ? 0 : 1; + } +} diff --git a/tools/SharpEmu.Tools.ShaderCompileBench/Program.cs b/tools/SharpEmu.Tools.ShaderCompileBench/Program.cs index e048a6848..0a8faaeb0 100644 --- a/tools/SharpEmu.Tools.ShaderCompileBench/Program.cs +++ b/tools/SharpEmu.Tools.ShaderCompileBench/Program.cs @@ -15,6 +15,7 @@ // Arguments are key=value: // dir=/metal-v2> (required; the list loader prunes files) // mode=translate|compute|libraries (default compute) +// mode=analysis hash= reports graphics IR opportunities only // workers=4 tworkers=4 priority=normal|below limit=0 order=list|bigfirst // salt= prefixes every source with a comment, so the front end misses its cache // opt=default|size lang=3.2|4.0 pipeline=1|0 csv= dumpmsl= @@ -161,6 +162,10 @@ private static int Main(string[] argv) { return Derivation.ComputeMatch(dir, Arg); } + else if (mode == "analysis") + { + return Observation.Analyze(dir, Arg); + } else if (mode == "graphics") { return GraphicsMatch(dir); From fb717072ad11ada9a53e27ac34219fdc91f1f5bb Mon Sep 17 00:00:00 2001 From: kuba Date: Tue, 27 Oct 2026 18:26:53 +0100 Subject: [PATCH 34/82] [GPU] Enable validated performance paths by default --- docs/performance/native-occlusion-runtime.md | 4 + docs/performance/performance-defaults.md | 79 +++++++++++++++++++ .../Native/DirectExecutionBackend.Imports.cs | 6 +- .../DeferredCommandThread.Recycling.cs | 2 +- .../Gen5MslCodegenOptions.cs | 24 +++--- .../V2/MetalCommandStream.cs | 6 +- .../V2/MetalRenderHost.Occlusion.cs | 4 +- .../SharpEmu.Gpu.Metal/V2/MetalRenderHost.cs | 2 +- .../SharpEmu.Gpu.Metal/V2/MetalV2Presenter.cs | 2 +- .../Scenes/NativeOcclusionSceneTests.cs | 22 ++++++ .../Scenes/OcclusionSceneTests.cs | 4 +- .../Audio/AudioOut2QueueDispatchTests.cs | 2 +- .../NggMeshRuntimeTests.cs | 12 ++- 13 files changed, 142 insertions(+), 27 deletions(-) create mode 100644 docs/performance/performance-defaults.md diff --git a/docs/performance/native-occlusion-runtime.md b/docs/performance/native-occlusion-runtime.md index 198b9328e..ac0911471 100644 --- a/docs/performance/native-occlusion-runtime.md +++ b/docs/performance/native-occlusion-runtime.md @@ -3,6 +3,10 @@ SPDX-License-Identifier: GPL-2.0-or-later --> # Native occlusion and Silent Hill room measurements +**Current default policy:** [performance-defaults.md](performance-defaults.md) +supersedes the historical opt-in decisions below. Per-run flags and results +remain unchanged; default promotion is not a new FPS measurement. + Native Metal visibility counters replace the conservative always-visible query results when `SHARPEMU_METAL_V2_NATIVE_OCCLUSION=1`. In a paired, same-binary Silent Hill first-room comparison, median guest FPS increased from 4.485 to diff --git a/docs/performance/performance-defaults.md b/docs/performance/performance-defaults.md new file mode 100644 index 000000000..2ea0c873a --- /dev/null +++ b/docs/performance/performance-defaults.md @@ -0,0 +1,79 @@ + + +# Performance defaults + +The maintainer requested default enablement based on correctness and positive +component benchmarks, without requiring an isolated Silent Hill FPS win for each +option. The following settings now turn on when their environment variable is +unset. Set the named variable to `0` to retain its reference path. No resolution, +VSync, guest quality, or unrelated application setting is changed. + +| Default enabled | Environment variable | Evidence and limits | +| --- | --- | --- | +| Native visibility counts | `SHARPEMU_METAL_V2_NATIVE_OCCLUSION` | Same-binary first-room median 4.485 → 13.43 FPS; ordered/reset/sparse/overlap device tests. The visibility audit requires native queries disabled. | +| Packed format word runs | `SHARPEMU_MSL_PACKED_FORMAT_LOADS` | Same-binary shadow encoders 40.768 → 34.326 ms; overall title median 7.49 → 7.50 FPS. Aligned, unaligned and partial-element device comparisons retain fallback decoding. | +| Static private scratch slots | `SHARPEMU_MSL_STATIC_SCRATCH` | Eligible compute dispatch roughly 16.35 → 7.2 ms across historical diagnostic runs; not an isolated FPS comparison. Alias, wrap, masks and base-lifetime proofs gate transformation. | +| Scalar-only mesh buffers in constant space | `SHARPEMU_MSL_SCALAR_CONSTANT_BUFFERS` | Same-binary diagnostic encoder totals 114.585 → 112.686 ms and 7.50 → 7.97 FPS; host-confounded, no broad speedup claim. Bounds, mixed-pointer ABI, coherent/write/vector refusal tests. | +| Graphics binding reuse | `SHARPEMU_METAL_V2_GRAPHICS_BINDING_REUSE` | 31.66% of eligible native bindings skipped in one room window; changed textures, encoder restart and stage-key tests. No isolated FPS attribution. | +| Unchanged-frame presentation reuse | `SHARPEMU_METAL_V2_REUSE_PRESENTED_FRAME` | Avoids repeated host presentations of an unchanged guest frame; movie updates, output changes, restoration and animated overlays invalidate the memo. Guest FPS is distinct from host Hz. | +| Audio queue query leaf dispatch | `SHARPEMU_AUDIO_OUT2_QUEUE_LEAF` | Synthetic dispatch about 369 → 84.5 ns and 40 → 0 B/call; guest audio thread remains busy. Result/error and shutdown tests retain full-path behavior. | +| Asynchronous command-item reuse | `SHARPEMU_COMMAND_ITEM_REUSE` | Native two-thread median 389.965 → 354.5175 ns; allocation 80.0008 → 0.026 B/item. Bounded 64-slot return ring; synchronous and failed items retain original ownership. No accepted runtime FPS gain. | +| Queued CP writes | `SHARPEMU_METAL_V2_QUEUE_CP_WRITES` | Removes synchronous handoffs for eligible writes when command threading is explicitly enabled. Pending ranges, prefetch gates and CPU-read fallbacks preserve ordering; producer-thread device tests cover byte/qword/sparse/predication cases. | + +The existing negative label page filter, exact materialization comparison, dense +register lookup, mip-layout construction and target memo fixes remain enabled. +The three promoted shader properties also default on in explicitly constructed +`Gen5MslCodegenOptions`; reference tests set those properties to `false`. + +## Retained opt-in experiments + +| Setting | Reason | +| --- | --- | +| `SHARPEMU_REGISTER_IDENTICAL_SCALARS` | Always-changing packets regress: native 101.40 → 103.08 ns; Rosetta 164.85 → 178.94 ns. Only 0.805% of eligible packets reused in the observed room. Identical-packet synthetic gains do not outweigh this demonstrated mixed-workload cost. | +| `SHARPEMU_MSL_GEOMETRY_CAPACITY` | Volume pass 39.888 → 57.027 ms, changed image, no accepted broad correctness/performance result. | +| `SHARPEMU_MSL_MESH_LDS_FORWARDING` | Same-binary shadow time 34.238 → 36.422 ms; title median 7.97 → 7.47 FPS. | +| `SHARPEMU_MSL_MESH_HOT_LDS` | Same-binary total encoder time 108.483 → 109.451 ms; no benefit established. | +| `SHARPEMU_MSL_MESH_EXIT_JOIN`, `SHARPEMU_MSL_UNOBSERVED_ALU` | Correctness coverage exists, but isolated performance evidence is inconclusive and their combined run had an unresolved mapping failure. | +| Command thread / parallel resolution / command-thread QoS | Earlier overlap runs were slower than serial resolution. Queued writes and recycling apply when overlap is requested; this change does not enable overlap globally. | +| Concurrent tiling and floating-point contraction | Representative writeback regressions or numerical differences remain; the original audit did not accept them. | + +Diagnostics, detailed tracing, scripted-only input and visibility auditing stay +off. Portable-buffer policy, device LDS placement threshold and asynchronous +library compilation policy are unchanged. These defaults do not establish +30 FPS or full-game compatibility. See [the runtime report](native-occlusion-runtime.md) +for source versions, sample windows, host-pressure limitations and failed runs. + + +## Validation of the new defaults + +Fresh serial Release runs with all `SHARPEMU_*` overrides removed: + +| Suite | Passed | Failed | Existing skips | +| --- | ---: | ---: | ---: | +| ShaderCompiler.Metal, required device | 3,589 | 0 | 0 | +| Gpu.Metal, required device, default synchronous mode | 1,635 | 0 | 7 | +| Libs, x64 | 9,052 | 0 | 5 | +| Total | 14,276 | 0 | 12 | + +Additional focused runs passed **56/56** with command threading and parallel +resolution enabled, then **56/56 Metal, 114/114 Metal shader and 29/29 Libs** +with all nine promoted switches set to `0`. These 255 executions are additional +configuration checks, not additional distinct tests. The query-default scene +checks actual sample counts in both modes; scalar-buffer tests check default +translation, explicit disablement and unchanged raw-read refusal semantics. +The synthetic-query audit/fallback fixture explicitly selects conservative mode. + +The first default-promotion shader run passed 3,587 cases and failed one stale +assertion that constructor defaults must disable scalar constant buffers. Its +reference case now explicitly disables the feature; the subsequent full run +above passed. No numerical or pixel oracle was relaxed. REUSE passed +2,864/2,864 files, and whitespace checks passed. Private command/TRX records: +`pr23-defaults-final-validation.json`, `pr23-defaults-final-results`, +`pr23-default-controls.json` and `pr23-default-controls-results` in the runtime +diagnostic directory. The earlier ShaderCompiler core suite passed 2,192/0/1 +before this defaults-only follow-up and was not rerun here. + +This validation establishes tested configuration paths, not a fresh game-FPS +measurement of the new default combination. Runtime investigation toward +30 FPS continues separately. diff --git a/src/SharpEmu.Core/Cpu/Native/DirectExecutionBackend.Imports.cs b/src/SharpEmu.Core/Cpu/Native/DirectExecutionBackend.Imports.cs index c0be8e4f1..937785aea 100644 --- a/src/SharpEmu.Core/Cpu/Native/DirectExecutionBackend.Imports.cs +++ b/src/SharpEmu.Core/Cpu/Native/DirectExecutionBackend.Imports.cs @@ -2162,10 +2162,10 @@ importStubEntry.Export is not { } export || // takes a host lock still qualifies, because the lock is the emulator's own // and cannot park the guest thread. // Queue-level reads only inspect the context/device counters and write two register- - // addressed outputs. They neither park the guest nor invoke guest callbacks. Keep the - // audio poll fast path opt-in until its runtime comparison is complete. + // addressed outputs. They neither park the guest nor invoke guest callbacks. The + // dispatch benchmark removes the gateway allocation; =0 retains the full path. private static readonly bool _audioQueueLeafEnabled = - Environment.GetEnvironmentVariable("SHARPEMU_AUDIO_OUT2_QUEUE_LEAF") == "1"; + Environment.GetEnvironmentVariable("SHARPEMU_AUDIO_OUT2_QUEUE_LEAF") != "0"; internal static bool IsTrivialLeafImport(string nid) => (_audioQueueLeafEnabled && nid is "R7d0F1g2qsU" or "Q8DZkKQ-SYc") || diff --git a/src/SharpEmu.Libs/Gpu/GpuCommands/DeferredCommandThread.Recycling.cs b/src/SharpEmu.Libs/Gpu/GpuCommands/DeferredCommandThread.Recycling.cs index 49e24e40e..5fe20e428 100644 --- a/src/SharpEmu.Libs/Gpu/GpuCommands/DeferredCommandThread.Recycling.cs +++ b/src/SharpEmu.Libs/Gpu/GpuCommands/DeferredCommandThread.Recycling.cs @@ -8,7 +8,7 @@ namespace SharpEmu.Libs.Gpu.GpuCommands; public sealed partial class DeferredCommandThread { private static readonly bool RecycleAsyncItemsEnabled = - Environment.GetEnvironmentVariable("SHARPEMU_COMMAND_ITEM_REUSE") == "1"; + Environment.GetEnvironmentVariable("SHARPEMU_COMMAND_ITEM_REUSE") != "0"; // A return channel, opposite to the work queue: only the render thread publishes items; // only the interpreter takes them. The existing producer/consumer ownership also applies diff --git a/src/SharpEmu.ShaderCompiler.Metal/Gen5MslCodegenOptions.cs b/src/SharpEmu.ShaderCompiler.Metal/Gen5MslCodegenOptions.cs index abda881da..46ed94c5e 100644 --- a/src/SharpEmu.ShaderCompiler.Metal/Gen5MslCodegenOptions.cs +++ b/src/SharpEmu.ShaderCompiler.Metal/Gen5MslCodegenOptions.cs @@ -156,9 +156,9 @@ public sealed record Gen5MslCodegenOptions( /// A single hardware wave may also publish its output at a loop-limit exit. public bool MeshExitJoin { get; init; } - /// Experimental constant indexing of provably invariant private scratch slots. - /// Enabled with SHARPEMU_MSL_STATIC_SCRATCH=1. - public bool StaticScratchSlots { get; init; } + /// Constant indexing of provably invariant private scratch slots. + /// SHARPEMU_MSL_STATIC_SCRATCH=0 retains dynamic addressing. + public bool StaticScratchSlots { get; init; } = true; /// Experimental removal of pure ALU guards when inactive values cannot /// be observed. SHARPEMU_MSL_UNOBSERVED_ALU=1 enables this for wave32 compute @@ -169,17 +169,17 @@ public sealed record Gen5MslCodegenOptions( /// with complete wave64 execution. SHARPEMU_MSL_GEOMETRY_CAPACITY=1. public bool GeometryOutputCapacity { get; init; } - /// Experimental single-run loads for statically known packed formats. - /// SHARPEMU_MSL_PACKED_FORMAT_LOADS=1; partial trailing elements retain the decoder. - public bool PackedFormatLoads { get; init; } + /// Single-run loads for statically known packed formats. + /// SHARPEMU_MSL_PACKED_FORMAT_LOADS=0 restores component loads; partial trailing elements retain the decoder. + public bool PackedFormatLoads { get; init; } = true; /// Experimental guarded forwarding of the last 128-bit LDS write in a /// single-wave mesh subgroup. SHARPEMU_MSL_MESH_LDS_FORWARDING=1. public bool MeshLdsForwarding { get; init; } - /// Experimental constant-address-space bindings for scalar-only buffers - /// in mesh shaders without device-memory writes. SHARPEMU_MSL_SCALAR_CONSTANT_BUFFERS=1. - public bool ScalarConstantBuffers { get; init; } + /// Constant-address-space bindings for scalar-only buffers + /// in mesh shaders without device-memory writes. SHARPEMU_MSL_SCALAR_CONSTANT_BUFFERS=0 disables this. + public bool ScalarConstantBuffers { get; init; } = true; /// Experimental compact threadgroup bank for frequently addressed LDS /// regions, retaining device storage for other addresses. SHARPEMU_MSL_MESH_HOT_LDS=1. @@ -215,11 +215,11 @@ public sealed record Gen5MslCodegenOptions( { UnobservedAluWrites = Environment.GetEnvironmentVariable("SHARPEMU_MSL_UNOBSERVED_ALU") == "1", GeometryOutputCapacity = Environment.GetEnvironmentVariable("SHARPEMU_MSL_GEOMETRY_CAPACITY") == "1", - PackedFormatLoads = Environment.GetEnvironmentVariable("SHARPEMU_MSL_PACKED_FORMAT_LOADS") == "1", + PackedFormatLoads = Environment.GetEnvironmentVariable("SHARPEMU_MSL_PACKED_FORMAT_LOADS") != "0", MeshLdsForwarding = Environment.GetEnvironmentVariable("SHARPEMU_MSL_MESH_LDS_FORWARDING") == "1", - ScalarConstantBuffers = Environment.GetEnvironmentVariable("SHARPEMU_MSL_SCALAR_CONSTANT_BUFFERS") == "1", + ScalarConstantBuffers = Environment.GetEnvironmentVariable("SHARPEMU_MSL_SCALAR_CONSTANT_BUFFERS") != "0", MeshHotLds = Environment.GetEnvironmentVariable("SHARPEMU_MSL_MESH_HOT_LDS") == "1", - StaticScratchSlots = Environment.GetEnvironmentVariable("SHARPEMU_MSL_STATIC_SCRATCH") == "1", + StaticScratchSlots = Environment.GetEnvironmentVariable("SHARPEMU_MSL_STATIC_SCRATCH") != "0", MeshExitJoin = Environment.GetEnvironmentVariable("SHARPEMU_MSL_MESH_EXIT_JOIN") == "1", MeshDeviceLdsThresholdBytes = uint.TryParse( Environment.GetEnvironmentVariable("SHARPEMU_MSL_MESH_LDS_THRESHOLD"), out var threshold) diff --git a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalCommandStream.cs b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalCommandStream.cs index f1a06a1f4..747d1e2a6 100644 --- a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalCommandStream.cs +++ b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalCommandStream.cs @@ -927,7 +927,7 @@ private ulong EncodeOrderedCopies( /// or a label is still owed by a completion - published by a completion behind it. /// internal bool QueueCommandProcessorWrites { get; set; } = - Environment.GetEnvironmentVariable("SHARPEMU_METAL_V2_QUEUE_CP_WRITES") == "1"; + Environment.GetEnvironmentVariable("SHARPEMU_METAL_V2_QUEUE_CP_WRITES") != "0"; public void RecordCommandProcessorWrite(ulong address, ReadOnlySpan source, string kind) { @@ -1895,8 +1895,8 @@ private void PublishClockAtCompletion(in EndOfPipeWrite label) /// public bool DefersEndOfPipeLabels => !PublishLabelsWhenParsed; - // Keep conservative pairs by default. Precise GPU snapshots are opt-in while - // native query ordering and runtime behavior are validated; the audit changes no dumps. + // Precise GPU snapshots preserve query ordering. The disable switch retains + // conservative pairs; the diagnostic audit changes no guest dumps. public bool PublishesSyntheticOcclusionDumps => !NativeOcclusionEnabled; // A deferred label is encoded into the command buffer being recorded, before it is diff --git a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.Occlusion.cs b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.Occlusion.cs index 434ec02a7..1b66e89da 100644 --- a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.Occlusion.cs +++ b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.Occlusion.cs @@ -5,13 +5,13 @@ namespace SharpEmu.Gpu.Metal.V2; -// Experimental precise queries. A dump closes a visibility interval, not the render +// Precise queries, enabled unless SHARPEMU_METAL_V2_NATIVE_OCCLUSION=0. A dump closes a visibility interval, not the render // encoder. At the encoder boundary a GPU prefix sum preserves each dump's snapshot; // the existing ordered-copy stream publishes them alongside guest WRITE_DATA stores. internal sealed partial class MetalRenderHost { internal bool NativeOcclusionEnabled { get; set; } = - Environment.GetEnvironmentVariable("SHARPEMU_METAL_V2_NATIVE_OCCLUSION") == "1"; + Environment.GetEnvironmentVariable("SHARPEMU_METAL_V2_NATIVE_OCCLUSION") != "0"; private const int OcclusionIntervalCapacity = 4096; internal int NativeOcclusionBatchBudget { get; set; } = 256; diff --git a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.cs b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.cs index bec20e0e3..3428aaeb7 100644 --- a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.cs +++ b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.cs @@ -36,7 +36,7 @@ internal sealed partial class MetalRenderHost : private readonly long[] _graphicsBindingAttempts = new long[3]; private readonly long[] _graphicsBindingSkipped = new long[3]; private bool _graphicsBindingReuseEnabled = - Environment.GetEnvironmentVariable("SHARPEMU_METAL_V2_GRAPHICS_BINDING_REUSE") == "1"; + Environment.GetEnvironmentVariable("SHARPEMU_METAL_V2_GRAPHICS_BINDING_REUSE") != "0"; internal bool GraphicsBindingReuseEnabled { diff --git a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalV2Presenter.cs b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalV2Presenter.cs index 698555a15..1ee40c47d 100644 --- a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalV2Presenter.cs +++ b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalV2Presenter.cs @@ -83,7 +83,7 @@ private readonly record struct PendingFrame( private static long _adoptedGuestFrameSequence; private static long _hostFrameSequence; private static readonly bool ReusePresentedFrame = - Environment.GetEnvironmentVariable("SHARPEMU_METAL_V2_REUSE_PRESENTED_FRAME") == "1"; + Environment.GetEnvironmentVariable("SHARPEMU_METAL_V2_REUSE_PRESENTED_FRAME") != "0"; private static readonly MetalPresentationCache PresentationCache = new(); private static int _firstGuestFrameNoted; private static long _publishedGuestFrames; diff --git a/tests/SharpEmu.Gpu.Metal.Tests/Scenes/NativeOcclusionSceneTests.cs b/tests/SharpEmu.Gpu.Metal.Tests/Scenes/NativeOcclusionSceneTests.cs index 502f00f0e..ccf0a60fd 100644 --- a/tests/SharpEmu.Gpu.Metal.Tests/Scenes/NativeOcclusionSceneTests.cs +++ b/tests/SharpEmu.Gpu.Metal.Tests/Scenes/NativeOcclusionSceneTests.cs @@ -25,6 +25,28 @@ private static ulong Delta(SceneRig rig, ulong query) return samples; } + [Fact] + public void DefaultQueryModePublishesNativeSamplesUnlessExplicitlyDisabled() + { + using var rig = CreateRig(); + if (rig is null) return; + var target = rig.ColorTarget(Size, Size); + var query = Query(rig); + var pm4 = new ScenePm4(); + var state = new SceneState(Size, Size).ColorTarget(0, target, Size, Size); + state.Context[Cx.DbCountControl] = 1u << 8; + Dump(pm4, query); + rig.DrawMesh(pm4, state, SceneDraws.Quad(0.1f, 1, 0, 0)); + Dump(pm4, query + 8); + rig.Submit(pm4); + var samples = Delta(rig, query); + if (Environment.GetEnvironmentVariable("SHARPEMU_METAL_V2_NATIVE_OCCLUSION") == "0") + Assert.Equal(16UL, samples); + else + Assert.Equal((ulong)Size * Size, samples); + Assert.Equal(0xFF0000FFu, SceneDraws.Pixel(rig.ReadImage(target, Size, Size), Size, Size / 2, Size / 2)); + } + [Fact] public void DisabledCountingPassesDoNotReduceOrChangeTheCounter() { diff --git a/tests/SharpEmu.Gpu.Metal.Tests/Scenes/OcclusionSceneTests.cs b/tests/SharpEmu.Gpu.Metal.Tests/Scenes/OcclusionSceneTests.cs index 8e01de4ad..c171cb50e 100644 --- a/tests/SharpEmu.Gpu.Metal.Tests/Scenes/OcclusionSceneTests.cs +++ b/tests/SharpEmu.Gpu.Metal.Tests/Scenes/OcclusionSceneTests.cs @@ -8,7 +8,7 @@ namespace SharpEmu.Gpu.Metal.Tests.Scenes; /// /// Occlusion queries (PIXEL_PIPE_STAT_DUMP begin/end pairs) and zpass predication -/// (SET_PREDICATION + the packet predicate bit). The emulator's dumps are synthetic: every +/// (SET_PREDICATION + the packet predicate bit). This suite explicitly tests the conservative fallback: every /// pair reads as visible, so occluded draws are never culled. That costs time, not pixels; /// what must hold is that a predicate built from real dump pairs never drops a draw whose /// query saw samples, whichever command buffer or queue carried the dumps. @@ -26,6 +26,7 @@ public void HardwareAuditDistinguishesHiddenDrawsWithoutChangingGuestQueries() { using var rig = CreateRig(); if (rig is null) return; + rig.Host.NativeOcclusionEnabled = false; rig.Host.OcclusionAuditEnabled = true; var depth = new DepthRig(rig, Size, Size, 3); var pm4 = depth.Pm4; @@ -81,6 +82,7 @@ public ScenePm4 DrawPredicated(ScenePm4 pm4, bool predicated = true) => private Scene? Create() { var rig = CreateRig(); + if (rig is not null) rig.Host.NativeOcclusionEnabled = false; return rig is null ? null : new Scene { Rig = rig, Target = rig.ColorTarget(Size, Size) }; } diff --git a/tests/SharpEmu.Libs.Tests/Audio/AudioOut2QueueDispatchTests.cs b/tests/SharpEmu.Libs.Tests/Audio/AudioOut2QueueDispatchTests.cs index 4b297421d..0f04f03b3 100644 --- a/tests/SharpEmu.Libs.Tests/Audio/AudioOut2QueueDispatchTests.cs +++ b/tests/SharpEmu.Libs.Tests/Audio/AudioOut2QueueDispatchTests.cs @@ -99,7 +99,7 @@ internal AudioQueueGateway(CpuContext context, bool lowLatency) entries.SetValue(Activator.CreateInstance(entryType, [0x200000UL, dispatchNid, export, false, false, false, false, 0UL]), index); } type.GetField("_importEntries", Instance)!.SetValue(_backend, entries); - Assert.Equal(Environment.GetEnvironmentVariable("SHARPEMU_AUDIO_OUT2_QUEUE_LEAF") == "1", DirectExecutionBackend.IsTrivialLeafImport(nid)); + Assert.Equal(Environment.GetEnvironmentVariable("SHARPEMU_AUDIO_OUT2_QUEUE_LEAF") != "0", DirectExecutionBackend.IsTrivialLeafImport(nid)); Assert.False(DirectExecutionBackend.IsTrivialLeafImport("test-full-audio-queue-query")); Assert.False((bool)type.GetField("_disableTrivialLeafDispatch", BindingFlags.Static | BindingFlags.NonPublic)!.GetValue(null)!); _dispatch = type.GetMethod("DispatchImportCore", Instance)!.CreateDelegate>(_backend); diff --git a/tests/SharpEmu.ShaderCompiler.Metal.Tests/NggMeshRuntimeTests.cs b/tests/SharpEmu.ShaderCompiler.Metal.Tests/NggMeshRuntimeTests.cs index 53e00804f..c3d9463dc 100644 --- a/tests/SharpEmu.ShaderCompiler.Metal.Tests/NggMeshRuntimeTests.cs +++ b/tests/SharpEmu.ShaderCompiler.Metal.Tests/NggMeshRuntimeTests.cs @@ -90,9 +90,17 @@ public void ScalarConstantBuffersRefuseCoherentVectorOrDeviceWriteAccess(bool co } [Fact] - public void ScalarConstantBuffersRequireOptInAndRawDwordSemantics() + public void DefaultScalarConstantBuffersUseTheConstantPointerForEligibleMeshLoads() { - foreach (var options in new[] { new Gen5MslCodegenOptions(), + Assert.True(Gen5MslTranslator.TryCompileProgram(ScalarConstantRequest(4, 0), + out var shader, out var error, new Gen5MslCodegenOptions()), error); + Assert.Contains("constant uint* b0 =", shader.Source, StringComparison.Ordinal); + } + + [Fact] + public void ScalarConstantBuffersHonorDisableAndRequireRawDwordSemantics() + { + foreach (var options in new[] { new Gen5MslCodegenOptions { ScalarConstantBuffers = false }, new Gen5MslCodegenOptions(RawDwordAccess: false) { ScalarConstantBuffers = true } }) { Assert.True(Gen5MslTranslator.TryCompileProgram(ScalarConstantRequest(4, 0), From eabcd9f659fcb218dd00bf8f8112f2628b4c48e8 Mon Sep 17 00:00:00 2001 From: kuba Date: Tue, 27 Oct 2026 18:55:05 +0100 Subject: [PATCH 35/82] [GPU] Remove redundant mesh register copy allocations --- docs/performance/native-occlusion-runtime.md | 90 +++++++++++++++++++ docs/performance/performance-defaults.md | 13 ++- .../Registers/UserConfigRegisters.cs | 13 ++- .../Gpu/Rendering/RenderExecutor.Prefetch.cs | 5 +- .../Registers/RegisterWriteTableTests.cs | 75 ++++++++++++++++ .../Rendering/RenderExecutorPrefetchTests.cs | 38 ++++++++ .../Rendering/RenderExecutorTestSupport.cs | 3 + 7 files changed, 231 insertions(+), 6 deletions(-) diff --git a/docs/performance/native-occlusion-runtime.md b/docs/performance/native-occlusion-runtime.md index ac0911471..373ad4424 100644 --- a/docs/performance/native-occlusion-runtime.md +++ b/docs/performance/native-occlusion-runtime.md @@ -1020,3 +1020,93 @@ one trailing blank line from the new scalar-constant MSL template; that final formatting-only difference was made after the suite build. Main remained clean. The publication contains source, tests, benchmark tools and this report, without raw commercial captures, game binaries or local runtime bundles. + + +### v19 defaults diagnostic: cold introduction, no matched room result + +Clean source `fb717072ad11ada9a53e27ac34219fdc91f1f5bb`, self-contained +x64 CLI/native ARM64 GPU host, with 471 verified runtime payload hashes. +Run `20261027-183237-silent hill short message.zar-room-defaults-v19-1` +used a fresh clone of `runtime-data-world-unprofiled-1`, all nine newly promoted +switches unset, and the existing default portable-buffer/LDS/library-deferral +policies. Prewarming was disabled for measurement; resolution, scale, VSync and +GeForce NOW were unchanged. Initial host load average was 13.16. + +The title menu was visually verified at 350 seconds. New Game was selected at +367 seconds; a long warning/loading phase followed, with zero-frame intervals +and Metal pipeline/library compilation visible in the GPU-host native sample. +Short confirmation pulses during stalled frames did not immediately advance +the warning. After a later confirmation the introduction advanced, and the +opening cutscene was visually verified at 832 and 872 seconds. The 900-second +cap ended the run before the stationary shelf/lamp room. Thus no FPS comparison +against the earlier room windows is valid. Process exit was 0 after a controlled +four-second termination; this does not certify guest teardown or full-game +compatibility. Main-process peak footprint was 9,460 MiB and peak RSS 2,095 MiB. + +The main-process native sample had only ten snapshots and is insufficient for +precise CPU attribution. Separate per-process unique-memory captures at +514.53–515.33 seconds recorded main task footprint 5,050,704,280 bytes and +compressed bytes 3,900,620,800. The helper attributed 4,155,510,784 logical +swapped/compressed bytes to stack/guard objects. This is a diagnostic lead, +not proof that reducing stack reservations is safe or would improve FPS. +The guest shared objects appear in both process maps and must not be added +together. Native samples and VM-object walks were outside any accepted FPS +window. Raw logs, samples and captures remain private diagnostic artifacts. + + +A follow-up synthetic check rejects treating that stack-region number as direct +physical footprint. Eight idle managed threads with 64-MiB reservations used +only 0.406 MiB of stack-resident pages on ARM64 and 0.516 MiB under Rosetta x64; +zero-thread and 4-MiB controls were also captured. After a one-second native +`sample`, the x64 VM-object walk reported **523.211 MiB** of stack-resident pages, +while task footprint changed only **13.233 → 13.339 MiB** (RSS 27.590 → 27.613 +MiB). ARM64's stack count did not change. All six probe processes exited 0. +Thus native sampling can strongly alter this Rosetta VM-object accounting; +the stack-category count cannot establish equivalent process-owned physical +memory or explain the game's FPS. The 64-MiB reservation remains unchanged. +Private source/results: `stack-reservation-probe/Program.cs`, +`run-stack-reservation-probe.py`, `stack-reservation-probe-results.json`. + +### Avoid redundant user-config and ordered-append copies + +Indexed triangle-list mesh draws already have the topology the resolver needs. +They now reuse the read-only user-config snapshot instead of deep-copying it. +Indexed triangle strips still get a private topology-conversion copy; tests +check source topology and independent ordered-append storage. Other topologies +and nonindexed paths keep their existing behavior. + +`OrderedAppendRegisters.Copy` now constructs directly with its cloned counter +array. The former public-constructor/object-initializer sequence allocated an +empty eight-counter array and immediately replaced it with the clone. The new +path still owns every counter and preserves all register values. + +Nine alternating-order rounds, 100,000 copies per arm after 10,000 warmup copies, +with matching checksums and allocation measured on the executing thread: + +| Runtime / configuration | Reference ns/copy | New ns/copy | Reference B/copy | New B/copy | +| --- | ---: | ---: | ---: | ---: | +| ARM64, tiering disabled | 33.331 | 28.991 | 208 | 120 | +| Rosetta x64, tiering disabled | 58.851 | 51.310 | 208 (one round +0.00056) | 120 | +| Rosetta x64, default tiering, full-suite run | 61.755 | 46.973 | 208 | 120 | + +These are component medians: about 13% less time with tiering disabled and 88 +fewer allocated bytes per counter-bank copy. They are not a game-FPS result. +The complete Release Libs run passed **9,060 / 0 / 5 existing skips**. Both +architecture-specific benchmark executions passed separately, using the same +compiled test/Libs assemblies. No Metal/shader source changed in this follow-up; +the earlier default-configuration device suites retain their own attribution. + +Reproduce the opt-in benchmark after building Release Libs tests: + +```sh +SHARPEMU_ORDERED_APPEND_BENCH_OUTPUT=/tmp/ordered-append-copy.json \ +DOTNET_TieredCompilation=0 dotnet test \ + tests/SharpEmu.Libs.Tests/SharpEmu.Libs.Tests.csproj \ + -c Release --no-build --no-restore \ + --settings docs/performance/shader-serial.runsettings \ + --filter FullyQualifiedName~MeasureOrderedAppendCopiesWhenRequested +``` + +Use the matching ARM64 or x64 `dotnet` host. JSON includes the actual process +architecture, each round's checksum, time and allocation. Raw records remain in +`ordered-append-copy-{arm64,x64}-notier.json` and `ordered-append-copy-x64.json`. diff --git a/docs/performance/performance-defaults.md b/docs/performance/performance-defaults.md index 2ea0c873a..c138641d9 100644 --- a/docs/performance/performance-defaults.md +++ b/docs/performance/performance-defaults.md @@ -74,6 +74,15 @@ above passed. No numerical or pixel oracle was relaxed. REUSE passed diagnostic directory. The earlier ShaderCompiler core suite passed 2,192/0/1 before this defaults-only follow-up and was not rerun here. -This validation establishes tested configuration paths, not a fresh game-FPS -measurement of the new default combination. Runtime investigation toward +This validation establishes tested configuration paths, not a first-room FPS +measurement of the new default combination. A clean `fb717072a` x64 CLI/native +ARM64 GPU-host runtime passed both publishes and verification of all 471 payload +hashes. Its bounded 900-second Silent Hill diagnostic reached the opening +cutscene, but did not reach the stationary comparison room. Cold pipeline +compilation and long zero-frame intervals remain visible; this run does not +establish a gain or regression against the earlier approximately 14-FPS room. +It ended at the harness deadline with process exit 0. Peak main-process physical +footprint was 9,460 MiB (RSS 2,095 MiB); these are not combined physical-memory +figures for the two processes. Prewarming was disabled for this diagnostic, +while all nine promoted switches were unset. Runtime investigation toward 30 FPS continues separately. diff --git a/src/SharpEmu.Libs/Gpu/GpuCommands/Registers/UserConfigRegisters.cs b/src/SharpEmu.Libs/Gpu/GpuCommands/Registers/UserConfigRegisters.cs index f4e2d76c2..0dc55976a 100644 --- a/src/SharpEmu.Libs/Gpu/GpuCommands/Registers/UserConfigRegisters.cs +++ b/src/SharpEmu.Libs/Gpu/GpuCommands/Registers/UserConfigRegisters.cs @@ -40,11 +40,20 @@ public sealed class OrderedAppendRegisters public const int CounterCount = 8; public uint Control; - public OrderedAppendCounter[] Counters = new OrderedAppendCounter[CounterCount]; + public OrderedAppendCounter[] Counters; + + public OrderedAppendRegisters() => Counters = new OrderedAppendCounter[CounterCount]; + + private OrderedAppendRegisters(uint control, OrderedAppendCounter[] counters) + { + Control = control; + Counters = counters; + } public uint Index => Control & (CounterCount - 1u); - public OrderedAppendRegisters Copy() => new() { Control = Control, Counters = (OrderedAppendCounter[])Counters.Clone() }; + // The public constructor would allocate an empty array that the clone immediately replaces. + public OrderedAppendRegisters Copy() => new(Control, (OrderedAppendCounter[])Counters.Clone()); } // The user-config bank: primitive assembly and the ordered-append counters. diff --git a/src/SharpEmu.Libs/Gpu/Rendering/RenderExecutor.Prefetch.cs b/src/SharpEmu.Libs/Gpu/Rendering/RenderExecutor.Prefetch.cs index 43eee0380..ab0c55742 100644 --- a/src/SharpEmu.Libs/Gpu/Rendering/RenderExecutor.Prefetch.cs +++ b/src/SharpEmu.Libs/Gpu/Rendering/RenderExecutor.Prefetch.cs @@ -137,8 +137,9 @@ public GraphicsPrograms ResolveGraphicsPrograms(RegisterBanks banks, in Graphics if (UsesMeshExecution(banks)) { var userConfig = banks.UserConfig; - if (inputs.Indexed && userConfig.PrimitiveType is - (uint)GuestPrimitiveType.TriangleList or (uint)GuestPrimitiveType.TriangleStrip) + // Indexed strips are expanded to lists. Lists already have the right topology; + // the pipeline resolver only reads these banks, so they need no private copy. + if (inputs.Indexed && userConfig.PrimitiveType == (uint)GuestPrimitiveType.TriangleStrip) { userConfig = userConfig.Copy(); userConfig.PrimitiveType = (uint)GuestPrimitiveType.TriangleList; diff --git a/tests/SharpEmu.Libs.Tests/Gpu/GpuCommands/Registers/RegisterWriteTableTests.cs b/tests/SharpEmu.Libs.Tests/Gpu/GpuCommands/Registers/RegisterWriteTableTests.cs index f6c3b8a23..d87c84339 100644 --- a/tests/SharpEmu.Libs.Tests/Gpu/GpuCommands/Registers/RegisterWriteTableTests.cs +++ b/tests/SharpEmu.Libs.Tests/Gpu/GpuCommands/Registers/RegisterWriteTableTests.cs @@ -1,6 +1,10 @@ // Copyright (C) 2026 SharpEmu Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later +using System.Diagnostics; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Text.Json; using SharpEmu.Libs.Gpu.GpuCommands; using SharpEmu.Libs.Gpu.GpuCommands.Registers; using SharpEmu.Libs.Gpu.Images; @@ -274,6 +278,77 @@ public void OrderedAppend_ControlSelectsTheCounterTheNextWritesLandIn() Assert.Equal(0u, banks.UserConfig.OrderedAppend.Counters[0].Counter); } + [Fact] + public void UserConfigCopyOwnsEveryOrderedAppendCounter() + { + var source = new UserConfigRegisters { PrimitiveType = 6, IndexOffset = 37, ObjectId = 41 }; + source.OrderedAppend.Control = 5; + for (var i = 0; i < OrderedAppendRegisters.CounterCount; i++) + source.OrderedAppend.Counters[i] = new() { Counter = (uint)i + 11, Address = (uint)i + 101 }; + var copy = source.Copy(); + Assert.NotSame(source.OrderedAppend, copy.OrderedAppend); + Assert.NotSame(source.OrderedAppend.Counters, copy.OrderedAppend.Counters); + Assert.Equal(source.OrderedAppend.Counters, copy.OrderedAppend.Counters); + Assert.Equal((6u, 37u, 41u, 5u), + (copy.PrimitiveType, copy.IndexOffset, copy.ObjectId, copy.OrderedAppend.Control)); + + source.OrderedAppend.Control = 3; + copy.PrimitiveType = 4; + for (var i = 0; i < OrderedAppendRegisters.CounterCount; i++) + { + source.OrderedAppend.Counters[i].Counter = 0; + copy.OrderedAppend.Counters[i].Address = 0; + Assert.Equal((uint)i + 11, copy.OrderedAppend.Counters[i].Counter); + Assert.Equal((uint)i + 101, source.OrderedAppend.Counters[i].Address); + } + Assert.Equal(5u, copy.OrderedAppend.Control); + Assert.Equal(6u, source.PrimitiveType); + } + + [Fact] + public void MeasureOrderedAppendCopiesWhenRequested() + { + var output = Environment.GetEnvironmentVariable("SHARPEMU_ORDERED_APPEND_BENCH_OUTPUT"); + if (string.IsNullOrEmpty(output)) return; + const int count = 100000; + var source = new OrderedAppendRegisters { Control = 5 }; + source.Counters[5] = new() { Counter = 17, Address = 23 }; + var rows = new List(); + foreach (var baseline in new[] { true, false }) ExerciseOrderedAppendCopies(source, baseline, 10000); + for (var trial = 0; trial < 9; trial++) + foreach (var baseline in trial % 2 == 0 ? new[] { true, false } : new[] { false, true }) + { + var bytes = GC.GetAllocatedBytesForCurrentThread(); + var start = Stopwatch.GetTimestamp(); + var checksum = ExerciseOrderedAppendCopies(source, baseline, count); + var duration = Stopwatch.GetElapsedTime(start); + var allocated = GC.GetAllocatedBytesForCurrentThread() - bytes; + Assert.Equal(45UL * count, checksum); + rows.Add(new { baseline, trial, checksum, nsPerCopy = duration.TotalNanoseconds / count, + bytesPerCopy = (double)allocated / count }); + } + File.WriteAllText(output, JsonSerializer.Serialize(new { + architecture = RuntimeInformation.ProcessArchitecture.ToString(), rows }, + new JsonSerializerOptions { WriteIndented = true })); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong ExerciseOrderedAppendCopies(OrderedAppendRegisters source, bool baseline, int count) + { + ulong checksum = 0; + for (var i = 0; i < count; i++) + { + // Exact former copy body: its public constructor allocates an array before replacement. + var copy = baseline + ? new OrderedAppendRegisters { Control = source.Control, + Counters = (OrderedAppendCounter[])source.Counters.Clone() } + : source.Copy(); + checksum += copy.Control + copy.Counters[5].Counter + copy.Counters[5].Address; + GC.KeepAlive(copy); + } + return checksum; + } + [Fact] public void Viewports_DecodeScaleScissorAndDepthRange() { diff --git a/tests/SharpEmu.Libs.Tests/Gpu/Rendering/RenderExecutorPrefetchTests.cs b/tests/SharpEmu.Libs.Tests/Gpu/Rendering/RenderExecutorPrefetchTests.cs index 9fed2055c..a34367353 100644 --- a/tests/SharpEmu.Libs.Tests/Gpu/Rendering/RenderExecutorPrefetchTests.cs +++ b/tests/SharpEmu.Libs.Tests/Gpu/Rendering/RenderExecutorPrefetchTests.cs @@ -55,6 +55,44 @@ public void MeshPrefetchPreservesDrawModeAndOwnership(bool indexed, bool request Assert.Equal(PrimitiveTriangleStrip, banks.UserConfig.PrimitiveType); } + [Theory] + [InlineData(4u, false)] + [InlineData(4u, true)] + [InlineData(6u, false)] + [InlineData(6u, true)] + [InlineData(2u, false)] + [InlineData(2u, true)] + public void MeshResolutionCopiesUserConfigOnlyForIndexedStripExpansion(uint primitive, bool indexed) + { + _pipelines.MeshShadersSupported = true; + var banks = Banks(primitive); + banks.Context.ShaderStages = 0x20; + banks.UserConfig.OrderedAppend.Control = 5; + banks.UserConfig.OrderedAppend.Counters[5] = new() { Counter = 17, Address = 23 }; + var snapshot = banks.Snapshot(); + var inputs = RenderExecutor.ProgramInputsOf(snapshot, _host.FormatSupport, _host.Fatal, indexed); + + _executor.ResolveGraphicsPrograms(snapshot, in inputs); + + var resolved = Assert.IsType(_pipelines.MeshUserConfig); + if (indexed && primitive == PrimitiveTriangleStrip) + { + Assert.NotSame(snapshot.UserConfig, resolved); + Assert.Equal(4u, resolved.PrimitiveType); + Assert.Equal(5u, resolved.OrderedAppend.Control); + Assert.Equal(17u, resolved.OrderedAppend.Counters[5].Counter); + Assert.Equal(23u, resolved.OrderedAppend.Counters[5].Address); + resolved.OrderedAppend.Counters[5].Counter = 99; + Assert.Equal(17u, snapshot.UserConfig.OrderedAppend.Counters[5].Counter); + } + else + { + Assert.Same(snapshot.UserConfig, resolved); + } + Assert.Equal(primitive, snapshot.UserConfig.PrimitiveType); + Assert.Equal(primitive, banks.UserConfig.PrimitiveType); + } + // Resolves ahead with the executor, as a worker would, and hands the result back once. private sealed class AheadPrefetch(RenderExecutor executor, IRenderHost host) : IProgramPrefetch { diff --git a/tests/SharpEmu.Libs.Tests/Gpu/Rendering/RenderExecutorTestSupport.cs b/tests/SharpEmu.Libs.Tests/Gpu/Rendering/RenderExecutorTestSupport.cs index 4a5fcaf36..a79351b3a 100644 --- a/tests/SharpEmu.Libs.Tests/Gpu/Rendering/RenderExecutorTestSupport.cs +++ b/tests/SharpEmu.Libs.Tests/Gpu/Rendering/RenderExecutorTestSupport.cs @@ -576,6 +576,8 @@ internal sealed class FakePipelineProvider : IShaderPipelineProvider public uint MeshPrimitiveType { get; private set; } + public UserConfigRegisters? MeshUserConfig { get; private set; } + public bool MeshDepthBound { get; private set; } public GraphicsPrograms GetMeshGraphicsPrograms( @@ -590,6 +592,7 @@ public GraphicsPrograms GetMeshGraphicsPrograms( { Calls.Add($"get_mesh_programs pixelActive={pixelActive}"); MeshPrimitiveType = userConfig.PrimitiveType; + MeshUserConfig = userConfig; MeshDepthBound = depthBound; return Graphics; } From 4281e488c9b647912a55ad30aafaf7513ff3aa2e Mon Sep 17 00:00:00 2001 From: kuba Date: Tue, 27 Oct 2026 19:18:31 +0100 Subject: [PATCH 36/82] [GPU] Record default-settings room controls and CPU bottlenecks --- docs/performance/native-occlusion-runtime.md | 91 ++++++++++++++++++++ docs/performance/performance-defaults.md | 23 ++++- 2 files changed, 112 insertions(+), 2 deletions(-) diff --git a/docs/performance/native-occlusion-runtime.md b/docs/performance/native-occlusion-runtime.md index 373ad4424..8f6dc3ba9 100644 --- a/docs/performance/native-occlusion-runtime.md +++ b/docs/performance/native-occlusion-runtime.md @@ -1110,3 +1110,94 @@ DOTNET_TieredCompilation=0 dotnet test \ Use the matching ARM64 or x64 `dotnet` host. JSON includes the actual process architecture, each round's checksum, time and allocation. Raw records remain in `ordered-append-copy-{arm64,x64}-notier.json` and `ordered-append-copy-x64.json`. + + +### v20 warmed defaults: stationary room and CPU attribution + +Clean source `eabcd9f659fcb218dd00bf8f8112f2628b4c48e8` passed both +self-contained publishes (x64 CLI, native ARM64 GPU host) and verification of +471 runtime payload hashes. Run +`20261027-190045-silent hill short message.zar-room-warm-defaults-v20-1` +used a fresh clone of v19's warmed user data. All nine promoted switches were +unset; background program resolution retained its existing default on, and +command-stream threading retained its default off. Portable buffers, LDS +threshold and deferred-library policy stayed at defaults. Detailed render-phase +and GPU-pass profiling were on; prewarming was off. Resolution stayed +1920×1080, scale 1, VSync on, and GeForce NOW was unchanged. + +The shelf/lamp room was visually verified after closing the phone; no movement +was sent during the accepted 375–410-second window. Seventeen guest-FPS reports +had median **11.95**, range **7.99–14.96**. This does not isolate the copy fix, +the default promotions, profiling overhead or host-load differences against +the earlier approximately 14-FPS run. **30 FPS remains unachieved.** + +Sixteen complete render-profile intervals covered 379 reported frames in +32 seconds. Leading GPU-worker self wall times per frame were: + +| Phase | ms/frame | +| --- | ---: | +| Program-prefetch wait | 12.311 | +| Command register writes | 7.338 | +| Command-stream interpretation | 7.332 | +| Draw-resource preparation | 7.273 | +| Label-copy encoding | 4.741 | +| Graphics-state application | 3.737 | + +These are instrumented wall times, including scheduling and wait costs. They +are not hardware CPU utilization. The window contained 7,123,581 register-write +phase entries, so per-scope instrumentation also warrants an unprofiled control. + +GPU-pass records contained 55,906 passes across 408 frame groups. Excluding +first and last groups, median summed encoder duration was **19.251 ms/frame** +(mean 19.508). Encoder durations can overlap and are not exclusive frame wall +time; they must not be added to CPU phase durations. The largest recurring +compute groups averaged 1.566 and 1.546 ms, and the leading fullscreen render +encoder averaged 1.115 ms. This room does not show the original large +shadow/volume encoder as its dominant single cost. + +A separate ten-second thread-CPU capture at 469.759–479.765 seconds, outside +the accepted FPS window, recorded **8.943 CPU-seconds** on the native GPU +worker and **2.897 CPU-seconds** on its program resolver. The guest audio event +thread used 9.686 CPU-seconds and RHI thread 6.067. Combined with the phase +profile, this supports investigating CPU draw processing and resolver handoffs; +it does not prove a particular optimization's gain. No native stack sampling +was used during this run. + +The run stopped at its own marker after 711 seconds, followed by controlled +four-second termination with process and wrapper exit 0. Peak main-process +footprint was 9,807 MiB and RSS 3,314 MiB; these are not combined physical usage +for both emulator processes. No full-game or teardown-correctness claim follows. +Private evidence includes `room-stationary-fps.json`, `render-phases-375-410.json`, +`pass-analysis-after-375s-before-410s.json` and `thread-cpu-deltas-0469.json`. + + +The same immutable v20 payload was then run with render-phase and GPU-pass +profiling disabled, using a fresh clone of the preceding run's warmed user data: +`20261027-191250-silent hill short message.zar-room-unprofiled-defaults-v20-2`. +The repeated title confirmations stopped at 132 seconds; cutscene skip finished +at 139 seconds and phone-close input at 162 seconds. The same stationary +shelf/lamp view was verified at 176 and 246 seconds, without movement. + +The preselected 205–240-second window contained **17 reports, median 15.48 FPS, +range 13.96–16.97**. The render/pass profiles were disabled; ordinary presentation +counters and harness screenshots remained enabled. All nine promoted switches +remained unset and all rendering/application constraints were unchanged. This +is a useful normal-settings control, but sequential cache warmth and changing +host load prevent attributing the full difference from 11.95 FPS to profiling. +It is also not an isolated before/after test of the copy fix or default changes. + +A separate thread-CPU capture at 259.940–269.945 seconds recorded 6.533 CPU-seconds +on the GPU worker, 2.523 on program resolution, 6.051 on guest RHI and 9.122 on +the audio event thread. The worker remains a substantial CPU cost even without +detailed phase instrumentation, while guest-side command production also needs +investigation. The result remains below 30 FPS. The profiler-off capture and +`thread-cpu-deltas-0259.json` are retained alongside the raw run records. + +The control stopped at its own marker after 274 seconds; process and wrapper +exited 0 after four seconds of controlled termination. Peak main-process +footprint was 8,901 MiB and RSS 4,395 MiB. Both emulator processes were gone +afterward; this is bounded-run cleanup, not full-game compatibility evidence. + +The only CPU-trap lines in the profiler-off capture occur at 275.367 seconds, +after the 274-second stop marker. They remain recorded as host-stop behavior; +process exit 0 does not establish a clean guest teardown. diff --git a/docs/performance/performance-defaults.md b/docs/performance/performance-defaults.md index c138641d9..f306b08e4 100644 --- a/docs/performance/performance-defaults.md +++ b/docs/performance/performance-defaults.md @@ -35,9 +35,14 @@ The three promoted shader properties also default on in explicitly constructed | `SHARPEMU_MSL_MESH_LDS_FORWARDING` | Same-binary shadow time 34.238 → 36.422 ms; title median 7.97 → 7.47 FPS. | | `SHARPEMU_MSL_MESH_HOT_LDS` | Same-binary total encoder time 108.483 → 109.451 ms; no benefit established. | | `SHARPEMU_MSL_MESH_EXIT_JOIN`, `SHARPEMU_MSL_UNOBSERVED_ALU` | Correctness coverage exists, but isolated performance evidence is inconclusive and their combined run had an unresolved mapping failure. | -| Command thread / parallel resolution / command-thread QoS | Earlier overlap runs were slower than serial resolution. Queued writes and recycling apply when overlap is requested; this change does not enable overlap globally. | +| Command thread / command-thread QoS | Earlier command-thread overlap runs were slower than serial command-stream interpretation. Queued writes and recycling apply when command threading is requested; this change does not enable it globally. | | Concurrent tiling and floating-point contraction | Representative writeback regressions or numerical differences remain; the original audit did not accept them. | +Background program resolution is a separate existing default: it remains enabled +unless `SHARPEMU_METAL_V2_PARALLEL_RESOLVE=0`. It can run while command-stream +interpretation stays on the GPU worker. The earlier table incorrectly grouped +it with opt-in command threading; no runtime policy is changed by this correction. + Diagnostics, detailed tracing, scripted-only input and visibility auditing stay off. Portable-buffer policy, device LDS placement threshold and asynchronous library compilation policy are unchanged. These defaults do not establish @@ -52,7 +57,7 @@ Fresh serial Release runs with all `SHARPEMU_*` overrides removed: | Suite | Passed | Failed | Existing skips | | --- | ---: | ---: | ---: | | ShaderCompiler.Metal, required device | 3,589 | 0 | 0 | -| Gpu.Metal, required device, default synchronous mode | 1,635 | 0 | 7 | +| Gpu.Metal, required device, serial command stream | 1,635 | 0 | 7 | | Libs, x64 | 9,052 | 0 | 5 | | Total | 14,276 | 0 | 12 | @@ -86,3 +91,17 @@ footprint was 9,460 MiB (RSS 2,095 MiB); these are not combined physical-memory figures for the two processes. Prewarming was disabled for this diagnostic, while all nine promoted switches were unset. Runtime investigation toward 30 FPS continues separately. + + +A subsequent clean `eabcd9f65` build reached the stationary shelf/lamp room with +all nine switches unset. Its detailed-profile window (375–410 seconds) reported +**11.95 FPS median, 7.99–14.96 range, 17 reports**. This is not a matched isolated +comparison with the earlier 14-FPS build. See the v20 section of the runtime +report for CPU/GPU attribution and measurement limits. Background program +resolution was enabled, while command threading remained disabled. + +A same-payload control with detailed render/pass profiling disabled then measured +**15.48 FPS median (13.96–16.97, 17 reports)** in the same stationary room. +Sequential cache warmth and host-load changes prevent assigning the entire +difference to profiling. This confirms operation of the default combination in +that scene; it still does not establish 30 FPS or isolate each setting's gain. From b4cc8292e6f0590794c41090836b331e78154f28 Mon Sep 17 00:00:00 2001 From: kuba Date: Tue, 27 Oct 2026 19:44:28 +0100 Subject: [PATCH 37/82] [GPU] Reuse completion values once submitted work is finished --- docs/performance/native-occlusion-runtime.md | 88 +++++++++++++ docs/performance/performance-defaults.md | 10 ++ .../SharpEmu.Gpu.Metal/V2/MetalRenderHost.cs | 24 +++- ...lEndOfPipeOrderTests.CompletionSnapshot.cs | 121 ++++++++++++++++++ .../MetalEndOfPipeOrderTests.cs | 2 +- 5 files changed, 239 insertions(+), 6 deletions(-) create mode 100644 tests/SharpEmu.Gpu.Metal.Tests/MetalEndOfPipeOrderTests.CompletionSnapshot.cs diff --git a/docs/performance/native-occlusion-runtime.md b/docs/performance/native-occlusion-runtime.md index 8f6dc3ba9..81bd83dd9 100644 --- a/docs/performance/native-occlusion-runtime.md +++ b/docs/performance/native-occlusion-runtime.md @@ -1201,3 +1201,91 @@ afterward; this is bounded-run cleanup, not full-game compatibility evidence. The only CPU-trap lines in the profiler-off capture occur at 275.367 seconds, after the 274-second stop marker. They remain recorded as host-stop behavior; process exit 0 does not establish a clean guest teardown. + + +### v20 serial program-resolution control + +The same `eabcd9f65` payload and the same warmed seed directory as the unprofiled +default run were used for +`20261027-192111-silent hill short message.zar-room-unprofiled-serial-resolve-v20-3`. +The captured environment differed only in the clone's destination path and +`SHARPEMU_METAL_V2_PARALLEL_RESOLVE=0`. Rendering, prewarm, profiling and other +application settings were unchanged. The stationary shelf/lamp view was verified +at 196 and 246 seconds. + +The same 205–240-second window had **17 reports, median 10.99 FPS, range +6.49–14.47**, versus 15.48 (13.96–16.97) with background resolution enabled. +Sequential host conditions still limit causal precision, but this control gives +no reason to disable the existing default. Thirty FPS remains unachieved. +A separate 271.936–281.946-second CPU capture recorded 7.253 CPU-seconds on the +GPU worker, 5.360 on guest RHI and 8.565 on the audio event thread. There was no +background program-resolver thread in this arm. + +Ten-second managed stack traces followed, outside the FPS/CPU windows, for the +GPU host and guest process. They completed and converted successfully. Their +leaf samples were dominated by `UNMANAGED_CODE_TIME`, with managed anchors at +monitor entry, GC polling, completion-event reads and resource rebasing. These +are sampled wall-time anchors, not exclusive CPU costs or proof of contention. +Earlier v14 event traces already showed why this distinction matters: actual +worker contention was 13.96 ms while sampled monitor-entry attribution was +1,680 ms. No lock or GC-policy change is justified by these stack totals. + +This control stopped at its own marker after 390 seconds; the CPU-trap lines +appeared at 391.201 seconds, after that stop. Process/wrapper exit 0 followed +four-second controlled termination. Peak main-process footprint was 8,877 MiB +and RSS 3,818 MiB. Both emulator processes exited; no full-game or clean guest +teardown claim follows. + +### Completion-event reads after all submitted work finishes + +The host already keeps the greatest completion-event value it has observed. +It now also publishes an upper bound on submitted serials immediately before +each native commit. When the observed completion reaches that bound, the lower +and upper bounds agree: an additional native event read cannot reveal more +completed work. A recorded but unsubmitted command buffer cannot advance the +event. If submitted work remains outstanding, the getter still reads the live +event and updates the monotonic hint. Concurrent readers can observe a valid +earlier completion snapshot, just as a native read racing submission can. +`SHARPEMU_METAL_V2_SERIAL_HINT=0` retains the native-query reference path. + +A device regression covers three successive externally gated submissions, +checks that recording and committing do not publish completion early, releases +each gate, then verifies the completed serial and GPU-written word in both +reading modes. The benchmark measures the actual getter on the same host after +its work finishes: 10,000 warmup pairs, nine alternating-order rounds of +1,000,000 reads per arm, and equal serial checksums. A second condition holds +a real GPU submission behind an external event, exercising the live-query +fallback while the completion hint is stale. + +| Completion condition | Native query, median ns/read | Guarded getter, median ns/read | B/read, both | +| --- | ---: | ---: | ---: | +| ARM64, completed | 3.020 | 1.319 | 0 | +| ARM64, pending | 3.241 | 2.981 | 0 | +| Rosetta x64, completed | 10.002 | 2.297 | 0 | +| Rosetta x64, pending | 5.864 | 6.209 | 0 | + +The x64 pending-work arm costs about 0.346 ns/read (5.9%) more; it does not take +the shortcut. The completed-work arm saves 7.705 ns/read in that harness. These +component medians do not establish the runtime hit ratio or an FPS gain. The +preliminary idle-only harness used 200,000 reads and a different loop shape; +its raw results are preserved separately rather than mixed with this table. The native getter's +measured nanosecond cost also cautions against interpreting its sampled wall +time as a large exclusive CPU cost. Fresh Release compilation and the focused +ARM64 device run passed **52/0/0**. The full x64 Metal suite using those rebuilt +assemblies passed **1,637/0/7 existing skips**. Shader/compiler/Libs production +code did not change in this follow-up, so their earlier suites were not repeated. + +Reproduce the optional benchmark with +`SHARPEMU_COMPLETION_SNAPSHOT_BENCH_OUTPUT=/absolute/path/results.json` and +`SHARPEMU_TEST_REQUIRE_DEVICE=1`, filtering Release Metal tests to +`MeasureCompletedSnapshotWhenRequested`. JSON records architecture, iterations, +every timing/allocation round and checksum. Command/TRX artifacts are retained +in `completion-snapshot-validation.json` and `completion-snapshot-results`; +timing files are `completion-snapshot-focused-arm64.json` and +`completion-snapshot-full-x64.json`. The runtime comparisons above predate this +getter optimization; it has not yet been measured in a game. + +Additional focused controls passed **52/0/0 each** with serial hints disabled +and with polling completions instead of native completion callbacks. These are +configuration executions of existing tests, not 104 distinct new tests. REUSE +passed **2,865/2,865** files, and whitespace checks passed. diff --git a/docs/performance/performance-defaults.md b/docs/performance/performance-defaults.md index f306b08e4..3609c6ab3 100644 --- a/docs/performance/performance-defaults.md +++ b/docs/performance/performance-defaults.md @@ -105,3 +105,13 @@ A same-payload control with detailed render/pass profiling disabled then measure Sequential cache warmth and host-load changes prevent assigning the entire difference to profiling. This confirms operation of the default combination in that scene; it still does not establish 30 FPS or isolate each setting's gain. + + +The subsequent same-seed control with background program resolution disabled +measured 10.99 FPS (6.49–14.47, 17 reports); its existing enabled default remains. +A completion-read follow-up uses the existing serial-hint switch to avoid native +queries once all submitted work is known complete. The guarded getter measured +3.020 → 1.319 ns on ARM64 and 10.002 → 2.297 ns on x64 for completed work. +Pending work retains live queries: ARM64 measured 3.241 → 2.981 ns, x64 +5.864 → 6.209 ns. Both paths allocate zero bytes; the runtime report documents its +device ordering tests and the absence of an isolated game-FPS result. diff --git a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.cs b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.cs index 3428aaeb7..acd7fe899 100644 --- a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.cs +++ b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.cs @@ -92,6 +92,9 @@ private string FormatGraphicsBindingCounters() => private nint _commandBuffer; private nint _previousCommandBuffer; private ulong _previousSerial; + // Published before commit: an upper bound on any value the completion event + // can reach. A recorded but unsubmitted command buffer cannot advance it. + private ulong _submittedSerial; private long _failedSubmissions; private long _reportedFailedSubmissions; private double _gpuBusySeconds; @@ -221,14 +224,24 @@ public ulong SubmissionSerial } } - public ulong CompletedSerial + public ulong CompletedSerial => ReadCompletedSerial(SerialHintEnabled); + + internal ulong ReadCompletedSerial(bool useCompletedSnapshot) { - get + if (useCompletedSnapshot && _completions is { } completions) { - var completed = MetalV2Native.SendValue64(_sharedEvent, SelSignaledValue); - _completions?.NoteSignaled(completed); - return completed; + var submitted = Volatile.Read(ref _submittedSerial); + var known = completions.CompletedHint; + // The hint is a lower bound on the event. Once it reaches the upper + // bound as well, another native read cannot discover more completed + // work. Pending submissions still query the live event as before. + if (known >= submitted) + return known; } + + var completed = MetalV2Native.SendValue64(_sharedEvent, SelSignaledValue); + _completions?.NoteSignaled(completed); + return completed; } public RenderHostLimits Limits { get; } = @@ -628,6 +641,7 @@ private ulong FlushCommandBuffer(PassEndCause cause, MetalFlushCause why) NoteCommandBufferForTelemetry(); MetalMemoryTelemetry.Commits.Add(); + Volatile.Write(ref _submittedSerial, serial); MetalNative.SendVoid(_commandBuffer, SelCommit); if (DsTrace.Gpu && DsTrace.Armed) { diff --git a/tests/SharpEmu.Gpu.Metal.Tests/MetalEndOfPipeOrderTests.CompletionSnapshot.cs b/tests/SharpEmu.Gpu.Metal.Tests/MetalEndOfPipeOrderTests.CompletionSnapshot.cs new file mode 100644 index 000000000..636edb51f --- /dev/null +++ b/tests/SharpEmu.Gpu.Metal.Tests/MetalEndOfPipeOrderTests.CompletionSnapshot.cs @@ -0,0 +1,121 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using System.Diagnostics; +using System.Text.Json; +using SharpEmu.Gpu.Metal.V2; +using Xunit; + +namespace SharpEmu.Gpu.Metal.Tests; + +public sealed unsafe partial class MetalEndOfPipeOrderTests +{ + [Fact] + public void CompletedSnapshotDoesNotCompleteRecordedOrBlockedSubmissions() + { + if (!RequireDevice()) return; + using var rig = new Rig(Device.Value); + var host = rig.Host; + var gate = MetalNative.Send(Device.Value, MetalNative.Selector("newSharedEvent")); + Assert.NotEqual(0, gate); + try + { + Assert.Equal(0UL, host.ReadCompletedSerial(true)); + var completed = 0UL; + for (var round = 1u; round <= 3; round++) + { + var transfer = ((IMetalImageCacheOwner)host).BeginTransfer(MetalTransferCause.GuestFill); + MetalV2Native.SendEventValue(transfer.CommandBuffer, + MetalNative.Selector("encodeWaitForEvent:value:"), gate, round); + host.FillBufferInOrder(WindowedLabel, sizeof(uint), round); + // Opening and recording the next buffer must not report it completed. + Assert.True(host.SubmissionSerial > completed); + Assert.Equal(completed, host.ReadCompletedSerial(true)); + var serial = host.FlushCommandBuffer(); + Assert.True(serial > completed); + // A stale hint must fall back to the live event while submitted work + // remains blocked. Neither reading mode may publish its serial early. + for (var read = 0; read < 20; read++) + { + Assert.Equal(completed, host.ReadCompletedSerial(true)); + Assert.Equal(completed, host.ReadCompletedSerial(false)); + } + MetalNative.SendVoid(gate, MetalNative.Selector("setSignaledValue:"), (nint)round); + WaitForGpu(host, serial); + Assert.Equal(serial, host.ReadCompletedSerial(true)); + Assert.Equal(serial, host.ReadCompletedSerial(false)); + Assert.Equal(round, rig.Backing.Read32(WindowedLabel)); + completed = serial; + } + WaitForCompletions(host); + } + finally + { + // Always unblock the queue before the rig drains it, including assertions. + MetalNative.SendVoid(gate, MetalNative.Selector("setSignaledValue:"), (nint)3); + MetalNative.SendVoid(gate, MetalNative.Selector("release")); + } + } + + [Fact] + public void MeasureCompletedSnapshotWhenRequested() + { + var path = Environment.GetEnvironmentVariable("SHARPEMU_COMPLETION_SNAPSHOT_BENCH_OUTPUT"); + if (path is null || !RequireDevice()) return; + using var rig = new Rig(Device.Value); + var host = rig.Host; + host.FillBufferInOrder(WindowedLabel, sizeof(uint), 0x1234); + var serial = host.FlushCommandBuffer(); + WaitForGpu(host, serial); + WaitForCompletions(host); + const int iterations = 1_000_000; + var rounds = new List(); + void Measure(string condition) + { + for (var warmup = 0; warmup < 10_000; warmup++) + { + _ = host.ReadCompletedSerial(false); + _ = host.ReadCompletedSerial(true); + } + for (var round = 0; round < 9; round++) + { + foreach (var snapshot in round % 2 == 0 ? new[] { false, true } : new[] { true, false }) + { + ulong checksum = 0; + var allocation = GC.GetAllocatedBytesForCurrentThread(); + var started = Stopwatch.GetTimestamp(); + for (var index = 0; index < iterations; index++) + checksum += host.ReadCompletedSerial(snapshot); + var elapsed = Stopwatch.GetTimestamp() - started; + allocation = GC.GetAllocatedBytesForCurrentThread() - allocation; + Assert.Equal(serial * iterations, checksum); + rounds.Add(new { condition, round, snapshot, checksum, + nanosecondsPerRead = elapsed * 1e9 / Stopwatch.Frequency / iterations, + bytesPerRead = (double)allocation / iterations }); + } + } + } + Measure("completed"); + var gate = MetalNative.Send(Device.Value, MetalNative.Selector("newSharedEvent")); + Assert.NotEqual(0, gate); + try + { + var transfer = ((IMetalImageCacheOwner)host).BeginTransfer(MetalTransferCause.GuestFill); + MetalV2Native.SendEventValue(transfer.CommandBuffer, + MetalNative.Selector("encodeWaitForEvent:value:"), gate, 1); + Assert.True(host.FlushCommandBuffer() > serial); + // Exercise the guard's cost when its shortcut is unavailable, too. + Measure("pending"); + } + finally + { + MetalNative.SendVoid(gate, MetalNative.Selector("setSignaledValue:"), (nint)1); + MetalNative.SendVoid(gate, MetalNative.Selector("release")); + } + WaitForGpu(host, host.LastWorkSerial); + WaitForCompletions(host); + File.WriteAllText(path, JsonSerializer.Serialize(new { + architecture = System.Runtime.InteropServices.RuntimeInformation.ProcessArchitecture.ToString(), + iterations, serial, rounds }, new JsonSerializerOptions { WriteIndented = true })); + } +} diff --git a/tests/SharpEmu.Gpu.Metal.Tests/MetalEndOfPipeOrderTests.cs b/tests/SharpEmu.Gpu.Metal.Tests/MetalEndOfPipeOrderTests.cs index dc1db1945..92873725f 100644 --- a/tests/SharpEmu.Gpu.Metal.Tests/MetalEndOfPipeOrderTests.cs +++ b/tests/SharpEmu.Gpu.Metal.Tests/MetalEndOfPipeOrderTests.cs @@ -26,7 +26,7 @@ namespace SharpEmu.Gpu.Metal.Tests; /// gap between the GPU finishing and the host publishing is as wide as it is in a slow /// frame. Device work is behind the exclusive device gate. /// -public sealed unsafe class MetalEndOfPipeOrderTests(ITestOutputHelper output) +public sealed unsafe partial class MetalEndOfPipeOrderTests(ITestOutputHelper output) { private const ulong GuestBase = 0x2_0000_0000; private const ulong WindowedLabel = GuestBase + 0x1000; From 37358ee0b8aee94537a3c5c8df92f052ea31d460 Mon Sep 17 00:00:00 2001 From: kuba Date: Tue, 27 Oct 2026 20:12:27 +0100 Subject: [PATCH 38/82] [Tests] Isolate mapped-copy fixture addresses and record runtime controls --- docs/performance/native-occlusion-runtime.md | 104 +++++++++++++++++- docs/performance/performance-defaults.md | 10 ++ .../MappedCopyDerivedProtectionTests.cs | 11 +- 3 files changed, 121 insertions(+), 4 deletions(-) diff --git a/docs/performance/native-occlusion-runtime.md b/docs/performance/native-occlusion-runtime.md index 81bd83dd9..4d9151081 100644 --- a/docs/performance/native-occlusion-runtime.md +++ b/docs/performance/native-occlusion-runtime.md @@ -1283,9 +1283,111 @@ every timing/allocation round and checksum. Command/TRX artifacts are retained in `completion-snapshot-validation.json` and `completion-snapshot-results`; timing files are `completion-snapshot-focused-arm64.json` and `completion-snapshot-full-x64.json`. The runtime comparisons above predate this -getter optimization; it has not yet been measured in a game. +getter optimization; the v21 captures below exercise it without establishing an +isolated FPS effect. Additional focused controls passed **52/0/0 each** with serial hints disabled and with polling completions instead of native completion callbacks. These are configuration executions of existing tests, not 104 distinct new tests. REUSE passed **2,865/2,865** files, and whitespace checks passed. + +### v21 runtime and GC-budget control + +Both Release publishes from clean `b4cc8292e6f0590794c41090836b331e78154f28` +completed successfully; all 471 immutable payload hashes were verified before +launch. These sequential captures used fresh clones of the same warmed seed as +the v20 controls, all nine promoted switches unset, background resolution on, +command threading off, 1920×1080, scale 1 and VSync on. Detailed render/pass and +memory profiling, prewarm and automatic stack sampling were disabled. GeForce +NOW and its settings were left unchanged. + +The private diagnostic launcher omitted `DOTNET_GCgen0size`, whereas the existing +app-bundle `LSEnvironment` and standard `scripts/macos/run-game.sh` already set it +to `0x4000000` (64 MiB). The first arm retained the earlier diagnostic environment. +The second explicitly supplied the existing application setting; process readback +confirmed inheritance by both the guest process and ARM64 GPU host. This is a +launcher-parity control, not a newly enabled production optimization. + +| Same v21 payload and cache seed | Stationary window | Reports | Median FPS | Range | +| --- | --- | ---: | ---: | ---: | +| Diagnostic launcher, GC budget unset | 250–285 s | 17 | 8.97 | 7.48–10.99 | +| Existing application GC budget, 64 MiB | 250–285 s | 17 | 7.97 | 5.98–9.50 | + +Both windows were selected before their FPS values were read and after closing +the phone; the shelf/lamp room was visually checked. Arrival was later in the +first arm, requiring a later window than the earlier v20 measurements. These +results establish neither a completion-read FPS gain nor a GC-budget FPS gain. +They also do not isolate a regression from v20's earlier 15.48 FPS result. +Host memory pressure was present in both that earlier run and these captures. +Seven system-memory samples covering the v20 and first v21 windows recorded, +respectively, 7,913 versus 12,638 swap-in pages and 7,148 versus 16,212 swap-out +pages. These are system-wide interval totals, not emulator-attributed I/O or a +proof of the slowdown's cause. Thirty FPS remains unachieved. + +Ten-second EventPipe GC/allocation/contention captures followed the FPS and +thread-CPU windows, without the managed sample profiler: + +| Post-window GPU-host events | Budget unset | 64 MiB budget | +| --- | ---: | ---: | +| Allocation ticks | 10,728 | 10,941 | +| Sampled weighted allocation bytes | 1,147,942,904 | 1,169,498,312 | +| Fully paired GC pauses | 137 | 20 | +| Total GC pause time | 477.127 ms | 233.208 ms | +| Median / maximum GC pause | 3.205 / 6.969 ms | 11.073 / 25.998 ms | + +Neither capture had unmatched suspension/restart pairs or `SuspendOther` pauses. +The larger budget reduced pause count and total traced pause time, with longer +individual pauses; allocation volume did not decrease. Tick weights estimate +allocation contributions and are not exact per-type allocation counts. The +largest sampled types include vertex-input records, vertex-stage registers, +user-scalar registers and resource snapshots. Frame-time conclusions cannot be +derived from these separately instrumented windows. The first capture's stacks +have frame records but unresolved method names; they are not useful allocation +call-site evidence. + +The budget-unset run's 293.069–303.081-second thread-CPU capture recorded 5.163 +CPU-seconds on the GPU worker, 1.822 on program resolution, 4.365 on guest RHI and +8.402 on the audio event thread. The 64 MiB arm's 292.258–302.269-second capture +recorded 5.433, 1.918, 4.651 and 8.221 CPU-seconds, respectively. These samples are +outside the FPS window and do not measure exclusive frame costs. + +Private run directories are +`20261027-195142-silent hill short message.zar-room-unprofiled-defaults-v21-1` +and `20261027-195726-silent hill short message.zar-room-unprofiled-gen0-64m-v21-2`. +They retain selected windows, runtime environments, CPU deltas, event traces, +allocation/GC summaries and system telemetry. They stopped at their own markers +at 315 and 313 seconds, followed by four-second controlled termination and +process/wrapper exit 0. Peak main-process footprint/RSS was 9,039/3,958 MiB and +8,894/4,098 MiB, respectively; shared mappings prevent adding process footprints +to infer combined usage. Post-stop CPU traps remain recorded as teardown behavior. + +A return control then launched the unchanged, hash-verified v20 `eabcd9f65` +payload with the original no-GC-override environment and another fresh clone of +the same seed. Run +`20261027-200336-silent hill short message.zar-room-unprofiled-v20-return-4` +measured **8.47 FPS median (7.47–9.50, 17 reports)** in the preselected +250–285-second window, with the same shelf/lamp view verified. Thus the lower +performance is not unique to v21. This does not prove the completion-read change +has no effect, or establish the exact cause of host/session variability. The +return run stopped at 303 seconds, exited 0 after four seconds of termination, +and peaked at 8,903 MiB main-process footprint / 4,838 MiB RSS. + +### Synthetic protection-fixture follow-up + +One of the two macOS CI executions for `b4cc8292e` failed the 64- and 65-view +cases in `AffectedViewBoundIsExactAndNeverPartiallyReopensOverflow`; the other +execution passed. The failing runner reported ARM64 / .NET 10.0.14 and an +`Assert.True` failure, without an address or source-line diagnostic. Inspection +found that the fixture's fixed guest range could overlap its pinned backing +array under ASLR. The ledger deliberately refuses effects on canonical aliases. +The fixture now places its synthetic guest range beyond its backing range and +logs both addresses. The exact overlap in the original CI process cannot be +reconstructed from that log; this change removes the collision possibility +without weakening the 64-view acceptance or 65-view whole-range rejection. +Production protection behavior is unchanged. + +Fresh serialized Release `SharpEmu.Host.Memory.Tests` runs passed **486/0/6 +existing skips each** on ARM64 and Rosetta x64. Launcher times were 13.794 and +51.073 seconds, including build/test startup. Commands and TRX files are retained +under `pr23-alias-fixture-{arm64,x64}-validation.json` and corresponding result +directories. No new production-code test claim is implied by this fixture fix. diff --git a/docs/performance/performance-defaults.md b/docs/performance/performance-defaults.md index 3609c6ab3..010b5f3bd 100644 --- a/docs/performance/performance-defaults.md +++ b/docs/performance/performance-defaults.md @@ -115,3 +115,13 @@ queries once all submitted work is known complete. The guarded getter measured Pending work retains live queries: ARM64 measured 3.241 → 2.981 ns, x64 5.864 → 6.209 ns. Both paths allocate zero bytes; the runtime report documents its device ordering tests and the absence of an isolated game-FPS result. + +Later v21 controls measured 8.97 FPS with the diagnostic GC budget unset and +7.97 FPS with the application's existing 64 MiB budget. Returning to the +unchanged v20 binary measured 8.47 FPS under the later host conditions. These +captures do not reproduce the earlier 15.48 FPS result or isolate a code +regression; all are retained in the runtime report. The private diagnostic +launcher had omitted the GC environment setting already supplied by the bundle +and standard game launcher. The parity control reduced traced GC pause count +and total pause time, but established no FPS gain. The nine promoted defaults +remain enabled; the 30-FPS objective remains open. diff --git a/tests/SharpEmu.Host.Memory.Tests/MappedCopyDerivedProtectionTests.cs b/tests/SharpEmu.Host.Memory.Tests/MappedCopyDerivedProtectionTests.cs index dba18be97..dbc405463 100644 --- a/tests/SharpEmu.Host.Memory.Tests/MappedCopyDerivedProtectionTests.cs +++ b/tests/SharpEmu.Host.Memory.Tests/MappedCopyDerivedProtectionTests.cs @@ -211,6 +211,11 @@ public void AffectedViewBoundIsExactAndNeverPartiallyReopensOverflow(int count, try { var alias = ((ulong)handle.AddrOfPinnedObject() + page - 1) & ~(page - 1); + // These are ledger-only guest addresses, not native mappings. A fixed guest + // range can collide with the pinned array under ASLR, in which case the + // ledger correctly rejects an effect on its canonical backing alias. + var guestBase = checked(alias + size + page); + output.WriteLine($"boundary fixture: alias=0x{alias:X}, guest=0x{guestBase:X}, bytes=0x{size:X}"); backing = new(alias, size, page, _ => new(HostBackingReleaseStatus.Released)); var ledger = new HostMappedCopyLedger(gate, page); lock (gate) @@ -218,15 +223,15 @@ public void AffectedViewBoundIsExactAndNeverPartiallyReopensOverflow(int count, ledger.RegisterBacking(backing); for (var index = 0; index < count; index++) { - var address = State.Address + (ulong)index * page; + var address = guestBase + (ulong)index * page; var prepared = ledger.PrepareView(backing, address, page, (ulong)index * page, HostPageProtection.ReadWrite); Assert.NotNull(prepared); Assert.True(ledger.BeforeNativeEffect(address, page)); ledger.PublishView(prepared); } - Assert.True(ledger.BeginDerivedProtection(State.Address, size, RW, HostPageProtection.ReadOnly, out var receipt)); + Assert.True(ledger.BeginDerivedProtection(guestBase, size, RW, HostPageProtection.ReadOnly, out var receipt)); ledger.CompleteDerivedProtection(receipt, true); } // Inspect both ends independently; overflow must not reopen its first64 rows. - foreach (var address in new[] { State.Address, State.Address + size - page }) + foreach (var address in new[] { guestBase, guestBase + size - page }) Assert.Equal(expectedAvailable ? CpuMappedCopyResult.Copied : CpuMappedCopyResult.Rejected, ledger.TryReadMappedCopy(backing, address, page, address, new byte[1], out _)); } From f70ce37a7fc2854ebc78b305fb0e427320bd7153 Mon Sep 17 00:00:00 2001 From: kuba Date: Tue, 27 Oct 2026 20:59:58 +0100 Subject: [PATCH 39/82] [GPU] Add bounded local draw reuse with allocation and runtime controls --- docs/performance/local-draw-reuse.md | 175 +++++++++++++++ docs/performance/performance-defaults.md | 1 + .../Agc/AgcExports.CommandStream.cs | 74 +++++-- .../LocalDeferredDrawReuseTests.Benchmark.cs | 92 ++++++++ .../Rendering/LocalDeferredDrawReuseTests.cs | 206 ++++++++++++++++++ 5 files changed, 535 insertions(+), 13 deletions(-) create mode 100644 docs/performance/local-draw-reuse.md create mode 100644 tests/SharpEmu.Libs.Tests/Gpu/Rendering/LocalDeferredDrawReuseTests.Benchmark.cs create mode 100644 tests/SharpEmu.Libs.Tests/Gpu/Rendering/LocalDeferredDrawReuseTests.cs diff --git a/docs/performance/local-draw-reuse.md b/docs/performance/local-draw-reuse.md new file mode 100644 index 000000000..91958e26b --- /dev/null +++ b/docs/performance/local-draw-reuse.md @@ -0,0 +1,175 @@ + + +# Local deferred draw allocation + +The interpreter's local deferred queue now retains at most 256 empty draw +wrappers. Completed wrappers are returned only after restoring the current +submission and resolver state, including when rendering throws. Every payload +reference is cleared before retention. Draw arguments and snapshots are assigned +again on acquisition. Producer/sink-owned work never borrows or returns these +wrappers; register snapshots, geometry snapshots and public resolver jobs retain +their original ownership and allocation behavior. + +`SHARPEMU_LOCAL_DRAW_REUSE=1` enables reuse; unset or `0` selects the allocating +reference path. The implementation reduces allocation pressure, but component +timings are mixed and the runtime comparison remains unresolved. It therefore +stays opt-in. The nine previously promoted performance defaults remain enabled. + +## Component measurements + +`LocalDeferredDrawReuseTests.MeasureLocalDrawReuseWhenRequested` exercises the +real translation queue and background resolver with unavailable fake graphics +programs, excluding native compilation and rendering. Every submitted job must +resolve and be consumed, with zero refused or abandoned jobs. Each arm gets 32 +warm-up batches, followed by nine alternating measurement rounds. Each round has +`max(16, 2048 / burst)` batches. Allocation is measured on the calling render +thread only; worker-thread allocations are excluded. Timing includes resolver +coordination and is not exclusive CPU time. + +The implementation was tested on .NET 10.0.3, native ARM64 and Rosetta x64. Median +nanoseconds per draw, allocating reference → reuse: + +| Burst | ARM64, normal tiering | x64, normal tiering | ARM64, tiering disabled | x64, tiering disabled | +| ---: | ---: | ---: | ---: | ---: | +| 1 | 9,816.711 → 9,437.541 | 12,942.932 → 13,854.004 | 7,042.114 → 7,082.479 | 9,430.339 → 9,611.166 | +| 5 | 4,337.143 → 4,351.426 | 5,711.899 → 5,644.947 | 2,046.414 → 2,052.404 | 2,782.885 → 2,801.650 | +| 32 | 1,737.956 → 1,995.056 | 4,103.455 → 3,788.086 | 1,129.903 → 1,134.196 | 2,005.229 → 2,045.919 | +| 512 | 663.589 → 714.071 | 2,486.888 → 1,894.023 | 995.204 → 985.464 | 1,650.589 → 1,661.560 | + +Normal tiering results are mixed, including regressions of approximately 7% for +x64 burst 1 and 15% for ARM64 burst 32. Disabling tiering with +`DOTNET_TieredCompilation=0` keeps median timing differences within approximately +2.1% in either direction. Neither set establishes a consistent timing win. + +With tiering disabled, both architectures measured **536 → 352 B/draw** for +bursts 1, 5 and 32: **184 B/draw removed**. Burst 512 measured +**536.169 → 444.169 B/draw**, removing 92 B/draw because only half the wrappers +fit in the 256-entry bound. Normal-tiering runs also removed approximately +184 B/draw within the bound and 92 B/draw above it, with additional JIT/measurement +variation. These are component allocations, not total emulator memory savings. + +To reproduce, run the Libs fixture serially with an absolute output path: + +```sh +SHARPEMU_LOCAL_DRAW_REUSE_BENCH_OUTPUT=/absolute/path/result.json \ + dotnet test tests/SharpEmu.Libs.Tests/SharpEmu.Libs.Tests.csproj -c Release \ + --filter FullyQualifiedName~LocalDeferredDrawReuseTests \ + --settings docs/performance/shader-serial.runsettings \ + --blame-hang-timeout 120s +``` + +Select the native or x64 `dotnet` installation explicitly and optionally set +`DOTNET_TieredCompilation=0`. Use the shared validation/game lock on this host. +The fixture supplies both reuse modes itself, independent of the process default. + +## Correctness and source evidence + +Serial Release validation of the initial default-enabled candidate passed: + +- Libs x64: **9,068 passed, 0 failed, 5 existing skips**. +- Required-device Metal x64: **1,637 passed, 0 failed, 7 existing skips**. +- Focused fixture: **8/0/0 each** on ARM64 with normal tiering, ARM64 with tiering + disabled, and x64 with tiering disabled. The normal-tiering x64 fixture is + included in the full Libs run; these are configuration executions, not distinct + added tests. + +Coverage includes indexed/automatic argument order, distinct register snapshots, +nested drain refusal, failure recovery with later queued work intact, actual +non-null geometry snapshot clearing, producer/sink ownership exclusion, bounded +retention, and render-thread allocation reduction. The optional benchmark was +explicitly enabled for the reported fixture executions. + +The benchmarked and runtime-tested candidate production file SHA-256 is +`9bb4e03a42320d21896200e1bfd8075a0c70a5a2bceba472c8cd96982c12bdef`; +the correctness fixture is +`c3fc0836ca1bb3899ab1c3c1371667d2ef751271355eb73e6ce6d8012d54171f`; +the benchmark fixture is +`7fe6f7bdaa650f78184005fe381086e0d895aa34df804265bcb21fadc6c49309`. +These are changes on parent `37358ee0b8aee94537a3c5c8df92f052ea31d460`. +After the runtime controls below, the only production difference from that +candidate is requiring `=1` instead of enabling reuse when unset. The benchmark +constructs both modes explicitly, so its measured implementation is unchanged. +The final opt-in production file SHA-256 is +`709d0165a6461a27910b5d25ed74aa6ad3ac2bafdf96e69229e834676167e345`. +Fresh serial Release Libs validation with all `SHARPEMU_*` overrides removed +again passed **9,068/0/5**, with a 232.769-second launcher duration including +build/test startup. The optional benchmark was not requested in that final run; +the unchanged explicit-mode benchmark measurements above remain separate. +Private commands, TRX files, source hashes and raw rounds are retained under +`artifacts/runtime-diagnosis-20261026/local-draw-reuse-*`. Raw filenames containing +`tier0` mean **tiering disabled**, not execution using the Tier 0 compiler. + +## Silent Hill enabled / disabled / enabled control + +One immutable v22 payload was published as an x64 CLI and native ARM64 GPU host; +both publishes passed and all 471 payload hashes were verified before each run. +Its source is the candidate hash above, with reuse enabled when unset. All three +runs cloned the same warmed seed, used 1920×1080 / scale 1 / VSync on, retained +the nine promoted defaults and background program resolution, and kept command +threading off. Both processes explicitly inherited the application's existing +64 MiB GC budget. Prewarming and detailed render/pass/memory profiling were off. +Only the second arm set `SHARPEMU_LOCAL_DRAW_REUSE=0`. GeForce NOW settings were +not altered. Builds, tests and games were serialized under the shared lock. + +The 250–285-second FPS window was selected before each game reached the room. +Saved frames confirm the same stationary shelf/lamp view before and during the +window. No movement was sent. Intro/phone dismissal occurred at different times; +the return arm's pre-window room screenshot was reviewed afterward. These are +sequential captures with different host conditions, not randomized repeated +measurements. + +| Arm | Median FPS | Range | Reports | System swap-in / swap-out pages in window | +| --- | ---: | ---: | ---: | ---: | +| Enabled first | 5.97 | 3.50–7.49 | 17 | 31,519 / 24,556 | +| Disabled | 7.98 | 6.98–8.98 | 18 | 12,712 / 7,056 | +| Enabled return | 6.485 | 3.49–9.99 | 18 | 27,312 / 110,664 | + +All seven host-memory samples per window reported pressure level 2. Compression +counts were 1,749,882 / 866,836 / 1,864,095 pages respectively. These are +system-wide measurements, not emulator-attributed I/O. The enabled return began +its window near 9–10 FPS and fell to 3.49 FPS; its lower whole-window median is +retained. Both enabled medians are lower than the disabled median. The paging, +warm-up and sequential-session differences prevent assigning that difference to +reuse, but **the return does not exonerate the change or establish an FPS win**. +The allocation benefit alone is insufficient to promote this candidate given +mixed component timings and unresolved runtime results. It remains opt-in. + +Event-only traces followed separate ten-second thread-CPU windows, starting +around 302 seconds, outside the FPS windows. The trace had no managed sampling +profiler. Weighted allocation bytes estimate contributions rather than exact +object counts; asynchronous allocations are not attributable to individual +frames from these records. + +| Post-window trace | Enabled first | Disabled | Enabled return | +| --- | ---: | ---: | ---: | +| Allocation ticks | 6,052 | 10,247 | 4,714 | +| Total sampled weighted bytes | 645,999,352 | 1,095,048,576 | 503,227,056 | +| PendingDraw weighted bytes | 5,561,120 | 51,362,560 | 5,769,584 | +| Guest frame timestamps within allocation-tick interval | 59 | 84 | 39 | +| Paired GC-related suspension intervals | 12 | 19 | 10 | +| Total suspension time, ms | 188.702 | 226.866 | 349.637 | +| Median / maximum interval, ms | 15.876 / 29.965 | 11.841 / 19.493 | 29.821 / 97.248 | + +The return includes nine `SuspendForGC` intervals totaling 315.577 ms and one +`SuspendForGCPrep` interval of 34.059 ms. No arm has unmatched suspension pairs +or `SuspendOther` intervals. Different traced throughput prevents comparing raw +total allocation volume as equal work. The component benchmark establishes the +exact bounded wrapper allocation saving; these traces only support the expected +reduction in that type's contribution. + +At approximately 292–302 seconds, CPU-seconds on the GPU worker / resolver / +guest RHI / audio-event thread were respectively **3.787 / 1.229 / 2.256 / 4.706**, +**4.944 / 1.726 / 4.114 / 7.981**, and **3.510 / 1.190 / 2.368 / 5.898**. These +post-window samples do not measure exclusive frame costs or identify the cause +of scheduling differences. + +Private run directory suffixes are `room-local-draw-reuse-v22-on-1`, +`room-local-draw-reuse-v22-off-2`, and `room-local-draw-reuse-v22-return-3`, with +timestamps `20261027-203642`, `20261027-204216` and `20261027-204756`. Runs stopped +at their own markers after 314 / 313 / 313 seconds, with four-second controlled +termination and process/wrapper exit 0. Peak main-process footprint/RSS was +8,829/2,986, 9,015/4,529 and 9,102/4,157 MiB; shared mappings prevent summing +process footprints as physical usage. CPU traps occurred after stop, at +316.032 / 314.836 / 317.387 seconds, and remain unresolved teardown evidence. +**30 FPS remains unachieved.** diff --git a/docs/performance/performance-defaults.md b/docs/performance/performance-defaults.md index 010b5f3bd..88f16f18c 100644 --- a/docs/performance/performance-defaults.md +++ b/docs/performance/performance-defaults.md @@ -30,6 +30,7 @@ The three promoted shader properties also default on in explicitly constructed | Setting | Reason | | --- | --- | +| `SHARPEMU_LOCAL_DRAW_REUSE` | Removes 184 B/draw within its bounded cache, but component timings are mixed. Enabled/disabled/enabled room medians were 5.97 / 7.98 / 6.485 FPS under differing paging pressure. This does not isolate a regression, but it also does not establish a speed benefit. Kept opt-in pending better timing evidence; [full measurements and fallback](local-draw-reuse.md). | | `SHARPEMU_REGISTER_IDENTICAL_SCALARS` | Always-changing packets regress: native 101.40 → 103.08 ns; Rosetta 164.85 → 178.94 ns. Only 0.805% of eligible packets reused in the observed room. Identical-packet synthetic gains do not outweigh this demonstrated mixed-workload cost. | | `SHARPEMU_MSL_GEOMETRY_CAPACITY` | Volume pass 39.888 → 57.027 ms, changed image, no accepted broad correctness/performance result. | | `SHARPEMU_MSL_MESH_LDS_FORWARDING` | Same-binary shadow time 34.238 → 36.422 ms; title median 7.97 → 7.47 FPS. | diff --git a/src/SharpEmu.Libs/Agc/AgcExports.CommandStream.cs b/src/SharpEmu.Libs/Agc/AgcExports.CommandStream.cs index 14d6d7b7f..037035ac8 100644 --- a/src/SharpEmu.Libs/Agc/AgcExports.CommandStream.cs +++ b/src/SharpEmu.Libs/Agc/AgcExports.CommandStream.cs @@ -176,9 +176,10 @@ public sealed class CommandStreamTranslation private readonly ShaderPipelineCache? _pipelines; private readonly RenderExecutor? _executor; - internal CommandStreamTranslation(ICpuMemory memory, ICommandStreamHost host) + internal CommandStreamTranslation(ICpuMemory memory, ICommandStreamHost host, bool? recycleLocalDraws = null) { _host = host; + _recycleLocalDraws = recycleLocalDraws ?? LocalDrawReuseEnabled; _context = new CpuContext(memory, Generation.Gen5); _gpuState = _submittedGpuStates.GetValue(CanonicalMemory(memory), static _ => new SubmittedGpuState()); if (host is IRenderHost renderHost && host is IShaderPipelineHost pipelineHost) @@ -230,16 +231,62 @@ private static void RecordKnownColorTargets(SubmittedDcbState state, RegisterBan } } - private sealed record PendingDraw( - ulong SubmitId, - SubmittedDcbState State, - RegisterBanks Banks, - bool Indexed, - DrawIndexedArguments IndexedArguments, - DrawAutoArguments AutoArguments, - SubmittedVertexSnapshot? VertexSnapshot, - SubmittedIndexSnapshot? IndexSnapshot, - ParallelProgramPrefetch.Job? Job) : DeferredWork; + private sealed record PendingDraw : DeferredWork + { + public ulong SubmitId; + public SubmittedDcbState State = null!; + public RegisterBanks Banks = null!; + public bool Indexed; + public DrawIndexedArguments IndexedArguments; + public DrawAutoArguments AutoArguments; + public SubmittedVertexSnapshot? VertexSnapshot; + public SubmittedIndexSnapshot? IndexSnapshot; + public ParallelProgramPrefetch.Job? Job; + } + + // Only the interpreter/render thread's local queue owns reusable wrappers. A sink + // may retain work on another thread, so its wrappers are never taken from or returned + // to this cache. Draw snapshots and resolver jobs themselves are never recycled. + private static readonly bool LocalDrawReuseEnabled = + Environment.GetEnvironmentVariable("SHARPEMU_LOCAL_DRAW_REUSE") == "1"; + private const int MaximumSpareLocalDraws = 256; + private readonly bool _recycleLocalDraws; + private readonly Stack _spareLocalDraws = new(); + + private PendingDraw TakePendingDraw(bool producing, ulong submitId, SubmittedDcbState state, + RegisterBanks banks, bool indexed, in DrawIndexedArguments indexedArguments, + in DrawAutoArguments autoArguments, ParallelProgramPrefetch.Job? job) + { + var draw = !producing && _recycleLocalDraws && _spareLocalDraws.TryPop(out var spare) + ? spare : new PendingDraw(); + draw.SubmitId = submitId; + draw.State = state; + draw.Banks = banks; + draw.Indexed = indexed; + draw.IndexedArguments = indexedArguments; + draw.AutoArguments = autoArguments; + draw.VertexSnapshot = state.CurrentVertexSnapshot; + draw.IndexSnapshot = state.CurrentIndexSnapshot; + draw.Job = job; + return draw; + } + + private void ReturnLocalDraw(PendingDraw draw) + { + if (!_recycleLocalDraws || _spareLocalDraws.Count >= MaximumSpareLocalDraws) return; + // Execution has finished and restored the current queue/prefetch state. Clear + // every payload, including on failure, before retaining the empty wrapper. + draw.SubmitId = 0; + draw.State = null!; + draw.Banks = null!; + draw.Indexed = false; + draw.IndexedArguments = default; + draw.AutoArguments = default; + draw.VertexSnapshot = null; + draw.IndexSnapshot = null; + draw.Job = null; + _spareLocalDraws.Push(draw); + } private sealed record PendingDispatch( ulong SubmitId, @@ -440,6 +487,7 @@ private void DrainPendingDrawsCore() { _parallel.Current = null; } + ReturnLocalDraw(pending); } } @@ -467,8 +515,8 @@ private bool TryRunExecutorDraw(ulong submitId, SubmittedDcbState state, bool in if (_parallel is not null || producing) { var snapshot = banks.Snapshot(); - var pending = new PendingDraw(submitId, state, snapshot, indexed, indexedArguments, autoArguments, - state.CurrentVertexSnapshot, state.CurrentIndexSnapshot, _parallel?.Submit(snapshot, producing ? Sink!.Gate : 0, indexed)); + var pending = TakePendingDraw(producing, submitId, state, snapshot, indexed, in indexedArguments, in autoArguments, + _parallel?.Submit(snapshot, producing ? Sink!.Gate : 0, indexed)); if (producing) { Sink!.EnqueueDeferred(pending, isDraw: true); diff --git a/tests/SharpEmu.Libs.Tests/Gpu/Rendering/LocalDeferredDrawReuseTests.Benchmark.cs b/tests/SharpEmu.Libs.Tests/Gpu/Rendering/LocalDeferredDrawReuseTests.Benchmark.cs new file mode 100644 index 000000000..0436d64d2 --- /dev/null +++ b/tests/SharpEmu.Libs.Tests/Gpu/Rendering/LocalDeferredDrawReuseTests.Benchmark.cs @@ -0,0 +1,92 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using System.Diagnostics; +using System.Runtime.InteropServices; +using System.Text.Json; +using SharpEmu.Libs.Gpu.Rendering; +using Xunit; + +namespace SharpEmu.Libs.Tests.Gpu.Rendering; + +public sealed partial class LocalDeferredDrawReuseTests +{ + private readonly record struct Measurement(double NanosecondsPerDraw, double RenderThreadBytesPerDraw, long Taken); + + // Exercise the real translation queue and background resolver. Unavailable fake programs + // keep native compilation/rendering out of this component benchmark; every job must still + // resolve and be consumed. Allocations measure this thread, not the background resolver. + private static Measurement Measure(Fixture fixture, int batches, int burst) + { + fixture.Prefetch.TakeCounters(); + var before = GC.GetAllocatedBytesForCurrentThread(); + var started = Stopwatch.GetTimestamp(); + for (var batch = 0; batch < batches; batch++) + { + for (var draw = 0; draw < burst; draw++) fixture.Draw(3); + fixture.Drain(); + fixture.Host.Calls.Clear(); + fixture.Host.DrawTargets.Clear(); + fixture.Pipelines.Calls.Clear(); + fixture.Pipelines.ExportMappings.Clear(); + } + var elapsed = Stopwatch.GetTimestamp() - started; + var bytes = GC.GetAllocatedBytesForCurrentThread() - before; + var counters = fixture.Prefetch.TakeCounters(); + Assert.Equal((long)batches * burst, counters.Taken); + Assert.Equal(0, counters.Refused); + Assert.Equal(0, counters.Abandoned); + return new(elapsed * 1e9 / Stopwatch.Frequency / (batches * burst), + (double)bytes / (batches * burst), counters.Taken); + } + + [Fact] + public void WarmLocalQueueAvoidsWrapperAllocations() + { + using var baseline = new Fixture(false); + using var reused = new Fixture(true); + baseline.Pipelines.Graphics = reused.Pipelines.Graphics = new GraphicsPrograms { Available = false }; + Measure(baseline, 16, 16); Measure(reused, 16, 16); + var before = Measure(baseline, 64, 16); + var after = Measure(reused, 64, 16); + Assert.True(before.RenderThreadBytesPerDraw - after.RenderThreadBytesPerDraw >= 128, + $"baseline={before.RenderThreadBytesPerDraw:F2} B/draw; reused={after.RenderThreadBytesPerDraw:F2} B/draw"); + reused.AssertCleared(); + } + + [Fact] + public void MeasureLocalDrawReuseWhenRequested() + { + var path = Environment.GetEnvironmentVariable("SHARPEMU_LOCAL_DRAW_REUSE_BENCH_OUTPUT"); + if (string.IsNullOrEmpty(path)) return; + var cases = new List(); + foreach (var burst in new[] { 1, 5, 32, 512 }) + { + using var baseline = new Fixture(false); + using var reused = new Fixture(true); + baseline.Pipelines.Graphics = reused.Pipelines.Graphics = new GraphicsPrograms { Available = false }; + Measure(baseline, 32, burst); Measure(reused, 32, burst); + var batches = Math.Max(16, 2048 / burst); + var rounds = new List(); + for (var round = 0; round < 9; round++) + { + Measurement before, after; + if ((round & 1) == 0) + { before = Measure(baseline, batches, burst); after = Measure(reused, batches, burst); } + else + { after = Measure(reused, batches, burst); before = Measure(baseline, batches, burst); } + Assert.Equal(before.Taken, after.Taken); + rounds.Add(new { round, before, after }); + } + reused.AssertCleared(); + cases.Add(new { burst, batches, rounds }); + } + File.WriteAllText(path, JsonSerializer.Serialize(new + { + architecture = RuntimeInformation.ProcessArchitecture.ToString(), + framework = RuntimeInformation.FrameworkDescription, + caveat = "Real translation queue/resolver with unavailable fake programs; current-thread allocation only; no GPU or FPS claim.", + cases, + }, new JsonSerializerOptions { WriteIndented = true })); + } +} diff --git a/tests/SharpEmu.Libs.Tests/Gpu/Rendering/LocalDeferredDrawReuseTests.cs b/tests/SharpEmu.Libs.Tests/Gpu/Rendering/LocalDeferredDrawReuseTests.cs new file mode 100644 index 000000000..9101c9e9c --- /dev/null +++ b/tests/SharpEmu.Libs.Tests/Gpu/Rendering/LocalDeferredDrawReuseTests.cs @@ -0,0 +1,206 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using System.Collections; +using System.Reflection; +using SharpEmu.Libs.Agc; +using SharpEmu.Libs.Gpu.GpuCommands; +using SharpEmu.Libs.Gpu.GpuCommands.Registers; +using SharpEmu.Libs.Gpu.Rendering; +using SharpEmu.Libs.Tests.Gpu.GpuCommands; +using SharpEmu.Libs.Tests.Gpu.Scheduling; +using Xunit; +using static SharpEmu.Libs.Tests.Gpu.Rendering.RenderExecutorFixtures; + +namespace SharpEmu.Libs.Tests.Gpu.Rendering; + +[Collection(SchedulingStateCollection.Name)] +public sealed partial class LocalDeferredDrawReuseTests +{ + private sealed class ConcurrentHost : IConcurrentResolveHost + { + public void BeginConcurrentResolve() { } + public object EndConcurrentResolve() => this; + public bool IsStillCurrent(object assumptions) => ReferenceEquals(this, assumptions); + } + + private sealed class Sink : AgcExports.IDeferredWorkSink + { + public bool IsProducerThread => true; + public long Gate => 0; + public readonly List Work = new(); + public void EnqueueDeferred(AgcExports.DeferredWork work, bool isDraw) => Work.Add(work); + } + + private sealed class Fixture : IDisposable + { + private readonly FatalScope _fatal = new(); + private static readonly FieldInfo ActiveHost = typeof(AgcExports).GetField("_activeCommandStreamHost", BindingFlags.Static | BindingFlags.NonPublic)!; + private readonly object? _previousActiveHost = ActiveHost.GetValue(null); + internal readonly RecordingCommandStreamHost Commands = new(); + internal readonly RecordingRenderHost Host = new(); + internal readonly FakePipelineProvider Pipelines = new(); + internal readonly GpuCommandInterpreter Interpreter; + internal readonly AgcExports.CommandStreamTranslation Translation; + internal readonly ParallelProgramPrefetch Prefetch; + + internal Fixture(bool reuse) + { + Interpreter = new(Commands, 0, 0); + var banks = Banks(); + foreach (var name in new[] { nameof(RegisterBanks.Context), nameof(RegisterBanks.Shader), nameof(RegisterBanks.UserConfig) }) + typeof(RegisterBanks).GetProperty(name)!.SetValue(Interpreter.TypedRegisters, + typeof(RegisterBanks).GetProperty(name)!.GetValue(banks)); + var executor = new RenderExecutor(Host, Pipelines); + Prefetch = new(executor, Host, new ConcurrentHost()); + executor.Prefetch = Prefetch; + Translation = new(Commands.Memory, Commands, recycleLocalDraws: reuse); + // Use real translation/executor/prefetch with the existing fake renderer, avoiding + // native shader compilation. The production constructor composes its own provider. + typeof(AgcExports.CommandStreamTranslation).GetField("_executor", BindingFlags.Instance | BindingFlags.NonPublic)!.SetValue(Translation, executor); + typeof(AgcExports.CommandStreamTranslation).GetField("_parallel", BindingFlags.Instance | BindingFlags.NonPublic)!.SetValue(Translation, Prefetch); + Translation.BeginSubmission(0, 1, null, Interpreter); + Host.WriteGuest(IndexBase, new byte[1024]); + } + + internal void Draw(uint count, ulong submit = 1) => Translation.DrawAuto(submit, Auto(count)); + internal void Index(uint count, ulong submit = 1) => Translation.DrawIndexed(submit, Indexed(count)); + internal void Drain() { Prefetch.WaitIdle(); Translation.DrainPendingDraws(); } + internal string[] Draws => Host.Calls.Where(x => x.StartsWith("draw ", StringComparison.Ordinal) || x.StartsWith("draw_indexed ", StringComparison.Ordinal)).ToArray(); + internal object[] Spares => ((IEnumerable)typeof(AgcExports.CommandStreamTranslation) + .GetField("_spareLocalDraws", BindingFlags.Instance | BindingFlags.NonPublic)! + .GetValue(Translation)!).Cast().ToArray(); + internal void AssertCleared() + { + foreach (var spare in Spares) + { + foreach (var name in new[] { "State", "Banks", "VertexSnapshot", "IndexSnapshot", "Job" }) + Assert.Null(spare.GetType().GetField(name)!.GetValue(spare)); + Assert.Equal(0UL, spare.GetType().GetField("SubmitId")!.GetValue(spare)); + Assert.Equal(default(DrawAutoArguments), spare.GetType().GetField("AutoArguments")!.GetValue(spare)); + Assert.Equal(default(DrawIndexedArguments), spare.GetType().GetField("IndexedArguments")!.GetValue(spare)); + } + } + public void Dispose() + { + try { Prefetch.Dispose(); } + finally { ActiveHost.SetValue(null, _previousActiveHost); _fatal.Dispose(); } + } + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void RepeatedBurstsPreserveArgumentsSnapshotsAndNestedDrainOrder(bool reuse) + { + using var f = new Fixture(reuse); + var expected = new List(); + for (var round = 0; round < 6; round++) + { + for (var i = 0; i < 12; i++) + { + var count = (uint)(3 + round * 12 + i); + var banks = f.Interpreter.TypedRegisters; + banks.PrepareUserConfigWrite(); + banks.UserConfig.PrimitiveType = PrimitiveTriangleList; + banks.UserConfig.ObjectId = count; + if ((i & 1) == 0) + { + f.Draw(count, count); + expected.Add($"draw {count} 1 0 0"); + } + else + { + f.Index(count, count); + expected.Add($"draw_indexed {count} 1 0 0 0"); + } + } + var queued = ((IEnumerable)typeof(AgcExports.CommandStreamTranslation) + .GetField("_pendingDraws", BindingFlags.Instance | BindingFlags.NonPublic)!.GetValue(f.Translation)!).Cast().ToArray(); + Assert.Equal(Enumerable.Range(3 + round * 12, 12).Select(x => (uint)x), queued.Select(x => + ((RegisterBanks)x.GetType().GetField("Banks")!.GetValue(x)!).UserConfig.ObjectId)); + f.Translation.EnqueuePendingAction(() => + { + Assert.True(f.Translation.IsDraining); + f.Translation.DrainPendingDraws(); // Must not execute the following draw recursively. + Assert.Equal(expected.ToArray(), f.Draws); + }); + f.Draw(1000); + f.Prefetch.WaitIdle(); + f.Translation.DrainPendingDraws(); + expected.Add("draw 1000 1 0 0"); + Assert.Equal(expected.ToArray(), f.Draws); + Assert.False(f.Translation.HasPendingDraws); + Assert.Null(f.Prefetch.Current); + f.AssertCleared(); + } + Assert.Equal(reuse ? 13 : 0, f.Spares.Length); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void FailureReturnsOnlyTheFinishedWrapperAndLeavesLaterDrawIntact(bool reuse) + { + using var f = new Fixture(reuse); + f.Draw(3); f.Draw(6); + f.Prefetch.WaitIdle(); + f.Host.FailPrepareBindings = true; + Assert.Throws(f.Translation.DrainPendingDraws); + Assert.False(f.Translation.IsDraining); + Assert.Equal(1, f.Translation.PendingDrawCount); + Assert.Null(f.Prefetch.Current); + Assert.Equal(reuse ? 1 : 0, f.Spares.Length); + f.AssertCleared(); + f.Host.FailPrepareBindings = false; + f.Draw(9); + f.Drain(); + Assert.Equal(new[] { "draw 6 1 0 0", "draw 9 1 0 0" }, f.Draws); + f.AssertCleared(); + } + + [Fact] + public void AReusedWrapperDoesNotCarryThePreviousDrawsGeometrySnapshot() + { + using var f = new Fixture(true); + var capture = AgcExports.CreateGeometrySnapshotsForTests(0, 0, 0, 3, 1, []); + f.Translation.BeginSubmission(0, 1, capture, f.Interpreter); + f.Draw(3); + var queue = (IEnumerable)typeof(AgcExports.CommandStreamTranslation) + .GetField("_pendingDraws", BindingFlags.Instance | BindingFlags.NonPublic)!.GetValue(f.Translation)!; + var wrapper = Assert.Single(queue.Cast()); + var snapshot = wrapper.GetType().GetField("VertexSnapshot")!.GetValue(wrapper); + Assert.NotNull(snapshot); + f.Drain(); + f.AssertCleared(); + f.Translation.BeginSubmission(0, 2, null, f.Interpreter); + f.Draw(6); + Assert.Same(wrapper, Assert.Single(queue.Cast())); + Assert.Null(wrapper.GetType().GetField("VertexSnapshot")!.GetValue(wrapper)); + f.Drain(); + Assert.Equal(new[] { "draw 3 1 0 0", "draw 6 1 0 0" }, f.Draws); + f.AssertCleared(); + } + + [Fact] + public void ProducerOwnedWrappersNeverEnterTheLocalCacheAndRetentionIsBounded() + { + using var f = new Fixture(true); + for (var i = 0; i < 300; i++) f.Draw(3); + f.Drain(); + var spares = f.Spares; + Assert.Equal(256, spares.Length); + var sink = new Sink(); f.Translation.Sink = sink; + f.Draw(6, 900); f.Draw(9, 901); + f.Prefetch.WaitIdle(); + Assert.Equal(2, sink.Work.Count); + Assert.DoesNotContain(sink.Work[0], spares); + Assert.DoesNotContain(sink.Work[1], spares); + foreach (var work in sink.Work) f.Translation.RunDeferred(work); + Assert.Equal(900UL, sink.Work[0].GetType().GetField("SubmitId")!.GetValue(sink.Work[0])); + Assert.Equal(901UL, sink.Work[1].GetType().GetField("SubmitId")!.GetValue(sink.Work[1])); + Assert.Equal(spares, f.Spares); + Assert.Equal(new[] { "draw 6 1 0 0", "draw 9 1 0 0" }, f.Draws[^2..]); + f.AssertCleared(); + } +} From 58996827ff647adc6734d1b9700162c300c92a38 Mon Sep 17 00:00:00 2001 From: kuba Date: Tue, 27 Oct 2026 21:52:33 +0100 Subject: [PATCH 40/82] test(perf): qualify default acceptance and fix polling timing --- .../dense-register-range-experiment.md | 115 ++++++++++++++++++ docs/performance/performance-defaults.md | 3 + .../RemoteWaitTests.cs | 18 ++- .../GpuCommandInterpreterRegisterTests.cs | 32 ++++- tools/SharpEmu.Tools.PerfAudit/Harness.cs | 21 ++++ tools/SharpEmu.Tools.PerfAudit/README.md | 9 ++ 6 files changed, 191 insertions(+), 7 deletions(-) create mode 100644 docs/performance/dense-register-range-experiment.md diff --git a/docs/performance/dense-register-range-experiment.md b/docs/performance/dense-register-range-experiment.md new file mode 100644 index 000000000..afd09df7e --- /dev/null +++ b/docs/performance/dense-register-range-experiment.md @@ -0,0 +1,115 @@ + + +# Dense register-range experiment: rejected as a default + +The nine validated [performance defaults](performance-defaults.md) remain enabled. +A separate raw-register range-copy prototype was tested and removed from production: +faster isolated warm writes did not establish a safe overall speed improvement. +No game was launched for this follow-up, and it establishes no FPS gain or 30-FPS result. + +The prototype proved all destination entries present before copying a contiguous +shader-register packet. Missing entries, short packets and invalid ranges retained +the scalar path, including partial raw writes before a fatal invalid tail. Variant 1 +used a variable-length span copy. Variant 2 specialized 4/8/16/32-word copies with +constant lengths. Both preserved insertion order, presence, counts and snapshots. +All pre-existing commits remain intact; neither prototype is a new runtime default +or a retained opt-in feature. + +## Method and limitations + +Baseline source: `f70ce37a7fc2854ebc78b305fb0e427320bd7153`. Each comparison +used frozen ARM64 .NET 10.0.3 self-contained payloads with ReadyToRun disabled; +hashes confirmed only `SharpEmu.Libs.dll` differed within each pair. Each arm ran +twice in A/B/B/A order. The five PM4 interpreter scenarios each used 5 seconds +of warmup followed by five approximately 150-ms samples. Shader blocks contain +eight changing words; fake host callbacks isolate interpretation from rendering. +The allocation result remained 0.1 B/synthetic draw in both arms. + +Initial wall-only comparisons were unsuitable for acceptance: one unchanged +control moved from 834.3 to 1770.0 ns/draw. Isolated map tests covered warm/cold +entries, 1/2/4/8/16/32 words and starts at 12/63, on ARM64 and x64, with normal +and disabled tiering. Warm range writes looked favorable, but even bypass controls +varied substantially. Those short elapsed-time samples are not treated as stable +speedup estimates. + +The benchmark now optionally records calling-thread CPU time alongside elapsed +time (`SHARPEMU_BENCH_THREAD_CPU=1`, macOS). This excludes descheduling and other +threads' work but does not eliminate core-placement, frequency, cache or JIT drift. +All 300 paired samples below had positive CPU intervals no greater than their +elapsed intervals. Separate disassembly captures confirmed optimized Tier1 code for the baseline +and variant 1; those captures are not timing evidence. + +## CPU-time results + +Values below are nanoseconds per synthetic draw. Each cell averages the two +per-run medians, giving the two process runs equal weight. Positive deltas are +slower. These are component measurements, not measured game-frame costs. + +| Scenario | Variant 1, tiering disabled: baseline → candidate | Delta | Variant 1, normal tiering: baseline → candidate | Delta | +| --- | ---: | ---: | ---: | ---: | +| 20 context registers, block reads | 887.7 → 845.0 | -4.8% | 557.7 → 620.7 | +11.3% | +| Same stream, per-packet reads | 1110.8 → 997.3 | -10.2% | 552.7 → 616.3 | +11.5% | +| Indexed draws, block reads | 1071.3 → 1014.6 | -5.3% | 578.8 → 589.1 | +1.8% | +| 5 context registers, block reads | 374.8 → 324.7 | -13.4% | 202.5 → 225.6 | +11.4% | +| 25 context registers, block reads | 1152.7 → 1109.4 | -3.8% | 639.4 → 700.4 | +9.5% | + +Variant 1 therefore did not qualify under the application's normal tiering policy. +Variant 2 requested constant-length copies for common packet sizes, but still did +not qualify. Its complete normal-tiering comparison follows; cells retain the +individual run medians in execution order within each arm. + +| Scenario | Baseline CPU ns (A1, A4) | Candidate CPU ns (B2, B3) | Delta of arm means | +| --- | ---: | ---: | ---: | +| 20 context registers, block reads | 634.1, 447.8 | 588.6, 461.5 | -2.9% | +| Same stream, per-packet reads | 643.3, 671.2 | 593.8, 658.8 | -4.7% | +| Indexed draws, block reads | 504.4, 489.6 | 552.8, 574.9 | +13.5% | +| 5 context registers, block reads | 256.2, 266.8 | 210.5, 224.7 | -16.8% | +| 25 context registers, block reads | 775.7, 558.7 | 548.7, 672.2 | -8.5% | + +The indexed case is 13.5% slower while its two controls differ by about 3%. +Other cases retain appreciable drift. This is sufficient to reject broad default +acceptance; it does not prove a game regression or identify a unique JIT cause. +No further default was enabled on the strength of the isolated copy benchmark. + +## Correctness and retained changes + +Variant 1 passed 76 focused register tests. Its 16 range-test cases also passed +on ARM64 and x64 with normal and disabled tiering (repeated configuration checks, +not 64 additional distinct tests). Variant 2 passed 84 focused tests, including +adjacent-register sentinels around every fixed-size copy. The rejected source, +tests, payload hashes, raw samples and commands are retained privately under +`artifacts/runtime-diagnosis-20261026/`: `dense-range-*.json`, associated logs, +`dense-range-v1.patch` and `rejected-dense-range-v2.patch`. + +The existing interpreter gained two additional regression cases for a longer +invalid-tail packet and repeated writes across a retained snapshot. Production +register-map and interpreter source were restored exactly to the baseline. The +rebuilt `SharpEmu.Libs.dll` also matches the baseline byte-for-byte (SHA-256 +`b487a9ed68a6424e3d8959bd038c66805046951aebc5bdcf107054695338e148`). +The new optional CPU-time diagnostic is retained in the synthetic benchmark tool; +see its [measurement instructions](../../tools/SharpEmu.Tools.PerfAudit/README.md). + +A separate [CI failure](https://github.com/sharpemu/sharpemu-mac/actions/runs/40574121112/job/122436124389) +reported a zero-millisecond polling interval against a 20-ms minimum. Its writer +was started before the stopwatch and could finish while the caller was descheduled. +The fixture now starts timing before starting the writer and joins it in `finally` +before freeing shared memory. Production polling behavior and timing bounds are +unchanged. An initial full GPU-remote run passed 56/0/0. + +Final restored-source Release validation, with no `SHARPEMU_*` overrides and +serial runsettings: **Libs 9,070 passed / 0 failed / 5 existing Opus skips**; +**GPU-remote 56 passed / 0 failed / 0 skipped**. Combined: **9,126/0/5**. +The Libs test phase took 3m17s; GPU-remote took 8s (build times excluded). +Commands used the shared machine lock and the x64 .NET host: + +```sh +dotnet test tests/SharpEmu.Libs.Tests/SharpEmu.Libs.Tests.csproj -c Release --no-restore --settings docs/performance/shader-serial.runsettings --blame-hang-timeout 120s +dotnet test tests/SharpEmu.GpuRemote.Tests/SharpEmu.GpuRemote.Tests.csproj -c Release --no-restore --settings docs/performance/shader-serial.runsettings --blame-hang-timeout 120s +``` + +Full commands, TRX hashes and counts: `dense-range-final-validation.json` in the +private diagnostic directory. These checks validate the retained tests/tooling +and restored library; they are separate from prototype timing and game FPS. + +REUSE passed **2,869/2,869 files**, and whitespace checks passed. diff --git a/docs/performance/performance-defaults.md b/docs/performance/performance-defaults.md index 88f16f18c..7c5298901 100644 --- a/docs/performance/performance-defaults.md +++ b/docs/performance/performance-defaults.md @@ -23,6 +23,9 @@ VSync, guest quality, or unrelated application setting is changed. The existing negative label page filter, exact materialization comparison, dense register lookup, mip-layout construction and target memo fixes remain enabled. +The later [dense register-range experiment](dense-register-range-experiment.md) +was rejected: isolated warm-copy gains did not carry across representative +interpreter workloads under normal tiering. It adds no new default or switch. The three promoted shader properties also default on in explicitly constructed `Gen5MslCodegenOptions`; reference tests set those properties to `false`. diff --git a/tests/SharpEmu.GpuRemote.Tests/RemoteWaitTests.cs b/tests/SharpEmu.GpuRemote.Tests/RemoteWaitTests.cs index db807280e..6e6d9779e 100644 --- a/tests/SharpEmu.GpuRemote.Tests/RemoteWaitTests.cs +++ b/tests/SharpEmu.GpuRemote.Tests/RemoteWaitTests.cs @@ -84,11 +84,21 @@ public void PollingFallbackSeesTheChangeAndHonoursTheTimeout() Thread.Sleep(30); Volatile.Write(ref *word, 2); }); - changer.Start(); + // Include thread startup: a descheduled caller may resume only after + // the writer has already changed the word, making Poll return at once. var watch = Stopwatch.StartNew(); - Assert.True(RemoteWait.Poll(word, 1, 10_000_000_000)); - Assert.InRange(watch.ElapsedMilliseconds, 20, 2_000); - changer.Join(); + changer.Start(); + try + { + Assert.True(RemoteWait.Poll(word, 1, 10_000_000_000)); + Assert.InRange(watch.ElapsedMilliseconds, 20, 2_000); + Assert.Equal(2u, Volatile.Read(ref *word)); + } + finally + { + // An assertion failure must not free memory still owned by the writer. + changer.Join(); + } } finally { diff --git a/tests/SharpEmu.Libs.Tests/Gpu/GpuCommands/GpuCommandInterpreterRegisterTests.cs b/tests/SharpEmu.Libs.Tests/Gpu/GpuCommands/GpuCommandInterpreterRegisterTests.cs index f811bc3c6..2827eb530 100644 --- a/tests/SharpEmu.Libs.Tests/Gpu/GpuCommands/GpuCommandInterpreterRegisterTests.cs +++ b/tests/SharpEmu.Libs.Tests/Gpu/GpuCommands/GpuCommandInterpreterRegisterTests.cs @@ -120,13 +120,19 @@ public void RegistersOutsideTheBank_AreFatal() Assert.Contains("range leaves the bank", runner.RunExpectingFatal(StreamRunner.Packet(PacketOpcode.SetContextRegister, 0x3FF, 1, 2)).Message); } - [Fact] - public void DirectShaderRangeWithInvalidTailPreservesOnlyItsRawPrefix() + [Theory] + [InlineData(2)] + [InlineData(8)] + public void DirectShaderRangeWithInvalidTailPreservesOnlyItsRawPrefix(int count) { var runner = new StreamRunner(); var before = runner.Interpreter.TypedRegisters.Snapshot(); var last = RegisterBankLayout.ShaderRegisterCount - 1; - var error = runner.RunExpectingFatal(StreamRunner.Packet(PacketOpcode.SetShaderRegister, last, 0x1234, 0x5678)); + runner.Interpreter.Registers.Shader[last] = 99; + var values = Enumerable.Repeat(0x5678u, count + 1).ToArray(); + values[0] = last; + values[1] = 0x1234; + var error = runner.RunExpectingFatal(StreamRunner.Packet(PacketOpcode.SetShaderRegister, values)); Assert.Contains("shader register range leaves the bank", error.Message); Assert.Equal(0x1234u, runner.Interpreter.Registers.Shader[last]); Assert.False(runner.Interpreter.Registers.Shader.ContainsKey(last + 1)); @@ -135,6 +141,26 @@ public void DirectShaderRangeWithInvalidTailPreservesOnlyItsRawPrefix() Assert.Same(before.Shader.Compute, runner.Interpreter.TypedRegisters.Shader.Compute); } + [Fact] + public void RepeatedDirectShaderRangeUpdatesRawAndTypedValuesWithoutChangingSnapshots() + { + var runner = new StreamRunner(); + var offset = ShaderRegisterOffset.SpiShaderUserDataPs0; + var values = new uint[] { offset, 1, 2, 3, 4, 5, 6, 7, 8 }; + runner.Run(StreamRunner.Packet(PacketOpcode.SetShaderRegister, values)); + var before = runner.Interpreter.TypedRegisters.Snapshot(); + var keys = runner.Interpreter.Registers.Shader.Keys.ToArray(); + for (var i = 1; i < values.Length; i++) values[i] += 20; + runner.Run(StreamRunner.Packet(PacketOpcode.SetShaderRegister, values)); + Assert.Equal(keys, runner.Interpreter.Registers.Shader.Keys); + for (var i = 0; i < 8; i++) + { + Assert.Equal((uint)(i + 1), before.Shader.Pixel.UserScalars.Values[i]); + Assert.Equal((uint)(i + 21), runner.Interpreter.TypedRegisters.Shader.Pixel.UserScalars.Values[i]); + Assert.Equal((uint)(i + 21), runner.Interpreter.Registers.Shader[offset + (uint)i]); + } + } + [Fact] public void DirectDepthWrites_ClearTheCompositeExtent() { diff --git a/tools/SharpEmu.Tools.PerfAudit/Harness.cs b/tools/SharpEmu.Tools.PerfAudit/Harness.cs index 53feb5dd5..7103f0c3f 100644 --- a/tools/SharpEmu.Tools.PerfAudit/Harness.cs +++ b/tools/SharpEmu.Tools.PerfAudit/Harness.cs @@ -3,6 +3,7 @@ using System.Diagnostics; using System.Globalization; +using System.Runtime.InteropServices; using System.Text; namespace SharpEmu.PerfAudit; @@ -27,6 +28,12 @@ internal static class Harness public static double TargetRunMs = 150; public static double WarmupMs = 700; public static readonly List Results = []; + private static readonly bool MeasureThreadCpu = Environment.GetEnvironmentVariable("SHARPEMU_BENCH_THREAD_CPU") == "1"; + + // Darwin's CLOCK_THREAD_CPUTIME_ID, from the macOS SDK's _time.h. This + // measures only the calling thread; it is not process CPU or elapsed time. + [DllImport("/usr/lib/libSystem.B.dylib", EntryPoint = "clock_gettime_nsec_np")] + private static extern ulong ThreadCpuNanoseconds(int clockId); /// operations the body performs per call (ns/op = elapsed / calls / ops). /// illustrative operation count, not a measured game frame (6331 draws, 2700 dispatches, ...). @@ -52,6 +59,9 @@ public static BenchResult Measure(string id, string name, double opsPerInvoke, d var invokes = (int)Math.Clamp(TargetRunMs / Math.Max(perInvokeMs, 1e-6), 1, maxInvokesPerRun); var times = new double[Runs]; var bytes = new double[Runs]; + if (MeasureThreadCpu && !OperatingSystem.IsMacOS()) + throw new PlatformNotSupportedException("Thread CPU diagnostics currently require macOS."); + var cpuTimes = MeasureThreadCpu ? new double[Runs] : null; var index = 0; for (var run = 0; run < Runs; run++) { @@ -60,14 +70,25 @@ public static BenchResult Measure(string id, string name, double opsPerInvoke, d GC.WaitForPendingFinalizers(); var allocatedBefore = processWideAllocations ? GC.GetTotalAllocatedBytes(true) : GC.GetAllocatedBytesForCurrentThread(); var start = Stopwatch.GetTimestamp(); + var cpuStart = cpuTimes is null ? 0 : ThreadCpuNanoseconds(16); for (var i = 0; i < invokes; i++) body(index++); + var cpuEnd = cpuTimes is null ? 0 : ThreadCpuNanoseconds(16); var elapsed = Stopwatch.GetElapsedTime(start); var allocatedAfter = processWideAllocations ? GC.GetTotalAllocatedBytes(true) : GC.GetAllocatedBytesForCurrentThread(); var ops = (double)invokes * opsPerInvoke; times[run] = elapsed.TotalNanoseconds / ops; bytes[run] = (allocatedAfter - allocatedBefore) / ops; + if (cpuTimes is not null) + { + if (cpuStart == 0 || cpuEnd < cpuStart) + throw new InvalidOperationException("The thread CPU clock did not return a valid interval."); + cpuTimes[run] = (cpuEnd - cpuStart) / ops; + } } + if (cpuTimes is not null) + Console.WriteLine($"thread_cpu id={id} ns_op=[{string.Join(",", cpuTimes.Select(x => x.ToString("F3", CultureInfo.InvariantCulture)))}] " + + $"wall_ns_op=[{string.Join(",", times.Select(x => x.ToString("F3", CultureInfo.InvariantCulture)))}]"); Array.Sort(times); Array.Sort(bytes); var median = times[Runs / 2]; diff --git a/tools/SharpEmu.Tools.PerfAudit/README.md b/tools/SharpEmu.Tools.PerfAudit/README.md index 06c01141e..e8b839cb8 100644 --- a/tools/SharpEmu.Tools.PerfAudit/README.md +++ b/tools/SharpEmu.Tools.PerfAudit/README.md @@ -52,6 +52,15 @@ The printed p95 is a percentile of run averages, not individual operation latenc The scaled-cost column uses illustrative counts (6,331 draws / 2,700 dispatches); it does not measure a game frame, predict FPS, or account for overlapping work. +On macOS, `SHARPEMU_BENCH_THREAD_CPU=1` additionally prints paired per-run +`ns_op` (calling-thread CPU time) and `wall_ns_op` samples. The regular summary +table continues to report elapsed wall time. CPU time uses Darwin's +`CLOCK_THREAD_CPUTIME_ID`; it excludes descheduling and work on other threads, +including background resolvers, GC workers and the GPU. It does not remove +frequency, cache, JIT or core-placement effects. Retain both measurements and +unchanged controls when investigating a busy host; lower CPU time alone is not +evidence of higher game FPS. This diagnostic currently rejects non-macOS hosts. + For the final cold-admission A/B experiment, use the dedicated [materialization harness](../SharpEmu.Tools.MaterializationBench/README.md), whose sampling and counters match the published final result. Historical measurements From c5ae56c56b55ba764e553b7d8994a3947f02624d Mon Sep 17 00:00:00 2001 From: kuba Date: Tue, 27 Oct 2026 22:49:34 +0100 Subject: [PATCH 41/82] test(perf): correct Rosetta thread CPU timing with Mach counters --- .../dense-register-range-experiment.md | 5 ++- .../SharpEmu.Libs.Tests.csproj | 1 + .../DarwinThreadCpuClock.cs | 38 +++++++++++++++++++ tools/SharpEmu.Tools.PerfAudit/Harness.cs | 10 +---- tools/SharpEmu.Tools.PerfAudit/README.md | 6 ++- 5 files changed, 50 insertions(+), 10 deletions(-) create mode 100644 tools/SharpEmu.Tools.PerfAudit/DarwinThreadCpuClock.cs diff --git a/docs/performance/dense-register-range-experiment.md b/docs/performance/dense-register-range-experiment.md index afd09df7e..b0811fe30 100644 --- a/docs/performance/dense-register-range-experiment.md +++ b/docs/performance/dense-register-range-experiment.md @@ -37,7 +37,10 @@ The benchmark now optionally records calling-thread CPU time alongside elapsed time (`SHARPEMU_BENCH_THREAD_CPU=1`, macOS). This excludes descheduling and other threads' work but does not eliminate core-placement, frequency, cache or JIT drift. All 300 paired samples below had positive CPU intervals no greater than their -elapsed intervals. Separate disassembly captures confirmed optimized Tier1 code for the baseline +elapsed intervals. These comparisons ran natively on ARM64. A later Rosetta +clock check found unscaled ticks from `CLOCK_THREAD_CPUTIME_ID`; the diagnostic +now uses Mach `THREAD_BASIC_INFO` in both architectures. That correction does +not change the native ARM64 measurements below. Separate disassembly captures confirmed optimized Tier1 code for the baseline and variant 1; those captures are not timing evidence. ## CPU-time results diff --git a/tests/SharpEmu.Libs.Tests/SharpEmu.Libs.Tests.csproj b/tests/SharpEmu.Libs.Tests/SharpEmu.Libs.Tests.csproj index d5cb719cd..9838d3c21 100644 --- a/tests/SharpEmu.Libs.Tests/SharpEmu.Libs.Tests.csproj +++ b/tests/SharpEmu.Libs.Tests/SharpEmu.Libs.Tests.csproj @@ -11,6 +11,7 @@ SPDX-License-Identifier: GPL-2.0-or-later + diff --git a/tools/SharpEmu.Tools.PerfAudit/DarwinThreadCpuClock.cs b/tools/SharpEmu.Tools.PerfAudit/DarwinThreadCpuClock.cs new file mode 100644 index 000000000..1b2f127bc --- /dev/null +++ b/tools/SharpEmu.Tools.PerfAudit/DarwinThreadCpuClock.cs @@ -0,0 +1,38 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using System.Runtime.InteropServices; + +namespace SharpEmu.PerfAudit; + +// Shared by optional component benchmarks. Rosetta on the measured host returns +// unscaled ticks from CLOCK_THREAD_CPUTIME_ID, including clock_gettime_nsec_np. +// Mach THREAD_BASIC_INFO explicitly reports seconds and microseconds in both ABIs. +internal static class DarwinThreadCpuClock +{ + [StructLayout(LayoutKind.Sequential)] + private struct ThreadBasicInfo + { + public int UserSeconds, UserMicroseconds, SystemSeconds, SystemMicroseconds; + public int CpuUsage, Policy, RunState, Flags, SuspendCount, SleepTime; + } + + [DllImport("/usr/lib/libSystem.B.dylib")] + private static extern nint pthread_self(); + [DllImport("/usr/lib/libSystem.B.dylib")] + private static extern uint pthread_mach_thread_np(nint thread); + [DllImport("/usr/lib/libSystem.B.dylib")] + private static extern int thread_info(uint thread, int flavor, out ThreadBasicInfo info, ref uint count); + + internal static ulong Nanoseconds() + { + if (!OperatingSystem.IsMacOS()) throw new PlatformNotSupportedException("Thread CPU diagnostics require macOS."); + uint count = 10; + // pthread_mach_thread_np borrows the existing port; mach_thread_self would + // create a send right requiring deallocation at every sample boundary. + if (thread_info(pthread_mach_thread_np(pthread_self()), 3, out var info, ref count) != 0 || count != 10) + throw new InvalidOperationException("Cannot read THREAD_BASIC_INFO."); + return ((ulong)info.UserSeconds + (ulong)info.SystemSeconds) * 1_000_000_000 + + ((ulong)info.UserMicroseconds + (ulong)info.SystemMicroseconds) * 1_000; + } +} diff --git a/tools/SharpEmu.Tools.PerfAudit/Harness.cs b/tools/SharpEmu.Tools.PerfAudit/Harness.cs index 7103f0c3f..6915b9a7d 100644 --- a/tools/SharpEmu.Tools.PerfAudit/Harness.cs +++ b/tools/SharpEmu.Tools.PerfAudit/Harness.cs @@ -3,7 +3,6 @@ using System.Diagnostics; using System.Globalization; -using System.Runtime.InteropServices; using System.Text; namespace SharpEmu.PerfAudit; @@ -30,11 +29,6 @@ internal static class Harness public static readonly List Results = []; private static readonly bool MeasureThreadCpu = Environment.GetEnvironmentVariable("SHARPEMU_BENCH_THREAD_CPU") == "1"; - // Darwin's CLOCK_THREAD_CPUTIME_ID, from the macOS SDK's _time.h. This - // measures only the calling thread; it is not process CPU or elapsed time. - [DllImport("/usr/lib/libSystem.B.dylib", EntryPoint = "clock_gettime_nsec_np")] - private static extern ulong ThreadCpuNanoseconds(int clockId); - /// operations the body performs per call (ns/op = elapsed / calls / ops). /// illustrative operation count, not a measured game frame (6331 draws, 2700 dispatches, ...). /// the measured work; receives a running iteration index. @@ -70,9 +64,9 @@ public static BenchResult Measure(string id, string name, double opsPerInvoke, d GC.WaitForPendingFinalizers(); var allocatedBefore = processWideAllocations ? GC.GetTotalAllocatedBytes(true) : GC.GetAllocatedBytesForCurrentThread(); var start = Stopwatch.GetTimestamp(); - var cpuStart = cpuTimes is null ? 0 : ThreadCpuNanoseconds(16); + var cpuStart = cpuTimes is null ? 0 : DarwinThreadCpuClock.Nanoseconds(); for (var i = 0; i < invokes; i++) body(index++); - var cpuEnd = cpuTimes is null ? 0 : ThreadCpuNanoseconds(16); + var cpuEnd = cpuTimes is null ? 0 : DarwinThreadCpuClock.Nanoseconds(); var elapsed = Stopwatch.GetElapsedTime(start); var allocatedAfter = processWideAllocations ? GC.GetTotalAllocatedBytes(true) : GC.GetAllocatedBytesForCurrentThread(); var ops = (double)invokes * opsPerInvoke; diff --git a/tools/SharpEmu.Tools.PerfAudit/README.md b/tools/SharpEmu.Tools.PerfAudit/README.md index e8b839cb8..2fea33109 100644 --- a/tools/SharpEmu.Tools.PerfAudit/README.md +++ b/tools/SharpEmu.Tools.PerfAudit/README.md @@ -55,11 +55,15 @@ it does not measure a game frame, predict FPS, or account for overlapping work. On macOS, `SHARPEMU_BENCH_THREAD_CPU=1` additionally prints paired per-run `ns_op` (calling-thread CPU time) and `wall_ns_op` samples. The regular summary table continues to report elapsed wall time. CPU time uses Darwin's -`CLOCK_THREAD_CPUTIME_ID`; it excludes descheduling and work on other threads, +`THREAD_BASIC_INFO` seconds/microseconds; it excludes descheduling and work on other threads, including background resolvers, GC workers and the GPU. It does not remove frequency, cache, JIT or core-placement effects. Retain both measurements and unchanged controls when investigating a busy host; lower CPU time alone is not evidence of higher game FPS. This diagnostic currently rejects non-macOS hosts. +The earlier `CLOCK_THREAD_CPUTIME_ID` implementation returned unscaled ticks +under Rosetta on the measured M4 host, despite its nanosecond API. Both native +and translated builds now use the explicit-unit Mach counters; do not interpret +old Rosetta clock samples as nanoseconds. For the final cold-admission A/B experiment, use the dedicated [materialization harness](../SharpEmu.Tools.MaterializationBench/README.md), whose From 4b6bad96b38cabfc82c936efd0f561572b02deb9 Mon Sep 17 00:00:00 2001 From: kuba Date: Tue, 27 Oct 2026 22:49:40 +0100 Subject: [PATCH 42/82] [GPU] Enable bounded vertex layout history with adaptive miss backoff --- docs/performance/performance-defaults.md | 1 + docs/performance/vertex-layout-history.md | 156 +++++++++++ .../Gpu/Pipelines/VertexInputResolver.cs | 52 +++- .../Gpu/Pipelines/VertexLayoutHistory.cs | 163 +++++++++++ .../Gpu/Pipelines/VertexInputResolverTests.cs | 2 +- .../VertexLayoutHistoryBenchmarks.cs | 93 +++++++ .../Gpu/Pipelines/VertexLayoutHistoryTests.cs | 256 ++++++++++++++++++ 7 files changed, 716 insertions(+), 7 deletions(-) create mode 100644 docs/performance/vertex-layout-history.md create mode 100644 src/SharpEmu.Libs/Gpu/Pipelines/VertexLayoutHistory.cs create mode 100644 tests/SharpEmu.Libs.Tests/Gpu/Pipelines/VertexLayoutHistoryBenchmarks.cs create mode 100644 tests/SharpEmu.Libs.Tests/Gpu/Pipelines/VertexLayoutHistoryTests.cs diff --git a/docs/performance/performance-defaults.md b/docs/performance/performance-defaults.md index 7c5298901..2b85bbe58 100644 --- a/docs/performance/performance-defaults.md +++ b/docs/performance/performance-defaults.md @@ -20,6 +20,7 @@ VSync, guest quality, or unrelated application setting is changed. | Audio queue query leaf dispatch | `SHARPEMU_AUDIO_OUT2_QUEUE_LEAF` | Synthetic dispatch about 369 → 84.5 ns and 40 → 0 B/call; guest audio thread remains busy. Result/error and shutdown tests retain full-path behavior. | | Asynchronous command-item reuse | `SHARPEMU_COMMAND_ITEM_REUSE` | Native two-thread median 389.965 → 354.5175 ns; allocation 80.0008 → 0.026 B/item. Bounded 64-slot return ring; synchronous and failed items retain original ownership. No accepted runtime FPS gain. | | Queued CP writes | `SHARPEMU_METAL_V2_QUEUE_CP_WRITES` | Removes synchronous handoffs for eligible writes when command threading is explicitly enabled. Pending ranges, prefetch gates and CPU-read fallbacks preserve ordering; producer-thread device tests cover byte/qword/sparse/predication cases. | +| Bounded vertex-layout history | `SHARPEMU_VERTEX_LAYOUT_HISTORY` | Exact byte-verified reuse across alternating layouts; adaptive backoff for low reuse, 512 slots and 2 MiB additional payload budget. Component timing/allocation evidence and miss costs are in [the measurements](vertex-layout-history.md); no isolated game-FPS claim. | The existing negative label page filter, exact materialization comparison, dense register lookup, mip-layout construction and target memo fixes remain enabled. diff --git a/docs/performance/vertex-layout-history.md b/docs/performance/vertex-layout-history.md new file mode 100644 index 000000000..64cc99eb4 --- /dev/null +++ b/docs/performance/vertex-layout-history.md @@ -0,0 +1,156 @@ + + +# Bounded vertex-layout history + +`SHARPEMU_VERTEX_LAYOUT_HISTORY` is enabled when unset. Set it to `0` to retain +only the previous last-layout memo. This is a component-benchmark acceptance; +no new game-FPS result or 30-FPS claim accompanies it. The nine earlier accepted +performance settings remain enabled, including component wins without an +isolated Silent Hill FPS improvement. + +## Behavior and ownership + +Alternating meshes previously decoded and allocated layout arrays whenever a +shader's current tables differed from its immediately preceding draw. The new +history reuses older arrays only after rereading guest attribute and buffer tables +and checking shader metadata identity, hash, and every table byte. A hash is only +a candidate selector. Moving equal tables can reuse their layout; modifying bytes, +changing shader identity, or losing memory readability cannot return stale data. +Each draw still gets its own mutable `VertexInputInfo` wrapper. Retained arrays +follow the existing immutable-layout contract and are never pooled or overwritten. +The pipeline's existing resolve lock owns the cache. + +There are 512 direct-mapped slots and a conservative 2 MiB retained-payload budget, +plus the approximately 12 KiB slot array. This limits the additional history, +not the pre-existing per-shader last-layout memos. History is allocated lazily and +cleared when the 4,096-entry shader-metadata cache is cleared. Each metadata entry +adds 24 bytes of hash/backoff fields on the measured runtimes. A successful cold +table parse now avoids a redundant metadata object; empty and failed parses +preserve the public non-null out-parameter contract. + +After 32 changed-layout probes, fewer than 16 history hits suppress the next 4,096 +history probes for that shader. Fresh guest reads and last-layout comparisons +continue. Probation then resumes, allowing a later repeating working set to +recover. This trades delayed recovery after churn for much lower miss overhead. +`SHARPEMU_PROFILE_VERTEX_LAYOUTS=1` reports hits, decodes, refusals, suppression, +entries, retained bytes and evictions every 4,096 lookups; diagnostics remain off +by default. + +## Measurement method + +The optional `VertexInputResolverTests.BenchmarkVertexLayoutHistory` exercises +real table reads, decoding, buffer merging and wrapper construction over fake +guest memory. Both modes run in one process. It warms the first scenario for ten +seconds and later scenarios for five seconds, then measures nine alternating +rounds of 200,000 resolutions per mode. Checksums verify stream addresses and +semantic mappings. Normal .NET 10.0.3 tiering is retained. Managed allocation and +CPU time count only the calling thread; elapsed time is recorded separately. +These measurements exclude native rendering and are not frame-time estimates. + +Scenarios cover one unchanged layout, cycles of 2/8/64/1,024 layouts, a descriptor +that changes every operation, and a new shader identity every operation. The +latter intentionally stresses cold metadata creation and bounded-cache clearing. +The default and explicit `0` path use the public constructor; no timed environment +changes occur. The full-suite benchmark still runs serially with other tests. + +The preserved pre-change library has SHA-256 +`b487a9ed68a6424e3d8959bd038c66805046951aebc5bdcf107054695338e148`. +Frozen 1,444-file test payloads were identical except for `SharpEmu.Libs.dll`. +A baseline run between two candidate runs confirmed repeated-layout savings and +exposed the redundant cold metadata allocation. Comparing separate runs remains +subject to host/core/frequency drift; even the unchanged last-layout path moved. +Earlier short-warmup measurements included tier transitions and are not acceptance +numbers. The first history design also penalized miss-heavy workloads; adaptive +backoff was added before promotion. + +## Final measurements + +Median nanoseconds per resolution, history disabled → enabled. Each cell uses +all nine rounds. All rows, including slowdowns and bypass controls, are retained. + +| Workload | ARM64 CPU ns | x64 CPU ns | ARM64 elapsed ns | x64 elapsed ns | B/op, both architectures | +| --- | ---: | ---: | ---: | ---: | ---: | +| Unchanged last layout | 126.35 → 116.91 | 88.53 → 89.45 | 249.57 → 227.78 | 88.59 → 89.49 | 216.00 → 216.00 | +| Cycle of 2 | 214.25 → 122.31 | 193.78 → 109.91 | 240.18 → 137.42 | 193.82 → 110.04 | 488.00 → 216.00 | +| Cycle of 8 | 249.32 → 148.12 | 191.52 → 108.61 | 358.67 → 195.95 | 192.55 → 109.32 | 488.00 → 216.00 | +| Cycle of 64 | 243.22 → 144.08 | 202.94 → 127.16 | 257.22 → 145.73 | 207.38 → 128.92 | 488.00 → 233.00 | +| Cycle of 1,024 | 270.69 → 270.91 | 199.28 → 209.76 | 308.18 → 483.72 | 200.74 → 212.99 | 488.00 → 487.46 | +| Always changing | 284.18 → 298.10 | 195.99 → 197.81 | 379.95 → 468.83 | 196.06 → 198.15 | 488.00 → 488.00 | +| Cold shader identity | 456.40 → 464.73 | 419.38 → 404.39 | 602.64 → 761.45 | 427.21 → 412.68 | 584.00 → 584.00 | + +Cycles of 2–64 reduce calling-thread CPU medians by **37–43%** and save +**255–272 B/resolution**. Larger working sets and unique descriptors establish +no speed win: CPU median changes in the final paired runs span approximately +**+0.1% to +5.3%**, with substantial per-round variation. Their elapsed medians +can be much worse under host contention, as the table shows. The unchanged-layout +and cold cases also drift although the history is not queried there. Acceptance +therefore rests on repeated-layout wins, bounded miss work and correctness, not a +claim that every workload improves. + +The native return run before the cold-parser cleanup measured 118.99 → 70.56 ns +for two layouts, 118.61 → 77.01 ns for 64, 121.39 → 122.20 ns for 1,024, and +126.63 → 128.96 ns for unique descriptors. The baseline library measured about +115–117 ns for cycling/changing cases in the intervening run. This shows a small +common-path cost can exist even with the new history disabled; the kill switch +is a functional reference, not a promise of byte-identical old-code timing. + +The final cold path allocates **584 B/resolution**, compared with **600 B** in +the preserved pre-change library and **648 B** before removing the redundant +metadata allocation. This is cold shader metadata/layout resolution, not the +separate resource-cache benchmark. Sequential CPU timings do not establish a +cold-path speedup. The final native run began at a host load average of about +18.7; host conditions +were not controlled by closing applications or changing GeForce NOW. + +## CPU clock correction + +The original optional CPU timer used `CLOCK_THREAD_CPUTIME_ID`. A native C busy-loop +control found that both its nanosecond and timespec APIs returned unscaled ticks +under Rosetta on this host. The same interval reported 3,459,532 purported ns but +144,153,000 ns through Mach thread counters, versus 148,448,000 ns elapsed. +Native ARM64 reported 261,802,584 ns versus 261,807,000 ns through Mach. + +The benchmark tool and this fixture now share `DarwinThreadCpuClock`, using +`THREAD_BASIC_INFO` seconds/microseconds on both architectures. It borrows the +pthread Mach port and creates no send-right leak. Old Rosetta clock samples must +not be interpreted as nanoseconds. The published dense-register CPU comparisons +ran natively on ARM64 and are unaffected. CPU timing excludes descheduling and +other threads, but still reflects JIT, cache, core-placement and frequency changes. + +## Reproduction + +```sh +SHARPEMU_VERTEX_LAYOUT_BENCH_OUTPUT=/absolute/path/layout-results.json \ + dotnet test tests/SharpEmu.Libs.Tests/SharpEmu.Libs.Tests.csproj -c Release \ + --filter FullyQualifiedName~VertexInputResolverTests \ + --settings docs/performance/shader-serial.runsettings \ + --blame-hang-timeout 180s +``` + +Use the native or x64 .NET host explicitly and the shared validation lock on this +machine. Raw rounds, full commands, source/payload hashes and TRX files are retained +under `artifacts/runtime-diagnosis-20261026/vertex-history-*`. + +## Final validation + +- Full serial Release Libs suite on x64: **9,084 passed, 0 failed, 5 existing + Opus skips**; test phase 2m45s. Performance switches were unset, with only the + optional benchmark output requested. +- Final focused ARM64 resolver/pipeline suite: **109 passed, 0 failed, 0 skipped**, + including the optional benchmark; test phase 52s. These overlap the full-suite + cases and are configuration checks, not 109 additional distinct tests. +- Coverage includes the public unset/1/0 selection, unchanged retained arrays, + fresh reads on history hits, unreadable-table refusal, moving descriptors, + shader identity, forced same-slot ownership/hash collisions, byte/entry bounds, + backoff recovery, metadata-cache clearing and non-null empty/error outputs. +- Release benchmark-tool build passed with four existing platform-analysis warnings. + Native and Rosetta PM4 timing smoke runs each produced 15 positive Mach CPU + intervals bounded by elapsed time. All 252 final layout benchmark intervals + were also positive and bounded by elapsed time. +- REUSE passed **2,874/2,874 files** and whitespace checks passed. + +No game was launched or new application bundle published for this follow-up. +The tests and component measurements do not demonstrate a Silent Hill FPS gain, +and **30 FPS remains unachieved**. GeForce NOW and other application settings +were unchanged. The PR remains draft and unmerged. diff --git a/src/SharpEmu.Libs/Gpu/Pipelines/VertexInputResolver.cs b/src/SharpEmu.Libs/Gpu/Pipelines/VertexInputResolver.cs index e07048471..b38a4acb1 100644 --- a/src/SharpEmu.Libs/Gpu/Pipelines/VertexInputResolver.cs +++ b/src/SharpEmu.Libs/Gpu/Pipelines/VertexInputResolver.cs @@ -19,6 +19,8 @@ public sealed class VertexTableMetadata // The attributes and buffers decoded from the tables the guest held at the last draw, with the // bytes they were decoded from. Replaced as a whole, never mutated. internal VertexLayoutMemo? Layout; + internal ulong LayoutHash; + internal int HistoryProbes, HistoryHits, HistorySkipsRemaining; } // A draw's decoded vertex layout and the guest bytes it came from: the attribute table words from the lowest @@ -32,12 +34,18 @@ internal sealed class VertexLayoutMemo(byte[] attributeBytes, byte[] bufferBytes public readonly byte[] BufferBytes = bufferBytes; public readonly VertexAttributeResource[] Attributes = attributes; public readonly VertexInputBuffer[] Buffers = buffers; + // Conservative object/array overhead and eight-byte alignment, including empty arrays. + internal long RetainedBytes => 64 + 4 * 32 + Align(AttributeBytes.LongLength) + Align(BufferBytes.LongLength) + + Align((long)Attributes.Length * System.Runtime.CompilerServices.Unsafe.SizeOf()) + + Align((long)Buffers.Length * System.Runtime.CompilerServices.Unsafe.SizeOf()); + + private static long Align(long bytes) => (bytes + 7) & ~7L; } // The header-derived tables of vertex programs already read, keyed by the registered header // and the code hash so a different binary at the same address never reuses an entry. -// Used under the pipeline cache's resolve lock; entries are never mutated after they are stored. -public sealed class VertexTableMetadataCache +// Header metadata stays fixed; the layout memo and history are maintained under that lock. +public sealed partial class VertexTableMetadataCache { private const int MaxEntries = 4096; private readonly Dictionary<(RegisteredShader Shader, ulong CodeHash), VertexTableMetadata> _entries = new(); @@ -50,6 +58,7 @@ internal void Add(RegisteredShader shader, ulong codeHash, VertexTableMetadata m if (_entries.Count >= MaxEntries) { _entries.Clear(); + _history?.Clear(); } _entries[(shader, codeHash)] = metadata; @@ -102,8 +111,9 @@ public static VertexInputInfo ResolveVertexInputs(CpuContext context, Registered } if (metadataCache is null || - !TryResolveLayout(context, metadata, attributeTable, bufferTable, shader.CodeAddress, out attributes, out buffers)) + !TryResolveLayout(context, metadataCache, metadata, attributeTable, bufferTable, shader.CodeAddress, out attributes, out buffers)) { + if (metadataCache is { CollectStatistics: true }) metadataCache.RecordLookup(VertexLayoutLookup.Refused); // Per-thread scratch: the list never outlives the call; the draw keeps the array. var resources = _attributeScratch ??= new List(VertexInputInfo.MaxBuffers); resources.Clear(); @@ -163,7 +173,17 @@ private sealed class ResolvedBuffer // Reads the direct resource registers and the input semantics; false with the reason when invalid. public static bool TryReadTables(CpuContext context, RegisteredShader shader, uint maxUserScalarRegisters, out VertexTableMetadata metadata, out string error) { - metadata = new VertexTableMetadata(); + var valid = TryReadTablesCore(context, shader, maxUserScalarRegisters, out var parsed, out error); + // Keep the public non-null out contract on empty/error paths without + // allocating a throwaway metadata object before every successful parse. + metadata = parsed ?? new VertexTableMetadata(); + return valid; + } + + private static bool TryReadTablesCore(CpuContext context, RegisteredShader shader, uint maxUserScalarRegisters, + out VertexTableMetadata? metadata, out string error) + { + metadata = null; if (shader.UserDataAddress == 0) { error = "the header has no user data block"; @@ -379,9 +399,10 @@ private static void AddResource(ShaderInputSemantic semantic, uint attribute, Re // The tables of a draw in two guest reads, one over the attribute words the program names and one over the buffer // descriptors they index, instead of two reads per attribute (each taking the memory read lock). A draw whose // bytes equal the last draw's gets that draw's attributes and buffers back without decoding or - // allocating them; any other byte decodes afresh, so a guest that rewrites a table is seen on the next draw. + // allocating them. The bounded history also requires an exact match of freshly read bytes. + // Any other byte decodes afresh, so a guest rewrite is seen on the next draw. // False leaves the word-by-word path to decode the draw, and to report whatever stopped this one. - private static bool TryResolveLayout(CpuContext context, VertexTableMetadata metadata, ulong attributeTable, ulong bufferTable, + private static bool TryResolveLayout(CpuContext context, VertexTableMetadataCache cache, VertexTableMetadata metadata, ulong attributeTable, ulong bufferTable, ulong shaderAddress, out VertexAttributeResource[] attributes, out VertexInputBuffer[] buffers) { attributes = []; @@ -441,9 +462,26 @@ private static bool TryResolveLayout(CpuContext context, VertexTableMetadata met { attributes = memo.Attributes; buffers = memo.Buffers; + if (cache.CollectStatistics) cache.RecordLookup(VertexLayoutLookup.LastHit); return true; } + ulong contentHash = 0; + if (memo is not null && cache.ReuseLayouts && cache.ShouldProbeHistory(metadata)) + { + var recalled = cache.FindPrevious(metadata, attributeBytes, bufferBytes, out contentHash); + cache.Remember(metadata, memo, metadata.LayoutHash); + if (recalled is not null) + { + metadata.Layout = recalled; + metadata.LayoutHash = contentHash; + attributes = recalled.Attributes; + buffers = recalled.Buffers; + if (cache.CollectStatistics) cache.RecordLookup(VertexLayoutLookup.HistoryHit); + return true; + } + } + var resources = _attributeScratch ??= new List(VertexInputInfo.MaxBuffers); resources.Clear(); Span descriptorWords = stackalloc uint[BufferDescriptorBytes / sizeof(uint)]; @@ -465,6 +503,8 @@ private static bool TryResolveLayout(CpuContext context, VertexTableMetadata met attributes = resources.ToArray(); resources.Clear(); metadata.Layout = new VertexLayoutMemo(attributeBytes.ToArray(), bufferBytes.ToArray(), attributes, buffers); + metadata.LayoutHash = contentHash; + if (cache.CollectStatistics) cache.RecordLookup(VertexLayoutLookup.Decoded); return true; } diff --git a/src/SharpEmu.Libs/Gpu/Pipelines/VertexLayoutHistory.cs b/src/SharpEmu.Libs/Gpu/Pipelines/VertexLayoutHistory.cs new file mode 100644 index 000000000..cf011c7e3 --- /dev/null +++ b/src/SharpEmu.Libs/Gpu/Pipelines/VertexLayoutHistory.cs @@ -0,0 +1,163 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using System.IO.Hashing; +using System.Numerics; +using System.Runtime.CompilerServices; + +namespace SharpEmu.Libs.Gpu.Pipelines; + +// A bounded history of immutable CPU layouts. Guest tables must be reread before +// probing, and hashes only select candidates: ownership and every byte are checked. +// Like the metadata cache, this is owned by ShaderPipelineCache's resolve lock. +internal sealed class VertexLayoutHistory +{ + internal const int Capacity = 512; + internal const long ByteBudget = 2 * 1024 * 1024; + private readonly Slot[] _slots = new Slot[Capacity]; + private int _evictionCursor; + internal int Count { get; private set; } + internal long RetainedBytes { get; private set; } + internal long Evictions { get; private set; } + + private struct Slot + { + internal VertexTableMetadata? Owner; + internal VertexLayoutMemo? Layout; + internal ulong Hash; + } + + internal static ulong Hash(ReadOnlySpan attributes, ReadOnlySpan buffers) => + XxHash3.HashToUInt64(attributes) ^ BitOperations.RotateLeft(XxHash3.HashToUInt64(buffers), 17); + + private static int Index(VertexTableMetadata owner, ulong hash) => + (int)((hash ^ (uint)RuntimeHelpers.GetHashCode(owner)) & (Capacity - 1)); + + internal VertexLayoutMemo? Find(VertexTableMetadata owner, ReadOnlySpan attributes, + ReadOnlySpan buffers, out ulong hash) + { + hash = Hash(attributes, buffers); + ref var slot = ref _slots[Index(owner, hash)]; + var layout = slot.Layout; + return ReferenceEquals(slot.Owner, owner) && layout is not null && slot.Hash == hash && + attributes.SequenceEqual(layout.AttributeBytes) && buffers.SequenceEqual(layout.BufferBytes) + ? layout : null; + } + + internal void Remember(VertexTableMetadata owner, VertexLayoutMemo layout, ulong hash) + { + var bytes = layout.RetainedBytes; + if (bytes > ByteBudget) return; + if (hash == 0) hash = Hash(layout.AttributeBytes, layout.BufferBytes); + var index = Index(owner, hash); + ref var slot = ref _slots[index]; + if (ReferenceEquals(slot.Owner, owner) && ReferenceEquals(slot.Layout, layout)) return; + Evict(index); + while (RetainedBytes + bytes > ByteBudget) + { + Evict(_evictionCursor); + _evictionCursor = (_evictionCursor + 1) & (Capacity - 1); + } + slot.Owner = owner; + slot.Layout = layout; + slot.Hash = hash; + RetainedBytes += bytes; + Count++; + } + + private void Evict(int index) + { + ref var slot = ref _slots[index]; + if (slot.Layout is not { } layout) return; + RetainedBytes -= layout.RetainedBytes; + Count--; + Evictions++; + slot = default; + } + + internal void Clear() + { + Array.Clear(_slots); + Count = 0; + RetainedBytes = 0; + _evictionCursor = 0; + } +} + +internal enum VertexLayoutLookup { LastHit, HistoryHit, Decoded, Refused } + +internal readonly record struct VertexLayoutStatistics(long Queries, long LastHits, long HistoryHits, + long Decoded, long Refused, int Entries, long RetainedBytes, long Evictions, long Suppressed); + +public sealed partial class VertexTableMetadataCache +{ + private VertexLayoutHistory? _history; + internal readonly bool ReuseLayouts; + internal readonly bool CollectStatistics; + private readonly bool _logStatistics; + private long _queries, _lastHits, _historyHits, _decoded, _refused, _suppressed; + + // Component-validated default; =0 retains only the existing last-layout memo. + public VertexTableMetadataCache() : this( + Environment.GetEnvironmentVariable("SHARPEMU_VERTEX_LAYOUT_HISTORY") != "0", + Environment.GetEnvironmentVariable("SHARPEMU_PROFILE_VERTEX_LAYOUTS") == "1") + { + _logStatistics = CollectStatistics; + if (_logStatistics) + Console.Error.WriteLine($"[VERTEX-LAYOUT] enabled={(ReuseLayouts ? 1 : 0)} capacity={VertexLayoutHistory.Capacity} budget={VertexLayoutHistory.ByteBudget}"); + } + + internal VertexTableMetadataCache(bool reuseLayouts, bool collectStatistics = false) + { + ReuseLayouts = reuseLayouts; + CollectStatistics = collectStatistics; + } + + // A shader with little reuse probes for 32 changed layouts, then skips 4096. + // Probation resumes automatically so a later stable working set can recover. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal bool ShouldProbeHistory(VertexTableMetadata owner) + { + if (owner.HistorySkipsRemaining == 0) return true; + owner.HistorySkipsRemaining--; + if (CollectStatistics) _suppressed++; + return false; + } + + internal VertexLayoutMemo? FindPrevious(VertexTableMetadata owner, ReadOnlySpan attributes, + ReadOnlySpan buffers, out ulong hash) + { + var result = (_history ??= new()).Find(owner, attributes, buffers, out hash); + owner.HistoryProbes++; + if (result is not null) owner.HistoryHits++; + if (owner.HistoryProbes == 32) + { + if (owner.HistoryHits < 16) owner.HistorySkipsRemaining = 4096; + owner.HistoryProbes = owner.HistoryHits = 0; + } + return result; + } + + internal void Remember(VertexTableMetadata owner, VertexLayoutMemo layout, ulong hash) => _history!.Remember(owner, layout, hash); + + internal VertexLayoutStatistics Statistics => new(_queries, _lastHits, _historyHits, _decoded, _refused, + _history?.Count ?? 0, _history?.RetainedBytes ?? 0, _history?.Evictions ?? 0, _suppressed); + + internal void RecordLookup(VertexLayoutLookup result) + { + if (!CollectStatistics) return; + _queries++; + switch (result) + { + case VertexLayoutLookup.LastHit: _lastHits++; break; + case VertexLayoutLookup.HistoryHit: _historyHits++; break; + case VertexLayoutLookup.Decoded: _decoded++; break; + case VertexLayoutLookup.Refused: _refused++; break; + } + if (_logStatistics && (_queries & 4095) == 0) + { + var stats = Statistics; + Console.Error.WriteLine($"[VERTEX-LAYOUT] enabled={(ReuseLayouts ? 1 : 0)} queries={stats.Queries} last_hits={stats.LastHits} history_hits={stats.HistoryHits} decoded={stats.Decoded} refused={stats.Refused} entries={stats.Entries} bytes={stats.RetainedBytes} evictions={stats.Evictions} suppressed={stats.Suppressed}"); + } + } +} diff --git a/tests/SharpEmu.Libs.Tests/Gpu/Pipelines/VertexInputResolverTests.cs b/tests/SharpEmu.Libs.Tests/Gpu/Pipelines/VertexInputResolverTests.cs index 63b750e89..3ae33da3b 100644 --- a/tests/SharpEmu.Libs.Tests/Gpu/Pipelines/VertexInputResolverTests.cs +++ b/tests/SharpEmu.Libs.Tests/Gpu/Pipelines/VertexInputResolverTests.cs @@ -12,7 +12,7 @@ namespace SharpEmu.Libs.Tests.Gpu.Pipelines; // The vertex tables: registers from the header's direct resources, attributes by semantic, buffers merged by stream. [Collection(SchedulingStateCollection.Name)] -public sealed class VertexInputResolverTests : IDisposable +public sealed partial class VertexInputResolverTests : IDisposable { private const ulong MemoryBase = 0x1_0000_0000; private const ulong ShaderAddress = MemoryBase + 0x100; diff --git a/tests/SharpEmu.Libs.Tests/Gpu/Pipelines/VertexLayoutHistoryBenchmarks.cs b/tests/SharpEmu.Libs.Tests/Gpu/Pipelines/VertexLayoutHistoryBenchmarks.cs new file mode 100644 index 000000000..6df6441b7 --- /dev/null +++ b/tests/SharpEmu.Libs.Tests/Gpu/Pipelines/VertexLayoutHistoryBenchmarks.cs @@ -0,0 +1,93 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using System.Buffers.Binary; +using System.Diagnostics; +using System.Runtime.InteropServices; +using System.Text.Json; +using SharpEmu.Libs.Gpu.Pipelines; +using SharpEmu.PerfAudit; +using Xunit; + +namespace SharpEmu.Libs.Tests.Gpu.Pipelines; + +public sealed partial class VertexInputResolverTests +{ + // Optional paired component benchmark. No elapsed-time assertions or game-FPS claims. + [Fact] + public void BenchmarkVertexLayoutHistory() + { + var output = Environment.GetEnvironmentVariable("SHARPEMU_VERTEX_LAYOUT_BENCH_OUTPUT"); + if (string.IsNullOrEmpty(output)) return; + var (shader, userData) = PrepareHistoryFixture(); + for (uint index = 0; index < 1024; index++) + { + var address = MemoryBase + 0x2000 + index * 16; + WriteWord(address, unchecked((uint)(StreamBase + index * 256))); + WriteWord(address + 4, (uint)(StreamBase >> 32) | (16u << 16)); + WriteWord(address + 8, index + 3); + WriteWord(address + 12, IdentitySelect | (BufferFormat32x4Float << 12)); + } + const int iterations = 200_000; + var results = new List(); + foreach (var scenario in new[] { "last", "cycle2", "cycle8", "cycle64", "cycle1024", "unique", "cold" }) + { + var variants = scenario switch { "cycle2" => 2, "cycle8" => 8, "cycle64" => 64, "cycle1024" => 1024, _ => 1 }; + VertexTableMetadataCache CreateCache(bool enabled) + { + var prior = Environment.GetEnvironmentVariable("SHARPEMU_VERTEX_LAYOUT_HISTORY"); + try + { + Environment.SetEnvironmentVariable("SHARPEMU_VERTEX_LAYOUT_HISTORY", enabled ? "1" : "0"); + return new VertexTableMetadataCache(); + } + finally { Environment.SetEnvironmentVariable("SHARPEMU_VERTEX_LAYOUT_HISTORY", prior); } + } + var caches = new[] { CreateCache(false), CreateCache(true) }; + long sequence = 0; + (double WallNs, double CpuNs, double Bytes, ulong Checksum) Run(int arm, int count) + { + Span record = stackalloc byte[4]; + ulong checksum = 0; + var bytesBefore = GC.GetAllocatedBytesForCurrentThread(); + var start = Stopwatch.GetTimestamp(); + var cpuStart = OperatingSystem.IsMacOS() ? DarwinThreadCpuClock.Nanoseconds() : 0; + for (var index = 0; index < count; index++) + { + var variant = index & (variants - 1); + userData[10] = (uint)(0x2000 + variant * 16); + if (scenario == "unique") + { + BinaryPrimitives.WriteUInt32LittleEndian(record, (uint)++sequence); + if (!_memory.TryWrite(MemoryBase + 0x2008, record)) throw new InvalidOperationException(); + } + var hash = scenario == "cold" ? (ulong)++sequence : 1; + var resolved = VertexInputResolver.ResolveVertexInputs(_context, shader, userData, + metadataCache: caches[arm], codeHash: hash); + checksum += resolved.Buffers[0].Address + (uint)resolved.Attributes[1].RegisterStart; + } + var cpuEnd = OperatingSystem.IsMacOS() ? DarwinThreadCpuClock.Nanoseconds() : 0; + var elapsed = Stopwatch.GetElapsedTime(start).TotalNanoseconds; + var allocated = GC.GetAllocatedBytesForCurrentThread() - bytesBefore; + return (elapsed / count, (cpuEnd - cpuStart) / (double)count, allocated / (double)count, checksum); + } + // Warm both configurations with ordinary tiering before alternating order. + var warmup = Stopwatch.StartNew(); + do { Run(0, iterations); Run(1, iterations); } while (warmup.ElapsedMilliseconds < (scenario == "last" ? 10_000 : 5_000)); + for (var round = 0; round < 9; round++) + { + foreach (var arm in round % 2 == 0 ? new[] { 0, 1 } : new[] { 1, 0 }) + { + var sample = Run(arm, iterations); + ulong expected = 0; + for (var index = 0; index < iterations; index++) expected += StreamBase + (ulong)(index & (variants - 1)) * 256 + 4; + Assert.Equal(expected, sample.Checksum); + results.Add(new { scenario, enabled = arm == 1, round, iterations, + wall_ns = sample.WallNs, cpu_ns = sample.CpuNs, bytes = sample.Bytes, checksum = sample.Checksum }); + } + } + } + File.WriteAllText(output, JsonSerializer.Serialize(new { architecture = RuntimeInformation.ProcessArchitecture.ToString(), + framework = RuntimeInformation.FrameworkDescription, results }, new JsonSerializerOptions { WriteIndented = true })); + } +} diff --git a/tests/SharpEmu.Libs.Tests/Gpu/Pipelines/VertexLayoutHistoryTests.cs b/tests/SharpEmu.Libs.Tests/Gpu/Pipelines/VertexLayoutHistoryTests.cs new file mode 100644 index 000000000..d9aeb24c0 --- /dev/null +++ b/tests/SharpEmu.Libs.Tests/Gpu/Pipelines/VertexLayoutHistoryTests.cs @@ -0,0 +1,256 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using System.Runtime.CompilerServices; +using SharpEmu.HLE; +using SharpEmu.Libs.Gpu.Pipelines; +using SharpEmu.Libs.Gpu.Rendering; +using SharpEmu.Libs.Tests.Gpu.Scheduling; +using Xunit; + +namespace SharpEmu.Libs.Tests.Gpu.Pipelines; + +public sealed partial class VertexInputResolverTests +{ + private (RegisteredShader Shader, uint[] UserData) PrepareHistoryFixture() + { + var shader = RegisterTables(semanticCount: 2); + WriteWord(Semantics, SemanticWord(0, 0, 2)); + WriteWord(Semantics + 4, SemanticWord(1, 4, 2)); + WriteWord(AttributeTable, AttributeWord(0, AttributeFormat32x2Float, 0)); + WriteWord(AttributeTable + 4, AttributeWord(0, AttributeFormat32x2Float, 8)); + WriteBuffer(0, StreamBase, 16, 3); + var userData = new uint[12]; + userData[8] = unchecked((uint)AttributeTable); + userData[9] = (uint)(AttributeTable >> 32); + userData[10] = unchecked((uint)BufferTable); + userData[11] = (uint)(BufferTable >> 32); + return (shader, userData); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void AlternatingLayoutsPreserveSnapshotsAndReuseOnlyWhenEnabled(bool reuse) + { + var (shader, userData) = PrepareHistoryFixture(); + var cache = new VertexTableMetadataCache(reuse, collectStatistics: true); + VertexInputInfo Resolve() => VertexInputResolver.ResolveVertexInputs(_context, shader, userData, metadataCache: cache, codeHash: 1); + var first = Resolve(); + WriteBuffer(0, StreamBase + 0x100, 16, 3); + var second = Resolve(); + WriteBuffer(0, StreamBase, 16, 3); + var returned = Resolve(); + Assert.NotSame(first, returned); + Assert.NotSame(first.Attributes, second.Attributes); + Assert.Equal(StreamBase + 0x100, second.Buffers[0].Address); + Assert.Equal(StreamBase, first.Buffers[0].Address); + Assert.Equal(first.Attributes, returned.Attributes); + Assert.Equal(first.Buffers, returned.Buffers); + if (reuse) + { + Assert.Same(first.Attributes, returned.Attributes); + Assert.Same(first.Buffers, returned.Buffers); + } + else Assert.NotSame(first.Attributes, returned.Attributes); + var repeated = Resolve(); + Assert.Same(returned.Attributes, repeated.Attributes); + Assert.Equal(reuse ? 1 : 0, cache.Statistics.HistoryHits); + Assert.Equal(reuse ? 2 : 3, cache.Statistics.Decoded); + Assert.Equal(1, cache.Statistics.LastHits); + } + + [Theory] + [InlineData(null, true)] + [InlineData("1", true)] + [InlineData("0", false)] + public void EnvironmentSelectionControlsReturningLayoutReuse(string? value, bool reuse) + { + var prior = Environment.GetEnvironmentVariable("SHARPEMU_VERTEX_LAYOUT_HISTORY"); + try + { + Environment.SetEnvironmentVariable("SHARPEMU_VERTEX_LAYOUT_HISTORY", value); + var (shader, userData) = PrepareHistoryFixture(); + var cache = new VertexTableMetadataCache(); + VertexInputInfo Resolve() => VertexInputResolver.ResolveVertexInputs(_context, shader, userData, metadataCache: cache, codeHash: 1); + var first = Resolve(); + WriteBuffer(0, StreamBase + 256, 16, 3); + Resolve(); + WriteBuffer(0, StreamBase, 16, 3); + Assert.Equal(reuse, ReferenceEquals(first.Attributes, Resolve().Attributes)); + } + finally { Environment.SetEnvironmentVariable("SHARPEMU_VERTEX_LAYOUT_HISTORY", prior); } + } + + [Fact] + public void EmptyAndFailedHeadersKeepNonNullDefaultMetadata() + { + var shader = RegisterTables(directCount: 0); + Assert.True(VertexInputResolver.TryReadTables(_context, shader, 32, out var empty, out _)); + Assert.NotNull(empty); + Assert.Equal(VertexTableMetadata.NoRegister, empty.VertexBufferRegister); + Assert.False(VertexInputResolver.TryReadTables(_context, shader with { UserDataAddress = 0 }, 32, out var failed, out _)); + Assert.NotNull(failed); + Assert.Empty(failed.InputSemantics); + } + + private sealed class ReadGate(ICpuMemory inner) : ICpuMemory + { + internal bool DenyTables; + internal int TableReads; + public bool TryRead(ulong address, Span destination) + { + var table = address >= AttributeTable && address < BufferTable + 16; + if (table) TableReads++; + return !(DenyTables && table) && inner.TryRead(address, destination); + } + public bool TryWrite(ulong address, ReadOnlySpan source) => inner.TryWrite(address, source); + } + + [Fact] + public void HistoryHitsRereadGuestTablesAndCannotHideAnUnreadableTable() + { + var (shader, userData) = PrepareHistoryFixture(); + var memory = new ReadGate(_memory); + var context = new CpuContext(memory, Generation.Gen5); + var cache = new VertexTableMetadataCache(true, collectStatistics: true); + VertexInputInfo Resolve() => VertexInputResolver.ResolveVertexInputs(context, shader, userData, metadataCache: cache, codeHash: 1); + var first = Resolve(); + WriteBuffer(0, StreamBase + 0x100, 16, 3); + Resolve(); + WriteBuffer(0, StreamBase, 16, 3); + memory.TableReads = 0; + Assert.Same(first.Attributes, Resolve().Attributes); + Assert.Equal(2, memory.TableReads); + memory.DenyTables = true; + Assert.Throws(() => Resolve()); + Assert.Equal(1, cache.Statistics.Refused); + } + + [Fact] + public void MovingEqualTablesReusesTheirLayoutButChangesAtTheNewAddressAreObserved() + { + var (shader, userData) = PrepareHistoryFixture(); + var cache = new VertexTableMetadataCache(true); + VertexInputInfo Resolve() => VertexInputResolver.ResolveVertexInputs(_context, shader, userData, metadataCache: cache, codeHash: 1); + var first = Resolve(); + WriteBuffer(0, StreamBase + 0x100, 16, 3); + Resolve(); + WriteBuffer(0, StreamBase, 16, 3); + Span bytes = stackalloc byte[16]; + Assert.True(_memory.TryRead(AttributeTable, bytes[..8])); + Assert.True(_memory.TryWrite(MemoryBase + 0x2000, bytes[..8])); + Assert.True(_memory.TryRead(BufferTable, bytes)); + Assert.True(_memory.TryWrite(MemoryBase + 0x2100, bytes)); + userData[8] = 0x2000; + userData[10] = 0x2100; + Assert.Same(first.Attributes, Resolve().Attributes); + WriteWord(MemoryBase + 0x2108, 7); + var changed = Resolve(); + Assert.NotSame(first.Attributes, changed.Attributes); + Assert.Equal(7u, changed.Buffers[0].RecordCount); + Assert.Equal(3u, first.Buffers[0].RecordCount); + } + + [Fact] + public void ANewCodeIdentityCannotRecallAnotherShadersSemanticMapping() + { + var (shader, userData) = PrepareHistoryFixture(); + var cache = new VertexTableMetadataCache(true); + var first = VertexInputResolver.ResolveVertexInputs(_context, shader, userData, metadataCache: cache, codeHash: 1); + WriteBuffer(0, StreamBase + 0x100, 16, 3); + VertexInputResolver.ResolveVertexInputs(_context, shader, userData, metadataCache: cache, codeHash: 1); + WriteBuffer(0, StreamBase, 16, 3); + WriteWord(Semantics, SemanticWord(0, 12, 2)); + var changed = VertexInputResolver.ResolveVertexInputs(_context, shader, userData, metadataCache: cache, codeHash: 2); + Assert.NotSame(first.Attributes, changed.Attributes); + Assert.Equal(12, changed.Attributes[0].RegisterStart); + Assert.Equal(0, first.Attributes[0].RegisterStart); + } + + [Fact] + public void HashCollisionsAndDifferentOwnersCannotSubstituteLayoutBytes() + { + var owner = new VertexTableMetadata(); + var history = new VertexLayoutHistory(); + byte[] wanted = [1, 2, 3, 4]; + byte[] buffers = [5, 6, 7, 8]; + var hash = VertexLayoutHistory.Hash(wanted, buffers); + var wrong = new VertexLayoutMemo([9, 2, 3, 4], buffers, [], []); + history.Remember(owner, wrong, hash); + Assert.Null(history.Find(owner, wanted, buffers, out _)); + var right = new VertexLayoutMemo(wanted, buffers, [], []); + history.Remember(owner, right, hash); + Assert.Same(right, history.Find(owner, wanted, buffers, out _)); + VertexTableMetadata? other = null; + for (var attempt = 0; attempt < 100_000; attempt++) + { + var candidate = new VertexTableMetadata(); + if ((RuntimeHelpers.GetHashCode(candidate) & (VertexLayoutHistory.Capacity - 1)) != + (RuntimeHelpers.GetHashCode(owner) & (VertexLayoutHistory.Capacity - 1))) continue; + other = candidate; + break; + } + Assert.NotNull(other); + Assert.Null(history.Find(other, wanted, buffers, out _)); + Assert.Null(history.Find(owner, wanted, [5, 6, 7, 9], out _)); + } + + [Fact] + public void HistoryBoundsRetentionAndRejectsAnOversizedEntry() + { + var history = new VertexLayoutHistory(); + var owner = new VertexTableMetadata(); + for (var index = 0; index < 2000; index++) + { + var bytes = new byte[8192]; + System.Buffers.Binary.BinaryPrimitives.WriteInt32LittleEndian(bytes, index); + history.Remember(owner, new VertexLayoutMemo(bytes, [], [], []), 0); + Assert.InRange(history.Count, 1, VertexLayoutHistory.Capacity); + Assert.InRange(history.RetainedBytes, 0, VertexLayoutHistory.ByteBudget); + } + Assert.True(history.Evictions > 0); + var before = (history.Count, history.RetainedBytes); + history.Remember(owner, new VertexLayoutMemo(new byte[VertexLayoutHistory.ByteBudget], [], [], []), 0); + Assert.Equal(before, (history.Count, history.RetainedBytes)); + history.Clear(); + Assert.Equal(0, history.Count); + Assert.Equal(0, history.RetainedBytes); + } + + [Fact] + public void LowReuseBacksOffAndLaterRepeatedLayoutsRecover() + { + var (shader, userData) = PrepareHistoryFixture(); + var cache = new VertexTableMetadataCache(true, collectStatistics: true); + VertexInputInfo Resolve() => VertexInputResolver.ResolveVertexInputs(_context, shader, userData, metadataCache: cache, codeHash: 1); + for (uint index = 0; index < 1100; index++) + { + WriteBuffer(0, StreamBase + index * 16, 16, 3); + Assert.Equal(StreamBase + index * 16, Resolve().Buffers[0].Address); + } + Assert.True(cache.Statistics.Suppressed >= 1024); + var hitsBefore = cache.Statistics.HistoryHits; + for (uint index = 0; index < 4500; index++) + { + var address = StreamBase + (index & 1) * 256; + WriteBuffer(0, address, 16, 3); + Assert.Equal(address, Resolve().Buffers[0].Address); + } + Assert.True(cache.Statistics.HistoryHits > hitsBefore + 50); + } + + [Fact] + public void MetadataLimitClearsTheHistoryAsWellAsHeaderEntries() + { + var (shader, userData) = PrepareHistoryFixture(); + var cache = new VertexTableMetadataCache(true, collectStatistics: true); + VertexInputResolver.ResolveVertexInputs(_context, shader, userData, metadataCache: cache, codeHash: 1); + WriteBuffer(0, StreamBase + 0x100, 16, 3); + VertexInputResolver.ResolveVertexInputs(_context, shader, userData, metadataCache: cache, codeHash: 1); + Assert.True(cache.Statistics.Entries > 0); + for (ulong hash = 2; hash <= 4097; hash++) cache.Add(shader, hash, new VertexTableMetadata()); + Assert.Equal(0, cache.Statistics.Entries); + Assert.Equal(0, cache.Statistics.RetainedBytes); + } +} From 500a1c1a09eccef58470f63ecf48aa7ed877b8ae Mon Sep 17 00:00:00 2001 From: kuba Date: Tue, 27 Oct 2026 23:12:56 +0100 Subject: [PATCH 43/82] docs(perf): record default-on runtime and occlusion drain evidence --- docs/performance/native-occlusion-runtime.md | 61 ++++++++++++++++++++ docs/performance/performance-defaults.md | 8 +++ 2 files changed, 69 insertions(+) diff --git a/docs/performance/native-occlusion-runtime.md b/docs/performance/native-occlusion-runtime.md index 4d9151081..0b278e2e7 100644 --- a/docs/performance/native-occlusion-runtime.md +++ b/docs/performance/native-occlusion-runtime.md @@ -1391,3 +1391,64 @@ existing skips each** on ARM64 and Rosetta x64. Launcher times were 13.794 and 51.073 seconds, including build/test startup. Commands and TRX files are retained under `pr23-alias-fixture-{arm64,x64}-validation.json` and corresponding result directories. No new production-code test claim is implied by this fixture fix. + +### v23 default-on vertex history and drain attribution + +The clean `4b6bad96b38cabfc82c936efd0f561572b02deb9` build passed the x64 CLI +and native ARM64 GPU-host publishes; the launcher verified all 471 payload +hashes. All ten promoted performance switches were unset, including +`SHARPEMU_VERTEX_LAYOUT_HISTORY`. Background program resolution remained on, +command threading and local draw reuse off. The existing application GC budget +was 64 MiB. The run used a fresh clone of the v20 warm seed, 1920×1080, scale 1 +and unchanged VSync/quality. GeForce NOW and its settings were untouched. +Vertex-layout counters and drain-caller counters were enabled; detailed render, +GPU-pass and memory profiling and shader prewarming were disabled. + +Run `20261027-225814-silent hill short message.zar-room-vertex-history-v23-on-1` +reached the same stationary shelf/lamp room, with screenshots confirming the +view before and during measurement. The **375–410-second** window was selected +at 193 seconds, before gameplay. Its **18 reports measured 1.0 FPS median, +0.0–4.49 range**. This is a poor absolute result, not an isolated comparison of +vertex history with its reference path. The launch load average was 36.09 on +the ten-core host. Seven system-memory samples during the FPS window recorded +304,736 swap-in and 310,316 swap-out pages, pressure levels 1 and 2, 27–112 MiB +free memory and 8,491–9,523 MiB swap used. Those are system-wide observations; +they neither attribute all paging to the emulator nor prove that paging alone +explains the frame rate. No 30-FPS result or runtime speedup is established. + +Layout snapshots at 372.595 and 409.655 seconds recorded 196,608 additional +queries: 72,170 last-layout hits, **12,645 history hits (6.43%)**, and 111,793 +decodes. The final snapshot held 512 entries / 190,560 accounted payload bytes; +110,787 changed-layout probes were suppressed by adaptive backoff during that +interval. This demonstrates real reuse and bounded retention, not an FPS gain. +The separate component results and full correctness validation remain in +[the vertex-history report](vertex-layout-history.md). + +The eleven PERF intervals ending at 377.792–408.475 seconds total 36.36 seconds; +their first interval begins before the FPS window. They recorded **31,565 +`DumpOcclusionCounter` drains out of 37,740 (83.6%)**, versus 4,424 global-barrier +drains. Program-prefetch wait totaled 28,478 ms, while resolver wall time totaled +2,442.5 ms. These overlapping wall-time counters are not exclusive CPU costs. +Explicit host GPU waits were zero in those intervals. The evidence directs the +next investigation toward frequent query-related drains and resolver handoff; +it does not justify removing query ordering. A deferred dump must still publish +the exact snapshot and prevent later operand or program-resolution reads from +using stale query bytes. No new query-deferral behavior was enabled here. + +After the FPS window, the ten-second thread sample measured 2.507 CPU-seconds +on the GPU worker, 0.953 on program resolution, 1.885 on guest RHI and 3.289 on +the audio event thread. The separate 427.273–438.048-second event capture had +six fully paired GC suspensions totaling 93.350 ms (24.339 ms maximum), plus +one unmatched restart at the capture boundary; the paired total is incomplete. +Neither later capture establishes per-frame costs in the FPS window. + +The stop marker was written at 438.106 seconds; the harness reported stopping +at 440 seconds, followed by four-second controlled termination and exit 0. +The guest CPU trap at 442.355 seconds followed the stop marker, matching the +unresolved teardown behavior in earlier captures. Peak main-process physical +footprint/RSS was 8,842/2,487 MiB, not combined process memory. Both game +processes ended. Private evidence includes `measurement-plan.json`, +`vertex-history-runtime-summary.json`, frame reports, screenshots, memory +telemetry and post-FPS CPU/event captures in the run directory; build source +and payload manifests are `vertex-history-v23-build-source.json` and +`vertex-history-v23-payload-hashes.json` in the runtime diagnostic directory. diff --git a/docs/performance/performance-defaults.md b/docs/performance/performance-defaults.md index 2b85bbe58..6054fd4b6 100644 --- a/docs/performance/performance-defaults.md +++ b/docs/performance/performance-defaults.md @@ -130,3 +130,11 @@ launcher had omitted the GC environment setting already supplied by the bundle and standard game launcher. The parity control reduced traced GC pause count and total pause time, but established no FPS gain. The nine promoted defaults remain enabled; the 30-FPS objective remains open. + +The later v23 capture used all ten promoted defaults, including vertex history. +It measured 1.0 FPS median (0.0–4.49, 18 reports) during heavy system paging; +there was no matched reference-path run, so this does not isolate a regression +or establish a gain. History reuse occurred in the game, while occlusion dumps +accounted for 83.6% of the sampled draw-queue drains. The runtime report retains +the exact source, windows, counters and limitations. The defaults retain their +component/test evidence; the 30-FPS objective remains open. From f2bd8c7a7717924d3252eb501f28ee7ef1ab2682 Mon Sep 17 00:00:00 2001 From: kuba Date: Wed, 28 Oct 2026 00:12:44 +0100 Subject: [PATCH 44/82] [GPU] Resolve short prefetch queues on the caller by default --- docs/performance/caller-resolution.md | 220 ++++++++++++++ docs/performance/native-occlusion-runtime.md | 13 + docs/performance/performance-defaults.md | 8 + .../Gpu/Rendering/ParallelProgramPrefetch.cs | 90 +++++- .../Gpu/Rendering/RenderExecutor.Prefetch.cs | 4 + .../V2/MetalRenderHost.ConcurrentResolve.cs | 12 +- .../MetalImageTransferTests.SrgbTwinViews.cs | 4 +- .../MetalImageTransferTests.TextureTiles.cs | 4 +- .../MetalImageTransferTests.VolumeViews.cs | 4 +- .../Scenes/CallerResolutionSceneTests.cs | 106 +++++++ ...elProgramPrefetchCallerTests.Benchmarks.cs | 111 +++++++ .../ParallelProgramPrefetchCallerTests.cs | 278 ++++++++++++++++++ .../Rendering/RenderExecutorTestSupport.cs | 2 + 13 files changed, 841 insertions(+), 15 deletions(-) create mode 100644 docs/performance/caller-resolution.md create mode 100644 tests/SharpEmu.Gpu.Metal.Tests/Scenes/CallerResolutionSceneTests.cs create mode 100644 tests/SharpEmu.Libs.Tests/Gpu/Rendering/ParallelProgramPrefetchCallerTests.Benchmarks.cs create mode 100644 tests/SharpEmu.Libs.Tests/Gpu/Rendering/ParallelProgramPrefetchCallerTests.cs diff --git a/docs/performance/caller-resolution.md b/docs/performance/caller-resolution.md new file mode 100644 index 000000000..1c44d7fd1 --- /dev/null +++ b/docs/performance/caller-resolution.md @@ -0,0 +1,220 @@ + + +# Caller resolution for short draw queues + +Frequent query and barrier drains can leave the render thread waiting for a +resolver thread to wake for only one or a few draws. The drawing thread can now +claim the untouched FIFO head when at most eight jobs remain. It refuses a +worker-owned job, an active resolution, a paused queue, an unmet dependency gate, +a stopped queue, or any other head. Longer queues retain background overlap. +Only one resolution runs at a time. The speculative read context, input checks, +ownership checks, exception policy and query ordering are retained. + +Metal enables this policy by default; set +`SHARPEMU_METAL_V2_CALLER_RESOLVE=0` to retain worker-only resolution. Other backends opt +in through `IConcurrentResolveHost.AllowsCallerResolution`; the interface default +is false. The `caller_jobs` and `caller_ms` counters account for work moved onto +the drawing thread. `resolve_ms` still measures background resolution, and +`wait_ms` excludes successful caller resolution. A shorter wait alone is not a +performance result. + +## Component measurements + +Apple M4, .NET 10.0.3, native ARM64 and Rosetta x64, Release, normal tiering. +The real prefetch/executor queue uses synthetic unavailable programs, without a +GPU. The caller and worker both use the native UserInteractive QoS class. +Seven alternating paired rounds follow a five-second initial warmup and +two-second warmups for later cases, with approximately 8,192 jobs per round. +The table reports paired-run medians, not differences between historical runs. +Whole-process CPU includes runtime activity; caller CPU alone would hide work +moved from the resolver. Allocation stays about 272 B/job in both modes. + +| Architecture | Added spins | Burst | Wall off ns/job | Wall on ns/job | Process CPU off ns/job | Process CPU on ns/job | Caller claims | +|---|---:|---:|---:|---:|---:|---:|---:| +| arm64 | 0 | 1 | 2874.1 | 542.1 | 4786.9 | 961.5 | 91.4% | +| arm64 | 0 | 5 | 732.1 | 508.7 | 1327.7 | 881.0 | 85.5% | +| arm64 | 0 | 32 | 329.3 | 356.2 | 654.5 | 691.2 | 0.0% | +| arm64 | 0 | 256 | 333.5 | 331.4 | 682.6 | 679.2 | 0.0% | +| arm64 | 512 | 1 | 26908.0 | 23027.0 | 41943.8 | 25838.3 | 100.0% | +| arm64 | 512 | 5 | 23084.3 | 23080.8 | 39583.4 | 25860.9 | 100.0% | +| arm64 | 512 | 32 | 21741.7 | 21812.6 | 38534.7 | 38630.0 | 0.0% | +| arm64 | 512 | 256 | 21546.5 | 21407.6 | 38444.2 | 38110.8 | 0.0% | +| x64 | 0 | 1 | 3160.1 | 972.2 | 5509.8 | 1710.6 | 88.4% | +| x64 | 0 | 5 | 917.2 | 889.0 | 1677.5 | 1605.3 | 65.8% | +| x64 | 0 | 32 | 541.4 | 526.3 | 1070.1 | 1039.4 | 0.0% | +| x64 | 0 | 256 | 427.8 | 403.3 | 874.6 | 824.1 | 0.0% | +| x64 | 512 | 1 | 45216.6 | 41632.8 | 71946.9 | 45248.0 | 100.0% | +| x64 | 512 | 5 | 41720.9 | 41654.9 | 71201.7 | 45246.4 | 99.9% | +| x64 | 512 | 32 | 40928.5 | 41014.9 | 71259.0 | 71510.5 | 0.0% | +| x64 | 512 | 256 | 40590.0 | 40811.4 | 71200.9 | 71170.3 | 0.0% | + +The single-job cases improve clearly. The empty-program ARM64 burst of 32 has +an 8.2% wall-time and 5.6% process-CPU cost (about 27 ns and 37 ns/job); this +cost is retained in the acceptance record. Larger/spinning batches are mostly +flat. An earlier uncapped implementation had mixed large-batch behavior and +was restricted to eight outstanding jobs. Host load changed substantially +between development versions, so their absolute timings are not code comparisons. + +To reproduce the component benchmark, set +`SHARPEMU_CALLER_RESOLVE_BENCH_OUTPUT` to an absolute JSON output path and run +`ParallelProgramPrefetchCallerTests` in `SharpEmu.Libs.Tests` with the serial +runsettings. The benchmark is dormant without that environment variable. + +## Correctness coverage + +Tests cover the FIFO completion prefix, the worker claim while a gate opens, +pause and stop refusal, the eight/nine-job boundary, indexed inputs and ownership +changes, exception policy restoration, 2,048-job competing claims, and a pause +while the caller resolves. Device tests check exact pixels and an earlier query +snapshot after later draws, plus speculative refusal of GPU-owned query bytes +without waiting on the GPU. Enabled, disabled and command-thread modes are +checked separately. + +The capped implementation passed the full x64 Libs suite (9,100 passed, +0 failed, five existing skips) and required-device ARM64 Metal suite (1,639 +passed, 0 failed, seven existing skips). The final early eligibility guard then +passed 31 focused tests on each architecture, plus 21 enabled, 21 disabled and +58 command-thread Metal checks. The final default-policy validation is recorded below. + +## Runtime comparison + +The v24 immutable payload was built from `500a1c1a09eccef58470f63ecf48aa7ed877b8ae` +plus tracked patch SHA-256 +`fa4bd3a6df60af685643ba4fe4de7a42ad3bae9f32bf3fb6f30ba1524be50e4d`. +Its 471 payload hashes are checked before each launch. Both arms use a fresh +clone of the same seed, 1920x1080, scale 1, VSync, unchanged quality, the ten +previously promoted defaults, parallel resolution, and a serial command stream. +Detailed render/pass/memory profiling and prewarming are disabled. Host load +and paging still limit causal attribution. GeForce NOW settings are unchanged. + +Enabled arm: `20261027-234802-silent hill short message.zar-room-caller-resolve-v24-on-1`. +The initially selected 250–285-second window still covered the intro and is not +room evidence. At elapsed 258 seconds, before gameplay, the window was moved to +375–410 seconds. The shelf/lamp room was visually verified before and during +that window. Its 18 presentation reports measured **16.49 FPS median, +11.98–18.98 range**. This remains below 30 FPS. + +Seventeen PERF reports ending in that window cover 34.35 seconds: 1,810,848 +resolutions, 341,756 caller jobs (18.9%), 984.7 ms caller resolution, +8,752.1 ms background resolution, 1,215.4 ms wait, 139 speculative abandons, +zero ownership/input refusals, 220.5 ms synchronous fallback and zero host-GPU +wait. These wall counters can overlap and must not be added as exclusive CPU +costs. GPU busy time was 21,100.8 ms. The later 417–427-second thread sample +recorded 8.051 CPU-seconds on the GPU worker, 3.053 on the resolver, 6.949 on +RHI and 9.823 on the guest audio event thread. + +Seven system-memory samples in the FPS window recorded 5,961 swap-in and +9,900 swap-out pages, 836,450 compressions and 778,447 decompressions; pressure +was 2, free memory 77–120 MiB, swap use 4,893–5,024 MiB. Launch load was 3.64, +far lower than v23's 36.09, so the v23-to-v24 FPS change is not an isolated code +win. Main-process peak footprint/RSS was 9,747/4,537 MiB, not combined process +memory. The stop marker was written at elapsed 427.160 seconds. CPU-trap status +appeared at 429.449 seconds during requested shutdown; launcher exit was zero. + +Disabled arm: `20261027-235523-silent hill short message.zar-room-caller-resolve-v24-off-2`. +The 375–410-second window was selected at elapsed 30 seconds. It replayed the +same intro-skip and phone-close timing, and the stationary room was verified +before and during measurement. Its 17 reports measured **16.48 FPS median, +14.00–17.93 range**. The enabled/disabled medians are effectively identical. + +Seventeen PERF reports cover 34.54 seconds: 1,835,475 resolutions, no caller +jobs, 9,954.4 ms background resolution, 4,233.4 ms wait, 299 speculative abandons, +zero ownership/input refusals, 5.2 ms synchronous fallback and zero host-GPU +wait. GPU busy time was 21,786.2 ms. Idle time rose from 1,934 ms disabled to +5,754 ms enabled while resolver wait fell; **this is not a game speedup**. +The later 417–427-second thread sample recorded 8.913 CPU-seconds on the GPU +worker, 3.270 on the resolver, 6.810 on RHI and 9.857 on the audio event thread. +A separate three-second native stack capture at 411 seconds was outside the +FPS window; managed JIT frames were not symbolized sufficiently to rank source +methods, so no source-level hotspot claim is made from it. + +Seven disabled-window memory samples recorded 9,665 swap-in and 6,924 swap-out +pages, 305,595 compressions and 382,481 decompressions; pressure was 1, free +memory 82–117 MiB, swap use 6,078–6,190 MiB. Launch load was 8.76. Main-process +peak footprint/RSS was 9,956/4,127 MiB. Its stop marker was elapsed 427.157 seconds; +CPU-trap status appeared at 428.632 seconds during requested shutdown, with +launcher exit zero. Neither run logged that trap before its stop marker. + +## Default decision + +Enable the bounded path based on its short-batch wall/process-CPU component +wins and ordering/device coverage, consistent with the maintainer's acceptance +of component improvements without an isolated Silent Hill gain. The ARM64 +burst-of-32 cost remains a limitation; this is not a universal speedup claim. +Both runtime arms rendered the intended room and had effectively identical +median FPS under different host paging conditions. All ten earlier accepted +switches remain enabled, bringing the documented promoted defaults to eleven. +No quality reduction or unrelated application setting is part of this change. +The 30-FPS goal is still unmet. + + +## Reproduction and evidence locations + +Run validations sequentially under the shared machine-wide game/test lock. +The final suite commands use Release, `--no-restore`, +`--settings docs/performance/shader-serial.runsettings`, +`--blame-hang-timeout 120s`, `-p:UseSharedCompilation=false -m:2 -nr:false`, +and TRX output. Run `tests/SharpEmu.Libs.Tests/SharpEmu.Libs.Tests.csproj` with +the x64 SDK and no `SHARPEMU_*` overrides. Then run +`tests/SharpEmu.Gpu.Metal.Tests/SharpEmu.Gpu.Metal.Tests.csproj` with the ARM64 +SDK and only `SHARPEMU_TEST_REQUIRE_DEVICE=1`. The explicit fallback control +adds `SHARPEMU_METAL_V2_CALLER_RESOLVE=0` and filters +`CallerResolutionSceneTests`, `NativeOcclusionSceneTests` and +`GpuWordNoWaitSceneTests` (OR each with `FullyQualifiedName~`). + +For the optional component benchmark, use each architecture's SDK in turn: + +```sh +SHARPEMU_CALLER_RESOLVE_BENCH_OUTPUT="$PWD/artifacts/caller-resolve-bench.json" \ + dotnet test tests/SharpEmu.Libs.Tests/SharpEmu.Libs.Tests.csproj \ + -c Release --settings docs/performance/shader-serial.runsettings \ + --filter 'FullyQualifiedName~ParallelProgramPrefetchCallerTests' \ + --blame-hang-timeout 120s +``` + +Private evidence is retained under `artifacts/runtime-diagnosis-20261026/`: +`caller-resolve-v3-source.json`, both `caller-resolve-v3-bench-*.json` files, +`caller-resolve-v24-build-source.json`, the working patch and untracked-test +archive, payload hashes, `caller-resolve-default-validation.json`, `caller-resolve-final-source.json`, +`caller-resolve-final-validation.json`, and +`caller-resolve-results/*.trx`. Each runtime directory contains its selected +window, screenshots, complete timestamped log, input/stop markers, memory +samples, thread CPU snapshots, `stationary-fps.json` and `caller-v24-summary.json`. +Commercial game data and raw game logs are not committed. + + +## Final suite triage + +The first final-policy Metal run reported **1,638 passed, one failed and seven +existing skips**. `ACachedImageKeepsOneTilerViewUntilItsTextureIsReleased` +observed a nonzero retained view where it expected none. Inspection found that +the volume and sRGB-view test fixtures constructed owning `MetalCachedImage` +objects without disposing them, while their harness released the texture. +That left stale registrations keyed by a reusable native handle. + +A temporary isolated registration-balance diagnostic called the volume fixture +with caller resolution explicitly **disabled**. It failed deterministically: +zero registrations before, one afterward. With the fixture ownership fixed it +passed (one test, zero failures). The temporary diagnostic source and both TRXs +are archived privately and removed from the shipped test assembly. The volume +and sRGB fixtures now retain a reference for the image and dispose it before +the harness; the existing tiler lifetime test also uses scoped disposal so an +assertion failure cannot leak its image registrations. Existing pixel/view and +lifetime assertions are retained. This is a test-fixture cleanup correction; +it does not change production texture ownership or claim a runtime memory win. + + +## Final validation result + +| Final configuration | Passed | Failed | Existing skips | +|---|---:|---:|---:| +| Libs x64, all feature overrides unset | 9,100 | 0 | 5 | +| Metal ARM64, required device, feature overrides unset | 1,639 | 0 | 7 | +| Metal ARM64, explicit caller-resolution fallback | 21 | 0 | 0 | + +These are test executions per configuration, not a count of unique tests. The +Libs run preceded the Metal-only fixture cleanup; production sources and Libs +tests did not change afterward. The final Metal assembly excludes the temporary +diagnostic. Both runtime publishes, REUSE and whitespace validation passed. +Earlier failure and reproduction results are retained above, not overwritten. diff --git a/docs/performance/native-occlusion-runtime.md b/docs/performance/native-occlusion-runtime.md index 0b278e2e7..79a77eeef 100644 --- a/docs/performance/native-occlusion-runtime.md +++ b/docs/performance/native-occlusion-runtime.md @@ -1452,3 +1452,16 @@ processes ended. Private evidence includes `measurement-plan.json`, telemetry and post-FPS CPU/event captures in the run directory; build source and payload manifests are `vertex-history-v23-build-source.json` and `vertex-history-v23-payload-hashes.json` in the runtime diagnostic directory. + + +### v24 bounded caller resolution + +The query-drain investigation led to caller resolution for short, unclaimed +prefetch queues, preserving FIFO, gates, pause and speculative-read validation. +The component benchmark supports enabling it by default, with an explicit +ARM64 burst-of-32 cost. The same-payload first-room comparison measured +**16.49 FPS enabled / 16.48 disabled** at 1920x1080, scale 1, unchanged VSync and +quality. Wait time fell while idle time rose; this is not an isolated FPS win. +See [the full source-bound benchmark and runtime report](caller-resolution.md) +for both run identifiers, windows, CPU counters, memory pressure and shutdown +trap timing. The previous ten defaults remain on; 30 FPS remains unestablished. diff --git a/docs/performance/performance-defaults.md b/docs/performance/performance-defaults.md index 6054fd4b6..5adbc2c24 100644 --- a/docs/performance/performance-defaults.md +++ b/docs/performance/performance-defaults.md @@ -21,6 +21,7 @@ VSync, guest quality, or unrelated application setting is changed. | Asynchronous command-item reuse | `SHARPEMU_COMMAND_ITEM_REUSE` | Native two-thread median 389.965 → 354.5175 ns; allocation 80.0008 → 0.026 B/item. Bounded 64-slot return ring; synchronous and failed items retain original ownership. No accepted runtime FPS gain. | | Queued CP writes | `SHARPEMU_METAL_V2_QUEUE_CP_WRITES` | Removes synchronous handoffs for eligible writes when command threading is explicitly enabled. Pending ranges, prefetch gates and CPU-read fallbacks preserve ordering; producer-thread device tests cover byte/qword/sparse/predication cases. | | Bounded vertex-layout history | `SHARPEMU_VERTEX_LAYOUT_HISTORY` | Exact byte-verified reuse across alternating layouts; adaptive backoff for low reuse, 512 slots and 2 MiB additional payload budget. Component timing/allocation evidence and miss costs are in [the measurements](vertex-layout-history.md); no isolated game-FPS claim. | +| Caller resolution for short queues | `SHARPEMU_METAL_V2_CALLER_RESOLVE` | At most eight outstanding jobs; FIFO, gates, pause and ownership checks retained. Single-job ARM64 2,874 → 542 ns, x64 3,160 → 972 ns; ARM64 empty-program burst 32 costs about 27 ns/job. Same-payload room medians 16.49 enabled / 16.48 disabled, no isolated FPS gain. [Full measurements](caller-resolution.md). | The existing negative label page filter, exact materialization comparison, dense register lookup, mip-layout construction and target memo fixes remain enabled. @@ -138,3 +139,10 @@ or establish a gain. History reuse occurred in the game, while occlusion dumps accounted for 83.6% of the sampled draw-queue drains. The runtime report retains the exact source, windows, counters and limitations. The defaults retain their component/test evidence; the 30-FPS objective remains open. + + +The v24 caller-resolution follow-up adds the eleventh default based on component +wins and correctness checks. Its same-payload Silent Hill medians were 16.49 FPS +enabled and 16.48 disabled; this establishes no game-FPS gain. The full component +table, larger-batch cost, run windows and paging limitations are retained in +[the caller-resolution report](caller-resolution.md). diff --git a/src/SharpEmu.Libs/Gpu/Rendering/ParallelProgramPrefetch.cs b/src/SharpEmu.Libs/Gpu/Rendering/ParallelProgramPrefetch.cs index a654314b2..9ecca5c9a 100644 --- a/src/SharpEmu.Libs/Gpu/Rendering/ParallelProgramPrefetch.cs +++ b/src/SharpEmu.Libs/Gpu/Rendering/ParallelProgramPrefetch.cs @@ -44,14 +44,28 @@ public sealed class Job(RegisterBanks banks, long gate, bool indexed = false) // Set while the worker sleeps because of a pause, under _lock. private bool _waitingOnPause; private bool _running; + // The worker owns the head even while it releases the lock to wait for its gate. + private bool _workerClaimed; + private readonly bool _allowCallerResolution; + // Long bursts give the resolver useful lead; racing it for each head adds handoffs. + // Short queues are the query/barrier pattern where the renderer otherwise waits + // for a newly woken thread to resolve only a handful of jobs. + private const int MaximumCallerOutstanding = 8; private bool _stopping; public ParallelProgramPrefetch(RenderExecutor executor, IRenderHost host, IConcurrentResolveHost concurrent) + : this(executor, host, concurrent, concurrent.AllowsCallerResolution) + { + } + + internal ParallelProgramPrefetch(RenderExecutor executor, IRenderHost host, IConcurrentResolveHost concurrent, + bool allowCallerResolution) { _executor = executor; _host = host; _fatal = host.Fatal; _concurrent = concurrent; + _allowCallerResolution = allowCallerResolution; // The worker waits on this thread per draw: it runs at the worker's class, with the // worker's stack, since it resolves what the worker would resolve itself. _thread = SharpEmu.HLE.Host.HostHelperThread.Start("SharpEmu program resolve", 16 * 1024 * 1024, Run); @@ -62,7 +76,7 @@ public ParallelProgramPrefetch(RenderExecutor executor, IRenderHost host, IConcu public long Submitted { get; private set; } - // The worker waits on an empty queue (read and written under _lock). + // The worker waits for jobs or an active caller (read and written under _lock). private bool _idleForJobs; // What became of the jobs since the last TakeCounters: resolved, abandoned (the resolution threw), @@ -195,8 +209,15 @@ public bool TryTakeGraphics(RegisterBanks banks, in GraphicsProgramInputs inputs Current = null; var waitStarted = System.Diagnostics.Stopwatch.GetTimestamp(); - WaitFor(job); - Interlocked.Add(ref _waitTicks, System.Diagnostics.Stopwatch.GetTimestamp() - waitStarted); + // Caller work is reported separately, not disguised as a shorter resolver wait. + // Keep completed jobs and long queues out of the claim routine's lock/finally + // frame. Almost every take in a well-prefetched burst is already complete. + if (!_allowCallerResolution || job.State != 0 || Volatile.Read(ref _outstanding) > MaximumCallerOutstanding || + !TryResolveOnCaller(job)) + { + WaitFor(job); + Interlocked.Add(ref _waitTicks, System.Diagnostics.Stopwatch.GetTimestamp() - waitStarted); + } using var profile = RenderPhaseProfile.MeasureDetail(RenderPhaseProfile.Phase.ProgramPrefetchValidation); if (job.State != 1) { @@ -255,6 +276,57 @@ private void WaitFor(Job job) } } + // Claim only the untouched FIFO head. Never race a worker that already peeked at it, + // bypass a dependency gate, or resolve concurrently against shared pipeline caches. + internal bool TryResolveOnCaller(Job job) + { + if (!_allowCallerResolution || job.State != 0 || Volatile.Read(ref _outstanding) > MaximumCallerOutstanding || + Volatile.Read(ref _running) || Volatile.Read(ref _workerClaimed)) + return false; + lock (_lock) + { + if (_stopping || _paused || _running || _workerClaimed || job.State != 0 || _outstanding > MaximumCallerOutstanding || + job.Gate > Volatile.Read(ref _executedGate) || !_queue.TryPeek(out var head) || !ReferenceEquals(head, job)) + return false; + _queue.Dequeue(); + _running = true; + } + + var speculative = GpuFatal.ThrowOnFatal; + var started = System.Diagnostics.Stopwatch.GetTimestamp(); + GpuFatal.ThrowOnFatal = true; + try + { + ResolveCore(job); + } + finally + { + GpuFatal.ThrowOnFatal = speculative; + Interlocked.Add(ref _callerResolveTicks, System.Diagnostics.Stopwatch.GetTimestamp() - started); + Interlocked.Increment(ref _callerResolves); + FinishJob(worker: false); + } + return true; + } + + private long _callerResolves, _callerResolveTicks; + + public (long Count, double Milliseconds) TakeCallerResolveCounters() => + (Interlocked.Exchange(ref _callerResolves, 0), + Interlocked.Exchange(ref _callerResolveTicks, 0) * 1000.0 / System.Diagnostics.Stopwatch.Frequency); + + private void FinishJob(bool worker) + { + lock (_lock) + { + _outstanding--; + _finished++; + _running = false; + if (worker) _workerClaimed = false; + Monitor.PulseAll(_lock); + } + } + private void Run() { GpuFatal.ThrowOnFatal = true; @@ -263,7 +335,7 @@ private void Run() Job job; lock (_lock) { - while (_queue.Count == 0 && !_stopping) + while ((_queue.Count == 0 || _running) && !_stopping) { _idleForJobs = true; Monitor.Wait(_lock); @@ -276,6 +348,7 @@ private void Run() } job = _queue.Peek(); + _workerClaimed = true; if (job.Gate > Volatile.Read(ref _executedGate)) { // The gate usually opens within a few items: spin before sleeping. @@ -317,14 +390,11 @@ private void Run() _running = true; } - Resolve(job); - lock (_lock) + try { - _outstanding--; - _finished++; - _running = false; - Monitor.PulseAll(_lock); + Resolve(job); } + finally { FinishJob(worker: true); } } } diff --git a/src/SharpEmu.Libs/Gpu/Rendering/RenderExecutor.Prefetch.cs b/src/SharpEmu.Libs/Gpu/Rendering/RenderExecutor.Prefetch.cs index ab0c55742..403c168c3 100644 --- a/src/SharpEmu.Libs/Gpu/Rendering/RenderExecutor.Prefetch.cs +++ b/src/SharpEmu.Libs/Gpu/Rendering/RenderExecutor.Prefetch.cs @@ -71,6 +71,10 @@ public interface IConcurrentResolveHost // so it can be turned on and off at run time. bool ResolvesConcurrently => true; + // Opt-in: the host's speculative-read context may also run on the drawing thread. + // The queue still permits only one resolution at a time, including that caller. + bool AllowsCallerResolution => false; + // On the resolving thread, around one resolution. void BeginConcurrentResolve(); diff --git a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.ConcurrentResolve.cs b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.ConcurrentResolve.cs index e01bfd4af..77d7f1a21 100644 --- a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.ConcurrentResolve.cs +++ b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.ConcurrentResolve.cs @@ -31,6 +31,14 @@ internal sealed partial class MetalRenderHost : IConcurrentResolveHost bool IConcurrentResolveHost.ResolvesConcurrently => ParallelResolve; + // Small unclaimed queues can resolve on the drawing thread without a worker wakeup. + // Component gains and ordering/device checks justify the default; the Silent Hill + // comparison was flat. Set SHARPEMU_METAL_V2_CALLER_RESOLVE=0 for the worker-only path. + private static readonly bool CallerResolve = + Environment.GetEnvironmentVariable("SHARPEMU_METAL_V2_CALLER_RESOLVE") != "0"; + + bool IConcurrentResolveHost.AllowsCallerResolution => CallerResolve; + void IConcurrentResolveHost.BeginConcurrentResolve() => BeginConcurrentResolve(); object IConcurrentResolveHost.EndConcurrentResolve() => EndConcurrentResolve(); @@ -117,12 +125,14 @@ private string FormatPrefetchCounters() } var (resolved, abandoned, taken, refused, waitMs) = prefetch.TakeCounters(); + var (callerResolves, callerMs) = prefetch.TakeCallerResolveCounters(); var drains = Interlocked.Exchange(ref _drains, 0); var deferredBarriers = Interlocked.Exchange(ref _deferredBarriers, 0); var operandReadsKept = Interlocked.Exchange(ref _operandReadsKept, 0); var drained = Interlocked.Exchange(ref _drainedDraws, 0); return $"resolved={resolved} abandoned={abandoned} taken={taken} refused={refused} drains={drains} drained={drained} deferred_barriers={deferredBarriers} operand_reads_kept={operandReadsKept} " + - $"wait_ms={waitMs.ToString("F1", culture)} resolve_ms={prefetch.TakeResolveMilliseconds().ToString("F1", culture)} {own}{TakeDrainCallers()}"; + $"wait_ms={waitMs.ToString("F1", culture)} resolve_ms={prefetch.TakeResolveMilliseconds().ToString("F1", culture)} " + + $"caller_jobs={callerResolves} caller_ms={callerMs.ToString("F1", culture)} {own}{TakeDrainCallers()}"; } internal enum ConcurrentQuery : byte diff --git a/tests/SharpEmu.Gpu.Metal.Tests/MetalImageTransferTests.SrgbTwinViews.cs b/tests/SharpEmu.Gpu.Metal.Tests/MetalImageTransferTests.SrgbTwinViews.cs index fca9ddbc0..329b83f4a 100644 --- a/tests/SharpEmu.Gpu.Metal.Tests/MetalImageTransferTests.SrgbTwinViews.cs +++ b/tests/SharpEmu.Gpu.Metal.Tests/MetalImageTransferTests.SrgbTwinViews.cs @@ -60,7 +60,9 @@ public void TheOtherEncodingOfABlockImageGetsARealView( pixelFormat, Mtl.TextureType2D, 64, 64, 1, 1, 1, 1, (ulong)MetalImageTransfer.UsageOf(description, (MtlPixelFormat)pixelFormat)); var texture = harness.Texture(shape); - var image = new MetalCachedImage(in description, in shape, texture); + // The harness and cached image each own a reference; dispose the image first. + MetalNative.SendVoid(texture, MetalNative.Selector("retain")); + using var image = new MetalCachedImage(in description, in shape, texture); var request = new ImageViewDescription( viewFormat, GpuImageViewType.Type2D, GpuImageAspect.ColorBit, 0, 1, 0, 1, default, GpuImageUsage.SampledBit); diff --git a/tests/SharpEmu.Gpu.Metal.Tests/MetalImageTransferTests.TextureTiles.cs b/tests/SharpEmu.Gpu.Metal.Tests/MetalImageTransferTests.TextureTiles.cs index f63d3e014..c30ddc73f 100644 --- a/tests/SharpEmu.Gpu.Metal.Tests/MetalImageTransferTests.TextureTiles.cs +++ b/tests/SharpEmu.Gpu.Metal.Tests/MetalImageTransferTests.TextureTiles.cs @@ -211,7 +211,7 @@ public void ACachedImageKeepsOneTilerViewUntilItsTextureIsReleased() // The image releases the texture it owns; the harness releases its own reference. MetalNative.SendVoid(texture, MetalNative.Selector("retain")); - var image = new MetalCachedImage(in description, in shape, texture); + using var image = new MetalCachedImage(in description, in shape, texture); nint kept = 0; for (var pass = 0; pass < 3; pass++) { @@ -230,8 +230,8 @@ public void ACachedImageKeepsOneTilerViewUntilItsTextureIsReleased() // An image retiring the texture (PromoteToOwned and its kin) refers to it too, so // the view survives until both have let go. - var retired = new MetalCachedImage(in description, in shape, texture); MetalNative.SendVoid(texture, MetalNative.Selector("retain")); + using var retired = new MetalCachedImage(in description, in shape, texture); image.Dispose(); Assert.Equal(kept, MetalTileViews.KeptView(texture)); retired.Dispose(); diff --git a/tests/SharpEmu.Gpu.Metal.Tests/MetalImageTransferTests.VolumeViews.cs b/tests/SharpEmu.Gpu.Metal.Tests/MetalImageTransferTests.VolumeViews.cs index a782dc80e..2167d4b16 100644 --- a/tests/SharpEmu.Gpu.Metal.Tests/MetalImageTransferTests.VolumeViews.cs +++ b/tests/SharpEmu.Gpu.Metal.Tests/MetalImageTransferTests.VolumeViews.cs @@ -31,7 +31,9 @@ public void AVolumeAttachmentBindsTheTextureRatherThanAnArrayViewOfIt() description.Pitch = 16; description.Type = GuestImageType.Color3D; description.Extent = new GpuExtent3D(16, 8, 4); - var image = new MetalCachedImage(in description, in shape, texture); + // The harness and cached image each own a reference; dispose the image first. + MetalNative.SendVoid(texture, MetalNative.Selector("retain")); + using var image = new MetalCachedImage(in description, in shape, texture); // One slice of the volume, as the colour-target builder describes it. var slice = new ImageViewDescription( diff --git a/tests/SharpEmu.Gpu.Metal.Tests/Scenes/CallerResolutionSceneTests.cs b/tests/SharpEmu.Gpu.Metal.Tests/Scenes/CallerResolutionSceneTests.cs new file mode 100644 index 000000000..0bc67bbba --- /dev/null +++ b/tests/SharpEmu.Gpu.Metal.Tests/Scenes/CallerResolutionSceneTests.cs @@ -0,0 +1,106 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using System.Reflection; +using SharpEmu.Gpu.Metal.V2; +using SharpEmu.Libs.Agc; +using SharpEmu.Libs.Gpu.Rendering; +using SharpEmu.Libs.Gpu.Scheduling; +using Xunit; +using Xunit.Abstractions; + +namespace SharpEmu.Gpu.Metal.Tests.Scenes; + +public sealed class CallerResolutionSceneTests(ITestOutputHelper output) : SceneTestBase(output) +{ + [Fact] + public void SpeculativeReadsOnTheDrawingThreadRefusePendingQueryBytes() + { + using var rig = CreateRig(); + if (rig is null) return; + var host = rig.Host; + host.NativeOcclusionEnabled = true; + var device = (nint)typeof(MetalRenderHost).GetField("_device", BindingFlags.Instance | BindingFlags.NonPublic)!.GetValue(host)!; + var gate = MetalNative.Send(device, MetalNative.Selector("newSharedEvent")); + Assert.NotEqual(0, gate); + var signal = MetalNative.Selector("setSignaledValue:"); + var query = rig.Memory.Allocate(8, 64); + var fired = 0; + using var cancel = new ManualResetEventSlim(); + var watchdog = new Thread(() => + { + if (cancel.Wait(TimeSpan.FromSeconds(10))) return; + Interlocked.Exchange(ref fired, 1); + MetalNative.SendVoid(gate, signal, (nint)1); + }) { IsBackground = true }; + watchdog.Start(); + try + { + MetalV2Native.SendEventValue(host.CommandBuffer, MetalNative.Selector("encodeWaitForEvent:value:"), gate, 1); + Assert.True(host.DumpOcclusionCounter(0, query, 1, 8)); + var waits = host.HostGpuWaits; + var concurrent = (IConcurrentResolveHost)host; + concurrent.BeginConcurrentResolve(); + try + { + Assert.Throws(() => host.TryReadGuestWord(query + 4, out _)); + Assert.Equal(waits, host.HostGpuWaits); + Assert.Equal(0, Volatile.Read(ref fired)); + } + finally { concurrent.ReleaseAssumptions(concurrent.EndConcurrentResolve()); } + MetalNative.SendVoid(gate, signal, (nint)1); + Assert.True(host.TryReadGuestWord(query + 4, out var ready)); + Assert.Equal(0x80000000u, ready); + rig.Drain(); + } + finally + { + MetalNative.SendVoid(gate, signal, (nint)1); + cancel.Set(); + watchdog.Join(); + MetalNative.SendVoid(gate, MetalNative.Selector("release")); + } + } + + [Fact] + public void CallerResolutionPreservesPixelsAndEarlierOcclusionSnapshots() + { + using var rig = CreateRig(); + if (rig is null) return; + rig.Host.NativeOcclusionEnabled = true; + const uint size = 32; + const ulong ready = 1UL << 63; + var target = rig.ColorTarget(size, size); + var query = rig.Memory.Allocate(256, 64); + var state = new SceneState(size, size).ColorTarget(0, target, size, size); + state.Context[Cx.DbCountControl] = 1u << 8; + var pm4 = new ScenePm4(); + pm4.EventWithAddress(ScenePm4.EventPixelPipeStatDump, 1, query); + rig.DrawMesh(pm4, state, SceneDraws.Quad(0.1f, 1, 0, 0)); + pm4.EventWithAddress(ScenePm4.EventPixelPipeStatDump, 1, query + 8); + rig.DrawMesh(pm4, state, SceneDraws.Quad(0.1f, 0, 1, 0)); + var translation = (AgcExports.CommandStreamTranslation)typeof(MetalRenderHost) + .GetField("_translation", BindingFlags.Instance | BindingFlags.NonPublic)!.GetValue(rig.Host)!; + var prefetch = translation.Prefetch; + var forceCaller = prefetch is not null && !rig.Host.UsesCommandThread && ((IConcurrentResolveHost)rig.Host).AllowsCallerResolution; + prefetch?.TakeCallerResolveCounters(); + if (forceCaller) + { + // Serial interpretation and drawing run on this test thread. Block the resolver + // from claiming the queued jobs so this exercises the caller path deterministically. + var queueLock = typeof(ParallelProgramPrefetch).GetField("_lock", BindingFlags.Instance | BindingFlags.NonPublic)!.GetValue(prefetch)!; + lock (queueLock) rig.Submit(pm4); + } + else rig.Submit(pm4); + var callers = prefetch?.TakeCallerResolveCounters().Count ?? 0; + if (forceCaller) Assert.True(callers >= 2, $"Expected both draws to resolve on the caller, got {callers}."); + Output.WriteLine($"forced_caller={forceCaller} caller_jobs={callers}"); + var values = rig.Read(query, 32); + Assert.Equal(ready, values[0]); + Assert.Equal(ready | (ulong)size * size, values[1]); + Assert.All(values.Skip(2), value => Assert.Equal(ready, value)); + Assert.True(rig.Host.TryReadOcclusionCounter(0, out var finalCount)); + Assert.Equal(2UL * size * size, finalCount); + Assert.Equal(0xFF00FF00u, SceneDraws.Pixel(rig.ReadImage(target, size, size), size, size / 2, size / 2)); + } +} diff --git a/tests/SharpEmu.Libs.Tests/Gpu/Rendering/ParallelProgramPrefetchCallerTests.Benchmarks.cs b/tests/SharpEmu.Libs.Tests/Gpu/Rendering/ParallelProgramPrefetchCallerTests.Benchmarks.cs new file mode 100644 index 000000000..5fbf9ac71 --- /dev/null +++ b/tests/SharpEmu.Libs.Tests/Gpu/Rendering/ParallelProgramPrefetchCallerTests.Benchmarks.cs @@ -0,0 +1,111 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using System.Diagnostics; +using System.Runtime.InteropServices; +using System.Runtime.ExceptionServices; +using System.Text.Json; +using SharpEmu.HLE.Host; +using SharpEmu.Libs.Gpu.Rendering; +using SharpEmu.PerfAudit; +using Xunit; +using static SharpEmu.Libs.Tests.Gpu.Rendering.RenderExecutorFixtures; + +namespace SharpEmu.Libs.Tests.Gpu.Rendering; + +public sealed partial class ParallelProgramPrefetchCallerTests +{ + private readonly record struct Measurement(double WallNs, double CallerCpuNs, double ProcessCpuNs, + double ProcessAllocatedBytes, long CallerJobs, long Taken, double WaitMs, double WorkerResolveMs, double CallerResolveMs); + + [Fact] + public void BenchmarkCallerResolutionWhenRequested() + { + var output = Environment.GetEnvironmentVariable("SHARPEMU_CALLER_RESOLVE_BENCH_OUTPUT"); + if (string.IsNullOrEmpty(output)) return; + object? result = null; + Exception? failure = null; + var callerQos = Environment.GetEnvironmentVariable("SHARPEMU_CALLER_RESOLVE_BENCH_QOS") != "0"; + if (callerQos) + { + // Match the native Metal GPU worker's class instead of comparing a normal + // xUnit caller with a UserInteractive resolver on a contended host. + var thread = HostHelperThread.Start("prefetch caller benchmark", 16 * 1024 * 1024, () => + { + try { result = MeasureBenchmark(callerQos); } + catch (Exception exception) { failure = exception; } + }); + Assert.True(thread.Join(TimeSpan.FromMinutes(3)), "Component benchmark exceeded its bound."); + Assert.True(thread.HasQos, "The requested caller QoS was unavailable."); + if (failure is not null) ExceptionDispatchInfo.Capture(failure).Throw(); + } + else result = MeasureBenchmark(callerQos); + File.WriteAllText(output, JsonSerializer.Serialize(result, new JsonSerializerOptions { WriteIndented = true })); + } + + private static object MeasureBenchmark(bool callerQos) + { + using var process = Process.GetCurrentProcess(); + var results = new List(); + foreach (var spins in new[] { 0, 512 }) + foreach (var burst in new[] { 1, 5, 32, 256 }) + { + using var baseline = new Fixture(false); + using var candidate = new Fixture(true); + var fixtures = new[] { baseline, candidate }; + foreach (var f in fixtures) + { + f.Pipelines.Graphics = new GraphicsPrograms { Available = false }; + f.Pipelines.BeforeGraphicsResolve = _ => { if (spins != 0) Thread.SpinWait(spins); }; + } + var banks = Enumerable.Range(0, burst).Select(_ => Banks().Snapshot()).ToArray(); + var jobs = new ParallelProgramPrefetch.Job[burst]; + var batches = Math.Max(32, 8192 / burst); + Measurement Run(int arm, int batchCount) + { + var f = fixtures[arm]; + f.Prefetch.WaitIdle(); + f.Prefetch.TakeCounters(); f.Prefetch.TakeCallerResolveCounters(); f.Prefetch.TakeResolveMilliseconds(); + process.Refresh(); + var processBefore = process.TotalProcessorTime.TotalNanoseconds; + var bytesBefore = GC.GetTotalAllocatedBytes(precise: true); + var cpuBefore = OperatingSystem.IsMacOS() ? DarwinThreadCpuClock.Nanoseconds() : 0; + var started = Stopwatch.GetTimestamp(); + for (var batch = 0; batch < batchCount; batch++) + { + for (var i = 0; i < burst; i++) jobs[i] = f.Prefetch.Submit(banks[i]); + for (var i = 0; i < burst; i++) + if (!f.Take(banks[i], jobs[i])) throw new InvalidOperationException("A benchmark resolution was refused."); + f.Prefetch.WaitIdle(); + f.Pipelines.Calls.Clear(); f.Pipelines.ExportMappings.Clear(); + } + var elapsed = Stopwatch.GetElapsedTime(started).TotalNanoseconds; + var cpuAfter = OperatingSystem.IsMacOS() ? DarwinThreadCpuClock.Nanoseconds() : 0; + var allocated = GC.GetTotalAllocatedBytes(precise: true) - bytesBefore; + process.Refresh(); + var processAfter = process.TotalProcessorTime.TotalNanoseconds; + var counters = f.Prefetch.TakeCounters(); + var caller = f.Prefetch.TakeCallerResolveCounters(); + var count = (long)batchCount * burst; + Assert.Equal(count, counters.Taken); + Assert.Equal(count, counters.Resolved); + Assert.Equal(0, counters.Refused); Assert.Equal(0, counters.Abandoned); + return new(elapsed / count, (cpuAfter - cpuBefore) / (double)count, + (processAfter - processBefore) / count, (double)allocated / count, + caller.Count, counters.Taken, counters.WaitMilliseconds, f.Prefetch.TakeResolveMilliseconds(), caller.Milliseconds); + } + var warm = Stopwatch.StartNew(); + do { Run(0, batches); Run(1, batches); } while (warm.ElapsedMilliseconds < (spins == 0 && burst == 1 ? 5000 : 2000)); + for (var round = 0; round < 7; round++) + foreach (var arm in round % 2 == 0 ? new[] { 0, 1 } : new[] { 1, 0 }) + results.Add(new { spins, burst, batches, round, caller_enabled = arm == 1, measurement = Run(arm, batches) }); + } + return new + { + architecture = RuntimeInformation.ProcessArchitecture.ToString(), framework = RuntimeInformation.FrameworkDescription, + caller_qos = callerQos, + caveat = "Real prefetch queue/executor with unavailable fake programs; optional fixed work uses SpinWait. Whole-process CPU/allocation includes runtime activity. Caller CPU alone omits background resolution. No GPU/FPS claim.", + results, + }; + } +} diff --git a/tests/SharpEmu.Libs.Tests/Gpu/Rendering/ParallelProgramPrefetchCallerTests.cs b/tests/SharpEmu.Libs.Tests/Gpu/Rendering/ParallelProgramPrefetchCallerTests.cs new file mode 100644 index 000000000..8caa97842 --- /dev/null +++ b/tests/SharpEmu.Libs.Tests/Gpu/Rendering/ParallelProgramPrefetchCallerTests.cs @@ -0,0 +1,278 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using System.Reflection; +using SharpEmu.Libs.Gpu.GpuCommands.Registers; +using SharpEmu.Libs.Gpu.Rendering; +using SharpEmu.Libs.Gpu.Scheduling; +using SharpEmu.Libs.Tests.Gpu.Scheduling; +using Xunit; +using static SharpEmu.Libs.Tests.Gpu.Rendering.RenderExecutorFixtures; + +namespace SharpEmu.Libs.Tests.Gpu.Rendering; + +[Collection(SchedulingStateCollection.Name)] +public sealed partial class ParallelProgramPrefetchCallerTests +{ + private sealed class ConcurrentHost : IConcurrentResolveHost + { + internal bool Current = true; + internal int Active, MaximumActive, Begins, Ends, Releases; + public void BeginConcurrentResolve() + { + var active = Interlocked.Increment(ref Active); + int old; + do { old = Volatile.Read(ref MaximumActive); } + while (active > old && Interlocked.CompareExchange(ref MaximumActive, active, old) != old); + Interlocked.Increment(ref Begins); + } + public object EndConcurrentResolve() + { + Interlocked.Increment(ref Ends); + Interlocked.Decrement(ref Active); + return this; + } + public bool IsStillCurrent(object assumptions) => ReferenceEquals(this, assumptions) && Current; + public void ReleaseAssumptions(object assumptions) + { + Assert.Same(this, assumptions); + Interlocked.Increment(ref Releases); + } + } + + private sealed class Fixture : IDisposable + { + private readonly FatalScope _fatal = new(); + internal readonly RecordingRenderHost Host = new(); + internal readonly FakePipelineProvider Pipelines = new(); + internal readonly ConcurrentHost Concurrent = new(); + internal readonly ParallelProgramPrefetch Prefetch; + // Holding the queue monitor on this test thread makes the unclaimed-head + // case deterministic, without sleeps or a production scheduler test hook. + internal readonly object QueueLock; + internal Fixture(bool enabled = true) + { + Prefetch = new(new RenderExecutor(Host, Pipelines), Host, Concurrent, enabled); + QueueLock = typeof(ParallelProgramPrefetch).GetField("_lock", BindingFlags.Instance | BindingFlags.NonPublic)!.GetValue(Prefetch)!; + } + internal bool Take(RegisterBanks banks, ParallelProgramPrefetch.Job job, bool indexed = false) + { + Prefetch.Current = job; + var inputs = RenderExecutor.ProgramInputsOf(banks, Host.FormatSupport, Host.Fatal, indexed); + return Prefetch.TryTakeGraphics(banks, in inputs, out _); + } + internal void Idle() => Assert.True(Task.Run(Prefetch.WaitIdle).Wait(TimeSpan.FromSeconds(10)), "Resolver did not become idle."); + public void Dispose() { Prefetch.Dispose(); _fatal.Dispose(); } + } + + [Fact] + public void CallerTakesOnlyTheHeadAndCompletesItsPrefixExactlyOnce() + { + using var f = new Fixture(); + var first = Banks(); var second = Banks(); + lock (f.QueueLock) + { + var a = f.Prefetch.Submit(first); var b = f.Prefetch.Submit(second); + Assert.False(f.Prefetch.TryResolveOnCaller(b)); + Assert.Equal(0, f.Concurrent.Begins); + Assert.True(f.Take(first, a)); + Assert.False(f.Prefetch.TryResolveOnCaller(a)); + Assert.Equal(0, b.State); + f.Prefetch.WaitFinished(1); + Assert.True(f.Take(second, b)); + f.Prefetch.WaitFinished(2); + } + f.Idle(); + Assert.Equal((2, 2, 2, 0, 1), (f.Concurrent.Begins, f.Concurrent.Ends, f.Concurrent.Releases, f.Concurrent.Active, f.Concurrent.MaximumActive)); + Assert.Equal(2, f.Prefetch.TakeCallerResolveCounters().Count); + Assert.Equal(2, f.Prefetch.TakeCounters().Taken); + } + + [Fact] + public void PauseAndDependencyGateBothPreventCallerResolution() + { + using var f = new Fixture(); + lock (f.QueueLock) + { + f.Prefetch.Pause(); + var banks = Banks(); var job = f.Prefetch.Submit(banks, gate: 2); + Assert.False(f.Prefetch.TryResolveOnCaller(job)); + f.Prefetch.Resume(); + Assert.False(f.Prefetch.TryResolveOnCaller(job)); + f.Prefetch.AdvanceGate(1); + Assert.False(f.Prefetch.TryResolveOnCaller(job)); + f.Prefetch.AdvanceGate(2); + f.Prefetch.Pause(); + Assert.False(f.Prefetch.TryResolveOnCaller(job)); + f.Prefetch.Resume(); + Assert.True(f.Take(banks, job)); + } + f.Idle(); + Assert.Equal(1, f.Concurrent.Begins); + } + + [Fact] + public void AWorkerWaitingAtAGateKeepsItsClaimWhenTheGateOpens() + { + using var f = new Fixture(); + var job = f.Prefetch.Submit(Banks(), gate: 1); + var claim = typeof(ParallelProgramPrefetch).GetField("_workerClaimed", BindingFlags.Instance | BindingFlags.NonPublic)!; + Assert.True(SpinWait.SpinUntil(() => { lock (f.QueueLock) return (bool)claim.GetValue(f.Prefetch)!; }, TimeSpan.FromSeconds(10))); + lock (f.QueueLock) + { + Assert.False(f.Prefetch.TryResolveOnCaller(job)); + f.Prefetch.AdvanceGate(1); + Assert.False(f.Prefetch.TryResolveOnCaller(job)); + } + f.Idle(); + Assert.Equal(1, job.State); + Assert.Equal(0, f.Prefetch.TakeCallerResolveCounters().Count); + Assert.Equal(1, f.Concurrent.Begins); + } + + [Theory] + [InlineData(false, false, true, true)] + [InlineData(true, true, true, true)] + [InlineData(true, false, true, false)] + [InlineData(false, true, true, false)] + [InlineData(false, false, false, false)] + public void CallerResultsStillRequireMatchingInputsAndCurrentOwnership(bool indexed, bool requested, bool current, bool accepted) + { + using var f = new Fixture(); + f.Concurrent.Current = current; + var banks = Banks(); + lock (f.QueueLock) + Assert.Equal(accepted, f.Take(banks, f.Prefetch.Submit(banks, indexed: indexed), requested)); + f.Idle(); + Assert.Equal(1, f.Concurrent.Releases); + Assert.Equal(1, f.Prefetch.TakeCallerResolveCounters().Count); + Assert.Null(f.Prefetch.Current); + } + + [Fact] + public void PauseWaitsForAnActiveCallerAndHoldsTheFollowingJob() + { + using var f = new Fixture(); + Task? pause = null; + var pausedField = typeof(ParallelProgramPrefetch).GetField("_paused", BindingFlags.Instance | BindingFlags.NonPublic)!; + ParallelProgramPrefetch.Job? second = null; + var invocations = 0; + f.Pipelines.BeforeGraphicsResolve = _ => + { + if (++invocations != 1) return; + // Release the test's outer monitor while the caller is inside resolution. + // Production releases its own claim lock before invoking this callback. + Monitor.Exit(f.QueueLock); + try + { + pause = Task.Run(f.Prefetch.Pause); + Assert.True(SpinWait.SpinUntil(() => { lock (f.QueueLock) return (bool)pausedField.GetValue(f.Prefetch)!; }, TimeSpan.FromSeconds(10))); + Assert.False(pause.IsCompleted); + Assert.Equal(0, second!.State); + } + finally { Monitor.Enter(f.QueueLock); } + }; + var firstBanks = Banks(); var secondBanks = Banks(); + lock (f.QueueLock) + { + var first = f.Prefetch.Submit(firstBanks); + second = f.Prefetch.Submit(secondBanks); + Assert.True(f.Take(firstBanks, first)); + } + Assert.NotNull(pause); + Assert.True(pause.Wait(TimeSpan.FromSeconds(10))); + lock (f.QueueLock) + { + Assert.Equal(0, second.State); + Assert.False(f.Prefetch.TryResolveOnCaller(second)); + } + f.Prefetch.Resume(); + Assert.True(f.Take(secondBanks, second)); + f.Idle(); + Assert.Equal(1, f.Concurrent.MaximumActive); + Assert.Equal(2, f.Concurrent.Releases); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void SpeculativeFailureAbandonsOnceAndRestoresCallerFatalPolicy(bool alreadySpeculative) + { + using var f = new Fixture(); + f.Pipelines.BeforeGraphicsResolve = _ => throw GpuFatal.Fatal("synthetic speculative failure"); + var previous = GpuFatal.ThrowOnFatal; + try + { + GpuFatal.ThrowOnFatal = alreadySpeculative; + var banks = Banks(); + lock (f.QueueLock) + Assert.False(f.Take(banks, f.Prefetch.Submit(banks))); + Assert.Equal(alreadySpeculative, GpuFatal.ThrowOnFatal); + } + finally { GpuFatal.ThrowOnFatal = previous; } + f.Idle(); + Assert.Equal(1, f.Prefetch.TakeCounters().Abandoned); + Assert.Equal((1, 1, 1), (f.Concurrent.Begins, f.Concurrent.Ends, f.Concurrent.Releases)); + } + + [Fact] + public void DisabledAndStoppedQueuesRefuseCallerClaims() + { + using var f = new Fixture(false); + lock (f.QueueLock) + Assert.False(f.Prefetch.TryResolveOnCaller(f.Prefetch.Submit(Banks()))); + f.Idle(); + using var stopped = new Fixture(); + lock (stopped.QueueLock) + { + var job = stopped.Prefetch.Submit(Banks()); + stopped.Prefetch.Dispose(); + Assert.False(stopped.Prefetch.TryResolveOnCaller(job)); + } + } + + [Fact] + public void ConcurrentClaimsRetainFifoAndNeverResolveTwoJobsAtOnce() + { + using var f = new Fixture(); + var resolved = new List(); + f.Pipelines.BeforeGraphicsResolve = vertex => { resolved.Add(vertex.ExportAddress); Thread.SpinWait(128); }; + const int count = 2048; + var banks = Enumerable.Range(0, count).Select(i => { var b = Banks(); b.Shader.Vertex.ExportAddress = (ulong)(i + 1) * 256; return b; }).ToArray(); + var jobs = new ParallelProgramPrefetch.Job[count]; + for (var start = 0; start < count; start += 8) + { + // Guarantee at least one caller claim, then let both threads race for the rest. + lock (f.QueueLock) + { + for (var i = start; i < start + 8; i++) jobs[i] = f.Prefetch.Submit(banks[i]); + if (start == 0) Assert.True(f.Take(banks[0], jobs[0])); + } + for (var i = start == 0 ? 1 : start; i < start + 8; i++) Assert.True(f.Take(banks[i], jobs[i])); + f.Prefetch.WaitFinished(start + 8); + } + f.Idle(); + Assert.Equal(banks.Select(b => b.Shader.Vertex.ExportAddress), resolved); + Assert.Equal((count, count, count, 0, 1), (f.Concurrent.Begins, f.Concurrent.Ends, f.Concurrent.Releases, f.Concurrent.Active, f.Concurrent.MaximumActive)); + Assert.True(f.Prefetch.TakeCallerResolveCounters().Count > 0); + } + + [Theory] + [InlineData(8, true)] + [InlineData(9, false)] + public void CallerClaimsAreBoundedToShortQueues(int count, bool claimed) + { + using var f = new Fixture(); + var banks = Banks(); + ParallelProgramPrefetch.Job[] jobs; + lock (f.QueueLock) + { + jobs = Enumerable.Range(0, count).Select(_ => f.Prefetch.Submit(banks)).ToArray(); + Assert.Equal(claimed, f.Prefetch.TryResolveOnCaller(jobs[0])); + } + f.Idle(); + foreach (var job in jobs) Assert.True(f.Take(banks, job)); + Assert.Equal(count, f.Concurrent.Begins); + Assert.Equal(count, f.Concurrent.Releases); + } +} diff --git a/tests/SharpEmu.Libs.Tests/Gpu/Rendering/RenderExecutorTestSupport.cs b/tests/SharpEmu.Libs.Tests/Gpu/Rendering/RenderExecutorTestSupport.cs index a79351b3a..9e0993419 100644 --- a/tests/SharpEmu.Libs.Tests/Gpu/Rendering/RenderExecutorTestSupport.cs +++ b/tests/SharpEmu.Libs.Tests/Gpu/Rendering/RenderExecutorTestSupport.cs @@ -567,6 +567,7 @@ private static string Describe(ReadOnlySpan mapping) => // Hands back the configured programs and records every pipeline request. internal sealed class FakePipelineProvider : IShaderPipelineProvider { + internal Action? BeforeGraphicsResolve { get; set; } public GraphicsPrograms Graphics { get; set; } = RenderExecutorFixtures.Programs(); // Stands in for a backend with mesh pipelines (Metal); Vulkan leaves it false. @@ -628,6 +629,7 @@ public GraphicsPrograms GetGraphicsPrograms( bool depthAspectBound, bool stencilAspectBound = false) { + BeforeGraphicsResolve?.Invoke(vertex); Calls.Add($"get_graphics_programs pixelActive={pixelActive}"); ExportMappings.Add(targetExportMapping.ToArray()); return Graphics; From d7d1f48aca5b26b3877c6dc31fc20729a30fd6b6 Mon Sep 17 00:00:00 2001 From: kuba Date: Wed, 28 Oct 2026 02:42:06 +0100 Subject: [PATCH 45/82] [GPU] Cache pixel-input instruction analysis per decoded program Each draw rescanned every instruction of the pixel program to build the per-vertex parameter mask. A decoded program never changes, so its instruction-only attribute mask is now cached next to the interpolated attribute list, and the draw's live interpolator controls are applied on every use. Warm 512-instruction sparse shader: ARM64 2,602 -> 8.5 ns, Rosetta x64 5,681 -> 13.7 ns. Cold entries cost 32 B and 0.9-2.3% on the smallest cases. Same-payload Silent Hill room medians were 16.93 fps on and 16.47 off; overlapping ranges and different paging do not establish an fps gain. Libs 9,106/0/5 and required-device Metal 1,639/0/7 passed. Set SHARPEMU_PIXEL_INPUT_ANALYSIS=0 for the per-draw scan. --- docs/performance/native-occlusion-runtime.md | 15 ++ docs/performance/performance-defaults.md | 9 + docs/performance/pixel-input-analysis.md | 180 ++++++++++++++++++ .../ShaderPipelineCache.PerVertexInputs.cs | 28 ++- .../Gpu/Pipelines/ShaderPipelineCache.cs | 23 ++- ...PixelInputAnalysisCacheTests.Benchmarks.cs | 98 ++++++++++ .../Pipelines/PixelInputAnalysisCacheTests.cs | 87 +++++++++ 7 files changed, 430 insertions(+), 10 deletions(-) create mode 100644 docs/performance/pixel-input-analysis.md create mode 100644 tests/SharpEmu.Libs.Tests/Gpu/Pipelines/PixelInputAnalysisCacheTests.Benchmarks.cs create mode 100644 tests/SharpEmu.Libs.Tests/Gpu/Pipelines/PixelInputAnalysisCacheTests.cs diff --git a/docs/performance/native-occlusion-runtime.md b/docs/performance/native-occlusion-runtime.md index 79a77eeef..89bb68ada 100644 --- a/docs/performance/native-occlusion-runtime.md +++ b/docs/performance/native-occlusion-runtime.md @@ -1465,3 +1465,18 @@ quality. Wait time fell while idle time rose; this is not an isolated FPS win. See [the full source-bound benchmark and runtime report](caller-resolution.md) for both run identifiers, windows, CPU counters, memory pressure and shutdown trap timing. The previous ten defaults remain on; 30 FPS remains unestablished. + + +## Pixel-input analysis cache (v25–v26) + +The native/JIT-mapped v25 profile identified repeated pixel-input instruction +analysis on the resolver. The follow-up caches immutable instruction facts and +keeps each draw's controls live. It is the twelfth promoted default, accepted +for component gains with correctness coverage. Same-payload v26 room medians +were 16.93 FPS enabled and 16.465 disabled; differing paging and overlapping +ranges prevent a reliable FPS-gain claim. Post-window resolver CPU fell from +3.052 to 2.510 s over ten seconds. The 30-FPS objective remains open. + +See [pixel-input analysis](pixel-input-analysis.md) for the source-bound +profiling method, warm/cold ARM64 and Rosetta measurements, tests, exact run +windows, host-pressure counters and shutdown limitations. diff --git a/docs/performance/performance-defaults.md b/docs/performance/performance-defaults.md index 5adbc2c24..80c6d7c65 100644 --- a/docs/performance/performance-defaults.md +++ b/docs/performance/performance-defaults.md @@ -22,6 +22,7 @@ VSync, guest quality, or unrelated application setting is changed. | Queued CP writes | `SHARPEMU_METAL_V2_QUEUE_CP_WRITES` | Removes synchronous handoffs for eligible writes when command threading is explicitly enabled. Pending ranges, prefetch gates and CPU-read fallbacks preserve ordering; producer-thread device tests cover byte/qword/sparse/predication cases. | | Bounded vertex-layout history | `SHARPEMU_VERTEX_LAYOUT_HISTORY` | Exact byte-verified reuse across alternating layouts; adaptive backoff for low reuse, 512 slots and 2 MiB additional payload budget. Component timing/allocation evidence and miss costs are in [the measurements](vertex-layout-history.md); no isolated game-FPS claim. | | Caller resolution for short queues | `SHARPEMU_METAL_V2_CALLER_RESOLVE` | At most eight outstanding jobs; FIFO, gates, pause and ownership checks retained. Single-job ARM64 2,874 → 542 ns, x64 3,160 → 972 ns; ARM64 empty-program burst 32 costs about 27 ns/job. Same-payload room medians 16.49 enabled / 16.48 disabled, no isolated FPS gain. [Full measurements](caller-resolution.md). | +| Cached pixel-input analysis | `SHARPEMU_PIXEL_INPUT_ANALYSIS` | Immutable decoded-program analysis with live draw controls. Warm 512-instruction sparse shader: ARM64 2,602 → 8.5 ns, x64 5,681 → 13.7 ns. Cold entries cost 32 extra bytes; smallest cold cases cost 0.9–2.3% time. [Full measurements and runtime limits](pixel-input-analysis.md). | The existing negative label page filter, exact materialization comparison, dense register lookup, mip-layout construction and target memo fixes remain enabled. @@ -146,3 +147,11 @@ wins and correctness checks. Its same-payload Silent Hill medians were 16.49 FPS enabled and 16.48 disabled; this establishes no game-FPS gain. The full component table, larger-batch cost, run windows and paging limitations are retained in [the caller-resolution report](caller-resolution.md). + + +The v26 follow-up adds cached pixel-input analysis as the twelfth promoted +default. Full Libs passed 9,106/0/5 and required-device Metal 1,639/0/7, with +79/0/0 focused executions on each architecture and another 79/0/0 with the +cache disabled. Same-payload room medians were 16.93 enabled and 16.465 disabled; +this does not establish a reliable FPS gain. The [full report](pixel-input-analysis.md) +retains the small cold-entry cost and the warm-path improvement. diff --git a/docs/performance/pixel-input-analysis.md b/docs/performance/pixel-input-analysis.md new file mode 100644 index 000000000..6883d11e7 --- /dev/null +++ b/docs/performance/pixel-input-analysis.md @@ -0,0 +1,180 @@ + + +# Cached pixel-input analysis + +The graphics resolver repeatedly scanned every decoded pixel instruction to find +which inputs need per-vertex values. The existing weak attribute cache now also +stores that instruction-only bitmask. Every draw still maps the set attributes +through its live input controls. Cache entries use decoded-program object +identity and weak lifetime; copied programs receive separate analysis. The public +compiler routine remains unchanged for callers with editable instruction arrays. + +`SHARPEMU_PIXEL_INPUT_ANALYSIS` defaults on; `0` restores the per-draw compiler +scan. The attribute cache still has the additional analysis field in that mode. +This adds 32 bytes per cold program in the measured cases, with no warm-path +allocation. It changes no shader arithmetic or rendering quality. + +## Why this path + +The v25 native sample of the existing caller-resolution payload resolved 5,028 +of 5,191 previously unnamed address lines using JIT method ranges from the same +process. The resolver had 3,621 snapshots, including 2,507 condition-wait samples; +`PerVertexParameterMask` had 190 self and 211 inclusive samples and was its +leading identified non-wait leaf. These are sampled stack counts, not exclusive +CPU percentages. The eight-second native sample and twelve-second JIT/Loader +capture followed the FPS window; no managed SampleProfiler was enabled. + +That unchanged-payload room window, 375–410 seconds, reported 16.935 FPS median +(10.47–17.99, 18 reports). It is diagnostic context, not a before/after result. +Its source was the immutable v24 candidate payload with caller resolution +explicitly enabled, equivalent to the later default policy at `f2bd8c7a7`; +the checkout hash in `run.env` alone does not identify the binary. + +## Component measurements + +Apple M4, .NET 10.0.3, Release with normal tiering. Seven alternating rounds +follow two seconds of warmup per case. Warm rounds use 100,000 operations for +16/512 instructions and 10,000 for 4,096; cold rounds use 1,024 distinct keys +after 64 warmup keys per mode. Sparse cases contain ALU instructions +and either zero or eight per-vertex interpolation inputs. Dense cases are an +interpolation-heavy stress test. Neither is a captured commercial shader. +Warm measurements compare the unchanged compiler scan to cached live mapping. +Cold measurements include the exact previous attribute-cache factory plus the +compiler scan, versus the combined cache plus live mapping. Each cold operation +has a distinct program identity; key creation is outside the measured region. +Wall time, current-thread CPU and allocation are recorded separately. These are +component measurements and establish no game-FPS improvement. + +### Warm sparse cases + +| Architecture | Instructions | Active inputs | Wall reference → cache ns/op | CPU reference → cache ns/op | Allocation reference → cache B/op | +| --- | ---: | ---: | ---: | ---: | ---: | +| arm64 | 16 | 0 | 97.74 → 4.45 | 97.65 → 4.46 | 0.00 → 0.00 | +| arm64 | 16 | 8 | 184.91 → 29.72 | 137.45 → 29.74 | 0.00 → 0.00 | +| arm64 | 512 | 0 | 2608.39 → 4.47 | 2608.21 → 4.48 | 0.00 → 0.00 | +| arm64 | 512 | 8 | 2601.72 → 8.53 | 2601.68 → 8.54 | 0.00 → 0.00 | +| arm64 | 4096 | 0 | 21035.27 → 4.54 | 21034.50 → 4.60 | 0.00 → 0.00 | +| arm64 | 4096 | 8 | 20413.52 → 8.54 | 20412.60 → 8.70 | 0.00 → 0.00 | +| x64 | 16 | 0 | 207.74 → 7.42 | 207.70 → 7.43 | 0.00 → 0.00 | +| x64 | 16 | 8 | 222.60 → 13.64 | 222.61 → 13.64 | 0.00 → 0.00 | +| x64 | 512 | 0 | 5730.62 → 7.56 | 5726.68 → 7.57 | 0.00 → 0.00 | +| x64 | 512 | 8 | 5680.88 → 13.66 | 5680.48 → 13.66 | 0.00 → 0.00 | +| x64 | 4096 | 0 | 44808.09 → 7.62 | 44807.60 → 7.70 | 0.00 → 0.00 | +| x64 | 4096 | 8 | 44856.74 → 13.60 | 44856.30 → 13.70 | 0.00 → 0.00 | + +### Cold sparse cases + +| Architecture | Instructions | Active inputs | Wall reference → cache ns/op | CPU reference → cache ns/op | Allocation reference → cache B/op | +| --- | ---: | ---: | ---: | ---: | ---: | +| arm64 | 16 | 0 | 988.28 → 1009.47 | 987.30 → 1009.77 | 344.00 → 376.00 | +| arm64 | 16 | 8 | 1978.03 → 1995.80 | 1978.52 → 1996.09 | 1000.00 → 1032.00 | +| arm64 | 512 | 0 | 4297.07 → 3818.75 | 4296.88 → 3819.34 | 344.00 → 376.00 | +| arm64 | 512 | 8 | 4500.39 → 4037.70 | 4500.98 → 4039.06 | 1000.00 → 1032.00 | +| arm64 | 4096 | 0 | 33679.00 → 30562.89 | 33678.71 → 30558.59 | 344.00 → 376.00 | +| arm64 | 4096 | 8 | 34802.54 → 30533.20 | 34802.73 → 30499.02 | 1000.00 → 1032.00 | +| x64 | 16 | 0 | 1832.13 → 1873.34 | 1832.03 → 1875.00 | 344.00 → 376.00 | +| x64 | 16 | 8 | 2773.73 → 2804.20 | 2776.37 → 2804.69 | 1000.00 → 1032.00 | +| x64 | 512 | 0 | 9051.46 → 7666.70 | 9031.25 → 7667.97 | 344.00 → 376.00 | +| x64 | 512 | 8 | 9706.54 → 8251.95 | 9708.01 → 8252.93 | 1000.00 → 1032.00 | +| x64 | 4096 | 0 | 70264.75 → 59116.21 | 70265.62 → 59115.23 | 344.00 → 376.00 | +| x64 | 4096 | 8 | 70745.51 → 59719.34 | 70731.45 → 59712.89 | 1000.00 → 1032.00 | + +The smallest cold cases cost 0.9–2.3% wall time and 32 bytes per +program. Larger cold cases improved in this run, but comparisons between the +old factory reproduced in the test and the production factory also include +JIT/call-layout differences. The robust result is removal of repeated scanning +on a warm cache, not a general shader-compilation speedup. Normal tiering and +host scheduling remain visible (especially the first ARM64 active-input case). + +The interpolation-dense stress cases also improved on the warm path. They are +retained in the raw JSON and are not treated as representative shader mixes. +The earlier v2 benchmark contained only that dense stress shape; v3 supersedes +it for acceptance. The cached arm allocated zero bytes in every warm case. The +reference arm allocated zero except its first 16-instruction round on each +architecture (0.0024 B/op, a one-time 240 B over 100,000 operations). + +Reproduce with `SHARPEMU_PIXEL_INPUT_BENCH_OUTPUT=` and +`dotnet test tests/SharpEmu.Libs.Tests -c Release --settings + docs/performance/shader-serial.runsettings --filter + FullyQualifiedName~PixelInputAnalysisCacheTests --blame-hang-timeout 300s`. +Use native ARM64 and Rosetta x64 hosts separately and serialize execution with +the machine-wide validation lock. Without the environment variable the optional +benchmark fact returns immediately; it is not run during ordinary CI. + +Private evidence lives under `artifacts/runtime-diagnosis-20261026/`: +`pixel-input-v3-source.json`, `pixel-input-v3-{arm64,x64}.{json,log}` and +`pixel-input-results/pixel-input-v3-{arm64,x64}.trx`. Source parent is +`f2bd8c7a7717924d3252eb501f28ee7ef1ab2682`; the source manifest records hashes +for both production files and both test files. + +| Raw benchmark | SHA-256 | +| --- | --- | +| `pixel-input-v3-arm64.json` | `396e3b1f7a24fcfb130c114a9129d44a38b64e6c14490a2f2c9d7fd9bf945e68` | +| `pixel-input-v3-x64.json` | `56b78d2b492d60aed5d4196335c95a5d340abba86c30b7d585bf86048d45e311` | + +## Validation and runtime + +Focused native and Rosetta runs each passed 79 tests, zero failures or skips. +Full default-on x64 Libs passed **9,106/0/5**, native required-device Metal passed +**1,639/0/7**, and the fallback (`SHARPEMU_PIXEL_INPUT_ANALYSIS=0`) pipeline +selection passed **79/0/0** (passed/failed/skipped). The five new semantic facts +cover changing controls, program copies, interpolation/modifier equivalence, +concurrent readers and the unchanged public editable-IR routine. The sixth fact +is the optional benchmark. Existing attribute-cache and pipeline tests also ran. +REUSE passed 2,881/2,881 including this report; `git diff --check` passed. +Full commands, environment overrides, timings and TRX hashes are retained in +`pixel-input-final-validation.json` and `pixel-input-final-results.json`. + +### Same-payload Silent Hill control + +Both publishes passed (x64 CLI and native ARM64 GPU host, self-contained Release, +ReadyToRun disabled). The immutable `runtime-pixel-input-v26` payload has 471 +verified file hashes. Binary source is parent `f2bd8c7a7` plus tracked patch +`8007ab029bd25281f28fec279133277f2d3cbf813d45616b8899c2b250fc47a7`; +source snapshot, patch and untracked-test archive are preserved. Production and +test hashes match the v3 benchmark manifest. Later documentation does not change +the tested executable. + +Both runs used fresh copies of `runtime-data-room-warm-defaults-v20-1`, 1920×1080, +scale 1, VSync, unchanged quality and the eleven previous effective defaults. +Command threading, local draw reuse, shader prewarming and detailed profiling +stayed off; the application's existing 64 MiB GC budget was applied to both +processes. Only `SHARPEMU_PIXEL_INPUT_ANALYSIS=1/0` differed. Automatic Enter +stopped at 258 s; Escape holds at 258.15, 261.20 and 309.59 s reproduced the +cutscene skip and phone-close sequence. The shelf/lamp room matched visually. +GeForce NOW and its settings were left unchanged. + +Each 375–410 s FPS window was selected at process startup, before reading its +results. Ten seconds of thread CPU sampling followed at approximately 417–427 s; +no native or managed stack profiler ran during either comparison. + +| Cache | Run directory prefix / suffix | FPS median | Range | Reports | Resolver CPU s | GPU-worker CPU s | Guest RHI CPU s | +| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | +| On | `20261028-004544` / `room-pixel-input-v26-on-1` | 16.930 | 11.97–18.97 | 17 | 2.510 | 8.241 | 6.791 | +| Off | `20261028-005314` / `room-pixel-input-v26-off-2` | 16.465 | 10.98–17.97 | 18 | 3.052 | 8.269 | 6.790 | + +The resolver's post-window CPU time fell by about 0.54 s over ten seconds, while +GPU-worker and guest RHI CPU were almost unchanged. Those samples are separate +from the FPS window and are not a fixed-work instruction benchmark. In-window +PERF reports summed 6,850.5 ms background resolution / 813.9 ms caller resolution +with the cache, versus 9,237.0 / 972.5 ms without it. Report durations differ +(34.28 versus 36.44 s), so raw totals are not equal-work comparisons. Both modes +reported zero host-wait milliseconds and zero refused resolutions. + +This pair does not establish a reliable FPS gain: the ranges overlap and host +paging differs. Seven system-memory samples in each FPS window recorded +10,354/11,156 swap-in pages and 30,256/0 swap-out pages (on/off), with memory +pressure levels 1–2 in both. Compression/decompression also differed; these are +whole-system counters, not emulator-only attribution. The cache is accepted as +a default for its measured warm-path benefit and tested semantics, with the +cold-entry cost retained. **30 FPS remains unachieved.** + +Both harnesses stopped at their own marker after 427 s and exited 0 after four +seconds. CPU_TRAP results occurred only after the requested stops (428.998 s on, +429.022 s off); they remain shutdown diagnostics, not clean-shutdown assertions. +No runtime fatal or unsolicited CPU trap preceded either stop. All four game +processes exited. Main-process peak footprint/RSS was 9,883/4,310 MiB on and +9,831/5,012 MiB off; these are not combined physical-memory figures for the split +processes. Full results are retained in `pixel-input-v26-runtime-results.json` +and each run's FPS, memory, thread-CPU, input, screenshot and result files. diff --git a/src/SharpEmu.Libs/Gpu/Pipelines/ShaderPipelineCache.PerVertexInputs.cs b/src/SharpEmu.Libs/Gpu/Pipelines/ShaderPipelineCache.PerVertexInputs.cs index 0c9c58fdd..238383715 100644 --- a/src/SharpEmu.Libs/Gpu/Pipelines/ShaderPipelineCache.PerVertexInputs.cs +++ b/src/SharpEmu.Libs/Gpu/Pipelines/ShaderPipelineCache.PerVertexInputs.cs @@ -39,6 +39,32 @@ internal enum PerVertexInputMode private readonly HashSet<(ulong Hash, string Path)> _loggedPerVertexPaths = new(); + private static readonly bool CachePixelInputAnalysis = + Environment.GetEnvironmentVariable("SHARPEMU_PIXEL_INPUT_ANALYSIS") != "0"; + + // Decoded programs have the same immutable lifetime as InterpolatedAttributeCache. + // Cache by object identity, not shader address or a draw's mutable control words. + // The public compiler analysis stays fresh for callers constructing or editing IR. + internal static uint PerVertexParametersOf(Gen5ShaderProgram program, ReadOnlySpan controls, bool cached) + { + if (!cached) + { + return Gen5PixelInputMapping.PerVertexParameterMask(program, controls); + } + + var attributes = PixelAnalysisOf(program).PerVertexAttributes; + var parameters = 0u; + while (attributes != 0) + { + var attribute = System.Numerics.BitOperations.TrailingZeroCount(attributes); + var offset = attribute < controls.Length ? controls[attribute] & 0x1Fu : (uint)attribute; + parameters |= 1u << (int)offset; + attributes &= attributes - 1; + } + + return parameters; + } + /// /// Where the per-vertex values of a pixel program's inputs come from: host interpolants /// (the plan), the mesh primitives (the returned param mask), or nowhere. @@ -54,7 +80,7 @@ private uint ResolvePerVertexInputs( { manualInterpolation = null; var controls = pixelInfo.InterpolatorSettings.AsSpan(0, (int)pixelInfo.InputCount); - var mask = Gen5PixelInputMapping.PerVertexParameterMask(pixelProgram, controls); + var mask = PerVertexParametersOf(pixelProgram, controls, CachePixelInputAnalysis); if (mask == 0) { return 0; diff --git a/src/SharpEmu.Libs/Gpu/Pipelines/ShaderPipelineCache.cs b/src/SharpEmu.Libs/Gpu/Pipelines/ShaderPipelineCache.cs index 1f8a70a63..47144f716 100644 --- a/src/SharpEmu.Libs/Gpu/Pipelines/ShaderPipelineCache.cs +++ b/src/SharpEmu.Libs/Gpu/Pipelines/ShaderPipelineCache.cs @@ -640,18 +640,23 @@ private HostViewportTransform HostViewport(ContextRegisters context) // The interpolated attributes a pixel program reads, ascending. A program never changes, and this // was rebuilt with LINQ on every draw. - private static readonly System.Runtime.CompilerServices.ConditionalWeakTable InterpolatedAttributeCache = new(); + private sealed record PixelProgramAnalysis(uint[] Attributes, uint PerVertexAttributes); + + private static readonly System.Runtime.CompilerServices.ConditionalWeakTable InterpolatedAttributeCache = new(); internal uint[] GetPixelAttributes(Gen5ShaderProgram program) => InterpolatedAttributesOf(program); - internal static uint[] InterpolatedAttributesOf(Gen5ShaderProgram program) => - InterpolatedAttributeCache.GetValue(program, static program => program.Instructions - .Select(static instruction => instruction.Control) - .OfType() - .Select(static control => control.Attribute) - .Distinct() - .Order() - .ToArray()); + internal static uint[] InterpolatedAttributesOf(Gen5ShaderProgram program) => PixelAnalysisOf(program).Attributes; + + private static PixelProgramAnalysis PixelAnalysisOf(Gen5ShaderProgram program) => + InterpolatedAttributeCache.GetValue(program, static program => new PixelProgramAnalysis( + program.Instructions.Select(static instruction => instruction.Control) + .OfType() + .Select(static control => control.Attribute) + .Distinct().Order().ToArray(), + // Empty controls map each input to itself: this is the instruction-only + // attribute mask. Draw controls still map those attributes on every use. + Gen5PixelInputMapping.PerVertexParameterMask(program, []))); // One past the highest parameter location the pixel program reads, resolved as its translator does. internal static uint ReadVertexOutputCount(ReadOnlySpan attributes, PixelInputInfo info) diff --git a/tests/SharpEmu.Libs.Tests/Gpu/Pipelines/PixelInputAnalysisCacheTests.Benchmarks.cs b/tests/SharpEmu.Libs.Tests/Gpu/Pipelines/PixelInputAnalysisCacheTests.Benchmarks.cs new file mode 100644 index 000000000..423aa6d55 --- /dev/null +++ b/tests/SharpEmu.Libs.Tests/Gpu/Pipelines/PixelInputAnalysisCacheTests.Benchmarks.cs @@ -0,0 +1,98 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using System.Diagnostics; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Text.Json; +using SharpEmu.Libs.Gpu.Pipelines; +using SharpEmu.PerfAudit; +using SharpEmu.ShaderCompiler; +using Xunit; + +namespace SharpEmu.Libs.Tests.Gpu.Pipelines; + +public sealed partial class PixelInputAnalysisCacheTests +{ + [Fact] + public void BenchmarkPixelInputAnalysisWhenRequested() + { + var output = Environment.GetEnvironmentVariable("SHARPEMU_PIXEL_INPUT_BENCH_OUTPUT"); + if (string.IsNullOrEmpty(output)) return; + var rows = new List(); + // Exact previous attribute-cache factory, paired with the unchanged compiler mask. + // This includes the old cold-cache work instead of comparing a cache fill to a mask alone. + var referenceAttributes = new ConditionalWeakTable(); + uint Cold(Gen5ShaderProgram program, uint[] controls, bool cached) + { + if (cached) ShaderPipelineCache.InterpolatedAttributesOf(program); + else referenceAttributes.GetValue(program, static p => p.Instructions + .Select(static i => i.Control).OfType() + .Select(static c => c.Attribute).Distinct().Order().ToArray()); + return ShaderPipelineCache.PerVertexParametersOf(program, controls, cached); + } + foreach (var length in new[] { 16, 512, 4096 }) + foreach (var dense in new[] { false, true }) + foreach (var active in new[] { false, true }) + { + var ordinary = dense ? Interpolation("VInterpP1F32", 1) : + new Gen5ShaderInstruction(0, Gen5ShaderEncoding.Vop2, "VAddF32", [0], + [Gen5Operand.Vector(0)], [Gen5Operand.Vector(1), Gen5Operand.Vector(2)], null); + var instructions = Enumerable.Repeat(ordinary, length).ToArray(); + if (active) + for (var i = 0; i < 8; i++) instructions[i * (length / 8)] = Interpolation("VInterpMovF32", (uint)i); + var program = new Gen5ShaderProgram(0, instructions); + uint[] controls = Enumerable.Range(0, 32).Select(i => (uint)i).ToArray(); + var expected = active ? 255u : 0u; + ShaderPipelineCache.InterpolatedAttributesOf(program); + uint Run(bool cached, int count) + { + var sum = 0u; + for (var i = 0; i < count; i++) + sum += ShaderPipelineCache.PerVertexParametersOf(program, controls, cached); + return sum; + } + var warm = Stopwatch.StartNew(); + while (warm.Elapsed < TimeSpan.FromSeconds(2)) { Run(false, 256); Run(true, 256); } + var count = length == 4096 ? 10000 : 100000; + for (var round = 0; round < 7; round++) + foreach (var cached in round % 2 == 0 ? new[] { false, true } : new[] { true, false }) + { + var bytes = GC.GetAllocatedBytesForCurrentThread(); + var cpu = DarwinThreadCpuClock.Nanoseconds(); + var started = Stopwatch.GetTimestamp(); + var sum = Run(cached, count); + var wallNs = Stopwatch.GetElapsedTime(started).TotalNanoseconds / count; + var cpuNs = (DarwinThreadCpuClock.Nanoseconds() - cpu) / (double)count; + var allocated = (GC.GetAllocatedBytesForCurrentThread() - bytes) / (double)count; + Assert.Equal(unchecked(expected * count), sum); + rows.Add(new { phase="warm", length, dense, active, round, cached, count, wallNs, cpuNs, allocated }); + } + // Precreate keys outside the measurement. Instructions are shared and immutable; + // each record instance has a distinct weak-cache identity, including the warmup keys. + foreach (var cached in new[] { false, true }) + for (var i = 0; i < 64; i++) Cold(program with { }, controls, cached); + const int coldCount = 1024; + for (var round = 0; round < 7; round++) + foreach (var cached in round % 2 == 0 ? new[] { false, true } : new[] { true, false }) + { + var keys = Enumerable.Range(0, coldCount).Select(_ => program with { }).ToArray(); + var bytes = GC.GetAllocatedBytesForCurrentThread(); + var cpu = DarwinThreadCpuClock.Nanoseconds(); + var started = Stopwatch.GetTimestamp(); + var sum = 0u; + foreach (var key in keys) sum += Cold(key, controls, cached); + var wallNs = Stopwatch.GetElapsedTime(started).TotalNanoseconds / coldCount; + var cpuNs = (DarwinThreadCpuClock.Nanoseconds() - cpu) / (double)coldCount; + var allocated = (GC.GetAllocatedBytesForCurrentThread() - bytes) / (double)coldCount; + Assert.Equal(unchecked(expected * coldCount), sum); + GC.KeepAlive(keys); + rows.Add(new { phase="cold", length, dense, active, round, cached, count=coldCount, wallNs, cpuNs, allocated }); + } + } + Directory.CreateDirectory(Path.GetDirectoryName(Path.GetFullPath(output))!); + File.WriteAllText(output, JsonSerializer.Serialize(new { architecture=RuntimeInformation.ProcessArchitecture.ToString(), + framework=RuntimeInformation.FrameworkDescription, warmupSeconds=2, rounds=7, + caveat="Synthetic sparse ALU and interpolation-dense stress programs, no GPU. Cold includes old attribute-cache factory plus compiler mask versus combined cache plus live mapping; key creation excluded. No game FPS inference.", rows }, new JsonSerializerOptions { WriteIndented=true })); + } +} diff --git a/tests/SharpEmu.Libs.Tests/Gpu/Pipelines/PixelInputAnalysisCacheTests.cs b/tests/SharpEmu.Libs.Tests/Gpu/Pipelines/PixelInputAnalysisCacheTests.cs new file mode 100644 index 000000000..aa5942878 --- /dev/null +++ b/tests/SharpEmu.Libs.Tests/Gpu/Pipelines/PixelInputAnalysisCacheTests.cs @@ -0,0 +1,87 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using SharpEmu.Libs.Gpu.Pipelines; +using SharpEmu.ShaderCompiler; +using Xunit; + +namespace SharpEmu.Libs.Tests.Gpu.Pipelines; + +public sealed partial class PixelInputAnalysisCacheTests +{ + private static Gen5ShaderInstruction Interpolation(string opcode, uint attribute, uint parameter = 0, + uint absolute = 0, uint negate = 0) => + new(0, Gen5ShaderEncoding.Vintrp, opcode, [0], [Gen5Operand.Vector(0)], [Gen5Operand.Vector(4)], + new Gen5InterpolationControl(attribute, 0) + { + Parameter = parameter, + Modifiers = absolute != 0 || negate != 0 ? new Gen5Vop3Control(absolute, negate, 0, false, 0, null) : null, + }); + + [Fact] + public void ChangingControlsAreMappedAfterTheInstructionAnalysis() + { + var program = new Gen5ShaderProgram(0x1000, + [Interpolation("VInterpMovF32", 1), Interpolation("VInterpMovF32", 31), Interpolation("VInterpMovF32", 2, 2)]); + uint[] controls = [0, 0x405]; + Assert.Equal((1u << 5) | (1u << 31), ShaderPipelineCache.PerVertexParametersOf(program, controls, true)); + controls[1] = 0x41f; + Assert.Equal(1u << 31, ShaderPipelineCache.PerVertexParametersOf(program, controls, true)); + Assert.Equal((1u << 1) | (1u << 31), ShaderPipelineCache.PerVertexParametersOf(program, [], true)); + Assert.Equal(Gen5PixelInputMapping.PerVertexParameterMask(program, controls), + ShaderPipelineCache.PerVertexParametersOf(program, controls, false)); + } + + [Fact] + public void RecordCopiesWithDifferentInstructionsDoNotInheritAnalysis() + { + var original = new Gen5ShaderProgram(0x1000, [Interpolation("VInterpMovF32", 1)]); + Assert.Equal(2u, ShaderPipelineCache.PerVertexParametersOf(original, [], true)); + var changed = original with { Instructions = [Interpolation("VInterpMovF32", 4)] }; + var empty = original with { Instructions = [] }; + Assert.Equal(16u, ShaderPipelineCache.PerVertexParametersOf(changed, [], true)); + Assert.Equal(0u, ShaderPipelineCache.PerVertexParametersOf(empty, [], true)); + Assert.Equal(2u, ShaderPipelineCache.PerVertexParametersOf(original, [], true)); + } + + [Fact] + public void CachedAnalysisMatchesTheCompilerAcrossInterpolationFormsAndControlLengths() + { + string[] opcodes = ["VInterpMovF32", "VInterpP1F32", "VInterpP2F32", "VInterpP1llF16", "VInterpP1lvF16", "VInterpP2F16"]; + var random = new System.Random(70413); + for (var trial = 0; trial < 64; trial++) + { + var instructions = Enumerable.Range(0, 128).Select(_ => Interpolation(opcodes[random.Next(opcodes.Length)], + (uint)random.Next(40), (uint)random.Next(3), (uint)random.Next(8), (uint)random.Next(8))).ToArray(); + var program = new Gen5ShaderProgram((ulong)trial, instructions); + foreach (var length in new[] { 0, 1, 7, 31, 32, 40 }) + { + var controls = Enumerable.Range(0, length).Select(_ => (uint)random.Next()).ToArray(); + Assert.Equal(Gen5PixelInputMapping.PerVertexParameterMask(program, controls), + ShaderPipelineCache.PerVertexParametersOf(program, controls, true)); + } + } + } + + [Fact] + public async Task SharedProgramAnalysisIsSafeForConcurrentReaders() + { + var program = new Gen5ShaderProgram(0, [Interpolation("VInterpMovF32", 3), Interpolation("VInterpP1F32", 3)]); + await Task.WhenAll(Enumerable.Range(0, 16).Select(index => Task.Run(() => + { + uint[] controls = [0, 1, 2, (uint)index]; + for (var iteration = 0; iteration < 1000; iteration++) + Assert.Equal(1u << index, ShaderPipelineCache.PerVertexParametersOf(program, controls, true)); + }))); + } + + [Fact] + public void PublicCompilerAnalysisStillObservesEditableInstructionArrays() + { + Gen5ShaderInstruction[] instructions = [Interpolation("VInterpMovF32", 1)]; + var program = new Gen5ShaderProgram(0, instructions); + Assert.Equal(2u, Gen5PixelInputMapping.PerVertexParameterMask(program, [])); + instructions[0] = Interpolation("VInterpMovF32", 4); + Assert.Equal(16u, Gen5PixelInputMapping.PerVertexParameterMask(program, [])); + } +} From 7558bab906e48d166955c27563046f5e3f47dc3a Mon Sep 17 00:00:00 2001 From: kuba Date: Wed, 28 Oct 2026 02:50:48 +0100 Subject: [PATCH 46/82] [GPU] Apply the pipeline look-ahead time limit only after its first block The 50 ms limit counted from before the first packet read, so a first scan slowed by JIT or a busy host stopped before reading anything. PipelineLookaheadTests then saw no prefetched dispatches and failed intermittently on CI (run 40600561034). The scan now always reads its first 2048-word block before checking the clock; the word and dispatch caps are unchanged. Libs 8,901 passed / 122 skipped. --- .../Gpu/GpuCommands/GpuCommandInterpreter.Lookahead.cs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/SharpEmu.Libs/Gpu/GpuCommands/GpuCommandInterpreter.Lookahead.cs b/src/SharpEmu.Libs/Gpu/GpuCommands/GpuCommandInterpreter.Lookahead.cs index 84c1dbac7..0633da621 100644 --- a/src/SharpEmu.Libs/Gpu/GpuCommands/GpuCommandInterpreter.Lookahead.cs +++ b/src/SharpEmu.Libs/Gpu/GpuCommands/GpuCommandInterpreter.Lookahead.cs @@ -25,7 +25,8 @@ public sealed partial class GpuCommandInterpreter Environment.GetEnvironmentVariable("SHARPEMU_GPU_PIPELINE_LOOKAHEAD") != "0"; // A look-ahead stops at whichever comes first; the next dispatch that suspends past where it - // stopped reads on from there. + // stopped reads on from there. The time limit applies only after the first block, so a first + // scan slowed by JIT or a busy host still reads the packets right after the waiting dispatch. private const uint LookaheadMaxDwords = 1u << 20; private const int LookaheadMaxDispatches = 2048; private const int LookaheadMaxDepth = 4; @@ -187,7 +188,7 @@ private uint ScanAhead(ref Lookahead state, ulong address, uint dwordCount, uint } if (state.Dwords >= LookaheadMaxDwords || state.Dispatches >= LookaheadMaxDispatches || - Stopwatch.GetTimestamp() > state.Deadline) + (state.Dwords >= LookaheadBlockDwords && Stopwatch.GetTimestamp() > state.Deadline)) { state.Stopped = true; return address == startAddress ? offset : uint.MaxValue; From cdb27f4b98e8ac7b6e5a388733b9af18b41382e1 Mon Sep 17 00:00:00 2001 From: kuba Date: Wed, 28 Oct 2026 03:24:56 +0100 Subject: [PATCH 47/82] [GPU] Share one wave64 exchange between back-to-back constant-lane V_READLANE_B32 Cause: each V_READLANE_B32 of a guest wave64 is its own threadgroup exchange (flip, owner writes, barrier, read). The wave-scan idiom reads lane 31 and lane 63 back to back, so it paid two barriers where one serves both: the owners write slots 2 and 3 of the same set. Two consecutive reads from inline-constant lanes, both vector source to scalar destination and neither a spill reload, now take one exchange. Everything else keeps one each. Round-2 k21 items 1-6 and 7 (default off) were already in this branch; of the rest, the get_num_mip_levels skip is in too, and this is the wave64 exchange item. Demon's Souls compute dumps have 24-93 exchanges per shader, 4-6 of them readlanes, so the real effect is small. Measured (MslTranslatorOverheadBenchTests.FusedLaneReads, 256 trips of lane 31 + 63, 4096 groups of 64, GPU start/end time, best of 3, two runs): 2.099 ms -> 2.009 ms (-4.3%) in both runs. Execution test on device returns each lane's values in both halves. Switch: SHARPEMU_MSL_READLANE_FUSE=0 keeps one exchange per read. --- .../Gen5MslCodegenOptions.cs | 11 ++++- .../Gen5MslTranslator.Alu.cs | 45 +++++++++++++++++++ .../Gen5MslTranslator.cs | 23 ++++++++-- .../Gen5ComputeFixtures.cs | 22 +++++++++ .../MetalRuntimeTests.cs | 18 ++++++++ .../MslTranslationTests.cs | 23 +++++++++- .../MslTranslatorOverheadBenchTests.cs | 41 ++++++++++++++--- 7 files changed, 171 insertions(+), 12 deletions(-) diff --git a/src/SharpEmu.ShaderCompiler.Metal/Gen5MslCodegenOptions.cs b/src/SharpEmu.ShaderCompiler.Metal/Gen5MslCodegenOptions.cs index 46ed94c5e..cce94657d 100644 --- a/src/SharpEmu.ShaderCompiler.Metal/Gen5MslCodegenOptions.cs +++ b/src/SharpEmu.ShaderCompiler.Metal/Gen5MslCodegenOptions.cs @@ -120,6 +120,11 @@ namespace SharpEmu.ShaderCompiler.Metal; /// the product on the guest, so a fused result can differ by an ULP; vertex and mesh programs, /// which feed positions, keep the pragma either way. Off by default; /// SHARPEMU_MSL_CONTRACT=1 turns it on. +/// Two V_READLANE_B32 in a row from constant lanes of a guest wave64 +/// share one exchange (two value slots, one barrier) instead of one each. The wave-scan and +/// reduction idiom reads lane 31 and lane 63 back to back. A pair whose second lane is not an +/// inline constant keeps one exchange each. SHARPEMU_MSL_READLANE_FUSE=0 keeps one +/// exchange per read. public sealed record Gen5MslCodegenOptions( bool StructuredControlFlow = true, bool RegisterAllocation = true, @@ -144,7 +149,8 @@ public sealed record Gen5MslCodegenOptions( bool ConstantTableSlots = true, bool FastReciprocals = true, bool ContractPixelCompute = false, - bool RuntimeDwordFormatLoads = true) + bool RuntimeDwordFormatLoads = true, + bool FusedLaneReads = true) { /// Diagnostic threshold for moving mesh LDS from threadgroup memory to the /// device pool, in bytes. The supported mesh allocation is at most 16 KiB. Zero @@ -211,7 +217,8 @@ public sealed record Gen5MslCodegenOptions( ConstantTableSlots: Environment.GetEnvironmentVariable("SHARPEMU_MSL_TABLEPAD") != "0", FastReciprocals: Environment.GetEnvironmentVariable("SHARPEMU_MSL_FASTRCP") != "0", ContractPixelCompute: Environment.GetEnvironmentVariable("SHARPEMU_MSL_CONTRACT") == "1", - RuntimeDwordFormatLoads: Environment.GetEnvironmentVariable("SHARPEMU_MSL_RUNTIME_DWORD_FORMAT") != "0") + RuntimeDwordFormatLoads: Environment.GetEnvironmentVariable("SHARPEMU_MSL_RUNTIME_DWORD_FORMAT") != "0", + FusedLaneReads: Environment.GetEnvironmentVariable("SHARPEMU_MSL_READLANE_FUSE") != "0") { UnobservedAluWrites = Environment.GetEnvironmentVariable("SHARPEMU_MSL_UNOBSERVED_ALU") == "1", GeometryOutputCapacity = Environment.GetEnvironmentVariable("SHARPEMU_MSL_GEOMETRY_CAPACITY") == "1", diff --git a/src/SharpEmu.ShaderCompiler.Metal/Gen5MslTranslator.Alu.cs b/src/SharpEmu.ShaderCompiler.Metal/Gen5MslTranslator.Alu.cs index 8eb4c423b..b9702a398 100644 --- a/src/SharpEmu.ShaderCompiler.Metal/Gen5MslTranslator.Alu.cs +++ b/src/SharpEmu.ShaderCompiler.Metal/Gen5MslTranslator.Alu.cs @@ -1846,6 +1846,51 @@ private string EmitWave64ReadFirstLane(string value) private string EmitWave64ReadLane(string value, string lane) => EmitWave64BroadcastFromLane(value, lane); + /// Whether and are two + /// consecutive V_READLANE_B32 of a wave64 that one exchange can serve: both read a + /// vector register at an inline-constant lane into a scalar register, and neither + /// is a reload of a parked scalar. + private bool CanFuseWave64ReadLanes(Gen5ShaderInstruction first, Gen5ShaderInstruction second) + { + if (!_options.FusedLaneReads || !IsWave64 || IsSingleLaneStage || + first.Opcode != "VReadlaneB32" || second.Opcode != "VReadlaneB32" || + !IsPlainConstantLaneRead(first) || !IsPlainConstantLaneRead(second)) + { + return false; + } + + if (LaneSpillPlan() is { } spills && + (spills.CopyReads.ContainsKey(first.Pc) || spills.CopyReads.ContainsKey(second.Pc))) + { + return false; + } + + return true; + } + + private static bool IsPlainConstantLaneRead(Gen5ShaderInstruction instruction) => + instruction.Destinations.Count != 0 && + instruction.Destinations[0].Kind == Gen5OperandKind.ScalarRegister && + instruction.Sources.Count >= 2 && + instruction.Sources[0].Kind == Gen5OperandKind.VectorRegister && + instruction.Sources[1].Kind is Gen5OperandKind.EncodedConstant or Gen5OperandKind.LiteralConstant; + + /// Emits two fused V_READLANE_B32: each lane's owner writes its own value slot, + /// one barrier, then both scalars read their slot. + private void EmitFusedWave64ReadLanes(Gen5ShaderInstruction first, Gen5ShaderInstruction second) + { + var valueA = RawSource(first, 0); + var laneA = Temp("uint", $"({RawSource(first, 1)}) & 63u"); + var valueB = RawSource(second, 0); + var laneB = Temp("uint", $"({RawSource(second, 1)}) & 63u"); + Line(WaveSetFlip); + Line($"if (sharpemu_lane == {laneA}) {{ {WaveSlot(2)} = {valueA}; }}"); + Line($"if (sharpemu_lane == {laneB}) {{ {WaveSlot(3)} = {valueB}; }}"); + Line(WaveExchangeBarrier); + StoreScalar(first.Destinations[0].Value, Temp("uint", WaveSlot(2))); + StoreScalar(second.Destinations[0].Value, Temp("uint", WaveSlot(3))); + } + /// Stores the wave ballot of into the /// mask register pair (low, low+1). Wave32 fills the low dword and clears /// the high; wave64 bridges both 32-wide halves through threadgroup diff --git a/src/SharpEmu.ShaderCompiler.Metal/Gen5MslTranslator.cs b/src/SharpEmu.ShaderCompiler.Metal/Gen5MslTranslator.cs index 6d26428aa..e0b4212a2 100644 --- a/src/SharpEmu.ShaderCompiler.Metal/Gen5MslTranslator.cs +++ b/src/SharpEmu.ShaderCompiler.Metal/Gen5MslTranslator.cs @@ -1850,13 +1850,30 @@ private bool TryEmitBlockBodyCore( return true; } - if (!TryEmitInstruction(instruction, out error)) + if (!isTerminator && CanFuseWave64ReadLanes(instruction, instructions[index + 1])) + { + // One exchange for both reads (see EmitFusedWave64ReadLanes). + var second = instructions[++index]; + isTerminator = index == block.EndIndex - 1; + if (Gen5MslDump.Enabled) + { + Line($"// 0x{second.Pc:X4} {second.Opcode} (fused)"); + } + + BeginInstruction(instruction); + EmitFusedWave64ReadLanes(instruction, second); + EmitInterpolationRewrites(instruction.Pc); + EmitInterpolationRewrites(second.Pc); + } + else if (!TryEmitInstruction(instruction, out error)) { error = $"pc=0x{instruction.Pc:X4} {instruction.Opcode}: {error}"; return false; } - - EmitInterpolationRewrites(instruction.Pc); + else + { + EmitInterpolationRewrites(instruction.Pc); + } if (isTerminator) { diff --git a/tests/SharpEmu.ShaderCompiler.Metal.Tests/Gen5ComputeFixtures.cs b/tests/SharpEmu.ShaderCompiler.Metal.Tests/Gen5ComputeFixtures.cs index 549900e42..2e0def03f 100644 --- a/tests/SharpEmu.ShaderCompiler.Metal.Tests/Gen5ComputeFixtures.cs +++ b/tests/SharpEmu.ShaderCompiler.Metal.Tests/Gen5ComputeFixtures.cs @@ -378,6 +378,28 @@ public static Gen5MslShader CompilePixelOrThrow( StoreScalarResourceBase: 8, StoreBackingBytes: 512); + // v1 = 3 * lane + 7; three constant-lane reads in a row: lanes 63 and 5 (one fused + // exchange) and lane 40 (its own). + public static readonly Gen5ComputeFixture Wave64ReadLaneTriple = new( + "wave64-readlane-triple", + [ + .. Wave64LaneIndex, + 0xD5430001, 0x021D0700, // v_mad_u32_u24 v1, v0, 3, 7 + 0xD7600000, 0x00017F01, // v_readlane_b32 s0, v1, 63 + 0xD7600001, 0x00010B01, // v_readlane_b32 s1, v1, 5 + 0xD7600002, 0x00015101, // v_readlane_b32 s2, v1, 40 + 0x7E040200, // v_mov_b32 v2, s0 + 0x7E060201, // v_mov_b32 v3, s1 + 0x7E0A0202, // v_mov_b32 v5, s2 + 0x34080084, // v_lshlrev_b32 v4, 4, v0 + 0xE0701000, 0x80020204, // buffer_store_dword v2, v4, s[8:11], 0 offen + 0xE0701004, 0x80020304, // buffer_store_dword v3, v4, s[8:11], 0 offen offset:4 + 0xE0701008, 0x80020504, // buffer_store_dword v5, v4, s[8:11], 0 offen offset:8 + 0xBF810000, // s_endpgm + ], + StoreScalarResourceBase: 8, + StoreBackingBytes: 1024); + private static readonly uint[] Wave64LaneSpillPark = [ 0xBE8203FF, 0x00001234, // s_mov_b32 s2, 0x1234 diff --git a/tests/SharpEmu.ShaderCompiler.Metal.Tests/MetalRuntimeTests.cs b/tests/SharpEmu.ShaderCompiler.Metal.Tests/MetalRuntimeTests.cs index 22ecaaf79..1cac8d4ab 100644 --- a/tests/SharpEmu.ShaderCompiler.Metal.Tests/MetalRuntimeTests.cs +++ b/tests/SharpEmu.ShaderCompiler.Metal.Tests/MetalRuntimeTests.cs @@ -264,6 +264,24 @@ public void Wave64ReadLaneReadsTheFullWaveInBothHalves() } } + [Fact] + public void Wave64FusedConstantLaneReadsReturnEachLanesValue() + { + if (SkipWithoutMetalDevice("the execution test")) return; + + var result = ExecuteRequestOrThrow(Gen5ComputeFixtures.Wave64ReadLaneTriple, new byte[1024], + waveLaneCount: 64, localSizeX: 64, threadsPerThreadgroup: 64); + + // Lane l holds 3 * l + 7: every lane sees lanes 63 and 5 from the fused exchange and + // lane 40 from its own. + for (var lane = 0; lane < 64; lane++) + { + Assert.Equal(3u * 63u + 7u, ReadDword(result, lane * 16)); + Assert.Equal(3u * 5u + 7u, ReadDword(result, lane * 16 + 4)); + Assert.Equal(3u * 40u + 7u, ReadDword(result, lane * 16 + 8)); + } + } + [Fact] public void Wave64SpillReloadsReadACopyInsteadOfTheExchange() { diff --git a/tests/SharpEmu.ShaderCompiler.Metal.Tests/MslTranslationTests.cs b/tests/SharpEmu.ShaderCompiler.Metal.Tests/MslTranslationTests.cs index 06417ffbd..ae3e222a4 100644 --- a/tests/SharpEmu.ShaderCompiler.Metal.Tests/MslTranslationTests.cs +++ b/tests/SharpEmu.ShaderCompiler.Metal.Tests/MslTranslationTests.cs @@ -552,7 +552,7 @@ public void OnlyTheLiveHalfOfAMixedWave64ProgramKeepsItsExchange() /// flips the slot set once, so flips and barriers match. [Theory] [InlineData("readfirstlane", 2)] - [InlineData("readlane", 2)] + [InlineData("readlane", 1)] [InlineData("loop", 2)] public void EveryWave64ExchangeCostsOneBarrier(string program, int exchanges) { @@ -568,6 +568,27 @@ public void EveryWave64ExchangeCostsOneBarrier(string program, int exchanges) Assert.Equal(exchanges, Occurrences(body, "sharpemu_wave_set ^= 4u;")); } + /// Back-to-back V_READLANE_B32 at inline-constant lanes share one exchange: the + /// owner of each lane writes its own value slot and one barrier serves both. A third read + /// after the pair takes its own, and SHARPEMU_MSL_READLANE_FUSE=0 keeps one per read. + [Fact] + public void ConsecutiveConstantLaneReadsShareOneWave64Exchange() + { + var request = Gen5ComputeFixtures.CreateComputeRequest( + Gen5ComputeFixtures.Wave64ReadLaneTriple, waveLaneCount: 64, localSizeX: 64); + Assert.True(Gen5MslTranslator.TryCompileProgram(request, out var fused, out var error), error); + Assert.Equal(2, Occurrences(Body(fused), "threadgroup_barrier")); + Assert.Equal(2, Occurrences(Body(fused), "sharpemu_wave_set ^= 4u;")); + Assert.Contains("sharpemu_wave_scratch[sharpemu_wave_set + 3u] =", fused.Source, StringComparison.Ordinal); + + Assert.True( + Gen5MslTranslator.TryCompileProgram( + request, out var each, out error, new Gen5MslCodegenOptions(FusedLaneReads: false)), + error); + Assert.Equal(3, Occurrences(Body(each), "threadgroup_barrier")); + Assert.Equal(3, Occurrences(Body(each), "sharpemu_wave_set ^= 4u;")); + } + /// DS_APPEND counts the wave's active lanes in one exchange and broadcasts the /// one atomic's result in a second: two barriers, where it used to take five. [Fact] diff --git a/tests/SharpEmu.ShaderCompiler.Metal.Tests/MslTranslatorOverheadBenchTests.cs b/tests/SharpEmu.ShaderCompiler.Metal.Tests/MslTranslatorOverheadBenchTests.cs index a59af67da..3b34343a8 100644 --- a/tests/SharpEmu.ShaderCompiler.Metal.Tests/MslTranslatorOverheadBenchTests.cs +++ b/tests/SharpEmu.ShaderCompiler.Metal.Tests/MslTranslatorOverheadBenchTests.cs @@ -146,9 +146,31 @@ public sealed class MslTranslatorOverheadBenchTests(ITestOutputHelper output) StoreScalarResourceBase: 8, StoreBackingBytes: 16); - private static Gen5MslShader Compile(Gen5ComputeFixture fixture, Gen5MslCodegenOptions options) + // 256 trips of the wave-scan idiom: V_READLANE_B32 of lanes 31 and 63, folded back into + // every lane. A guest wave64, so each read is a threadgroup exchange. + private static readonly Gen5ComputeFixture LaneReads = new( + "bench-readlane", + [ + 0xD7650001, 0x000100C1, // v_mbcnt_lo_u32_b32 v1, -1, 0 + 0xD7660001, 0x000202C1, // v_mbcnt_hi_u32_b32 v1, -1, v1 + 0xBE8003FF, 0x00000100, // s_mov_b32 s0, 256 + // loop: + 0xD7600001, 0x00013F01, // v_readlane_b32 s1, v1, 31 + 0xD7600002, 0x00017F01, // v_readlane_b32 s2, v1, 63 + 0x4A020201, // v_add_nc_u32 v1, s1, v1 + 0x4A020202, // v_add_nc_u32 v1, s2, v1 + 0x81808100, // s_sub_i32 s0, s0, 1 + 0xBF078000, // s_cmp_lg_u32 s0, 0 + 0xBF85FFF7, // s_cbranch_scc1 loop + 0xE0700000, 0x80020100, // buffer_store_dword v1, off, s[8:11], 0 + 0xBF810000, // s_endpgm + ], + StoreScalarResourceBase: 8, + StoreBackingBytes: 16); + + private static Gen5MslShader Compile(Gen5ComputeFixture fixture, Gen5MslCodegenOptions options, uint waveLanes = 32) { - var request = Gen5ComputeFixtures.CreateComputeRequest(fixture, localSizeX: Threads); + var request = Gen5ComputeFixtures.CreateComputeRequest(fixture, waveLanes, localSizeX: Threads); Assert.True(Gen5MslTranslator.TryCompileProgram(request, out var shader, out var error, options), error); return shader; } @@ -164,10 +186,10 @@ private double Time(Gen5MslShader shader, string label, int bufferBytes, bool mu return result.GpuMilliseconds; } - private void Compare(Gen5ComputeFixture fixture, string name, Gen5MslCodegenOptions before, Gen5MslCodegenOptions after, int bufferBytes) + private void Compare(Gen5ComputeFixture fixture, string name, Gen5MslCodegenOptions before, Gen5MslCodegenOptions after, int bufferBytes, uint waveLanes = 32) { - var old = Compile(fixture, before); - var current = Compile(fixture, after); + var old = Compile(fixture, before, waveLanes); + var current = Compile(fixture, after, waveLanes); Assert.NotEqual(old.Source, current.Source); var (bestOld, bestNew) = (double.MaxValue, double.MaxValue); for (var round = 0; round < Rounds; round++) @@ -181,7 +203,7 @@ private void Compare(Gen5ComputeFixture fixture, string name, Gen5MslCodegenOpti private static readonly Gen5MslCodegenOptions Baseline = new( VectorDwordLoads: false, ThreadgroupSizeHint: false, NativeFloatMinMax: false, ConstantTableSlots: false, - FastReciprocals: false, ContractPixelCompute: false); + FastReciprocals: false, ContractPixelCompute: false, FusedLaneReads: false); [Fact] public void VectorDwordLoads() @@ -211,6 +233,13 @@ public void ContractPixelCompute() Compare(MultiplyAdds, "mad/mul+add f32", Baseline, Baseline with { ContractPixelCompute = true }, 16); } + [Fact] + public void FusedLaneReads() + { + if (!Enabled) return; + Compare(LaneReads, "wave64 readlane 31 + 63", Baseline, Baseline with { FusedLaneReads = true }, 16, waveLanes: 64); + } + [Fact] public void ThreadgroupSizeHintOnTranslatedPrograms() { From c07ecdd14a26f98f836437035dcc7794410c1dee Mon Sep 17 00:00:00 2001 From: kuba Date: Wed, 28 Oct 2026 04:17:10 +0100 Subject: [PATCH 48/82] [Loader] Let the spin-nap pass patch pause loops behind a hoisted jump table or a pause-jmp tail The pass skipped any pause in a function whose jump table it could not resolve (clang hoists the table base into a callee-saved register and bounds-checks with a 64-bit cmp, which the resolver rejected) and any `pause; jmp short loop` (4 bytes, one short of the patch jump). A pure spin-wait loop of that shape never napped, so 13 such workers plus the producer kept every core busy and a loaded host starved the producer (test eboot group 3, "TEB TIMEOUT stage=3-sync-cross"). The spin pass now (only it; the red-zone pass is unchanged) trusts a hoisted table base when one RIP-relative lea is the sole write of that callee-saved register in the function (the epilogue pop is ignored), accepts a 64-bit bounds compare, and takes the alignment nops after an unconditional jump to reach 5 bytes unless something branches to them. Off with SHARPEMU_SPIN_NAP_SHAPES=0. Test eboot built with an eh_frame_hdr (the stock variants have none, so the pass never runs on them), group 3 spin only, release bundles, 2 runs per arm, same session, host load average 11-35: quiet N13.wake rounds 2401-3291 -> 10000/10000, mean 2.4-3.3 ms -> 0.29-0.38 ms, p50 667-959 -> 583-667 ns quiet N13.fanout rounds 734-830 -> 4544-5000 of 5000, p50 9.0-9.5 ms -> 0.84-1.0 ms quiet N13 cpu_per_round 20.5-26.1 ms -> 2.7-3.5 ms 12 yes N13.wake rounds 292-340 -> 967-1253, p50 10-18.8 ms -> 1.4-1.95 ms 12 yes N13.fanout rounds 103-125 -> 220-328, N13 cpu_per_round 67-98 ms -> 13-17 ms N4 wake/fanout p50 unchanged (208-209 ns / 375-500 ns); the nap adds tail (max up to ~0.33 ms, fanout p99 up to ~0.5 ms in 3 of 8 runs) as it does for any patched loop. --- .../Loader/GuestRedZonePatcher.SpinLoops.cs | 10 +- .../Loader/GuestRedZonePatcher.cs | 121 ++++++++++- src/SharpEmu.Core/Loader/SpinNapRuntime.cs | 8 + .../Loader/SpinLoopNapPatcherTests.cs | 195 +++++++++++++++++- 4 files changed, 322 insertions(+), 12 deletions(-) diff --git a/src/SharpEmu.Core/Loader/GuestRedZonePatcher.SpinLoops.cs b/src/SharpEmu.Core/Loader/GuestRedZonePatcher.SpinLoops.cs index 7a2741a30..6c618949c 100644 --- a/src/SharpEmu.Core/Loader/GuestRedZonePatcher.SpinLoops.cs +++ b/src/SharpEmu.Core/Loader/GuestRedZonePatcher.SpinLoops.cs @@ -83,9 +83,10 @@ public static SpinPatchResult PatchSpinLoops( } var includeLocked = !lockedProbeCode.IsEmpty; + var wideShapes = SpinNapRuntime.WideShapesEnabled; var lockedAddresses = new HashSet(); var sites = CollectSpinSites( - memory, programHeaders, imageBase, functionStarts, includeLocked, lockedAddresses, out var result); + memory, programHeaders, imageBase, functionStarts, includeLocked, wideShapes, lockedAddresses, out var result); if (sites.Count == 0) { return result; @@ -170,6 +171,7 @@ private static List CollectSpinSites( ulong imageBase, IReadOnlyList functionStarts, bool includeLocked, + bool wideShapes, HashSet lockedAddresses, out SpinPatchResult result) { @@ -249,7 +251,7 @@ private static List CollectSpinSites( branchTargets.Clear(); CollectBranchTargets(decoded, branchTargets); var canBorrow = TryCollectJumpTableTargets( - memory, programHeaders, imageBase, decoded, functionStart, functionEnd, branchTargets); + memory, programHeaders, imageBase, decoded, functionStart, functionEnd, branchTargets, wideShapes); var lastSiteEnd = 0UL; for (var index = 0; index < decoded.Count; index++) @@ -276,7 +278,9 @@ private static List CollectSpinSites( continue; } - if (!TryBuildPatchSpan(decoded, index, branchTargets, canBorrow, groupInstructions: false, out var span, out _)) + if (!TryBuildPatchSpan( + decoded, index, branchTargets, canBorrow, groupInstructions: false, out var span, out _, + absorbJumpPadding: wideShapes)) { unrelocatable++; continue; diff --git a/src/SharpEmu.Core/Loader/GuestRedZonePatcher.cs b/src/SharpEmu.Core/Loader/GuestRedZonePatcher.cs index eeb57664b..a566ca6dc 100644 --- a/src/SharpEmu.Core/Loader/GuestRedZonePatcher.cs +++ b/src/SharpEmu.Core/Loader/GuestRedZonePatcher.cs @@ -463,7 +463,8 @@ private static bool TryCollectJumpTableTargets( IReadOnlyList decoded, ulong functionStart, ulong functionEnd, - HashSet branchTargets) + HashSet branchTargets, + bool hoistedTables = false) { List? resolvedTargets = null; HashSet? instructionBoundaries = null; @@ -488,7 +489,8 @@ private static bool TryCollectJumpTableTargets( functionEnd, instructionBoundaries, branchTargets, - resolvedTargets)) + resolvedTargets, + hoistedTables)) { return false; } @@ -512,7 +514,8 @@ private static bool TryResolveJumpTable( ulong functionEnd, IReadOnlySet instructionBoundaries, IReadOnlySet knownBranchTargets, - List targets) + List targets, + bool hoistedTables = false) { if (jumpIndex < 4) { @@ -588,7 +591,9 @@ private static bool TryResolveJumpTable( boundsCompare.Mnemonic != Mnemonic.Cmp || boundsCompare.OpCount != 2 || boundsCompare.GetOpKind(0) != OpKind.Register || boundsCompare.GetOpRegister(0).GetFullRegister() != selectorRegister || - boundsCompare.GetOpRegister(0).GetSize() != sizeof(int) || + // A 64-bit compare bounds every bit of the selector, so the sign-extending table load stays in range too. + (boundsCompare.GetOpRegister(0).GetSize() != sizeof(int) && + !(hoistedTables && boundsCompare.GetOpRegister(0).GetSize() == sizeof(ulong))) || !TryGetNonNegativeImmediate(boundsCompare, 1, out var maximumIndex) || maximumIndex >= MaximumJumpTableEntries) { @@ -631,6 +636,11 @@ private static bool TryResolveJumpTable( tableDefinitionFound = true; } + if (!tableDefinitionFound && hoistedTables) + { + tableDefinitionFound = TryFindHoistedTableBase(decoded, tableRegister, out tableAddress); + } + if (!tableDefinitionFound) { return false; @@ -667,6 +677,89 @@ private static bool TryResolveJumpTable( return true; } + // A compiler hoists the jump table's base address out of a loop into a callee-saved register + // (`lea r12,[rip+table]` before the loop, `movsxd rax,[r12+rax*4]; add rax,r12; jmp rax` inside it). + // The base is trusted only when that one RIP-relative lea is the sole write of the register in the + // whole function: then every execution of the dispatch sees the lea's value. Writes the instruction + // info reports (explicit or implicit) all count; the epilogue pop that restores the caller's value + // does not, since it only runs on the way out. + private static bool TryFindHoistedTableBase( + IReadOnlyList decoded, + Register tableRegister, + out ulong tableAddress) + { + tableAddress = 0; + if (tableRegister is not (Register.RBX or Register.RBP or Register.R12 or Register.R13 or Register.R14 or Register.R15)) + { + return false; + } + + var infoFactory = new InstructionInfoFactory(); + var found = false; + for (var index = 0; index < decoded.Count; index++) + { + var instruction = decoded[index].Instruction; + var writes = false; + foreach (var used in infoFactory.GetInfo(instruction, InstructionInfoOptions.NoMemoryUsage).GetUsedRegisters()) + { + if (used.Access is OpAccess.Write or OpAccess.CondWrite or OpAccess.ReadWrite or OpAccess.ReadCondWrite && + used.Register.GetFullRegister() == tableRegister) + { + writes = true; + break; + } + } + + if (!writes || IsEpiloguePop(decoded, index, tableRegister)) + { + continue; + } + + if (found || + instruction.Mnemonic != Mnemonic.Lea || instruction.OpCount != 2 || + instruction.GetOpKind(0) != OpKind.Register || instruction.GetOpRegister(0) != tableRegister || + !instruction.IsIPRelativeMemoryOperand) + { + return false; + } + + tableAddress = instruction.IPRelativeMemoryAddress; + found = true; + } + + return found; + } + + // `pop reg` that only restores a callee-saved register on the way out: nothing but pops, stack + // cleanup and the final ret or jmp follow it. + private static bool IsEpiloguePop(IReadOnlyList decoded, int index, Register register) + { + var instruction = decoded[index].Instruction; + if (instruction.Mnemonic != Mnemonic.Pop || instruction.GetOpKind(0) != OpKind.Register || + instruction.GetOpRegister(0) != register) + { + return false; + } + + for (var next = index + 1; next < decoded.Count; next++) + { + var following = decoded[next].Instruction; + if (following.Mnemonic is Mnemonic.Ret or Mnemonic.Jmp) + { + return true; + } + + if (following.Mnemonic is not (Mnemonic.Pop or Mnemonic.Leave or Mnemonic.Add) || + following.Mnemonic == Mnemonic.Add && + (following.GetOpKind(0) != OpKind.Register || following.GetOpRegister(0) != Register.RSP)) + { + return false; + } + } + + return false; + } + private static bool IsStoreToMemory(in Instruction instruction) => instruction.Mnemonic == Mnemonic.Mov && instruction.OpCount == 2 && instruction.GetOpKind(0) == OpKind.Memory; @@ -994,7 +1087,8 @@ private static bool TryBuildPatchSpan( bool canBorrowFollowingInstructions, bool groupInstructions, out PatchSite site, - out SpanRefusal refusal) + out SpanRefusal refusal, + bool absorbJumpPadding = false) { refusal = SpanRefusal.None; var instructions = new List(3); @@ -1012,6 +1106,23 @@ private static bool TryBuildPatchSpan( // span with that branch; the trampoline runs it once RSP is restored. terminalBranch = instruction; byteLength += instruction.Length; + if (absorbJumpPadding && byteLength < MinimumJumpBytes && + instruction.FlowControl == FlowControl.UnconditionalBranch) + { + // `pause; jmp short loop` is 4 bytes: take the alignment nops after the jump. Execution + // never falls through an unconditional jump, so they are dead unless something branches there. + for (var padding = index + 1; padding < decoded.Count && byteLength < MinimumJumpBytes; padding++) + { + var filler = decoded[padding].Instruction; + if (filler.Mnemonic != Mnemonic.Nop || branchTargets.Contains(filler.IP)) + { + break; + } + + byteLength += filler.Length; + } + } + break; } diff --git a/src/SharpEmu.Core/Loader/SpinNapRuntime.cs b/src/SharpEmu.Core/Loader/SpinNapRuntime.cs index a656dafc6..f166abf45 100644 --- a/src/SharpEmu.Core/Loader/SpinNapRuntime.cs +++ b/src/SharpEmu.Core/Loader/SpinNapRuntime.cs @@ -28,6 +28,7 @@ internal static unsafe class SpinNapRuntime public const string NapEnvironmentVariable = "SHARPEMU_SPIN_NAP_US"; public const string GapEnvironmentVariable = "SHARPEMU_SPIN_NAP_GAP_US"; public const string AbEnvironmentVariable = "SHARPEMU_SPIN_NAP_AB_S"; + public const string WideShapesEnvironmentVariable = "SHARPEMU_SPIN_NAP_SHAPES"; public const string LockedEnvironmentVariable = "SHARPEMU_SPIN_NAP_LOCKED"; public const string LockedSitesEnvironmentVariable = "SHARPEMU_SPIN_NAP_LOCKED_SITES"; public const string LockedDensityEnvironmentVariable = "SHARPEMU_SPIN_NAP_LOCKED_DENSITY_US"; @@ -84,6 +85,13 @@ private sealed record NapTarget(ulong SleepAddress, uint SleepArgument, int AbSe internal static bool IsEnabled => !string.Equals(Environment.GetEnvironmentVariable(EnvironmentVariable), "0", StringComparison.Ordinal); + /// On by default (SHARPEMU_SPIN_NAP_SHAPES=0 turns it off): the spin pass also patches + /// a pause; jmp short loop (taking the alignment nops after the jump) and loops in functions + /// whose jump table base is hoisted out of the loop or bounds-checked with a 64-bit compare. Without + /// it the pass leaves those pauses alone, so a pure spin-wait loop of that shape never naps. + internal static bool WideShapesEnabled => + !string.Equals(Environment.GetEnvironmentVariable(WideShapesEnvironmentVariable), "0", StringComparison.Ordinal); + /// Opt-in (SHARPEMU_SPIN_NAP_LOCKED=1): also instrument short backward loops /// that poll with xchg mem / lock-prefixed read-modify-write ops. internal static bool LockedEnabled => diff --git a/tests/SharpEmu.Libs.Tests/Loader/SpinLoopNapPatcherTests.cs b/tests/SharpEmu.Libs.Tests/Loader/SpinLoopNapPatcherTests.cs index e8dc875fb..85ec81380 100644 --- a/tests/SharpEmu.Libs.Tests/Loader/SpinLoopNapPatcherTests.cs +++ b/tests/SharpEmu.Libs.Tests/Loader/SpinLoopNapPatcherTests.cs @@ -341,6 +341,188 @@ public void LockedOpOutsideAnyLoopOrInALongLoopIsNotRecognized() } } + // do { dec ecx ; jz exit ; pause ; jmp short loop ; } : the compiled shape of `while (!flag) pause;`. + // The jump is 4 bytes with the pause, one short of the 5-byte patch jump, so the pass takes the nop after it. + private static byte[] PauseJmpLoop() => + [ + 0xB9, 0x04, 0x00, 0x00, 0x00, // 0: mov ecx, 4 + 0xFF, 0xC9, // 5: loop: dec ecx + 0x74, 0x07, // 7: jz exit (next ip 9, +7 -> 16) + 0xF3, 0x90, // 9: pause + 0xEB, 0xF8, // 11: jmp short loop (next ip 13, -8 -> 5) + 0x0F, 0x1F, 0x00, // 13: nop dword [rax] (alignment padding) + 0xB8, 0x44, 0x33, 0x22, 0x11, // 16: exit: mov eax, 0x11223344 + 0xC3, // 21: ret + ]; + + [Fact] + public void PatchesAPauseJmpLoopByTakingTheAlignmentNopsAfterTheJump() + { + if (RuntimeInformation.ProcessArchitecture != Architecture.X64) + return; + + using var image = SpinImage.Create(PauseJmpLoop()); + Assert.Equal(1, image.Result.PauseSites); + Assert.Equal(1, image.Result.PatchedSites); + Assert.Equal(0, image.Result.UnrelocatableSites); + Assert.Equal(0xE9, image.ReadByte(9)); // the pause is now a jmp into the trampoline + Assert.Equal(0x11223344u, ((delegate* unmanaged)image.Base)()); + } + + [Fact] + public void LeavesAPauseJmpLoopAloneWhenTheShapeSwitchIsOff() + { + if (RuntimeInformation.ProcessArchitecture != Architecture.X64) + return; + + using var _ = EnvironmentVariableScope.Set(SpinNapRuntime.WideShapesEnvironmentVariable, "0"); + using var image = SpinImage.Create(PauseJmpLoop()); + Assert.Equal(0, image.Result.PatchedSites); + Assert.Equal(1, image.Result.UnrelocatableSites); + Assert.Equal(0xF3, image.ReadByte(9)); + Assert.Equal(0x11223344u, ((delegate* unmanaged)image.Base)()); + } + + [Fact] + public void DoesNotTakeABranchTargetAsLoopPadding() + { + if (RuntimeInformation.ProcessArchitecture != Architecture.X64) + return; + + // The same loop, but the exit branch lands on the nop that follows the jump: it is live code, not padding. + byte[] function = + [ + 0xB9, 0x04, 0x00, 0x00, 0x00, // 0: mov ecx, 4 + 0xFF, 0xC9, // 5: loop: dec ecx + 0x74, 0x04, // 7: jz 13 (next ip 9, +4) + 0xF3, 0x90, // 9: pause + 0xEB, 0xF8, // 11: jmp short loop + 0x0F, 0x1F, 0x00, // 13: nop dword [rax] (a branch target) + 0xB8, 0x44, 0x33, 0x22, 0x11, // 16: mov eax, 0x11223344 + 0xC3, // 21: ret + ]; + + using var image = SpinImage.Create(function); + Assert.Equal(0, image.Result.PatchedSites); + Assert.Equal(0xF3, image.ReadByte(9)); + Assert.Equal(0x11223344u, ((delegate* unmanaged)image.Base)()); + } + + private const int HoistedTableOffset = 0x2000; + private const int HoistedPauseOffset = 40; + + // switch (kind) inside a spin loop, as clang -O2 compiles it: the table base is hoisted out of the loop into + // a callee-saved register, and the bounds check is a 64-bit compare of the zero-extended selector. The + // dispatch is only recognised by the wide-shape rule. `clobberAfterEpiloguePop` adds a second write of r12 + // (a mov after the pop), which must make the pass distrust the hoisted base. + private static (byte[] Code, byte[] Table) HoistedSwitchSpinLoop(bool clobberAfterEpiloguePop = false) + { + var code = new List(); + code.AddRange([0x41, 0x54]); // push r12 + var tableLoad = code.Count; + code.AddRange([0x4C, 0x8D, 0x25, 0, 0, 0, 0]); // lea r12, [rip + table] + code.AddRange([0xB9, 0x03, 0x00, 0x00, 0x00]); // mov ecx, 3 + var loop = code.Count; + code.AddRange([0xB8, 0x01, 0x00, 0x00, 0x00]); // loop: mov eax, 1 (the selector) + code.AddRange([0x48, 0x83, 0xF8, 0x01]); // cmp rax, 1 + var boundsBranch = code.Count; + code.AddRange([0x77, 0x00]); // ja exit + code.AddRange([0x49, 0x63, 0x04, 0x84]); // movsxd rax, [r12 + rax*4] + code.AddRange([0x4C, 0x01, 0xE0]); // add rax, r12 + code.AddRange([0xFF, 0xE0]); // jmp rax + var case0 = code.Count; + var case0Jump = code.Count; + code.AddRange([0xEB, 0x00]); // case 0: jmp short exit + var case1 = code.Count; + code.AddRange([0xFF, 0xC9]); // case 1: dec ecx + var exitBranch = code.Count; + code.AddRange([0x74, 0x00]); // jz exit + var pause = code.Count; + code.AddRange([0xF3, 0x90]); // pause + code.AddRange([0xEB, unchecked((byte)(loop - (code.Count + 2)))]); // jmp short loop + code.AddRange([0x0F, 0x1F, 0x00]); // nop dword [rax] + var exit = code.Count; + code.AddRange([0xB8, 0x44, 0x33, 0x22, 0x11]); // exit: mov eax, 0x11223344 + code.AddRange([0x41, 0x5C]); // pop r12 (epilogue) + if (clobberAfterEpiloguePop) + { + code.AddRange([0x45, 0x31, 0xE4]); // xor r12d, r12d + } + + code.Add(0xC3); // ret + code[boundsBranch + 1] = (byte)(exit - (boundsBranch + 2)); + code[case0Jump + 1] = (byte)(exit - (case0Jump + 2)); + code[exitBranch + 1] = (byte)(exit - (exitBranch + 2)); + var bytes = code.ToArray(); + BinaryPrimitives.WriteInt32LittleEndian(bytes.AsSpan(tableLoad + 3), HoistedTableOffset - (tableLoad + 7)); + Assert.Equal(HoistedPauseOffset, pause); + var table = new byte[2 * sizeof(int)]; + BinaryPrimitives.WriteInt32LittleEndian(table.AsSpan(0), case0 - HoistedTableOffset); + BinaryPrimitives.WriteInt32LittleEndian(table.AsSpan(4), case1 - HoistedTableOffset); + return (bytes, table); + } + + [Fact] + public void PatchesASpinLoopInAFunctionWithAHoistedJumpTableBase() + { + if (RuntimeInformation.ProcessArchitecture != Architecture.X64) + return; + + var (code, table) = HoistedSwitchSpinLoop(); + using var image = SpinImage.Create(code, readOnlyData: (HoistedTableOffset, table)); + Assert.Equal(1, image.Result.PauseSites); + Assert.Equal(1, image.Result.PatchedSites); + Assert.Equal(0xE9, image.ReadByte(HoistedPauseOffset)); + Assert.Equal(0x11223344u, ((delegate* unmanaged)image.Base)()); + } + + [Fact] + public void LeavesTheSpinLoopAloneWhenTheHoistedTableRegisterIsWrittenAgain() + { + if (RuntimeInformation.ProcessArchitecture != Architecture.X64) + return; + + var (code, table) = HoistedSwitchSpinLoop(clobberAfterEpiloguePop: true); + using var image = SpinImage.Create(code, readOnlyData: (HoistedTableOffset, table)); + Assert.Equal(0, image.Result.PatchedSites); + Assert.Equal(1, image.Result.UnrelocatableSites); + Assert.Equal(0x11223344u, ((delegate* unmanaged)image.Base)()); + } + + [Fact] + public void LeavesTheSpinLoopAloneWhenTheShapeSwitchIsOff() + { + if (RuntimeInformation.ProcessArchitecture != Architecture.X64) + return; + + using var _ = EnvironmentVariableScope.Set(SpinNapRuntime.WideShapesEnvironmentVariable, "0"); + var (code, table) = HoistedSwitchSpinLoop(); + using var image = SpinImage.Create(code, readOnlyData: (HoistedTableOffset, table)); + Assert.Equal(0, image.Result.PatchedSites); + Assert.Equal(0x11223344u, ((delegate* unmanaged)image.Base)()); + } + + private sealed class EnvironmentVariableScope : IDisposable + { + private readonly string _name; + private readonly string? _previous; + + private EnvironmentVariableScope(string name, string? previous) + { + _name = name; + _previous = previous; + } + + public static EnvironmentVariableScope Set(string name, string value) + { + var scope = new EnvironmentVariableScope(name, Environment.GetEnvironmentVariable(name)); + Environment.SetEnvironmentVariable(name, value); + return scope; + } + + public void Dispose() => Environment.SetEnvironmentVariable(_name, _previous); + } + private sealed class SpinImage : IDisposable { private const ulong ImageSize = 0x10000; @@ -362,7 +544,7 @@ private SpinImage(PhysicalVirtualMemory memory, ulong imageBase, GuestRedZonePat public GuestRedZonePatcher.SpinPatchResult Result { get; } public static SpinImage Create(byte[] function, bool locked = false, byte[]? lockedProbe = null, - ulong reserveAt = 0, ulong reservedSize = 0) + ulong reserveAt = 0, ulong reservedSize = 0, (ulong Offset, byte[] Bytes)? readOnlyData = null) { var memory = new PhysicalVirtualMemory(); const ulong holeSize = 0x4000000; @@ -378,12 +560,17 @@ public static SpinImage Create(byte[] function, bool locked = false, byte[]? loc BinaryPrimitives.WriteUInt32LittleEndian(exceptionFrameHeader.AsSpan(12), 1); BinaryPrimitives.WriteUInt64LittleEndian(exceptionFrameHeader.AsSpan(16), imageBase); Assert.True(memory.TryWrite(imageBase + 0x1000, exceptionFrameHeader)); - ProgramHeader[] headers = - [ + var headers = new List + { CreateProgramHeader(ProgramHeaderType.Load, ProgramHeaderFlags.Read | ProgramHeaderFlags.Execute, 0, (ulong)function.Length), CreateProgramHeader(ProgramHeaderType.GnuEhFrame, ProgramHeaderFlags.Read, 0x1000, (ulong)exceptionFrameHeader.Length), - ]; + }; + if (readOnlyData is { } table) + { + Assert.True(memory.TryWrite(imageBase + table.Offset, table.Bytes)); + headers.Add(CreateProgramHeader(ProgramHeaderType.Load, ProgramHeaderFlags.Read, table.Offset, (ulong)table.Bytes.Length)); + } GuestRedZonePatcher.SpinNapArena? arena = null; if (reservedSize != 0) From 89f89c55f790276e397139f9dcf41b81f9bbf780 Mon Sep 17 00:00:00 2001 From: kuba Date: Wed, 28 Oct 2026 04:18:57 +0100 Subject: [PATCH 49/82] [Loader] Absorb jump padding into a spin-nap patch only when the function's jump tables are resolved A pause-jmp spin tail takes the alignment nops after its jump to reach the 5-byte patch. Those nops are dead only if nothing branches to them. With an unresolved jump table an indirect target is unknown, so the padding is left alone there. SHARPEMU_SPIN_NAP_SHAPES=0 still turns the wider shapes off. --- src/SharpEmu.Core/Loader/GuestRedZonePatcher.cs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/SharpEmu.Core/Loader/GuestRedZonePatcher.cs b/src/SharpEmu.Core/Loader/GuestRedZonePatcher.cs index a566ca6dc..6f03b3c9f 100644 --- a/src/SharpEmu.Core/Loader/GuestRedZonePatcher.cs +++ b/src/SharpEmu.Core/Loader/GuestRedZonePatcher.cs @@ -1106,11 +1106,12 @@ private static bool TryBuildPatchSpan( // span with that branch; the trampoline runs it once RSP is restored. terminalBranch = instruction; byteLength += instruction.Length; - if (absorbJumpPadding && byteLength < MinimumJumpBytes && + if (absorbJumpPadding && canBorrowFollowingInstructions && byteLength < MinimumJumpBytes && instruction.FlowControl == FlowControl.UnconditionalBranch) { // `pause; jmp short loop` is 4 bytes: take the alignment nops after the jump. Execution - // never falls through an unconditional jump, so they are dead unless something branches there. + // never falls through an unconditional jump, so they are dead unless something branches there; + // only when every jump table here is resolved, so no unknown indirect target can land on one. for (var padding = index + 1; padding < decoded.Count && byteLength < MinimumJumpBytes; padding++) { var filler = decoded[padding].Instruction; From 9aba2fe4da4f2ac5f9eff85db816519f8c2e8447 Mon Sep 17 00:00:00 2001 From: kuba Date: Wed, 28 Oct 2026 15:50:25 +0100 Subject: [PATCH 50/82] [Audio] A guest thread that just read a full AudioOut2 queue waits for a free slot instead of re-polling it. Wwise's AK::EventManager thread loops: Perform (sink IsDataNeeded = sceAudioOut2ContextGetQueueLevel, render the available grains, Advance + non-blocking Push), then sceKernelWaitEventFlag(flag, 1, OR|CLEAR_ALL, NULL, &timeout) with a timeout that is 0 for the sample-rate/frame-size configuration the title ships. Once the queue is full the level cannot change before the next grain period, so that loop re-polled the same answer at over a hundred thousand times a second and held a core. GetQueueLevel now records, per guest thread, when the queue it read as full gets a free slot (clock and device paced), and the next timed sceKernelWaitEventFlag of that thread is lengthened to that time (at most 50 ms, once per read). A wait the flag ends early is unaffected, a longer timeout is kept, and the queue is refilled the moment a slot frees. SHARPEMU_AUDIO_OUT2_IDLE_WAIT=0 restores the zero-timeout poll. The test drives the Wwise loop against a context: 92 polls per second instead of 170603, and 52 ms instead of 481 ms of thread CPU per second, with the queue kept fed. --- src/SharpEmu.Libs/Audio/AudioOut2Exports.cs | 71 +++++++ .../Kernel/KernelEventFlagCompatExports.cs | 5 + .../Audio/AudioOut2QueueTests.IdlePoll.cs | 189 ++++++++++++++++++ 3 files changed, 265 insertions(+) create mode 100644 tests/SharpEmu.Libs.Tests/Audio/AudioOut2QueueTests.IdlePoll.cs diff --git a/src/SharpEmu.Libs/Audio/AudioOut2Exports.cs b/src/SharpEmu.Libs/Audio/AudioOut2Exports.cs index 9a12e2676..b027cfb39 100644 --- a/src/SharpEmu.Libs/Audio/AudioOut2Exports.cs +++ b/src/SharpEmu.Libs/Audio/AudioOut2Exports.cs @@ -95,6 +95,22 @@ private static bool AllowStackOut(string which) => private static readonly bool _asyncPushNeverWaits = !string.Equals( Environment.GetEnvironmentVariable("SHARPEMU_AUDIO_OUT2_ASYNC_PUSH"), "0", StringComparison.Ordinal); + // A thread that has just read a full queue (no free slot) cannot get a different answer before the + // next slot frees, so its next timed event-flag wait is held until then, or until the flag is set. + // Wwise's audio thread polls the queue level and then waits on an event flag with a zero timeout; + // without this it re-polls the full queue millions of times a second. SHARPEMU_AUDIO_OUT2_IDLE_WAIT=0 + // restores the zero-timeout poll. + private static readonly bool _idleQueueWait = !string.Equals( + Environment.GetEnvironmentVariable("SHARPEMU_AUDIO_OUT2_IDLE_WAIT"), "0", StringComparison.Ordinal); + + // Longest a full queue holds one wait, as WaitForDeviceSlot's longest sleep does. + private const long IdleQueueWaitCapMicroseconds = 50_000; + + // Per guest thread: when the queue this thread last read as full gets a free slot (Stopwatch + // ticks), 0 when its last read found a free slot or the wait it covers has been taken. + [ThreadStatic] + private static long _queueFullUntilTimestamp; + // libSceAudioOut2 has its own error space (0x80268xxx); // the generic ORBIS_GEN2_* codes are not what a title compares against. private const int AudioOut2ErrorInvalidParam = unchecked((int)0x8026_8001); @@ -176,6 +192,28 @@ public uint ClockQueuedGrains() return ahead <= 0 ? 0 : (uint)Math.Min((ahead + GrainTicks - 1) / GrainTicks, QueueDepth); } + // Stopwatch ticks until the queue has a free slot again, 0 when it already has one: the later + // of the clock's and the device's, because the level reports the larger of the two. + public long QueueFullTicks(bool includeDevice) + { + var ticks = Volatile.Read(ref _nextAdvanceTimestamp) - Stopwatch.GetTimestamp() - + (GrainTicks * Math.Max((long)QueueDepth - 1, 0)); + if (includeDevice && DevicePaced && DeviceBackend is { } backend) + { + var queuedMilliseconds = backend.QueuedMilliseconds; + if (queuedMilliseconds >= 0) + { + var overMilliseconds = queuedMilliseconds - DeviceSlotFreeMilliseconds; + if (overMilliseconds > 0) + { + ticks = Math.Max(ticks, (long)Math.Ceiling(overMilliseconds * Stopwatch.Frequency / 1000.0)); + } + } + } + + return Math.Max(ticks, 0); + } + public double DeviceSlotFreeMilliseconds => DeviceSlotFreeThreshold(GrainSamples, Frequency, QueueDepth); @@ -709,6 +747,13 @@ public static int AudioOut2ContextGetQueueLevel(CpuContext ctx) // full queue every update and block in Push while holding its audio lock. var hasContext = Contexts.TryGetValue(ctx[CpuRegister.Rdi], out var context); var queued = hasContext ? QueuedGrains(context!) : 0u; + if (_idleQueueWait && _asyncPushNeverWaits) + { + _queueFullUntilTimestamp = hasContext && queued >= context!.QueueDepth + ? Stopwatch.GetTimestamp() + context.QueueFullTicks(includeDevice: true) + : 0; + } + Span level = stackalloc byte[sizeof(uint)]; if (outLevelAddress != 0) { @@ -1462,6 +1507,32 @@ internal static ulong ShortenedProducerSleep(ulong microseconds, uint grainSampl : microseconds; } + /// + /// Lengthens a timed event-flag wait taken right after a queue-level read that found no free slot + /// to the time until a slot frees (at most 50 ms), once per read. A wait the flag ends early is + /// unaffected; a wait that is already longer is kept. + /// + internal static uint IdleQueueWaitMicroseconds(uint requestedMicroseconds) + { + var until = _queueFullUntilTimestamp; + if (until == 0) + { + return requestedMicroseconds; + } + + _queueFullUntilTimestamp = 0; + var remaining = until - Stopwatch.GetTimestamp(); + if (remaining <= 0) + { + return requestedMicroseconds; + } + + var microseconds = Math.Min( + (long)Math.Ceiling(remaining * 1_000_000.0 / Stopwatch.Frequency), + IdleQueueWaitCapMicroseconds); + return microseconds > requestedMicroseconds ? (uint)microseconds : requestedMicroseconds; + } + private static void WaitForDeviceSlot(ContextState context) { var backend = ResolveContextBackend(context, out _); diff --git a/src/SharpEmu.Libs/Kernel/KernelEventFlagCompatExports.cs b/src/SharpEmu.Libs/Kernel/KernelEventFlagCompatExports.cs index 3d7620e90..87d3d5e39 100644 --- a/src/SharpEmu.Libs/Kernel/KernelEventFlagCompatExports.cs +++ b/src/SharpEmu.Libs/Kernel/KernelEventFlagCompatExports.cs @@ -341,6 +341,11 @@ public static int KernelWaitEventFlag(CpuContext ctx) return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); } + if (timeoutAddress != 0) + { + timeoutUsec = Audio.AudioOut2Exports.IdleQueueWaitMicroseconds(timeoutUsec); + } + var deadline = timeoutAddress != 0 ? GuestTime.DeadlineAfterMicroseconds(timeoutUsec) : 0; EventFlagWaiter? waiter = null; ulong observed; diff --git a/tests/SharpEmu.Libs.Tests/Audio/AudioOut2QueueTests.IdlePoll.cs b/tests/SharpEmu.Libs.Tests/Audio/AudioOut2QueueTests.IdlePoll.cs new file mode 100644 index 000000000..af69a4b45 --- /dev/null +++ b/tests/SharpEmu.Libs.Tests/Audio/AudioOut2QueueTests.IdlePoll.cs @@ -0,0 +1,189 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using System.Diagnostics; +using System.Text.Json; +using SharpEmu.HLE; +using SharpEmu.Libs.Audio; +using SharpEmu.Libs.Kernel; +using SharpEmu.PerfAudit; +using Xunit; + +namespace SharpEmu.Libs.Tests.Audio; + +public sealed partial class AudioOut2QueueTests +{ + private const ulong PollFsBase = MemoryBase + 0x9000; + private const ulong PollEventHandleSlot = MemoryBase + 0x700; + private const ulong PollEventName = MemoryBase + 0x720; + private const ulong PollTimeoutSlot = MemoryBase + 0x740; + private const ulong PollResultSlot = MemoryBase + 0x750; + private const ulong PollLevelAddress = MemoryBase + 0x760; + private const ulong PollAvailableAddress = MemoryBase + 0x764; + + private static readonly bool IdleWaitDisabled = + Environment.GetEnvironmentVariable("SHARPEMU_AUDIO_OUT2_IDLE_WAIT") == "0"; + + private readonly record struct PollRun( + long Iterations, long Pushes, long EmptyPolls, uint MaxLevel, long WallMilliseconds, long ThreadCpuNanoseconds); + + // Wwise's audio thread: read the queue level, render and push as many grains as are available + // (Advance, then a non-blocking Push), then wait on an event flag with a zero timeout. + private PollRun RunWwiseAudioThread(uint queueDepth, uint grainSamples, int milliseconds) + { + var context = CreateContext(queueDepth, grainSamples); + + var setup = new CpuContext(_memory, Generation.Gen5) { FsBase = PollFsBase }; + Assert.True(_memory.TryWrite(PollEventName, "idle-poll\0"u8)); + setup[CpuRegister.Rdi] = PollEventHandleSlot; + setup[CpuRegister.Rsi] = PollEventName; + setup[CpuRegister.Rdx] = 0x11; + setup[CpuRegister.Rcx] = 0; + setup[CpuRegister.R8] = 0; + Assert.Equal(0, KernelEventFlagCompatExports.KernelCreateEventFlag(setup)); + Assert.True(setup.TryReadUInt64(PollEventHandleSlot, out var flag)); + + PollRun run = default; + var thread = new Thread(() => + { + var ctx = new CpuContext(_memory, Generation.Gen5) { FsBase = PollFsBase }; + long iterations = 0, pushes = 0, emptyPolls = 0; + uint maxLevel = 0; + var cpuStart = OperatingSystem.IsMacOS() ? DarwinThreadCpuClock.Nanoseconds() : 0; + var stopwatch = Stopwatch.StartNew(); + while (stopwatch.ElapsedMilliseconds < milliseconds) + { + iterations++; + ctx[CpuRegister.Rdi] = context; + ctx[CpuRegister.Rsi] = PollLevelAddress; + ctx[CpuRegister.Rdx] = PollAvailableAddress; + Assert.Equal(0, AudioOut2Exports.AudioOut2ContextGetQueueLevel(ctx)); + var level = ReadUInt32(PollLevelAddress); + var available = ReadUInt32(PollAvailableAddress); + maxLevel = Math.Max(maxLevel, level); + if (level == 0 && pushes >= queueDepth) + { + emptyPolls++; + } + + for (var i = 0u; i < available; i++) + { + ctx[CpuRegister.Rdi] = context; + Assert.Equal(0, AudioOut2Exports.AudioOut2ContextAdvance(ctx)); + ctx[CpuRegister.Rdi] = context; + ctx[CpuRegister.Rsi] = NonBlocking; + Assert.Equal(0, AudioOut2Exports.AudioOut2ContextPush(ctx)); + pushes++; + } + + Assert.True(ctx.TryWriteUInt32(PollTimeoutSlot, 0)); + ctx[CpuRegister.Rdi] = flag; + ctx[CpuRegister.Rsi] = 1; + ctx[CpuRegister.Rdx] = 0x12; + ctx[CpuRegister.Rcx] = PollResultSlot; + ctx[CpuRegister.R8] = PollTimeoutSlot; + // Nothing sets the flag: every wait ends by timing out. + Assert.Equal((int)OrbisGen2Result.ORBIS_GEN2_ERROR_TIMED_OUT, KernelEventFlagCompatExports.KernelWaitEventFlag(ctx)); + } + + var cpuEnd = OperatingSystem.IsMacOS() ? DarwinThreadCpuClock.Nanoseconds() : 0; + run = new PollRun(iterations, pushes, emptyPolls, maxLevel, stopwatch.ElapsedMilliseconds, (long)(cpuEnd - cpuStart)); + }) { IsBackground = true }; + thread.Start(); + Assert.True(thread.Join(TimeSpan.FromSeconds(30))); + return run; + } + + [Fact] + public void AWwiseAudioThreadThatFindsTheQueueFullBlocksUntilASlotFrees() + { + if (IdleWaitDisabled) + { + return; + } + + // 0x200 samples: 10.7 ms grains, four slots. + var run = RunWwiseAudioThread(queueDepth: 4, grainSamples: 0x200, milliseconds: 200); + + var grainPeriods = run.WallMilliseconds / (0x200 * 1000.0 / 48000); + // About one poll per grain period; a zero-timeout poll loop makes hundreds of thousands. + Assert.True(run.Iterations <= 2 * grainPeriods + 8, $"{run.Iterations} polls in {run.WallMilliseconds} ms"); + // The queue is kept fed: the four slots, then a grain per grain period. + Assert.True(run.Pushes >= 4 + grainPeriods - 3, $"{run.Pushes} pushes"); + Assert.True(run.Pushes <= 4 + grainPeriods + 2, $"{run.Pushes} pushes"); + Assert.Equal(4u, run.MaxLevel); + Assert.True(run.EmptyPolls <= 1, $"{run.EmptyPolls} polls found the queue empty"); + } + + [Fact] + public void AnEventFlagWaitAfterAFullQueueReadStillEndsWhenTheFlagIsSet() + { + if (IdleWaitDisabled) + { + return; + } + + var context = CreateContext(queueDepth: 2, grainSamples: LongGrain); + Assert.Equal(0, Push(context, NonBlocking)); + Assert.Equal(0, Push(context, NonBlocking)); + Assert.Equal((2u, 0u), GetQueueLevel(context)); + + var ctx = new CpuContext(_memory, Generation.Gen5) { FsBase = PollFsBase }; + Assert.True(_memory.TryWrite(PollEventName, "idle-set\0"u8)); + ctx[CpuRegister.Rdi] = PollEventHandleSlot; + ctx[CpuRegister.Rsi] = PollEventName; + ctx[CpuRegister.Rdx] = 0x11; + ctx[CpuRegister.Rcx] = 0; + ctx[CpuRegister.R8] = 0; + Assert.Equal(0, KernelEventFlagCompatExports.KernelCreateEventFlag(ctx)); + Assert.True(ctx.TryReadUInt64(PollEventHandleSlot, out var flag)); + + // Another thread sets the flag 5 ms in: the wait ends then, not at the 50 ms cap. + var setter = Task.Run(() => + { + Thread.Sleep(5); + var setCtx = new CpuContext(_memory, Generation.Gen5) { FsBase = PollFsBase + 0x400 }; + setCtx[CpuRegister.Rdi] = flag; + setCtx[CpuRegister.Rsi] = 1; + Assert.Equal(0, KernelEventFlagCompatExports.KernelSetEventFlag(setCtx)); + }); + _ctx[CpuRegister.Rdi] = context; + _ctx[CpuRegister.Rsi] = PollLevelAddress; + _ctx[CpuRegister.Rdx] = PollAvailableAddress; + Assert.Equal(0, AudioOut2Exports.AudioOut2ContextGetQueueLevel(_ctx)); + _ctx.FsBase = PollFsBase; + Assert.True(_ctx.TryWriteUInt32(PollTimeoutSlot, 0)); + _ctx[CpuRegister.Rdi] = flag; + _ctx[CpuRegister.Rsi] = 1; + _ctx[CpuRegister.Rdx] = 0x12; + _ctx[CpuRegister.Rcx] = PollResultSlot; + _ctx[CpuRegister.R8] = PollTimeoutSlot; + var stopwatch = Stopwatch.StartNew(); + Assert.Equal(0, KernelEventFlagCompatExports.KernelWaitEventFlag(_ctx)); + Assert.True(stopwatch.ElapsedMilliseconds < 40, $"wait took {stopwatch.ElapsedMilliseconds} ms"); + setter.Wait(); + } + + // Optional benchmark: SHARPEMU_AUDIO_IDLE_WAIT_BENCH_OUTPUT=. Run it once as is and once + // with SHARPEMU_AUDIO_OUT2_IDLE_WAIT=0 to compare the audio thread's CPU time per second. + [Fact] + public void BenchmarkWwiseAudioThreadPollLoop() + { + var output = Environment.GetEnvironmentVariable("SHARPEMU_AUDIO_IDLE_WAIT_BENCH_OUTPUT"); + if (string.IsNullOrEmpty(output)) + { + return; + } + + var run = RunWwiseAudioThread(queueDepth: 4, grainSamples: 0x400, milliseconds: 1000); + File.WriteAllText(output, JsonSerializer.Serialize(new + { + idleWait = !IdleWaitDisabled, + run.WallMilliseconds, + run.Iterations, + run.Pushes, + run.EmptyPolls, + threadCpuMilliseconds = run.ThreadCpuNanoseconds / 1e6, + })); + } +} From 4be8a89dc478c1a0f3d48b5af20ee13dbc47c2db Mon Sep 17 00:00:00 2001 From: kuba Date: Wed, 28 Oct 2026 16:23:08 +0100 Subject: [PATCH 51/82] [GPU] Land a held batch of label copies through one compute dispatch instead of a blit per copy Silent Hill's first room holds about 19,000 label, occlusion-dump and command-processor copies a frame, up to 16,887 at one encoder boundary, and encoding a copyFromBuffer per copy took about 9% of the GPU worker (most of it AGX checking each blit against the earlier ones). A batch of 16 or more copies now becomes a list of 32-bit word moves (device addresses) that one compute dispatch performs. A word several copies write keeps the last copy's source, so the result is what the in-order blits left. A batch with a copy that is not whole aligned words, or in which one buffer is both source and destination, keeps the blits. The encoder waits on and updates the encoder fence, and the label barrier precedes it, as for the blit encoder. Synthetic device benchmark (arm64, median of 10 alternating rounds), worker encode / GPU land: 1,024 copies 733 -> 207 us / 1,396 -> 1,370 us; 4,096 copies 2,733 -> 551 us / 2,124 -> 1,430 us. SHARPEMU_METAL_V2_LABEL_SCATTER=0 keeps the blits. --- .../V2/MetalCommandStream.cs | 5 + .../V2/MetalRenderHost.LabelScatter.cs | 213 ++++++++++++++++++ .../SharpEmu.Gpu.Metal/V2/MetalRenderHost.cs | 1 + .../MetalEndOfPipeOrderTests.LabelScatter.cs | 135 +++++++++++ 4 files changed, 354 insertions(+) create mode 100644 src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.LabelScatter.cs create mode 100644 tests/SharpEmu.Gpu.Metal.Tests/MetalEndOfPipeOrderTests.LabelScatter.cs diff --git a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalCommandStream.cs b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalCommandStream.cs index 747d1e2a6..0bed9595c 100644 --- a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalCommandStream.cs +++ b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalCommandStream.cs @@ -1557,6 +1557,11 @@ private void EncodeLabelCopies(nint commandBuffer, ReadOnlySpan +/// Held label copies landed by one compute dispatch instead of one blit each. +/// Silent Hill's first room holds about 19,000 eight-byte copies a frame (occlusion dumps, +/// end-of-pipe labels, command-processor writes), up to 16,887 at one boundary, and a +/// copyFromBuffer per copy was about a tenth of the GPU worker's time, most of it the driver +/// checking each blit against the ones before it. +/// The dispatch copies 32-bit words. A word several copies write takes the last copy's +/// source, which is what the blits left in it, so the order of the copies is kept without +/// running them in order. A batch whose words do not divide into 32-bit units, or in which a +/// buffer is both copied from and copied into, keeps the blits. +/// SHARPEMU_METAL_V2_LABEL_SCATTER=0 keeps the blits for every batch. +/// +internal sealed partial class MetalRenderHost +{ + internal bool LabelScatterEnabled { get; set; } = + Environment.GetEnvironmentVariable("SHARPEMU_METAL_V2_LABEL_SCATTER") != "0"; + + // Fewer copies than this cost less as blits than a compute encoder does. + private const int LabelScatterMinCopies = 16; + private const int LabelScatterMaxBuffers = 32; + private const int LabelScatterThreads = 64; + + private const string LabelScatterSource = """ + #include + using namespace metal; + struct sharpemu_label_word { device const uint* source; device uint* destination; }; + kernel void sharpemu_label_scatter(device const sharpemu_label_word* words [[buffer(0)]], + constant uint& count [[buffer(1)]], uint index [[thread_position_in_grid]]) { + if (index >= count) return; + *words[index].destination = *words[index].source; + } + """; + + private nint _labelScatterPipeline; + private readonly Dictionary _labelScatterWords = []; + private readonly nint[] _labelScatterBuffers = new nint[LabelScatterMaxBuffers]; + private readonly ulong[] _labelScatterAddresses = new ulong[LabelScatterMaxBuffers]; + private LabelScatterWord[] _labelScatterList = new LabelScatterWord[1024]; + + // sharpemu_label_word: the device addresses of a source word and the word it lands in. + [System.Runtime.InteropServices.StructLayout(System.Runtime.InteropServices.LayoutKind.Sequential)] + private struct LabelScatterWord + { + public ulong Source; + public ulong Destination; + } + private long _labelScatterBatches, _labelScatterCopies, _labelScatterDeclined; + + internal (long Batches, long Copies, long Declined) LabelScatterTotals => + (_labelScatterBatches, _labelScatterCopies, _labelScatterDeclined); + + private unsafe bool TryEncodeLabelScatter(nint commandBuffer, ReadOnlySpan copies) + { + if (!LabelScatterEnabled || copies.Length < LabelScatterMinCopies) + { + return false; + } + + // The buffers copied from come first, then those copied into; none may be both. + var sources = 0; + foreach (ref readonly var copy in copies) + { + if (copy.Bytes <= 0 || (copy.Bytes & 3) != 0 || ((copy.StagingOffset | copy.DestinationOffset) & 3) != 0 || + copy.StagingBuffer == 0 || copy.DestinationBuffer == 0 || + !NoteLabelScatterBuffer(copy.StagingBuffer, 0, ref sources)) + { + _labelScatterDeclined++; + return false; + } + } + + var buffers = sources; + foreach (ref readonly var copy in copies) + { + if (Array.IndexOf(_labelScatterBuffers, copy.DestinationBuffer, 0, sources) >= 0 || + !NoteLabelScatterBuffer(copy.DestinationBuffer, sources, ref buffers)) + { + _labelScatterDeclined++; + return false; + } + } + + for (var index = 0; index < buffers; index++) + { + _labelScatterAddresses[index] = MetalNative.SendGpuAddress(_labelScatterBuffers[index], SelLabelScatterGpuAddress); + if (_labelScatterAddresses[index] == 0) + { + _labelScatterDeclined++; + return false; + } + } + + // One entry per destination word, holding the source of the last copy into it. + _labelScatterWords.Clear(); + var count = 0; + foreach (ref readonly var copy in copies) + { + var source = _labelScatterAddresses[Array.IndexOf(_labelScatterBuffers, copy.StagingBuffer, 0, sources)] + copy.StagingOffset; + var destination = _labelScatterAddresses[Array.IndexOf(_labelScatterBuffers, copy.DestinationBuffer, sources, buffers - sources)] + + copy.DestinationOffset; + for (var offset = 0UL; offset < (ulong)copy.Bytes; offset += sizeof(uint)) + { + if (_labelScatterWords.TryGetValue(destination + offset, out var existing)) + { + _labelScatterList[existing].Source = source + offset; + continue; + } + + if (count == _labelScatterList.Length) + { + Array.Resize(ref _labelScatterList, count * 2); + } + + _labelScatterWords.Add(destination + offset, count); + _labelScatterList[count++] = new LabelScatterWord { Source = source + offset, Destination = destination + offset }; + } + } + + var listBytes = (ulong)count * 16; + var list = _guestMemory.Allocate(listBytes, 16, out var payload); + if (!list.IsValid) + { + _labelScatterDeclined++; + return false; + } + + System.Runtime.InteropServices.MemoryMarshal.AsBytes(_labelScatterList.AsSpan(0, count)).CopyTo(payload); + var pipeline = LabelScatterPipeline(); + EncodeLabelBarrier(commandBuffer); + var encoder = RetainEncoder(MetalNative.Send(commandBuffer, SelComputeEncoder)); + if (encoder == 0) + { + throw Fatal("A Metal compute encoder could not be created for held label copies."); + } + + WaitForPreviousEncoder(encoder, render: false); + MetalNative.SendVoid(encoder, SelOcclusionPipeline, pipeline); + MetalNative.SendSetBuffer(encoder, SelOcclusionBuffer, list.Buffer, (nuint)list.Offset, 0); + var words = (uint)count; + MetalNative.SendSetBytes(encoder, SelOcclusionBytes, (nint)(&words), sizeof(uint), 1); + fixed (nint* resources = _labelScatterBuffers) + { + if (sources != 0) + { + MetalNative.SendUseResources(encoder, SelUseResourcesUsage, resources, (nuint)sources, Mtl.ResourceUsageRead); + } + + MetalNative.SendUseResources(encoder, SelUseResourcesUsage, resources + sources, (nuint)(buffers - sources), + Mtl.ResourceUsageWrite); + } + + MetalNative.SendDispatch(encoder, SelOcclusionDispatch, + new MtlSize { Width = (nuint)((count + LabelScatterThreads - 1) / LabelScatterThreads), Height = 1, Depth = 1 }, + new MtlSize { Width = LabelScatterThreads, Height = 1, Depth = 1 }); + EndAndReleaseEncoder(encoder, render: false); + _encodersInBuffer++; + _labelScatterBatches++; + _labelScatterCopies += copies.Length; + return true; + } + + private bool NoteLabelScatterBuffer(nint buffer, int first, ref int count) + { + if (Array.IndexOf(_labelScatterBuffers, buffer, first, count - first) >= 0) + { + return true; + } + + if (count == LabelScatterMaxBuffers) + { + return false; + } + + _labelScatterBuffers[count++] = buffer; + return true; + } + + private static readonly nint SelLabelScatterGpuAddress = MetalNative.Selector("gpuAddress"); + + private nint LabelScatterPipeline() + { + if (_labelScatterPipeline != 0) + { + return _labelScatterPipeline; + } + + var options = MetalNative.Send(MetalNative.Send(MetalNative.Class("MTLCompileOptions"), MetalNative.Selector("alloc")), MetalNative.Selector("init")); + MetalNative.ApplyMslCompileOptions(options); + nint error = 0; + var library = MetalNative.Send(_device, MetalNative.Selector("newLibraryWithSource:options:error:"), + MetalNative.NsString(LabelScatterSource), options, ref error); + MetalNative.SendVoid(options, SelRelease); + if (library == 0) throw Fatal($"The label scatter kernel did not compile: {MetalNative.DescribeError(error)}."); + var function = MetalNative.Send(library, MetalNative.Selector("newFunctionWithName:"), MetalNative.NsString("sharpemu_label_scatter")); + MetalNative.SendVoid(library, SelRelease); + if (function == 0) throw Fatal("The label scatter function is missing."); + _labelScatterPipeline = MetalNative.Send(_device, MetalNative.Selector("newComputePipelineStateWithFunction:error:"), function, ref error); + MetalNative.SendVoid(function, SelRelease); + if (_labelScatterPipeline == 0) throw Fatal($"The label scatter pipeline did not build: {MetalNative.DescribeError(error)}."); + return _labelScatterPipeline; + } + + private void DisposeLabelScatter() + { + if (_labelScatterPipeline != 0) MetalNative.SendVoid(_labelScatterPipeline, SelRelease); + _labelScatterPipeline = 0; + } +} diff --git a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.cs b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.cs index acd7fe899..daae738c3 100644 --- a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.cs +++ b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.cs @@ -2809,6 +2809,7 @@ public void Dispose() _completions.Dispose(); DisposeNativeOcclusion(); + DisposeLabelScatter(); DisposeOcclusionAudit(); DisposeWriteFences(); DisposeHostMovie(); diff --git a/tests/SharpEmu.Gpu.Metal.Tests/MetalEndOfPipeOrderTests.LabelScatter.cs b/tests/SharpEmu.Gpu.Metal.Tests/MetalEndOfPipeOrderTests.LabelScatter.cs new file mode 100644 index 000000000..ce8c6a88d --- /dev/null +++ b/tests/SharpEmu.Gpu.Metal.Tests/MetalEndOfPipeOrderTests.LabelScatter.cs @@ -0,0 +1,135 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using System.Diagnostics; +using System.Text.Json; +using Xunit; + +namespace SharpEmu.Gpu.Metal.Tests; + +public sealed unsafe partial class MetalEndOfPipeOrderTests +{ + // Held copies land through one compute dispatch (MetalRenderHost.LabelScatter): every word + // ends up with what the last copy into it carried, as the blits left it. + [Fact] + public void ManyHeldCopiesLandThroughOneDispatchAndTheLastCopyIntoAWordWins() + { + if (!RequireDevice()) return; + using var rig = new Rig(Device.Value); + var host = rig.Host; + Assert.True(host.DefersCopiesInCompute); + + _ = host.EnsureComputeEncoder(); + for (var index = 0; index < 48; index++) + { + host.RecordCommandProcessorWrite(LaterLabel + (ulong)index * 16, BitConverter.GetBytes(0x1000UL + (ulong)index), "WRITE_DATA"); + } + + // Rewrites of earlier qwords, one of them twice, and a dword over half of another. + host.RecordCommandProcessorWrite(LaterLabel + 3 * 16, BitConverter.GetBytes(0xAAAA_0003UL), "WRITE_DATA"); + host.RecordCommandProcessorWrite(LaterLabel + 7 * 16, BitConverter.GetBytes(0xBBBB_0007UL), "WRITE_DATA"); + host.RecordCommandProcessorWrite(LaterLabel + 7 * 16, BitConverter.GetBytes(0xCCCC_0007UL), "WRITE_DATA"); + host.RecordCommandProcessorWrite(LaterLabel + 9 * 16 + 4, BitConverter.GetBytes(0xDDDDu), "WRITE_DATA"); + var before = host.LabelScatterTotals; + + host.OrderAfterDeferredCopies(0, ulong.MaxValue); + Assert.Equal(0, host.DeferredLabelCopyCount); + Assert.Equal(before.Batches + 1, host.LabelScatterTotals.Batches); + + host.FlushCommandBuffer(); + WaitForGpu(host, host.LastWorkSerial); + WaitForCompletions(host); + for (var index = 0; index < 48; index++) + { + var expected = index switch + { + 3 => 0xAAAA_0003UL, + 7 => 0xCCCC_0007UL, + 9 => (0xDDDDUL << 32) | (0x1000UL + 9), + _ => 0x1000UL + (ulong)index, + }; + Assert.Equal(expected, rig.Backing.Read64(LaterLabel + (ulong)index * 16)); + } + } + + // A copy that is not whole 32-bit words keeps the blits for its batch. + [Fact] + public void AHeldBatchWithAnUnalignedCopyLandsThroughBlits() + { + if (!RequireDevice()) return; + using var rig = new Rig(Device.Value); + var host = rig.Host; + + _ = host.EnsureComputeEncoder(); + for (var index = 0; index < 32; index++) + { + host.RecordCommandProcessorWrite(LaterLabel + (ulong)index * 8, BitConverter.GetBytes(0x2000UL + (ulong)index), "WRITE_DATA"); + } + + host.RecordCommandProcessorWrite(LaterLabel + 0x400, [0x11, 0x22, 0x33], "WRITE_DATA"); + var before = host.LabelScatterTotals; + + host.OrderAfterDeferredCopies(0, ulong.MaxValue); + Assert.Equal(before.Batches, host.LabelScatterTotals.Batches); + Assert.Equal(before.Declined + 1, host.LabelScatterTotals.Declined); + + host.FlushCommandBuffer(); + WaitForGpu(host, host.LastWorkSerial); + WaitForCompletions(host); + for (var index = 0; index < 32; index++) + { + Assert.Equal(0x2000UL + (ulong)index, rig.Backing.Read64(LaterLabel + (ulong)index * 8)); + } + + Assert.Equal(0x332211UL, rig.Backing.Read64(LaterLabel + 0x400) & 0xFF_FFFF); + } + + // SHARPEMU_LABEL_SCATTER_BENCH_OUTPUT=: the worker's cost of landing a held batch, and the + // GPU's time to run it, through blits and through the scatter, alternating per round. + [Fact] + public void MeasureLabelScatterWhenRequested() + { + var path = Environment.GetEnvironmentVariable("SHARPEMU_LABEL_SCATTER_BENCH_OUTPUT"); + if (path is null || !RequireDevice()) return; + using var rig = new Rig(Device.Value); + var host = rig.Host; + var rounds = new List(); + var value = new byte[8]; + // Past the page no window reaches. + const ulong benchBase = GuestBase + 0x10000; + foreach (var copies in new[] { 64, 1024, 4096 }) + { + for (var round = 0; round < 11; round++) + { + foreach (var scatter in round % 2 == 0 ? new[] { false, true } : new[] { true, false }) + { + host.LabelScatterEnabled = scatter; + _ = host.EnsureComputeEncoder(); + for (var index = 0; index < copies; index++) + { + BitConverter.TryWriteBytes(value, (ulong)(round * 100_000 + index)); + host.RecordCommandProcessorWrite(benchBase + (ulong)index * 16, value, "WRITE_DATA"); + } + + var started = Stopwatch.GetTimestamp(); + host.OrderAfterDeferredCopies(0, ulong.MaxValue); + var encoded = Stopwatch.GetTimestamp(); + var serial = host.FlushCommandBuffer(); + WaitForGpu(host, serial); + var landed = Stopwatch.GetTimestamp(); + WaitForCompletions(host); + Assert.Equal((ulong)(round * 100_000 + copies - 1), rig.Backing.Read64(benchBase + (ulong)(copies - 1) * 16)); + if (round == 0) continue; + rounds.Add(new { copies, round, scatter, + encodeMicroseconds = (encoded - started) * 1e6 / Stopwatch.Frequency, + landMicroseconds = (landed - encoded) * 1e6 / Stopwatch.Frequency }); + } + } + } + + host.LabelScatterEnabled = true; + File.WriteAllText(path, JsonSerializer.Serialize(new { + architecture = System.Runtime.InteropServices.RuntimeInformation.ProcessArchitecture.ToString(), rounds }, + new JsonSerializerOptions { WriteIndented = true })); + } +} From 203086942b1c9032d8813be9d13c872eff71641f Mon Sep 17 00:00:00 2001 From: kuba Date: Wed, 28 Oct 2026 15:27:25 +0100 Subject: [PATCH 52/82] [Tools] The perf-audit suite measures an sceAgc builder in the split guest process, hook by hook The agc selector runs the real builder bodies over a PhysicalVirtualMemory whose GuestGpuMemory holds the split's page filter (a real SplitSharedState), and times the per-write hooks, the command-buffer reads and writes and whole draw sequences on their own. --- .../Benches.AgcBuilders.cs | 146 ++++++++++++++++++ tools/SharpEmu.Tools.PerfAudit/Program.cs | 1 + .../SharpEmu.Tools.PerfAudit.csproj | 2 + 3 files changed, 149 insertions(+) create mode 100644 tools/SharpEmu.Tools.PerfAudit/Benches.AgcBuilders.cs diff --git a/tools/SharpEmu.Tools.PerfAudit/Benches.AgcBuilders.cs b/tools/SharpEmu.Tools.PerfAudit/Benches.AgcBuilders.cs new file mode 100644 index 000000000..aa9032c2c --- /dev/null +++ b/tools/SharpEmu.Tools.PerfAudit/Benches.AgcBuilders.cs @@ -0,0 +1,146 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using SharpEmu.Core.Cpu; +using SharpEmu.Core.Memory; +using SharpEmu.GpuRemote; +using SharpEmu.GpuSplit; +using SharpEmu.HLE; +using SharpEmu.HLE.GpuMemory; +using SharpEmu.HLE.Host; +using SharpEmu.Host.MacOS; +using SharpEmu.Libs.Agc; + +namespace SharpEmu.PerfAudit; + +/// +/// The guest-side cost of an sceAgc command builder in the split's guest process: the builder body over +/// a real PhysicalVirtualMemory with a GuestGpuMemory whose image store is the split's page filter +/// (a real SplitSharedState), and each per-write hook on its own. Select with agc. +/// +internal static unsafe class AgcBuilderBenches +{ + private const ulong GuestBase = 0x2_0000_0000; + private const ulong Dwords = 0x8000; + + // The guest process's image store minus the transport: the interest filter of RemoteImageStore. + private sealed class FilterStore(SplitSharedState state) : IGuestImageStore + { + public long Landed; + public bool MarkCpuWrite(ulong address, ulong size) => false; + public bool MarkPreciseCpuWrite(ulong address, ulong size) => state.MayInterest(address, size) && false; + public bool MarkCpuWriteLanded(ulong address, ulong size) + { + if (!state.MayInterest(address, size)) return false; + Landed++; + return true; + } + + public void Unregister(ulong address, ulong size) { } + } + + public static void Run() + { + Console.WriteLine("--- AGC builder cost in the split guest process ---"); + HostMemoryAuthority.Initialize(MacOSHost.CreateMemory); + var views = MacOSHost.CreateViewMemory(); + using var pvm = new PhysicalVirtualMemory(viewHost: views, backingBytes: 32UL * 1024 * 1024); + const ulong Region = 8UL << 20; + var hole = (Region + views.Granularity - 1) / views.Granularity * views.Granularity; + if (!pvm.TryHoldRangeAtOrAbove(GuestBase, hole, views.Granularity, out var address) || + !pvm.TryMapBacked(address, Region, 0, GuestPageProtection.Read | GuestPageProtection.Write, out _)) + { + throw new InvalidOperationException("PVM map failed"); + } + + using var channel = GpuRemoteChannel.Create(SharedMemoryRegion.CreateUniqueName("agcbench"), SplitSharedState.Layout); + var state = new SplitSharedState(channel); + // A watched page far from the command buffer, as the images a title draws to are. + state.SetPage(address + 0x40_0000); + using var gpuMemory = new GuestGpuMemory(pvm); + gpuMemory.Register(address, Region, GuestPageProtection.Read | GuestPageProtection.Write); + var store = new FilterStore(state); + gpuMemory.AttachStores(null, store); + GuestGpuMemoryHook.Attach(gpuMemory); + var memory = new TrackedCpuMemory(pvm); + var ctx = new CpuContext(memory, Generation.Gen5); + + var cb = address + 0x1000; + var packets = address + 0x10_0000; + var cursorSlot = cb + 0x10; + var resetState = new byte[0x38]; + void ResetCb() + { + BitConverter.TryWriteBytes(resetState.AsSpan(0x10), packets); + BitConverter.TryWriteBytes(resetState.AsSpan(0x18), packets + Dwords * 4); + memory.TryWrite(cb, resetState); + } + + ResetCb(); + var eight = new byte[8]; + var forty = new byte[40]; + var statebuf = new byte[0x24]; + long sink = 0; + + // ---- the hooks on their own + Harness.Measure("a1", "GuestImageWriteTracker.NotifyManagedWrite (no tracked images)", 1, 1, i => + GuestImageWriteTracker.NotifyManagedWrite(packets + (ulong)((i & 255) << 3), 8), unit: "call"); + Harness.Measure("a2", "GuestGpuMemoryHook.MarkCpuWrite (covered, no interest)", 1, 1, i => + GuestGpuMemoryHook.MarkCpuWrite(packets + (ulong)((i & 255) << 3), 8), unit: "call"); + Harness.Measure("a3", "GuestGpuMemoryHook.MarkCpuWriteLanded (covered, no interest)", 1, 1, i => + GuestGpuMemoryHook.MarkCpuWriteLanded(packets + (ulong)((i & 255) << 3), 8), unit: "call"); + Harness.Measure("a4", "GuestGpuMemory.Covers", 1, 1, i => + sink += gpuMemory.Covers(packets + (ulong)((i & 255) << 3), 8) ? 1 : 0, unit: "call"); + Harness.Measure("a5", "Interlocked.MemoryBarrier", 1, 1, i => Interlocked.MemoryBarrier(), unit: "call"); + Harness.Measure("a6", "SplitSharedState.MayInterest (1 page, clear)", 1, 1, i => + sink += state.MayInterest(packets + (ulong)((i & 255) << 3), 8) ? 1 : 0, unit: "call"); + Harness.Measure("a7", "PVM.TryWriteBacking 8 B (no hooks)", 1, 1, i => + pvm.TryWriteBacking(packets + (ulong)((i & 255) << 3), eight), unit: "call"); + + // ---- the guest accesses a builder makes + Harness.Measure("b1", "TrackedCpuMemory.TryRead 36 B (command-buffer state)", 1, 1, i => + memory.TryRead(cursorSlot, statebuf), unit: "call"); + Harness.Measure("b2", "ctx.TryWriteUInt64 (cursor store, with hooks)", 1, 1, i => + ctx.TryWriteUInt64(packets + (ulong)((i & 255) << 3), 1), unit: "call"); + Harness.Measure("b3", "TrackedCpuMemory.TryWrite 40 B (packet, with hooks)", 1, 1, i => + memory.TryWrite(packets + (ulong)((i & 255) << 3), forty), unit: "call"); + Harness.Measure("b4", "TrackedCpuMemory.TryWrite 8 B at a page with interest (hooks fire, filter hit)", 1, 1, i => + memory.TryWrite(address + 0x40_0000 + (ulong)((i & 255) << 3), eight), unit: "call"); + + // ---- whole builders, leaf-style (callbacks declined) as the import dispatcher runs them + ctx.DeclineGuestCallbacks = true; + var calls = 0; + void Bench(string id, string name, Action body, int perReset) + { + Harness.Measure(id, name, 1, 1, i => + { + if ((calls++ % perReset) == 0) ResetCb(); + body(ctx); + }, unit: "call"); + } + + ulong Reg(uint offset, uint value) => offset | ((ulong)value << 32); + Bench("c0", "CbNop(2)", c => { c[CpuRegister.Rdi] = cb; c[CpuRegister.Rsi] = 2; AgcExports.CbNop(c); }, 4000); + Bench("c1", "DcbSetShRegisterDirect", c => { c[CpuRegister.Rdi] = cb; c[CpuRegister.Rsi] = Reg(0x8C, 0xDEADBEEF); AgcExports.SetShaderRegisterDirect(c); }, 4000); + Bench("c2", "DcbSetCxRegisterDirect", c => { c[CpuRegister.Rdi] = cb; c[CpuRegister.Rsi] = Reg(0x10F, 0x3F800000); AgcExports.DcbSetCxRegisterDirect(c); }, 4000); + Bench("c3", "DcbSetNumInstances", c => { c[CpuRegister.Rdi] = cb; c[CpuRegister.Rsi] = 1; AgcExports.DcbSetNumInstances(c); }, 4000); + Bench("c4", "DcbDrawIndexAuto", c => { c[CpuRegister.Rdi] = cb; c[CpuRegister.Rsi] = 3; c[CpuRegister.Rdx] = 0; AgcExports.DcbDrawIndexAuto(c); }, 4000); + Bench("c5", "DcbDrawIndex", c => { c[CpuRegister.Rdi] = cb; c[CpuRegister.Rsi] = 3; c[CpuRegister.Rdx] = 0x9_0000_0000; c[CpuRegister.Rcx] = 0; AgcExports.DcbDrawIndex(c); }, 2500); + Bench("c6", "indexed draw sequence (10 builders)", c => + { + c[CpuRegister.Rdi] = cb; + c[CpuRegister.Rsi] = Reg(0x8C, 1); AgcExports.SetShaderRegisterDirect(c); + c[CpuRegister.Rdi] = cb; c[CpuRegister.Rsi] = Reg(0x8D, 2); AgcExports.SetShaderRegisterDirect(c); + c[CpuRegister.Rdi] = cb; c[CpuRegister.Rsi] = Reg(0x0C, 3); AgcExports.SetShaderRegisterDirect(c); + c[CpuRegister.Rdi] = cb; c[CpuRegister.Rsi] = Reg(0x10F, 0x3F800000); AgcExports.DcbSetCxRegisterDirect(c); + c[CpuRegister.Rdi] = cb; c[CpuRegister.Rsi] = Reg(0x08E, 0xF); AgcExports.DcbSetCxRegisterDirect(c); + c[CpuRegister.Rdi] = cb; c[CpuRegister.Rsi] = Reg(0x205, 0); AgcExports.DcbSetCxRegisterDirect(c); + c[CpuRegister.Rdi] = cb; c[CpuRegister.Rsi] = Reg(0x242, 4); AgcExports.DcbSetUcRegisterDirect(c); + c[CpuRegister.Rdi] = cb; c[CpuRegister.Rsi] = 1; c[CpuRegister.Rdx] = 0; AgcExports.DcbSetIndexSize(c); + c[CpuRegister.Rdi] = cb; c[CpuRegister.Rsi] = 1; AgcExports.DcbSetNumInstances(c); + c[CpuRegister.Rdi] = cb; c[CpuRegister.Rsi] = 3; c[CpuRegister.Rdx] = 0x9_0000_0000; c[CpuRegister.Rcx] = 0; AgcExports.DcbDrawIndex(c); + }, 400); + if (sink == long.MinValue) Console.WriteLine(sink); + GuestGpuMemoryHook.Attach(null); + } +} diff --git a/tools/SharpEmu.Tools.PerfAudit/Program.cs b/tools/SharpEmu.Tools.PerfAudit/Program.cs index b87e32b8d..e1800313a 100644 --- a/tools/SharpEmu.Tools.PerfAudit/Program.cs +++ b/tools/SharpEmu.Tools.PerfAudit/Program.cs @@ -27,6 +27,7 @@ if (On("misc")) MiscBenches.Run(); if (On("w8")) W8Benches.Run(); if (filter.Split(',').Contains("xlate")) TranslatorBenches.Run(); +if (filter.Split(',').Contains("agc")) AgcBuilderBenches.Run(); if (filter.Split(',').Contains("gword")) MetalRawBenches.GpuWordWait(); if (filter.Split(',').Contains("wranges")) WriteRangeBenches.Run(); if (filter.Split(',').Contains("labels")) LabelFilterBenches.Run(); diff --git a/tools/SharpEmu.Tools.PerfAudit/SharpEmu.Tools.PerfAudit.csproj b/tools/SharpEmu.Tools.PerfAudit/SharpEmu.Tools.PerfAudit.csproj index 6c46cdeb0..627f188b1 100644 --- a/tools/SharpEmu.Tools.PerfAudit/SharpEmu.Tools.PerfAudit.csproj +++ b/tools/SharpEmu.Tools.PerfAudit/SharpEmu.Tools.PerfAudit.csproj @@ -21,5 +21,7 @@ SPDX-License-Identifier: GPL-2.0-or-later --> + + From 3cc787584efdd2e55a85f3bee8a257bf67efd6ff Mon Sep 17 00:00:00 2001 From: kuba Date: Wed, 28 Oct 2026 16:14:10 +0100 Subject: [PATCH 53/82] [HLE] Write an AGC builder's cursor word and packet as one fused guest write Every managed sceAgc builder wrote the 8-byte cursor and then the packet through two separate TryWrite calls, each paying the alias pin, the managed-write interval, the history record and a full MarkCpuWriteLanded (Covers, MemoryBarrier, store checks). ICpuMemory gains TryWritePair. PhysicalVirtualMemory implements it as two ranges under one view lookup and one write operation, notifying the stores per range as before, then landing both ranges with a single barrier (the interest-bit read still follows the stores, so the GPU host's set-bit-then-read protocol is unchanged). TryEmitCommandDwords and the reserve/commit helpers use it. SHARPEMU_PAIR_WRITE=0 restores the two writes. The packets and cursor the GPU sees are byte-identical; tests cover both ranges landing with the stores hearing of two writes and a failure when either range is unmapped. The perf-audit agc selector gains a b5 bench for the pair write. --- src/SharpEmu.Core/Cpu/TrackedCpuMemory.cs | 12 +++ .../Memory/PhysicalVirtualMemory.cs | 36 +++++++ src/SharpEmu.HLE/GpuMemory/GuestGpuMemory.cs | 30 ++++++ .../GpuMemory/GuestGpuMemoryHook.cs | 11 +++ .../GuestMemory/GuestSpaceOwner.cs | 3 + .../GuestMemory/SharedBackingViews.cs | 52 ++++++++++ src/SharpEmu.HLE/ICpuMemory.cs | 7 ++ .../Agc/AgcExports.DrawPackets.cs | 8 +- .../Agc/AgcExports.RegisterPackets.cs | 4 +- src/SharpEmu.Libs/Agc/AgcExports.cs | 33 +++++-- .../PhysicalVirtualMemoryBackedTests.cs | 96 +++++++++++++++++++ .../Benches.AgcBuilders.cs | 2 + 12 files changed, 281 insertions(+), 13 deletions(-) diff --git a/src/SharpEmu.Core/Cpu/TrackedCpuMemory.cs b/src/SharpEmu.Core/Cpu/TrackedCpuMemory.cs index 74e42cbb7..9989d29a7 100644 --- a/src/SharpEmu.Core/Cpu/TrackedCpuMemory.cs +++ b/src/SharpEmu.Core/Cpu/TrackedCpuMemory.cs @@ -147,6 +147,18 @@ public bool TryWrite(ulong virtualAddress, ReadOnlySpan source) return result; } + public bool TryWritePair(ulong firstAddress, ReadOnlySpan first, ulong secondAddress, ReadOnlySpan second) + { + var result = _inner.TryWritePair(firstAddress, first, secondAddress, second); + if (!result) + { + // Diagnostic only: the pair does not say which of its writes failed. + LastFailure = new CpuMemoryAccessFailure(firstAddress, first.Length, isWrite: true); + } + + return result; + } + public bool TryCompare( ulong virtualAddress, ReadOnlySpan expected, diff --git a/src/SharpEmu.Core/Memory/PhysicalVirtualMemory.cs b/src/SharpEmu.Core/Memory/PhysicalVirtualMemory.cs index 60177765b..53f57d12f 100644 --- a/src/SharpEmu.Core/Memory/PhysicalVirtualMemory.cs +++ b/src/SharpEmu.Core/Memory/PhysicalVirtualMemory.cs @@ -2714,6 +2714,42 @@ public bool TryWrite(ulong virtualAddress, ReadOnlySpan source) } } + // SHARPEMU_PAIR_WRITE=0 sends every pair through two ordinary writes. + private static readonly bool PairWriteEnabled = + Environment.GetEnvironmentVariable("SHARPEMU_PAIR_WRITE") != "0"; + + // Two writes as TryWriteCore's lock-free single-view path does them, under one alias pin and one + // write admission, with one pass of the hooks: + // - the pre-write hooks run for each range before any byte lands, as they do for each write; + // - both ranges land, first then second, and only then does the landed hook run for both: + // one full barrier, ordering both new byte ranges before the page state of either is read. + // It is the barrier of two MarkCpuWriteLanded calls, which each order their own bytes (and + // the earlier ones') before their read, so no read of GPU-interest state moves ahead of bytes + // it has to see; + // - anything the single-view path would refuse (a range with no single view, a refused + // admission, an armed GuestWriteWatch) declines with nothing written, and the two + // ordinary writes run instead. + public bool TryWritePair(ulong firstAddress, ReadOnlySpan first, ulong secondAddress, ReadOnlySpan second) + { + if (PairWriteEnabled && !first.IsEmpty && !second.IsEmpty && !GuestWriteWatch.Armed && !_disposed && + _backedSpace is { } backed) + { + GuestImageWriteTracker.NotifyManagedWrite(firstAddress, (ulong)first.Length); + GuestImageWriteTracker.NotifyManagedWrite(secondAddress, (ulong)second.Length); + GuestGpuMemoryHook.MarkCpuWrite(firstAddress, (ulong)first.Length); + GuestGpuMemoryHook.MarkCpuWrite(secondAddress, (ulong)second.Length); + if (backed.TryWritePairSingleView(firstAddress, first, secondAddress, second)) + { + GuestGpuMemoryHook.MarkCpuWriteLanded(firstAddress, (ulong)first.Length, secondAddress, (ulong)second.Length); + GuestImageWriteTracker.TraceManagedDirtyCompletion(firstAddress, (ulong)first.Length); + GuestImageWriteTracker.TraceManagedDirtyCompletion(secondAddress, (ulong)second.Length); + return true; + } + } + + return TryWrite(firstAddress, first) && TryWrite(secondAddress, second); + } + // In-place counterpart of TryWriteCore's lock-free single-view path: fill writes the // canonical alias of a backed range directly. The alias is not the guest-visible // mapping, so guest page guards cannot fault the fill (exactly as for the memcpy diff --git a/src/SharpEmu.HLE/GpuMemory/GuestGpuMemory.cs b/src/SharpEmu.HLE/GpuMemory/GuestGpuMemory.cs index cc919db04..05f6184fb 100644 --- a/src/SharpEmu.HLE/GpuMemory/GuestGpuMemory.cs +++ b/src/SharpEmu.HLE/GpuMemory/GuestGpuMemory.cs @@ -235,6 +235,36 @@ public void MarkCpuWriteLanded(ulong address, ulong size) _ = Images?.MarkCpuWriteLanded(address, size); } + // MarkCpuWriteLanded for two writes that have both landed: one full barrier orders both byte + // ranges before the page state of either is read, which is what each call's own barrier does for + // its bytes (and for any landed before it); the stores then hear of the first and of the second, + // in that order, as they do from two calls. + public void MarkCpuWriteLanded(ulong firstAddress, ulong firstSize, ulong secondAddress, ulong secondSize) + { + var firstCovered = firstSize != 0 && Covers(firstAddress, firstSize); + var secondCovered = secondSize != 0 && Covers(secondAddress, secondSize); + if (!firstCovered && !secondCovered) + { + return; + } + + // The new bytes must be visible before the store reads its page state. + Interlocked.MemoryBarrier(); + if (firstCovered) + { + SplitCrossingStats.Count(SplitCrossing.ManagedWriteLanded); + Buffers?.MarkCpuWriteLanded(firstAddress, firstSize); + _ = Images?.MarkCpuWriteLanded(firstAddress, firstSize); + } + + if (secondCovered) + { + SplitCrossingStats.Count(SplitCrossing.ManagedWriteLanded); + Buffers?.MarkCpuWriteLanded(secondAddress, secondSize); + _ = Images?.MarkCpuWriteLanded(secondAddress, secondSize); + } + } + public bool Covers(ulong address, ulong size) { if (!new GuestSpan(address, size).IsValid) diff --git a/src/SharpEmu.HLE/GpuMemory/GuestGpuMemoryHook.cs b/src/SharpEmu.HLE/GpuMemory/GuestGpuMemoryHook.cs index f990afb13..dff54849b 100644 --- a/src/SharpEmu.HLE/GpuMemory/GuestGpuMemoryHook.cs +++ b/src/SharpEmu.HLE/GpuMemory/GuestGpuMemoryHook.cs @@ -163,6 +163,17 @@ public static void MarkCpuWriteLanded(ulong address, ulong size) current.MarkCpuWriteLanded(address, size); } + // Two managed writes that have both landed (see ICpuMemory.TryWritePair). + public static void MarkCpuWriteLanded(ulong firstAddress, ulong firstSize, ulong secondAddress, ulong secondSize) + { + if (_current is not { } current) + { + return; + } + + current.MarkCpuWriteLanded(firstAddress, firstSize, secondAddress, secondSize); + } + // Shutdown can skip disposal. Report the counts once for each attached session. public static bool TryTakeShutdownSummary(out string summary) { diff --git a/src/SharpEmu.HLE/GuestMemory/GuestSpaceOwner.cs b/src/SharpEmu.HLE/GuestMemory/GuestSpaceOwner.cs index e34ee1ac9..fb62394e9 100644 --- a/src/SharpEmu.HLE/GuestMemory/GuestSpaceOwner.cs +++ b/src/SharpEmu.HLE/GuestMemory/GuestSpaceOwner.cs @@ -236,6 +236,9 @@ public void RequireOrdinaryWriteWaitOutsideGates() } public bool TryWriteSingleView(ulong address, ReadOnlySpan data) => _views.TryWriteSingleView(address, data); + public bool TryWritePairSingleView(ulong firstAddress, ReadOnlySpan first, ulong secondAddress, ReadOnlySpan second) => + _views.TryWritePairSingleView(firstAddress, first, secondAddress, second); + public bool TryFillSingleView(ulong address, int length, TState state, GuestSpanFill fill, out int filled) => _views.TryFillSingleView(address, length, state, fill, out filled); diff --git a/src/SharpEmu.HLE/GuestMemory/SharedBackingViews.cs b/src/SharpEmu.HLE/GuestMemory/SharedBackingViews.cs index 6b4317cfd..ad3264f5d 100644 --- a/src/SharpEmu.HLE/GuestMemory/SharedBackingViews.cs +++ b/src/SharpEmu.HLE/GuestMemory/SharedBackingViews.cs @@ -658,6 +658,58 @@ public bool TryWriteSingleView(ulong address, ReadOnlySpan data) } } + // Two TryWriteSingleView writes (first, then second) under one alias pin and one operation + // admission; each still brackets its own bytes with its own interval, so overlapping read watches + // are invalidated per range before that range's bytes land, as for separate writes. False, with + // nothing written, when either range has no single view or the admission is refused. As in + // TryWriteSingleView, a lost interval admission after the first range landed throws. + public bool TryWritePairSingleView(ulong firstAddress, ReadOnlySpan first, ulong secondAddress, ReadOnlySpan second) + { + if (first.IsEmpty || second.IsEmpty || (!SingleViewWriteFastPath && _backing?.ManagedWrites is not null)) + { + return false; + } + + if (ulong.MaxValue - firstAddress < (ulong)first.Length || ulong.MaxValue - secondAddress < (ulong)second.Length || + !TryEnterAliasAccess()) + { + return false; + } + + try + { + var snapshot = Volatile.Read(ref _snapshot); + if (!TryFindAlias(snapshot, firstAddress, (ulong)first.Length, out var firstTarget) || + !TryFindAlias(snapshot, secondAddress, (ulong)second.Length, out var secondTarget)) + { + return false; + } + + var interval = _backing?.BeginManagedWriteOperation() ?? default; + if (!HostManagedWriteIntervals.AllowsBytes(interval)) + { + return false; + } + + var succeeded = false; + try + { + CopyToCanonicalAlias(firstTarget, first); + CopyToCanonicalAlias(secondTarget, second); + succeeded = true; + return true; + } + finally + { + HostBackingObject.EndManagedWrite(interval, succeeded); + } + } + finally + { + ExitAliasAccess(); + } + } + // TryWriteSingleView with the bytes produced in place by fill instead of copied from a // source span: same alias pin, ordinary operation admission and per-chunk interval // (overlapping read watches are invalidated before fill runs), and false with fill diff --git a/src/SharpEmu.HLE/ICpuMemory.cs b/src/SharpEmu.HLE/ICpuMemory.cs index 2bc05be6f..de0b1dbb3 100644 --- a/src/SharpEmu.HLE/ICpuMemory.cs +++ b/src/SharpEmu.HLE/ICpuMemory.cs @@ -13,6 +13,13 @@ public interface ICpuMemory bool TryWrite(ulong virtualAddress, ReadOnlySpan source); + // Two writes, first then second, as one unit when this memory can: the same bytes, in the same + // order and with the same write tracking as two TryWrite calls, but one guest-memory admission + // and one pass of the GPU write hooks. A command builder's cursor store and its packet are such + // a pair. False when either write failed (the first may then have landed, as with two calls). + bool TryWritePair(ulong firstAddress, ReadOnlySpan first, ulong secondAddress, ReadOnlySpan second) => + TryWrite(firstAddress, first) && TryWrite(secondAddress, second); + bool TryCompare(ulong virtualAddress, ReadOnlySpan expected) => TryCompare(virtualAddress, expected, out var equal) && equal; diff --git a/src/SharpEmu.Libs/Agc/AgcExports.DrawPackets.cs b/src/SharpEmu.Libs/Agc/AgcExports.DrawPackets.cs index 9871ef295..8adc1e26c 100644 --- a/src/SharpEmu.Libs/Agc/AgcExports.DrawPackets.cs +++ b/src/SharpEmu.Libs/Agc/AgcExports.DrawPackets.cs @@ -137,12 +137,12 @@ public static int DcbDrawIndex(CpuContext ctx) // With the room for all of them, the index base, buffer size and draw are one allocation and // one write; otherwise each packet is allocated on its own, as the buffer's callback expects. ulong baseCommand, drawCommand; - if (TryAllocateCommandDwordsInPlace(ctx, commandBufferAddress, 11, out baseCommand)) + if (TryReserveCommandDwordsInPlace(ctx, commandBufferAddress, 11, out baseCommand, out var nextCursor)) { drawCommand = baseCommand + 5 * sizeof(uint); - if (!TryWriteCommandDwords(ctx, baseCommand, - Pm4(3, ItIndexBase, 0), (uint)indexAddress, (uint)(indexAddress >> 32), Pm4(2, ItIndexBufferSize, 0), indexCount, - Pm4(6, ItDrawIndex2, 0), indexCount, (uint)indexAddress, (uint)(indexAddress >> 32), indexCount, drawInitiator)) + if (!TryCommitCommandDwords(ctx, commandBufferAddress, nextCursor, baseCommand, + [Pm4(3, ItIndexBase, 0), (uint)indexAddress, (uint)(indexAddress >> 32), Pm4(2, ItIndexBufferSize, 0), indexCount, + Pm4(6, ItDrawIndex2, 0), indexCount, (uint)indexAddress, (uint)(indexAddress >> 32), indexCount, drawInitiator])) { return ReturnPointer(ctx, 0); } diff --git a/src/SharpEmu.Libs/Agc/AgcExports.RegisterPackets.cs b/src/SharpEmu.Libs/Agc/AgcExports.RegisterPackets.cs index 118e3b0fb..0959b146a 100644 --- a/src/SharpEmu.Libs/Agc/AgcExports.RegisterPackets.cs +++ b/src/SharpEmu.Libs/Agc/AgcExports.RegisterPackets.cs @@ -130,7 +130,7 @@ public static int CbSetShRegisterRangeDirect(CpuContext ctx) // leading dwords one write - with a small payload, read first, in it too: each guest write // and cursor update is a backing admission, and this runs once per draw on GTA V's render // threads. - if (TryAllocateCommandDwordsInPlace(ctx, commandBufferAddress, 2 + packetDwords, out var markerAddress)) + if (TryReserveCommandDwordsInPlace(ctx, commandBufferAddress, 2 + packetDwords, out var markerAddress, out var nextCursor)) { commandAddress = markerAddress + 2 * sizeof(uint); Span packet = stackalloc uint[4 + (int)Math.Min(valueCount, InlinePayloadDwords)]; @@ -142,7 +142,7 @@ public static int CbSetShRegisterRangeDirect(CpuContext ctx) (valuesAddress >= markerAddress + (2 + (ulong)packetDwords) * sizeof(uint) || valuesAddress + (ulong)valueCount * sizeof(uint) <= markerAddress) && ctx.Memory.TryRead(valuesAddress, System.Runtime.InteropServices.MemoryMarshal.AsBytes(packet[4..])); - if (!TryWriteCommandDwords(ctx, markerAddress, payloadWritten ? packet : packet[..4])) + if (!TryCommitCommandDwords(ctx, commandBufferAddress, nextCursor, markerAddress, payloadWritten ? packet : packet[..4])) { AgcRegisterPacketProfile.RecordResult(false); return ReturnPointer(ctx, 0); diff --git a/src/SharpEmu.Libs/Agc/AgcExports.cs b/src/SharpEmu.Libs/Agc/AgcExports.cs index 0f98b15f5..93ba74d36 100644 --- a/src/SharpEmu.Libs/Agc/AgcExports.cs +++ b/src/SharpEmu.Libs/Agc/AgcExports.cs @@ -493,26 +493,45 @@ private static ulong ComputeFingerprint(ReadOnlySpan bytes) private static bool TryAllocateCommandDwords(CpuContext ctx, ulong commandBufferAddress, uint sizeDwords, out ulong commandAddress) => TryPrepareCommandDwords(ctx, commandBufferAddress, sizeDwords, true, out commandAddress); - // Allocates only when the buffer already has the room, never calling its full callback: a caller - // that emits several packets takes them in one allocation then, and packet by packet otherwise, so - // each packet still reaches the callback with the size it asks for alone. - private static bool TryAllocateCommandDwordsInPlace(CpuContext ctx, ulong commandBufferAddress, uint sizeDwords, out ulong commandAddress) + // Whether the buffer already has the room, without calling its full callback or moving the + // cursor: a caller that emits several packets takes them in one allocation then, and packet by + // packet otherwise, so each packet still reaches the callback with the size it asks for alone. + // The packet goes to commandAddress and the cursor to nextCursor, together, in TryCommitCommandDwords. + private static bool TryReserveCommandDwordsInPlace(CpuContext ctx, ulong commandBufferAddress, uint sizeDwords, out ulong commandAddress, out ulong nextCursor) { - commandAddress = 0; + commandAddress = nextCursor = 0; if (sizeDwords == 0 || !TryReadCommandBufferState(ctx, commandBufferAddress, out var cursorUp, out var cursorDown, out _, out _, out var reservedDwords) || - sizeDwords > GetRemainingCommandDwords(cursorUp, cursorDown, reservedDwords) || - !ctx.TryWriteUInt64(commandBufferAddress + CommandBufferCursorUpOffset, cursorUp + (ulong)sizeDwords * sizeof(uint))) + sizeDwords > GetRemainingCommandDwords(cursorUp, cursorDown, reservedDwords)) { return false; } commandAddress = cursorUp; + nextCursor = cursorUp + (ulong)sizeDwords * sizeof(uint); return true; } + // The cursor store and the packet write of a reservation, as one guest-memory operation (the + // cursor first, as when they were two writes; SHARPEMU_PAIR_WRITE=0 makes them two again). + private static bool TryCommitCommandDwords(CpuContext ctx, ulong commandBufferAddress, ulong nextCursor, ulong commandAddress, ReadOnlySpan dwords) + { + Span cursor = stackalloc byte[sizeof(ulong)]; + BinaryPrimitives.WriteUInt64LittleEndian(cursor, nextCursor); + var packet = System.Runtime.InteropServices.MemoryMarshal.AsBytes(dwords); + var cursorAddress = commandBufferAddress + CommandBufferCursorUpOffset; + return ctx.Memory.TryWritePair(cursorAddress, cursor, commandAddress, packet); + } + private static bool TryEmitCommandDwords(CpuContext ctx, ulong commandBufferAddress, out ulong commandAddress, params ReadOnlySpan dwords) { + if (TryReserveCommandDwordsInPlace(ctx, commandBufferAddress, (uint)dwords.Length, out commandAddress, out var nextCursor)) + { + return TryCommitCommandDwords(ctx, commandBufferAddress, nextCursor, commandAddress, dwords); + } + + // No room (or no readable buffer): the buffer's callback, a leaf's decline and the failure + // paths are TryAllocateCommandDwords's. return TryAllocateCommandDwords(ctx, commandBufferAddress, (uint)dwords.Length, out commandAddress) && TryWriteCommandDwords(ctx, commandAddress, dwords); } diff --git a/tests/SharpEmu.Libs.Tests/Memory/GuestMemory/PhysicalVirtualMemoryBackedTests.cs b/tests/SharpEmu.Libs.Tests/Memory/GuestMemory/PhysicalVirtualMemoryBackedTests.cs index 98f0267df..a44e5cdb6 100644 --- a/tests/SharpEmu.Libs.Tests/Memory/GuestMemory/PhysicalVirtualMemoryBackedTests.cs +++ b/tests/SharpEmu.Libs.Tests/Memory/GuestMemory/PhysicalVirtualMemoryBackedTests.cs @@ -334,6 +334,102 @@ public void ManagedWritesMarkTheGpuStoresAgainOnceTheBytesLand() } } + // An image store that records what it is told about, with every page of [interestStart, interestEnd) of interest. + private sealed class RecordingImageStore(ulong interestStart, ulong interestEnd) : IGuestImageStore + { + public List<(ulong Address, ulong Size)> Precise { get; } = []; + + public List<(ulong Address, ulong Size)> Landed { get; } = []; + + private bool Interested(ulong address, ulong size) => address < interestEnd && interestStart < address + size; + + public bool MarkCpuWrite(ulong address, ulong size) => false; + + public bool MarkPreciseCpuWrite(ulong address, ulong size) + { + if (Interested(address, size)) Precise.Add((address, size)); + return false; + } + + public bool MarkCpuWriteLanded(ulong address, ulong size) + { + if (!Interested(address, size)) return false; + Landed.Add((address, size)); + return true; + } + + public void Unregister(ulong address, ulong size) { } + } + + [NativePageProtectionFact] + public void AWritePairLandsBothRangesAndTheStoresHearOfThemAsOfTwoWrites() + { + if (!Supported) return; + const GuestPageProtection readWrite = GuestPageProtection.Read | GuestPageProtection.Write; + var host = HostViewMemory.Create(); + using var memory = new PhysicalVirtualMemory(viewHost: host, backingBytes: BackingSize); + var view = Hold(memory, host); + Assert.True(memory.TryMapBacked(view, Segment, 0, readWrite, out _)); + using var gpu = new GuestGpuMemory(memory); + gpu.Register(view, Segment, readWrite); + // The packet's page is of interest to the GPU side; the cursor's is not. + var cursor = view + 0x10; + var packet = view + 0x2000; + var store = new RecordingImageStore(packet & ~0xFFFUL, (packet & ~0xFFFUL) + 0x1000); + gpu.AttachStores(null, store); + GuestGpuMemoryHook.Attach(gpu); + try + { + var cursorBytes = BitConverter.GetBytes(0x1122334455667788UL); + var packetBytes = Enumerable.Range(1, 40).Select(value => (byte)(value * 3)).ToArray(); + Assert.True(memory.TryWritePair(cursor, cursorBytes, packet, packetBytes)); + + var readBack = new byte[8]; + Assert.True(memory.TryRead(cursor, readBack)); + Assert.Equal(cursorBytes, readBack); + var packetBack = new byte[40]; + Assert.True(memory.TryRead(packet, packetBack)); + Assert.Equal(packetBytes, packetBack); + Assert.Equal([(packet, 40UL)], store.Precise); + Assert.Equal([(packet, 40UL)], store.Landed); + + // Both ranges of interest: each is heard of, the first one first. + store.Landed.Clear(); + var wide = new RecordingImageStore(view, view + Segment); + gpu.AttachStores(null, wide); + Assert.True(memory.TryWritePair(cursor, cursorBytes, packet, packetBytes)); + Assert.Equal([(cursor, 8UL), (packet, 40UL)], wide.Landed); + + // A range outside every registered span is not heard of, and the pair still writes. + var outside = memory.AllocateAt(0, 0x4000, false); + Assert.NotEqual(0UL, outside); + wide.Landed.Clear(); + Assert.True(memory.TryWritePair(cursor, cursorBytes, outside, packetBytes)); + Assert.Equal([(cursor, 8UL)], wide.Landed); + var outsideBack = new byte[40]; + Assert.True(memory.TryRead(outside, outsideBack)); + Assert.Equal(packetBytes, outsideBack); + } + finally + { + GuestGpuMemoryHook.Attach(null); + } + } + + [NativePageProtectionFact] + public void AWritePairFailsWhenEitherRangeIsUnmapped() + { + if (!Supported) return; + var host = HostViewMemory.Create(); + using var memory = new PhysicalVirtualMemory(viewHost: host, backingBytes: BackingSize); + var view = Hold(memory, host); + Assert.True(memory.TryMapBacked(view, Segment, 0, GuestPageProtection.Read | GuestPageProtection.Write, out _)); + var bytes = new byte[8]; + Assert.False(memory.TryWritePair(view, bytes, 0x10, bytes)); + Assert.False(memory.TryWritePair(0x10, bytes, view, bytes)); + Assert.True(memory.TryWritePair(view, bytes, view + 8, bytes)); + } + [Fact] public void MixedCopyWithProtectedPrivateMemoryDoesNotUpgradeUnderTheReadLock() { diff --git a/tools/SharpEmu.Tools.PerfAudit/Benches.AgcBuilders.cs b/tools/SharpEmu.Tools.PerfAudit/Benches.AgcBuilders.cs index aa9032c2c..438e2e163 100644 --- a/tools/SharpEmu.Tools.PerfAudit/Benches.AgcBuilders.cs +++ b/tools/SharpEmu.Tools.PerfAudit/Benches.AgcBuilders.cs @@ -104,6 +104,8 @@ void ResetCb() ctx.TryWriteUInt64(packets + (ulong)((i & 255) << 3), 1), unit: "call"); Harness.Measure("b3", "TrackedCpuMemory.TryWrite 40 B (packet, with hooks)", 1, 1, i => memory.TryWrite(packets + (ulong)((i & 255) << 3), forty), unit: "call"); + Harness.Measure("b5", "TrackedCpuMemory.TryWritePair 8 B + 40 B (cursor + packet; SHARPEMU_PAIR_WRITE=0 makes it two writes)", 1, 1, i => + memory.TryWritePair(packets + (ulong)((i & 255) << 3), eight, packets + 0x1000 + (ulong)((i & 255) << 3), forty), unit: "call"); Harness.Measure("b4", "TrackedCpuMemory.TryWrite 8 B at a page with interest (hooks fire, filter hit)", 1, 1, i => memory.TryWrite(address + 0x40_0000 + (ulong)((i & 255) << 3), eight), unit: "call"); From 33d37c4681061ad347cafb38e4bbfef5d5f736ed Mon Sep 17 00:00:00 2001 From: kuba Date: Wed, 28 Oct 2026 16:14:10 +0100 Subject: [PATCH 54/82] [CPU] Answer fixed-shape AGC packet builders with a native stub in the guest For 14 leaf builders whose whole effect is "append these dwords at the command buffer's cursor and advance it" (register sets, index state, draws, dispatch) the guest-side import now jumps to a generated x86-64 stub instead of crossing into the managed export. The stub reads cursor/end/reserve from the 0x24-byte buffer state, stores the dwords with plain guest stores and bumps the cursor. Plain stores are what the title itself would do, so write-protected GPU-interest pages are still caught by the fault handler. It declines to the normal trampoline for a null buffer, an inverted buffer, or without room for the packet plus the buffer's reserve, which is exactly where the export would run the buffer's callback. Gated like the other stubs (native thunks allowed, write-watch armed, no AGC tracing). SHARPEMU_AGC_PACKET_STUBS=0 keeps every builder on the managed exports. AgcPacketStubTests runs each stub as machine code against its export with randomized arguments and buffer room and compares bytes, cursor and return value. --- .../Native/DirectExecutionBackend.FastPath.cs | 196 +++++++++++++- .../Agc/AgcExports.PacketStubs.cs | 117 ++++++++ .../Cpu/AgcPacketStubTests.cs | 252 ++++++++++++++++++ 3 files changed, 560 insertions(+), 5 deletions(-) create mode 100644 src/SharpEmu.Libs/Agc/AgcExports.PacketStubs.cs create mode 100644 tests/SharpEmu.Libs.Tests/Cpu/AgcPacketStubTests.cs diff --git a/src/SharpEmu.Core/Cpu/Native/DirectExecutionBackend.FastPath.cs b/src/SharpEmu.Core/Cpu/Native/DirectExecutionBackend.FastPath.cs index aeaf989e3..7e76224f2 100644 --- a/src/SharpEmu.Core/Cpu/Native/DirectExecutionBackend.FastPath.cs +++ b/src/SharpEmu.Core/Cpu/Native/DirectExecutionBackend.FastPath.cs @@ -41,6 +41,12 @@ internal enum GuestFastPathStub /// sceKernelGetProcessTime: microseconds since process start. ProcessTimeMicroseconds, + + /// + /// An sceAgc command builder that appends one fixed-shape packet (see ): + /// the stub stores the dwords and the cursor itself, as the title's own code does. + /// + AgcPacket, } // clock_gettime / sceKernelClockGettime / gettimeofday / sceKernelGetProcessTime. @@ -56,6 +62,9 @@ internal enum GuestFastPathStub /// internal readonly record struct ClockStubTargets(nint MachAbsoluteTime, long ProcessStartCounter, nint ClockGettime); + // SHARPEMU_AGC_PACKET_STUBS=0 keeps the sceAgc command builders on the managed exports. + private static readonly bool AgcPacketStubsEnabled = SharpEmu.Libs.Agc.AgcExports.PacketStubsEnabled; + // SHARPEMU_HLE_FAST_CLOCK=0 keeps the clock reads on the managed exports. private static readonly bool ClockStubsEnabled = !string.Equals(Environment.GetEnvironmentVariable("SHARPEMU_HLE_FAST_CLOCK"), "0", StringComparison.Ordinal); @@ -226,7 +235,9 @@ private void ConfigureGuestFastPath() ClockGettimeNid or KernelClockGettimeNid => GuestFastPathStub.ClockGettime, GettimeofdayNid => GuestFastPathStub.GetTimeOfDay, KernelGetProcessTimeNid => GuestFastPathStub.ProcessTimeMicroseconds, - _ => IsConstantZeroImport(nid) ? GuestFastPathStub.ConstantZero : GuestFastPathStub.None, + _ => IsConstantZeroImport(nid) + ? GuestFastPathStub.ConstantZero + : SharpEmu.Libs.Agc.AgcExports.TryGetPacketStubSpec(nid, out _) ? GuestFastPathStub.AgcPacket : GuestFastPathStub.None, }; /// @@ -269,15 +280,39 @@ private unsafe bool TryCreateGuestFastPathStub(string nid, nint fallbackTrampoli return false; } } + else if (kind == GuestFastPathStub.AgcPacket) + { + // Stores into the command buffer from guest-side code, so the write watch (which + // inspects managed writes only) and any mode that has to see every builder call + // (import and AGC tracing) rule it out; it needs no per-thread block. + if (!AgcPacketStubsEnabled || + !NativeImportThunksEnabled || + !_nativeConstantThunksAllowed || + GuestWriteWatch.Armed || + SharpEmu.Libs.Agc.AgcExports.PacketStubsBlockedByTracing || + !SharpEmu.Libs.Agc.AgcExports.TryGetPacketStubSpec(nid, out _)) + { + return false; + } + } else if (!_guestFastPathEnabled) { return false; } - var code = kind is GuestFastPathStub.ClockGettime or GuestFastPathStub.GetTimeOfDay or GuestFastPathStub.ProcessTimeMicroseconds - ? EmitGuestClockStub(kind, _clockStubTargets!.Value, fallbackTrampoline) - : EmitGuestFastPathStub(kind, _fastPathBlockTlsIndex, fallbackTrampoline); - const uint stubAllocationSize = 192u; + List code; + if (kind == GuestFastPathStub.AgcPacket) + { + SharpEmu.Libs.Agc.AgcExports.TryGetPacketStubSpec(nid, out var packetSpec); + code = EmitGuestAgcPacketStub(packetSpec, fallbackTrampoline); + } + else + { + code = kind is GuestFastPathStub.ClockGettime or GuestFastPathStub.GetTimeOfDay or GuestFastPathStub.ProcessTimeMicroseconds + ? EmitGuestClockStub(kind, _clockStubTargets!.Value, fallbackTrampoline) + : EmitGuestFastPathStub(kind, _fastPathBlockTlsIndex, fallbackTrampoline); + } + var stubAllocationSize = kind == GuestFastPathStub.AgcPacket ? 512u : 192u; if (code.Count > stubAllocationSize) { return false; @@ -572,6 +607,157 @@ void EmitRealtimeIntoStack() return code; } + /// + /// An sceAgc builder that appends one fixed-shape packet, as guest-side code: with the command + /// buffer in RDI it reads the cursor, the end and the reserve, declines (jumps to the import's + /// trampoline, with every argument register intact) unless the buffer is non-null and has the room + /// AgcExports.GetRemainingCommandDwords demands, stores the packet's dwords at the cursor, + /// advances the cursor, and returns the packet's address. The stores are plain guest stores, the + /// kind the title makes into the same buffer: a page the GPU process watches faults and is + /// resolved from the fault address, which the managed write hooks exist to emulate. Only RAX, R10, + /// R11 and the flags change, all caller-saved. + /// + internal static List EmitGuestAgcPacketStub(SharpEmu.Libs.Agc.AgcPacketStubSpec spec, nint fallbackTrampoline) + { + const int CursorUpOffset = 0x10; + const int CursorDownOffset = 0x18; + const int ReservedDwordsOffset = 0x30; + if (spec.SizeDwords is < 1 or > 15 || spec.Words.Length != spec.SizeDwords || spec.ReturnDwordOffset >= spec.SizeDwords) + { + throw new ArgumentException("Unsupported AGC packet stub shape.", nameof(spec)); + } + + var code = new List(256); + var slowPathFixups = new List(); + void Emit(params byte[] bytes) => code.AddRange(bytes); + void EmitImm32(uint value) => code.AddRange(BitConverter.GetBytes(value)); + void EmitBranchToSlowPath(byte shortOpcode) + { + Emit(0x0F, (byte)(shortOpcode + 0x10), 0, 0, 0, 0); + slowPathFixups.Add(code.Count - 4); + } + + // test rdi, rdi / jz slow: a null buffer is the export's to refuse. + Emit(0x48, 0x85, 0xFF); + EmitBranchToSlowPath(0x74); + Emit(0x48, 0x8B, 0x47, CursorUpOffset); // mov rax, [rdi+0x10] cursor + Emit(0x4C, 0x8B, 0x57, CursorDownOffset); // mov r10, [rdi+0x18] end + Emit(0x49, 0x29, 0xC2); // sub r10, rax + EmitBranchToSlowPath(0x72); // jb slow: end below the cursor + Emit(0x49, 0xC1, 0xEA, 0x02); // shr r10, 2 available dwords + Emit(0x41, 0xBB); EmitImm32(0xFFFF_FFFFu); // mov r11d, 0xFFFFFFFF + Emit(0x4D, 0x39, 0xDA); // cmp r10, r11 + EmitBranchToSlowPath(0x77); // ja slow: the export clamps the span to 32 bits + Emit(0x44, 0x8B, 0x5F, ReservedDwordsOffset); // mov r11d, [rdi+0x30] reserve + Emit(0x49, 0x83, 0xC3, checked((byte)spec.SizeDwords)); // add r11, size + Emit(0x4D, 0x39, 0xDA); // cmp r10, r11 + EmitBranchToSlowPath(0x72); // jb slow: size > available - reserve + + // Loads one part of a word into r10, or into r11 when intoR11. + void EmitPart(SharpEmu.Libs.Agc.AgcPacketStubPart part, bool intoR11) + { + var destination = intoR11 ? (byte)3 : (byte)2; // r11 / r10 low three bits + var source = (byte)part.Register; + var rex = (byte)(0x40 | (source >= 8 ? 0x04 : 0) | 0x01); // REX.R for r8, REX.B for r10/r11 + var modrm = (byte)(0xC0 | ((source & 7) << 3) | destination); + if (part.ShiftRight == 0) + { + Emit(rex, 0x89, modrm); // mov r10d/r11d, reg32 + } + else + { + Emit((byte)(rex | 0x08), 0x89, modrm); // mov r10/r11, reg64 + Emit(0x49, 0xC1, (byte)(0xE8 | destination), checked((byte)part.ShiftRight)); // shr r10/r11, n + } + + Emit(0x41, 0x81, (byte)(0xE0 | destination)); EmitImm32(part.Mask); // and r10d/r11d, mask + if (part.ShiftLeft != 0) + { + Emit(0x41, 0xC1, (byte)(0xE0 | destination), checked((byte)part.ShiftLeft)); // shl r10d/r11d, n + } + } + + for (var index = 0; index < spec.Words.Length; index++) + { + var word = spec.Words[index]; + var displacement = checked((byte)(index * sizeof(uint))); + if (word.Kind == SharpEmu.Libs.Agc.AgcPacketStubWordKind.Parts && word.Parts.Length == 0) + { + Emit(0xC7, 0x40, displacement); EmitImm32(word.Constant); // mov dword [rax+disp], imm32 + continue; + } + + switch (word.Kind) + { + case SharpEmu.Libs.Agc.AgcPacketStubWordKind.Parts: + EmitPart(word.Parts[0], intoR11: false); + for (var part = 1; part < word.Parts.Length; part++) + { + EmitPart(word.Parts[part], intoR11: true); + Emit(0x45, 0x09, 0xDA); // or r10d, r11d + } + + if (word.Constant != 0) + { + Emit(0x41, 0x81, 0xCA); EmitImm32(word.Constant); // or r10d, imm32 + } + + break; + + case SharpEmu.Libs.Agc.AgcPacketStubWordKind.AtLeastOne: + EmitPart(word.Parts[0], intoR11: false); + Emit(0x45, 0x85, 0xD2); // test r10d, r10d + Emit(0x75, 0x06); // jnz +6 + Emit(0x41, 0xBA); EmitImm32(1); // mov r10d, 1 + break; + + case SharpEmu.Libs.Agc.AgcPacketStubWordKind.DrawInitiator: + { + // bit 32 of the modifier set: 0; else (uint)modifier >> 3 & 0x20. + var source = (byte)word.Parts[0].Register; + var rex = (byte)(0x40 | (source >= 8 ? 0x04 : 0) | 0x01); + Emit(rex, 0x89, (byte)(0xC0 | ((source & 7) << 3) | 2)); // mov r10d, reg32 + Emit(0x41, 0xC1, 0xEA, 0x03); // shr r10d, 3 + Emit(0x41, 0x83, 0xE2, 0x20); // and r10d, 0x20 + Emit((byte)(rex | 0x08), 0x89, (byte)(0xC0 | ((source & 7) << 3) | 3)); // mov r11, reg64 + Emit(0x49, 0xC1, 0xEB, 0x20); // shr r11, 32 + Emit(0x41, 0xF7, 0xC3); EmitImm32(1); // test r11d, 1 + Emit(0x74, 0x03); // jz +3 + Emit(0x45, 0x31, 0xD2); // xor r10d, r10d + break; + } + + default: + throw new ArgumentOutOfRangeException(nameof(spec), word.Kind, "Unknown packet word kind."); + } + + Emit(0x44, 0x89, 0x50, displacement); // mov [rax+disp], r10d + } + + Emit(0x4C, 0x8D, 0x58, checked((byte)(spec.SizeDwords * sizeof(uint)))); // lea r11, [rax + size*4] + Emit(0x4C, 0x89, 0x5F, CursorUpOffset); // mov [rdi+0x10], r11 + if (spec.ReturnDwordOffset != 0) + { + Emit(0x48, 0x83, 0xC0, checked((byte)(spec.ReturnDwordOffset * sizeof(uint)))); // add rax, offset*4 + } + + Emit(0xC3); + + foreach (var fixup in slowPathFixups) + { + var delta = BitConverter.GetBytes(code.Count - (fixup + 4)); + for (var i = 0; i < 4; i++) + { + code[fixup + i] = delta[i]; + } + } + + // jmp qword [rip+0] with the absolute trampoline address after it. + Emit(0xFF, 0x25, 0x00, 0x00, 0x00, 0x00); + code.AddRange(BitConverter.GetBytes((long)fallbackTrampoline)); + return code; + } + /// /// memcpy/memmove(dst, src, n) as a forward 32-byte SSE loop (16-byte step, then /// rep movsb for the last 0-15 bytes), returning dst. Rosetta translates diff --git a/src/SharpEmu.Libs/Agc/AgcExports.PacketStubs.cs b/src/SharpEmu.Libs/Agc/AgcExports.PacketStubs.cs new file mode 100644 index 000000000..1cf3528f1 --- /dev/null +++ b/src/SharpEmu.Libs/Agc/AgcExports.PacketStubs.cs @@ -0,0 +1,117 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +namespace SharpEmu.Libs.Agc; + +/// The x86-64 register encodings an may read (SysV integer arguments 2-5). +public enum AgcStubRegister : byte +{ + Rcx = 1, + Rdx = 2, + Rsi = 6, + R8 = 8, +} + +/// One term of a packet dword: ((register >> ShiftRight) & Mask) << ShiftLeft, in 32 bits. +public readonly record struct AgcPacketStubPart(AgcStubRegister Register, int ShiftRight, uint Mask, int ShiftLeft); + +public enum AgcPacketStubWordKind : byte +{ + /// The constant ORed with every part. + Parts, + + /// max(first part, 1) (the constant is 0). + AtLeastOne, + + /// + /// The draw initiator of a modifier in the first part's register: 0 when bit 32 is set, else + /// ((uint)modifier >> 3) & 0x20. + /// + DrawInitiator, +} + +public sealed record AgcPacketStubWord(uint Constant, AgcPacketStubPart[] Parts, AgcPacketStubWordKind Kind = AgcPacketStubWordKind.Parts); + +/// +/// A command builder whose whole effect is "append these dwords at the command buffer's cursor and +/// advance the cursor", for a buffer that has the room: the one case a native stub in the guest can +/// do exactly as the export does. Every dword is a function of the integer arguments in registers. +/// The stub declines (the export runs) for a null buffer or one without room for the packet and the +/// buffer's reserve, which includes everything that would call the buffer's full callback. +/// +/// The dwords the builder advances the cursor by (1 to 15). +/// The packet address the builder returns, in dwords from the cursor it started at. +public sealed record AgcPacketStubSpec(uint SizeDwords, uint ReturnDwordOffset, AgcPacketStubWord[] Words); + +public static partial class AgcExports +{ + // SHARPEMU_AGC_PACKET_STUBS=0 keeps every builder on the managed exports. + public static readonly bool PacketStubsEnabled = + Environment.GetEnvironmentVariable("SHARPEMU_AGC_PACKET_STUBS") != "0"; + + /// True while AGC call tracing wants every builder call in the export. + public static bool PacketStubsBlockedByTracing => _traceAgc; + + private static AgcPacketStubWord Const(uint value) => new(value, []); + + private static AgcPacketStubWord Arg(AgcStubRegister register, int shiftRight = 0, uint mask = 0xFFFF_FFFFu, int shiftLeft = 0, uint constant = 0) => + new(constant, [new AgcPacketStubPart(register, shiftRight, mask, shiftLeft)]); + + private static AgcPacketStubSpec DirectRegisterSpec(uint op) => new(3, 0, + [Const(Pm4(3, op, 0)), Arg(AgcStubRegister.Rsi, 0, 0xFFFFu), Arg(AgcStubRegister.Rsi, 32)]); + + private static AgcPacketStubSpec IndirectRegistersSpec(uint op) => new(5, 0, + [Const(Pm4(5, op, 0)), Arg(AgcStubRegister.Rsi, 0, 0xFFFF_FFFCu), Arg(AgcStubRegister.Rsi, 32), Const(0x8000_0000u), Arg(AgcStubRegister.Rdx, 0, 0x3FFFu)]); + + private static AgcPacketStubWord DrawInitiatorOfRcx() => + new(0, [new AgcPacketStubPart(AgcStubRegister.Rcx, 0, 0xFFFF_FFFFu, 0)], AgcPacketStubWordKind.DrawInitiator); + + // The leaf builders that append one fixed-shape packet, by NID. Each entry mirrors its export's + // TryEmitCommandDwords call word for word (AgcPacketStubTests runs each stub against its export). + private static readonly Dictionary PacketStubSpecs = new() + { + ["pFLArOT53+w"] = DirectRegisterSpec(ItSetShReg), // sceAgcDcbSetShRegisterDirect + ["LHFXRrlTPD8"] = DirectRegisterSpec(ItSetContextReg), // sceAgcDcbSetCxRegisterDirect + ["w4-d0n60hdo"] = DirectRegisterSpec(ItSetUconfigReg), // sceAgcDcbSetUcRegisterDirect + ["-HOOCn0JY48"] = IndirectRegistersSpec(ItSetShRegIndirect), // sceAgcDcbSetShRegistersIndirect + ["ZvwO9euwYzc"] = IndirectRegistersSpec(ItSetContextRegIndirect), // sceAgcDcbSetCxRegistersIndirect + ["hvUfkUIQcOE"] = IndirectRegistersSpec(ItSetUconfigRegIndirect), // sceAgcDcbSetUcRegistersIndirect + ["tSBxhAPyytQ"] = new(2, 0, [Const(Pm4(2, ItNumInstances, 0)), Arg(AgcStubRegister.Rsi)]), // sceAgcDcbSetNumInstances + ["8N2tmT3jmC8"] = new(2, 0, [Const(Pm4(2, ItNop, RIndexCount)), Arg(AgcStubRegister.Rsi)]), // sceAgcDcbSetIndexCount + ["GIIW2J37e70"] = new(3, 0, // sceAgcDcbSetIndexSize + [ + Const(Pm4(3, ItSetUconfigRegIndex, 0)), + Const(0x2000_0000u | VgtIndexType), + new AgcPacketStubWord(0x400u, [new(AgcStubRegister.Rsi, 0, 3, 0), new(AgcStubRegister.Rdx, 0, 3, 6)]), + ]), + ["l4fM9K-Lyks"] = new(5, 0, // sceAgcDcbSetIndexBuffer + [ + Const(Pm4(3, ItIndexBase, 0)), Arg(AgcStubRegister.Rsi), Arg(AgcStubRegister.Rsi, 32), + Const(Pm4(2, ItIndexBufferSize, 0)), Arg(AgcStubRegister.Rdx), + ]), + ["Yw0jKSqop+E"] = new(7, 0, // sceAgcDcbDrawIndexAuto + [Const(Pm4(7, ItNop, RDrawIndexAuto)), Arg(AgcStubRegister.Rsi), Const(0), Const(0), Const(0), Const(0), Const(0)]), + ["B+aG9DUnTKA"] = new(5, 0, // sceAgcDcbDrawIndexOffset + [ + Const(Pm4(5, ItDrawIndexOffset2, 0)), + new AgcPacketStubWord(0, [new(AgcStubRegister.Rdx, 0, 0xFFFF_FFFFu, 0)], AgcPacketStubWordKind.AtLeastOne), + Arg(AgcStubRegister.Rsi), Arg(AgcStubRegister.Rdx), DrawInitiatorOfRcx(), + ]), + ["q88lQ+GP5Yk"] = new(11, 5, // sceAgcDcbDrawIndex: INDEX_BASE, INDEX_BUFFER_SIZE, DRAW_INDEX_2 + [ + Const(Pm4(3, ItIndexBase, 0)), Arg(AgcStubRegister.Rdx), Arg(AgcStubRegister.Rdx, 32), + Const(Pm4(2, ItIndexBufferSize, 0)), Arg(AgcStubRegister.Rsi), + Const(Pm4(6, ItDrawIndex2, 0)), Arg(AgcStubRegister.Rsi), Arg(AgcStubRegister.Rdx), Arg(AgcStubRegister.Rdx, 32), Arg(AgcStubRegister.Rsi), + DrawInitiatorOfRcx(), + ]), + ["k3GhuSNmBLU"] = new(5, 0, // sceAgcCbDispatch + [ + Const(Pm4(5, ItDispatchDirect, 0)), Arg(AgcStubRegister.Rsi), Arg(AgcStubRegister.Rdx), Arg(AgcStubRegister.Rcx), + Arg(AgcStubRegister.R8, 0, 0xA038u, 0, 0x41u), + ]), + }; + + /// The packet stub for a builder import, when it has one. + public static bool TryGetPacketStubSpec(string nid, out AgcPacketStubSpec spec) => + PacketStubSpecs.TryGetValue(nid, out spec!); +} diff --git a/tests/SharpEmu.Libs.Tests/Cpu/AgcPacketStubTests.cs b/tests/SharpEmu.Libs.Tests/Cpu/AgcPacketStubTests.cs new file mode 100644 index 000000000..2719e6e33 --- /dev/null +++ b/tests/SharpEmu.Libs.Tests/Cpu/AgcPacketStubTests.cs @@ -0,0 +1,252 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using SharpEmu.Core.Cpu.Native; +using SharpEmu.Core.Cpu.Native.Platform; +using SharpEmu.HLE; +using SharpEmu.Libs.Agc; +using SharpEmu.Libs.Tests.Agc; +using System.Runtime.InteropServices; +using Xunit; + +namespace SharpEmu.Libs.Tests.Cpu; + +/// +/// Runs the emitted AGC packet stubs as machine code against the exports they stand in for: the same +/// arguments, the same command buffer bytes in, and the same bytes, cursor and return value out - or +/// a decline (the fallback answers a sentinel) exactly where the export would have to run the buffer's +/// callback or refuse. +/// +[Collection(AgcCommandBufferChainCollection.Name)] +public sealed unsafe class AgcPacketStubTests +{ + private const ulong Sentinel = 0x5EEDFACE_0BADF00DUL; + private const int BlockBytes = 0x1000; + private const int PacketOffset = 0x100; + + public static IEnumerable Nids() => + new[] + { + "pFLArOT53+w", "LHFXRrlTPD8", "w4-d0n60hdo", "-HOOCn0JY48", "ZvwO9euwYzc", "hvUfkUIQcOE", "tSBxhAPyytQ", + "8N2tmT3jmC8", "GIIW2J37e70", "l4fM9K-Lyks", "Yw0jKSqop+E", "B+aG9DUnTKA", "q88lQ+GP5Yk", "k3GhuSNmBLU", + }.Select(nid => new object[] { nid }); + + [Theory] + [MemberData(nameof(Nids))] + public void TheStubWritesWhatTheExportWritesAndDeclinesWhereTheExportWouldCallBack(string nid) + { + if (!OperatingSystem.IsMacOS() || RuntimeInformation.ProcessArchitecture != Architecture.X64) + { + return; + } + + Assert.True(AgcExports.TryGetPacketStubSpec(nid, out var spec)); + using var harness = new StubHarness(spec); + var manager = new ModuleManager(); + manager.RegisterExports(SharpEmu.Generated.SysAbiExportRegistry.CreateExports(Generation.Gen5)); + var random = new System.Random(unchecked(nid.GetHashCode() ^ 0x5A17)); + var interesting = new ulong[] + { + 0, 1, 2, 3, 7, 0x28, 0x40, 0xA038, 0xFFFF, 0x1_0000, 0xFFFF_FFFC, 0xFFFF_FFFF, 0x1_0000_0000, 0x1_0000_0001, 0x2_6000_0A08, + 0x7FFF_FFFF_FFFF_FFFF, 0xFFFF_FFFF_FFFF_FFFF, 0x8000_0000_0000_0000, 0xDEAD_BEEF_CAFE_F00D, + }; + ulong Pick() => random.Next(3) == 0 ? (ulong)random.NextInt64() | ((ulong)random.Next(2) << 63) : interesting[random.Next(interesting.Length)]; + + var packetBytes = (int)spec.SizeDwords * 4; + var stubRan = 0; + for (var iteration = 0; iteration < 600; iteration++) + { + var reserved = (uint)random.Next(3) * 3; + // Room: exactly the packet and the reserve, one dword short, one dword over, plenty, none, or an inverted buffer. + var room = random.Next(6) switch + { + 0 => packetBytes + (int)reserved * 4, + 1 => packetBytes + (int)reserved * 4 - 4, + 2 => packetBytes + (int)reserved * 4 + 4, + 3 => 0x700, + 4 => 0, + _ => -64, + }; + var arguments = new[] { Pick(), Pick(), Pick(), Pick() }; + var nullBuffer = random.Next(25) == 0; + + using var managedBlock = new Block(); + using var stubBlock = new Block(); + foreach (var block in new[] { managedBlock, stubBlock }) + { + block.Reset(reserved, room, random.Next(1 << 30)); + } + + // Both blocks start byte for byte alike (Reset is seeded the same way). + stubBlock.CopyFrom(managedBlock); + + var buffer = nullBuffer ? 0UL : (ulong)managedBlock.Pointer; + var context = new CpuContext(new RawMemory(managedBlock), Generation.Gen5); + context[CpuRegister.Rdi] = buffer; + context[CpuRegister.Rsi] = arguments[0]; + context[CpuRegister.Rdx] = arguments[1]; + context[CpuRegister.Rcx] = arguments[2]; + context[CpuRegister.R8] = arguments[3]; + // The block's cursor words are absolute addresses: rebase them for each block. + managedBlock.Rebase(); + var previousScheduler = GuestThreadExecution.Scheduler; + GuestThreadExecution.Scheduler = null; + try + { + context.DeclineGuestCallbacks = true; + manager.Dispatch(nid, context); + context.DeclineGuestCallbacks = false; + } + finally + { + GuestThreadExecution.Scheduler = previousScheduler; + } + + stubBlock.Rebase(); + var stubRax = harness.Call(nullBuffer ? 0UL : (ulong)stubBlock.Pointer, arguments[0], arguments[1], arguments[2], arguments[3]); + if (stubRax == Sentinel) + { + Assert.True( + context.GuestCallbackDeclined || nullBuffer, + $"{nid}: the stub declined with room (room={room} reserved={reserved}) where the export built the packet"); + Assert.Equal(managedBlock.Normalized(), stubBlock.Normalized()); + continue; + } + + stubRan++; + Assert.False(context.GuestCallbackDeclined, $"{nid}: the stub built a packet where the export had to call back (room={room} reserved={reserved})"); + Assert.False(nullBuffer, nid + ": the stub accepted a null buffer"); + Assert.Equal(context[CpuRegister.Rax] - (ulong)managedBlock.Pointer, stubRax - (ulong)stubBlock.Pointer); + Assert.Equal(managedBlock.Normalized(), stubBlock.Normalized()); + } + + Assert.True(stubRan > 100, $"{nid}: the stub took the fast path only {stubRan} times"); + } + + [Fact] + public void EveryStubbedBuilderIsAlsoALeafBuilder() + { + foreach (var nid in Nids().Select(row => (string)row[0])) + { + Assert.True(DirectExecutionBackend.IsAgcBuilderLeafImport(nid), nid); + Assert.Equal(DirectExecutionBackend.GuestFastPathStub.AgcPacket, DirectExecutionBackend.ClassifyGuestFastPathStub(nid)); + } + } + + /// A guest "command buffer" in native memory: the buffer structure at +0, the packet area at +0x100. + private sealed class Block : IDisposable + { + private readonly byte* _memory = (byte*)NativeMemory.AlignedAlloc(BlockBytes, 16); + private uint _reserved; + private int _room; + + public Block() => NativeMemory.Clear(_memory, BlockBytes); + + public byte* Pointer => _memory; + + public void Reset(uint reserved, int room, int seed) + { + NativeMemory.Clear(_memory, BlockBytes); + var random = new System.Random(seed); + // Garbage in the packet area, so a stub that writes too little or too much shows. + for (var offset = PacketOffset; offset < BlockBytes; offset++) + { + _memory[offset] = (byte)random.Next(256); + } + + _reserved = reserved; + _room = room; + } + + public void CopyFrom(Block other) + { + Buffer.MemoryCopy(other._memory, _memory, BlockBytes, BlockBytes); + _reserved = other._reserved; + _room = other._room; + } + + // The cursor, the end and the reserve, relative to this block. + public void Rebase() + { + *(ulong*)(_memory + 0x10) = (ulong)(_memory + PacketOffset); + *(ulong*)(_memory + 0x18) = (ulong)((long)(_memory + PacketOffset) + _room); + *(ulong*)(_memory + 0x20) = 0x1234; + *(ulong*)(_memory + 0x28) = 0x5678; + *(uint*)(_memory + 0x30) = _reserved; + } + + // The block with the absolute addresses it holds made relative, so two blocks compare. + public byte[] Normalized() + { + var bytes = new byte[BlockBytes]; + Marshal.Copy((nint)_memory, bytes, 0, BlockBytes); + foreach (var field in new[] { 0x10, 0x18 }) + { + var value = BitConverter.ToUInt64(bytes, field) - (ulong)_memory; + BitConverter.TryWriteBytes(bytes.AsSpan(field), value); + } + + return bytes; + } + + public void Dispose() => NativeMemory.AlignedFree(_memory); + } + + private sealed class RawMemory(Block block) : ICpuMemory + { + public bool TryRead(ulong virtualAddress, Span destination) + { + var offset = virtualAddress - (ulong)block.Pointer; + if (offset > BlockBytes || (ulong)destination.Length > BlockBytes - offset) + { + return false; + } + + new ReadOnlySpan(block.Pointer + offset, destination.Length).CopyTo(destination); + return true; + } + + public bool TryWrite(ulong virtualAddress, ReadOnlySpan source) + { + var offset = virtualAddress - (ulong)block.Pointer; + if (offset > BlockBytes || (ulong)source.Length > BlockBytes - offset) + { + return false; + } + + source.CopyTo(new Span(block.Pointer + offset, source.Length)); + return true; + } + } + + private sealed class StubHarness : IDisposable + { + private readonly byte* _page; + private readonly delegate* unmanaged _entry; + + public StubHarness(AgcPacketStubSpec spec) + { + _page = (byte*)HostMemory.Alloc(null, 4096, HostMemory.MEM_COMMIT | HostMemory.MEM_RESERVE, HostMemory.PAGE_EXECUTE_READWRITE); + Assert.True(_page != null); + // Fallback trampoline at +0x100: mov rax, sentinel / ret. + _page[0x100] = 0x48; + _page[0x101] = 0xB8; + *(ulong*)(_page + 0x102) = Sentinel; + _page[0x10A] = 0xC3; + var stub = DirectExecutionBackend.EmitGuestAgcPacketStub(spec, (nint)(_page + 0x100)); + Assert.True(stub.Count < 0x400); + for (var i = 0; i < stub.Count; i++) + { + _page[0x200 + i] = stub[i]; + } + + Assert.True(HostMemory.Protect(_page, 4096, HostMemory.PAGE_EXECUTE_READ, out _)); + HostMemory.FlushInstructionCache(_page, 4096); + _entry = (delegate* unmanaged)(_page + 0x200); + } + + public ulong Call(ulong buffer, ulong rsi, ulong rdx, ulong rcx, ulong r8) => _entry(buffer, rsi, rdx, rcx, r8); + + public void Dispose() => HostMemory.Free(_page, 0, HostMemory.MEM_RELEASE); + } +} From b50c83c38a8a5027787ade79c08aea7c8597cc8c Mon Sep 17 00:00:00 2001 From: kuba Date: Wed, 28 Oct 2026 16:44:45 +0100 Subject: [PATCH 55/82] [CPU] Stub sceAgcCbSetShRegisterRangeDirect and sceAgcCbSetShRegistersDirect in the guest too These are the two per-draw user-data builders left after the fixed-shape packet stubs: RangeDirect (offset, values pointer, count) and RegistersDirect ((offset, value) pairs). Each gets a guest-side x86-64 stub that copies the payload into the command buffer with plain stores and advances the cursor, on the same terms as the fixed-shape stubs (decline to the import trampoline with the arguments intact). RangeDirect: offset 1..0x3FF, 1..64 values, non-null source that does not overlap the packet, room for marker NOP + SET_SH_REG + values + the buffer's reserve; returns the SET_SH_REG packet's address exactly as the export's in-place path does. RegistersDirect: 1..32 pairs whose offsets are consecutive ascending, where the export's sort is a no-op and it emits one packet. The pairs are verified before anything is written, so a decline leaves the buffer untouched; anything else (unsorted, runs, large) is the export's. AgcPacketStubTests runs both as machine code against their exports over randomized counts, offsets, room, reserves, overlapping and null sources, and checks bytes, cursor and return value, and that a decline never touches the buffer. SHARPEMU_AGC_PACKET_STUBS=0 turns them off. --- .../Native/DirectExecutionBackend.FastPath.cs | 194 +++++++++++++++++- .../Agc/AgcExports.PacketStubs.cs | 42 ++++ .../Cpu/AgcPacketStubTests.cs | 169 ++++++++++++++- 3 files changed, 400 insertions(+), 5 deletions(-) diff --git a/src/SharpEmu.Core/Cpu/Native/DirectExecutionBackend.FastPath.cs b/src/SharpEmu.Core/Cpu/Native/DirectExecutionBackend.FastPath.cs index 7e76224f2..49dee9220 100644 --- a/src/SharpEmu.Core/Cpu/Native/DirectExecutionBackend.FastPath.cs +++ b/src/SharpEmu.Core/Cpu/Native/DirectExecutionBackend.FastPath.cs @@ -237,7 +237,9 @@ private void ConfigureGuestFastPath() KernelGetProcessTimeNid => GuestFastPathStub.ProcessTimeMicroseconds, _ => IsConstantZeroImport(nid) ? GuestFastPathStub.ConstantZero - : SharpEmu.Libs.Agc.AgcExports.TryGetPacketStubSpec(nid, out _) ? GuestFastPathStub.AgcPacket : GuestFastPathStub.None, + : SharpEmu.Libs.Agc.AgcExports.TryGetPacketStubSpec(nid, out _) || SharpEmu.Libs.Agc.AgcExports.TryGetVariablePacketStub(nid, out _) + ? GuestFastPathStub.AgcPacket + : GuestFastPathStub.None, }; /// @@ -290,7 +292,7 @@ private unsafe bool TryCreateGuestFastPathStub(string nid, nint fallbackTrampoli !_nativeConstantThunksAllowed || GuestWriteWatch.Armed || SharpEmu.Libs.Agc.AgcExports.PacketStubsBlockedByTracing || - !SharpEmu.Libs.Agc.AgcExports.TryGetPacketStubSpec(nid, out _)) + !(SharpEmu.Libs.Agc.AgcExports.TryGetPacketStubSpec(nid, out _) || SharpEmu.Libs.Agc.AgcExports.TryGetVariablePacketStub(nid, out _))) { return false; } @@ -303,8 +305,9 @@ private unsafe bool TryCreateGuestFastPathStub(string nid, nint fallbackTrampoli List code; if (kind == GuestFastPathStub.AgcPacket) { - SharpEmu.Libs.Agc.AgcExports.TryGetPacketStubSpec(nid, out var packetSpec); - code = EmitGuestAgcPacketStub(packetSpec, fallbackTrampoline); + code = SharpEmu.Libs.Agc.AgcExports.TryGetVariablePacketStub(nid, out var variableKind) + ? EmitGuestAgcVariablePacketStub(variableKind, fallbackTrampoline) + : EmitGuestAgcPacketStub(SharpEmu.Libs.Agc.AgcExports.TryGetPacketStubSpec(nid, out var packetSpec) ? packetSpec : throw new InvalidOperationException(nid), fallbackTrampoline); } else { @@ -758,6 +761,189 @@ void EmitPart(SharpEmu.Libs.Agc.AgcPacketStubPart part, bool intoR11) return code; } + /// + /// The two variable-length register-set builders as guest-side code, on the same terms as + /// (plain stores, decline to the trampoline with the arguments + /// intact; only RAX, R8-R11 and the flags change, all caller-saved). Both decline for a null + /// pointer, a count outside the stub's range, a buffer without room for the packet plus its + /// reserve, and a source that overlaps the packet being written; the export handles the rest. + /// + /// ShRegisterRange(cb, offset, values, count): offset 1..0x3FF, count 1..64; writes the marker + /// NOP, the SET_SH_REG header, the offset and the values, and returns the SET_SH_REG packet's address. + /// ShRegisters(cb, pairs, count): count 1..32 and the pairs' offsets consecutive ascending (the + /// export's sort is then a no-op and it emits exactly one packet); verified before anything is + /// written, so a decline leaves the buffer untouched. + /// + /// + internal static List EmitGuestAgcVariablePacketStub(SharpEmu.Libs.Agc.AgcVariablePacketStub kind, nint fallbackTrampoline) + { + const int CursorUpOffset = 0x10; + const int CursorDownOffset = 0x18; + const int ReservedDwordsOffset = 0x30; + var range = kind == SharpEmu.Libs.Agc.AgcVariablePacketStub.ShRegisterRange; + var code = new List(320); + var slowPathFixups = new List(); + void Emit(params byte[] bytes) => code.AddRange(bytes); + void EmitImm32(uint value) => code.AddRange(BitConverter.GetBytes(value)); + void EmitBranchToSlowPath(byte shortOpcode) + { + Emit(0x0F, (byte)(shortOpcode + 0x10), 0, 0, 0, 0); + slowPathFixups.Add(code.Count - 4); + } + + // A backwards short jump (opcode jb 0x72) to a code offset. + void EmitLoopBack(int target) + { + var delta = target - (code.Count + 2); + if (delta < -128) + { + throw new InvalidOperationException("Stub loop too long."); + } + + Emit(0x72, unchecked((byte)(sbyte)delta)); + } + + // Arguments: rdi = buffer, then (range) rsi = offset, rdx = values, rcx = count + // or (registers) rsi = pairs, rdx = count. + Emit(0x48, 0x85, 0xFF); // test rdi, rdi + EmitBranchToSlowPath(0x74); // jz slow + if (range) + { + Emit(0x8D, 0x46, 0xFF); // lea eax, [rsi-1] + Emit(0x3D); EmitImm32(0x3FF); // cmp eax, 0x3FF + EmitBranchToSlowPath(0x73); // jae slow: offset outside 1..0x3FF + Emit(0x8D, 0x41, 0xFF); // lea eax, [rcx-1] + Emit(0x83, 0xF8, (byte)(SharpEmu.Libs.Agc.AgcExports.RangeStubMaxValues - 1)); // cmp eax, max-1 + EmitBranchToSlowPath(0x77); // ja slow: count 0 or too large + Emit(0x48, 0x85, 0xD2); // test rdx, rdx + EmitBranchToSlowPath(0x74); // jz slow: a null source reserves the payload for the caller + } + else + { + Emit(0x48, 0x85, 0xF6); // test rsi, rsi + EmitBranchToSlowPath(0x74); // jz slow + Emit(0x8D, 0x42, 0xFF); // lea eax, [rdx-1] + Emit(0x83, 0xF8, (byte)(SharpEmu.Libs.Agc.AgcExports.RegistersStubMaxRegisters - 1)); // cmp eax, max-1 + EmitBranchToSlowPath(0x77); // ja slow + } + + Emit(0x48, 0x8B, 0x47, CursorUpOffset); // mov rax, [rdi+0x10] cursor + Emit(0x4C, 0x8B, 0x57, CursorDownOffset); // mov r10, [rdi+0x18] end + Emit(0x49, 0x29, 0xC2); // sub r10, rax + EmitBranchToSlowPath(0x72); // jb slow + Emit(0x49, 0xC1, 0xEA, 0x02); // shr r10, 2 available dwords + Emit(0x41, 0xBB); EmitImm32(0xFFFF_FFFFu); // mov r11d, 0xFFFFFFFF + Emit(0x4D, 0x39, 0xDA); // cmp r10, r11 + EmitBranchToSlowPath(0x77); // ja slow + Emit(0x44, 0x8B, 0x5F, ReservedDwordsOffset); // mov r11d, [rdi+0x30] reserve + if (range) + { + Emit(0x44, 0x8D, 0x49, 0x04); // lea r9d, [rcx+4] dwords: marker 2 + header 2 + values + } + else + { + Emit(0x44, 0x8D, 0x4A, 0x02); // lea r9d, [rdx+2] dwords: header 2 + values + } + + Emit(0x4D, 0x01, 0xCB); // add r11, r9 + Emit(0x4D, 0x39, 0xDA); // cmp r10, r11 + EmitBranchToSlowPath(0x72); // jb slow: no room + + // The source must not overlap the packet area [rax, rax + r9*4). + Emit(0x4E, 0x8D, 0x04, 0x88); // lea r8, [rax + r9*4] end of the packet + if (range) + { + Emit(0x4C, 0x39, 0xC2); // cmp rdx, r8 + Emit(0x73, 0x0D); // jae ok source starts at or after the packet's end + Emit(0x4C, 0x8D, 0x04, 0x8A); // lea r8, [rdx + rcx*4] end of the source + } + else + { + Emit(0x4C, 0x39, 0xC6); // cmp rsi, r8 + Emit(0x73, 0x0D); // jae ok + Emit(0x4C, 0x8D, 0x04, 0xD6); // lea r8, [rsi + rdx*8] end of the pairs + } + + Emit(0x49, 0x39, 0xC0); // cmp r8, rax + EmitBranchToSlowPath(0x77); // ja slow: the source ends after the packet starts + // ok: + if (!range) + { + Emit(0x44, 0x8B, 0x06); // mov r8d, [rsi] first offset + Emit(0x45, 0x89, 0xC2); // mov r10d, r8d + Emit(0x41, 0x01, 0xD2); // add r10d, edx + EmitBranchToSlowPath(0x72); // jc slow: the offsets would wrap + Emit(0x45, 0x31, 0xC9); // xor r9d, r9d + var verify = code.Count; + Emit(0x46, 0x8B, 0x14, 0xCE); // mov r10d, [rsi + r9*8] + Emit(0x45, 0x89, 0xC3); // mov r11d, r8d + Emit(0x45, 0x01, 0xCB); // add r11d, r9d + Emit(0x45, 0x39, 0xDA); // cmp r10d, r11d + EmitBranchToSlowPath(0x75); // jne slow: not consecutive ascending + Emit(0x41, 0xFF, 0xC1); // inc r9d + Emit(0x41, 0x39, 0xD1); // cmp r9d, edx + EmitLoopBack(verify); // jb verify + } + + if (range) + { + Emit(0xC7, 0x00); EmitImm32(SharpEmu.Libs.Agc.AgcExports.RangePacketNopHeader); // mov dword [rax], nop header + Emit(0xC7, 0x40, 0x04); EmitImm32(SharpEmu.Libs.Agc.AgcExports.RangePacketMarker); // mov dword [rax+4], marker + Emit(0x41, 0x89, 0xCA); // mov r10d, ecx + Emit(0x41, 0xC1, 0xE2, 0x10); // shl r10d, 16 + Emit(0x41, 0x81, 0xCA); EmitImm32(SharpEmu.Libs.Agc.AgcExports.SetShRegHeaderWithoutCount); // or r10d, header + Emit(0x44, 0x89, 0x50, 0x08); // mov [rax+8], r10d + Emit(0x89, 0x70, 0x0C); // mov [rax+12], esi the register offset + Emit(0x45, 0x31, 0xC0); // xor r8d, r8d + var copy = code.Count; + Emit(0x46, 0x8B, 0x14, 0x82); // mov r10d, [rdx + r8*4] + Emit(0x46, 0x89, 0x54, 0x80, 0x10); // mov [rax + r8*4 + 16], r10d + Emit(0x41, 0xFF, 0xC0); // inc r8d + Emit(0x41, 0x39, 0xC8); // cmp r8d, ecx + EmitLoopBack(copy); // jb copy + Emit(0x44, 0x8D, 0x49, 0x04); // lea r9d, [rcx+4] + } + else + { + Emit(0x41, 0x89, 0xD2); // mov r10d, edx + Emit(0x41, 0xC1, 0xE2, 0x10); // shl r10d, 16 + Emit(0x41, 0x81, 0xCA); EmitImm32(SharpEmu.Libs.Agc.AgcExports.SetShRegHeaderWithoutCount); // or r10d, header + Emit(0x44, 0x89, 0x10); // mov [rax], r10d + Emit(0x45, 0x0F, 0xB7, 0xD0); // movzx r10d, r8w first offset & 0xFFFF + Emit(0x44, 0x89, 0x50, 0x04); // mov [rax+4], r10d + Emit(0x45, 0x31, 0xC9); // xor r9d, r9d + var copy = code.Count; + Emit(0x46, 0x8B, 0x54, 0xCE, 0x04); // mov r10d, [rsi + r9*8 + 4] + Emit(0x46, 0x89, 0x54, 0x88, 0x08); // mov [rax + r9*4 + 8], r10d + Emit(0x41, 0xFF, 0xC1); // inc r9d + Emit(0x41, 0x39, 0xD1); // cmp r9d, edx + EmitLoopBack(copy); // jb copy + Emit(0x44, 0x8D, 0x4A, 0x02); // lea r9d, [rdx+2] + } + + Emit(0x4E, 0x8D, 0x1C, 0x88); // lea r11, [rax + r9*4] the new cursor + Emit(0x4C, 0x89, 0x5F, CursorUpOffset); // mov [rdi+0x10], r11 + if (range) + { + Emit(0x48, 0x83, 0xC0, 0x08); // add rax, 8 the SET_SH_REG packet, after the marker + } + + Emit(0xC3); + + foreach (var fixup in slowPathFixups) + { + var delta = BitConverter.GetBytes(code.Count - (fixup + 4)); + for (var i = 0; i < 4; i++) + { + code[fixup + i] = delta[i]; + } + } + + Emit(0xFF, 0x25, 0x00, 0x00, 0x00, 0x00); + code.AddRange(BitConverter.GetBytes((long)fallbackTrampoline)); + return code; + } + /// /// memcpy/memmove(dst, src, n) as a forward 32-byte SSE loop (16-byte step, then /// rep movsb for the last 0-15 bytes), returning dst. Rosetta translates diff --git a/src/SharpEmu.Libs/Agc/AgcExports.PacketStubs.cs b/src/SharpEmu.Libs/Agc/AgcExports.PacketStubs.cs index 1cf3528f1..b908e2a70 100644 --- a/src/SharpEmu.Libs/Agc/AgcExports.PacketStubs.cs +++ b/src/SharpEmu.Libs/Agc/AgcExports.PacketStubs.cs @@ -114,4 +114,46 @@ private static AgcPacketStubWord DrawInitiatorOfRcx() => /// The packet stub for a builder import, when it has one. public static bool TryGetPacketStubSpec(string nid, out AgcPacketStubSpec spec) => PacketStubSpecs.TryGetValue(nid, out spec!); + + /// The marker NOP header and payload sceAgcCbSetShRegisterRangeDirect puts in front of its packet. + public static readonly uint RangePacketNopHeader = Pm4(2, ItNop, RZero); + + public const uint RangePacketMarker = CbSetShRegisterRangeMarker; + + /// The SET_SH_REG header of a zero-length payload; the stub ORs the value count shifted left by 16. + public static readonly uint SetShRegHeaderWithoutCount = Pm4(2, ItSetShReg, 0); + + /// The most values a variable-length packet stub copies; a larger packet is the export's. + public const int RangeStubMaxValues = 64; + + public const int RegistersStubMaxRegisters = 32; + + private const string CbSetShRegisterRangeDirectNid = "n2fD4A+pb+g"; + private const string CbSetShRegistersDirectNid = "UZbQjYAwwXM"; + + /// The variable-length packet stub for a builder import, when it has one. + public static bool TryGetVariablePacketStub(string nid, out AgcVariablePacketStub kind) + { + switch (nid) + { + case CbSetShRegisterRangeDirectNid: + kind = AgcVariablePacketStub.ShRegisterRange; + return true; + case CbSetShRegistersDirectNid: + kind = AgcVariablePacketStub.ShRegisters; + return true; + default: + kind = default; + return false; + } + } +} + +public enum AgcVariablePacketStub +{ + /// sceAgcCbSetShRegisterRangeDirect(cb, offset, values, count): marker NOP + SET_SH_REG with the values copied in. + ShRegisterRange, + + /// sceAgcCbSetShRegistersDirect(cb, (offset, value) pairs, count) for one run of consecutive ascending offsets. + ShRegisters, } diff --git a/tests/SharpEmu.Libs.Tests/Cpu/AgcPacketStubTests.cs b/tests/SharpEmu.Libs.Tests/Cpu/AgcPacketStubTests.cs index 2719e6e33..72d15cb39 100644 --- a/tests/SharpEmu.Libs.Tests/Cpu/AgcPacketStubTests.cs +++ b/tests/SharpEmu.Libs.Tests/Cpu/AgcPacketStubTests.cs @@ -123,6 +123,155 @@ public void TheStubWritesWhatTheExportWritesAndDeclinesWhereTheExportWouldCallBa Assert.True(stubRan > 100, $"{nid}: the stub took the fast path only {stubRan} times"); } + private const int ValuesOffset = 0xC00; + + public static IEnumerable VariableNids() => + new[] { "n2fD4A+pb+g", "UZbQjYAwwXM" }.Select(nid => new object[] { nid }); + + [Theory] + [MemberData(nameof(VariableNids))] + public void TheVariableLengthStubWritesWhatTheExportWritesAndLeavesTheBufferAloneWhenItDeclines(string nid) + { + if (!OperatingSystem.IsMacOS() || RuntimeInformation.ProcessArchitecture != Architecture.X64) + { + return; + } + + Assert.True(AgcExports.TryGetVariablePacketStub(nid, out var kind)); + var isRange = kind == AgcVariablePacketStub.ShRegisterRange; + using var harness = new StubHarness(kind); + var manager = new ModuleManager(); + manager.RegisterExports(SharpEmu.Generated.SysAbiExportRegistry.CreateExports(Generation.Gen5)); + var random = new System.Random(unchecked(nid.GetHashCode() ^ 0x77A1)); + var stubRan = 0; + var declinedWithRoom = 0; + for (var iteration = 0; iteration < 1500; iteration++) + { + var reserved = (uint)random.Next(3) * 3; + var count = random.Next(12) switch + { + 0 => 0, + 1 => 1, + 2 => 65, + 3 => 64, + 4 => 33, + 5 => 32, + _ => random.Next(1, 20), + }; + var packetDwords = (isRange ? 4 : 2) + count; + var room = random.Next(6) switch + { + 0 => packetDwords * 4 + (int)reserved * 4, + 1 => packetDwords * 4 + (int)reserved * 4 - 4, + 2 => packetDwords * 4 + (int)reserved * 4 + 4, + 3 => 0x700, + 4 => 0, + _ => -64, + }; + var nullBuffer = random.Next(30) == 0; + + using var managedBlock = new Block(); + using var stubBlock = new Block(); + managedBlock.Reset(reserved, room, random.Next(1 << 30)); + + // The source: the block's tail (clear of the packet area) or, now and then, inside the packet area. + var valuesInside = random.Next(8) == 0; + var valuesNull = random.Next(40) == 0; + var sourceOffset = valuesInside ? PacketOffset + 4 * random.Next(0, 24) : ValuesOffset; + var firstOffset = random.Next(6) switch + { + 0 => 0u, + 1 => 0x3FFu, + 2 => 0x400u, + 3 => 0xFFFF_FFF0u, + _ => (uint)random.Next(1, 0x3F0), + }; + var consecutive = random.Next(4) != 0; + unsafe + { + var source = (uint*)(managedBlock.Pointer + ValuesOffset); + if (!isRange) + { + for (var index = 0; index < 64; index++) + { + source[index * 2] = consecutive ? firstOffset + (uint)index : (uint)random.Next(0, 0x400); + source[index * 2 + 1] = (uint)random.NextInt64(); + } + + if (consecutive && random.Next(5) == 0 && count > 1) + { + source[2 * random.Next(1, count)] += 1; + consecutive = false; + } + + // The stub leaves a run that would wrap past 32 bits to the export. + consecutive &= (ulong)firstOffset + (ulong)count < 0x1_0000_0000UL; + } + } + + stubBlock.CopyFrom(managedBlock); + managedBlock.Rebase(); + stubBlock.Rebase(); + var pristine = stubBlock.Normalized(); + + ulong Source(Block block) => valuesNull ? 0UL : (ulong)(block.Pointer + sourceOffset); + var offsetArgument = isRange ? (ulong)(random.Next(8) == 0 ? random.Next(0, 0x500) : random.Next(1, 0x3FF)) : 0UL; + var buffer = nullBuffer ? 0UL : (ulong)managedBlock.Pointer; + var context = new CpuContext(new RawMemory(managedBlock), Generation.Gen5); + context[CpuRegister.Rdi] = buffer; + if (isRange) + { + context[CpuRegister.Rsi] = offsetArgument; + context[CpuRegister.Rdx] = Source(managedBlock); + context[CpuRegister.Rcx] = (ulong)count; + } + else + { + context[CpuRegister.Rsi] = Source(managedBlock); + context[CpuRegister.Rdx] = (ulong)count; + } + + var previousScheduler = GuestThreadExecution.Scheduler; + GuestThreadExecution.Scheduler = null; + try + { + context.DeclineGuestCallbacks = true; + manager.Dispatch(nid, context); + context.DeclineGuestCallbacks = false; + } + finally + { + GuestThreadExecution.Scheduler = previousScheduler; + } + + var stubSource = valuesNull ? 0UL : (ulong)(stubBlock.Pointer + sourceOffset); + var stubRax = isRange + ? harness.Call(nullBuffer ? 0UL : (ulong)stubBlock.Pointer, offsetArgument, stubSource, (ulong)count, 0) + : harness.Call(nullBuffer ? 0UL : (ulong)stubBlock.Pointer, stubSource, (ulong)count, 0, 0); + if (stubRax == Sentinel) + { + // A decline never touches the buffer; the export then runs. + Assert.Equal(pristine, stubBlock.Normalized()); + if (!nullBuffer && room >= packetDwords * 4 + (int)reserved * 4 && count is >= 1 and <= 32 && !valuesNull && !valuesInside && + (isRange ? offsetArgument is >= 1 and <= 0x3FF : consecutive)) + { + declinedWithRoom++; + } + + continue; + } + + stubRan++; + Assert.False(context.GuestCallbackDeclined, $"{nid}: the stub built a packet where the export had to call back"); + Assert.False(nullBuffer, nid + ": the stub accepted a null buffer"); + Assert.Equal(context[CpuRegister.Rax] - (ulong)managedBlock.Pointer, stubRax - (ulong)stubBlock.Pointer); + Assert.Equal(managedBlock.Normalized(), stubBlock.Normalized()); + } + + Assert.True(stubRan > 150, $"{nid}: the stub took the fast path only {stubRan} times"); + Assert.True(declinedWithRoom < 40, $"{nid}: the stub declined {declinedWithRoom} calls the export builds as one in-place packet"); + } + [Fact] public void EveryStubbedBuilderIsAlsoALeafBuilder() { @@ -131,6 +280,12 @@ public void EveryStubbedBuilderIsAlsoALeafBuilder() Assert.True(DirectExecutionBackend.IsAgcBuilderLeafImport(nid), nid); Assert.Equal(DirectExecutionBackend.GuestFastPathStub.AgcPacket, DirectExecutionBackend.ClassifyGuestFastPathStub(nid)); } + + Assert.True(DirectExecutionBackend.IsAgcBuilderLeafImport("n2fD4A+pb+g")); + foreach (var nid in VariableNids().Select(row => (string)row[0])) + { + Assert.Equal(DirectExecutionBackend.GuestFastPathStub.AgcPacket, DirectExecutionBackend.ClassifyGuestFastPathStub(nid)); + } } /// A guest "command buffer" in native memory: the buffer structure at +0, the packet area at +0x100. @@ -225,6 +380,16 @@ private sealed class StubHarness : IDisposable private readonly delegate* unmanaged _entry; public StubHarness(AgcPacketStubSpec spec) + : this(spec, null) + { + } + + public StubHarness(AgcVariablePacketStub kind) + : this(null, kind) + { + } + + private StubHarness(AgcPacketStubSpec? spec, AgcVariablePacketStub? kind) { _page = (byte*)HostMemory.Alloc(null, 4096, HostMemory.MEM_COMMIT | HostMemory.MEM_RESERVE, HostMemory.PAGE_EXECUTE_READWRITE); Assert.True(_page != null); @@ -233,7 +398,9 @@ public StubHarness(AgcPacketStubSpec spec) _page[0x101] = 0xB8; *(ulong*)(_page + 0x102) = Sentinel; _page[0x10A] = 0xC3; - var stub = DirectExecutionBackend.EmitGuestAgcPacketStub(spec, (nint)(_page + 0x100)); + var stub = spec is not null + ? DirectExecutionBackend.EmitGuestAgcPacketStub(spec, (nint)(_page + 0x100)) + : DirectExecutionBackend.EmitGuestAgcVariablePacketStub(kind!.Value, (nint)(_page + 0x100)); Assert.True(stub.Count < 0x400); for (var i = 0; i < stub.Count; i++) { From 880bb8b9e6967df68508879cb9d46f8f0aa4a040 Mon Sep 17 00:00:00 2001 From: kuba Date: Wed, 28 Oct 2026 17:38:04 +0100 Subject: [PATCH 56/82] [GPU] Build a render encoder's debug label only when encoder labels are on BeginRendering formatted "render WxH" for every pass and LabelEncoder then dropped it unless SHARPEMU encoder status labels were enabled: about 0.25% of the Silent Hill GPU worker's samples. The string is now built only when it is used. --- .../SharpEmu.Gpu.Metal/V2/MetalRenderHost.Rendering.cs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.Rendering.cs b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.Rendering.cs index 443d4fed7..3666aea10 100644 --- a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.Rendering.cs +++ b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.Rendering.cs @@ -1140,7 +1140,10 @@ public void BeginRendering(in RenderingState state) OcclusionAuditAttach(descriptor); _renderEncoder = RetainEncoder(MetalNative.Send(commandBuffer, SelRenderEncoder, descriptor)); MetalMemoryTelemetry.RenderEncoders.Add(); - LabelEncoder(_renderEncoder, $"render {state.Width}x{state.Height}"); + if (EncoderStatusEnabled) + { + LabelEncoder(_renderEncoder, $"render {state.Width}x{state.Height}"); + } if (_renderEncoder == 0) { throw Fatal( From 5778913a7d45061ed48dfce17233a383343e4f07 Mon Sep 17 00:00:00 2001 From: kuba Date: Wed, 28 Oct 2026 18:03:11 +0100 Subject: [PATCH 57/82] [GPU] A prefetched draw that is already resolved takes no clock reads. TryTakeGraphics read the clock before and after every take to account the render thread's wait, though a completed job (nearly every take of a well-prefetched burst) waits for nothing. Only a job still queued or running reads the clock now; the wait and caller-resolve counters are unchanged. --- .../Gpu/Rendering/ParallelProgramPrefetch.cs | 20 +++++++++------- .../ParallelProgramPrefetchCallerTests.cs | 24 +++++++++++++++++++ 2 files changed, 36 insertions(+), 8 deletions(-) diff --git a/src/SharpEmu.Libs/Gpu/Rendering/ParallelProgramPrefetch.cs b/src/SharpEmu.Libs/Gpu/Rendering/ParallelProgramPrefetch.cs index 9ecca5c9a..108bcf8a8 100644 --- a/src/SharpEmu.Libs/Gpu/Rendering/ParallelProgramPrefetch.cs +++ b/src/SharpEmu.Libs/Gpu/Rendering/ParallelProgramPrefetch.cs @@ -208,16 +208,20 @@ public bool TryTakeGraphics(RegisterBanks banks, in GraphicsProgramInputs inputs } Current = null; - var waitStarted = System.Diagnostics.Stopwatch.GetTimestamp(); - // Caller work is reported separately, not disguised as a shorter resolver wait. - // Keep completed jobs and long queues out of the claim routine's lock/finally - // frame. Almost every take in a well-prefetched burst is already complete. - if (!_allowCallerResolution || job.State != 0 || Volatile.Read(ref _outstanding) > MaximumCallerOutstanding || - !TryResolveOnCaller(job)) + // A completed job waits for nothing, so it reads no clock: almost every take in a + // well-prefetched burst is one, and two clock reads per draw were measurable. + if (job.State == 0) { - WaitFor(job); - Interlocked.Add(ref _waitTicks, System.Diagnostics.Stopwatch.GetTimestamp() - waitStarted); + var waitStarted = System.Diagnostics.Stopwatch.GetTimestamp(); + // Caller work is reported separately, not disguised as a shorter resolver wait. + // Keep long queues out of the claim routine's lock/finally frame. + if (!_allowCallerResolution || Volatile.Read(ref _outstanding) > MaximumCallerOutstanding || !TryResolveOnCaller(job)) + { + WaitFor(job); + Interlocked.Add(ref _waitTicks, System.Diagnostics.Stopwatch.GetTimestamp() - waitStarted); + } } + using var profile = RenderPhaseProfile.MeasureDetail(RenderPhaseProfile.Phase.ProgramPrefetchValidation); if (job.State != 1) { diff --git a/tests/SharpEmu.Libs.Tests/Gpu/Rendering/ParallelProgramPrefetchCallerTests.cs b/tests/SharpEmu.Libs.Tests/Gpu/Rendering/ParallelProgramPrefetchCallerTests.cs index 8caa97842..9281824e2 100644 --- a/tests/SharpEmu.Libs.Tests/Gpu/Rendering/ParallelProgramPrefetchCallerTests.cs +++ b/tests/SharpEmu.Libs.Tests/Gpu/Rendering/ParallelProgramPrefetchCallerTests.cs @@ -275,4 +275,28 @@ public void CallerClaimsAreBoundedToShortQueues(int count, bool claimed) Assert.Equal(count, f.Concurrent.Begins); Assert.Equal(count, f.Concurrent.Releases); } + + [Fact] + public void TakingACompletedJobReportsNoWaitAndAnUnfinishedOneDoes() + { + using var f = new Fixture(false); + var banks = Banks(); + var done = f.Prefetch.Submit(banks); + f.Idle(); + Assert.True(f.Take(banks, done)); + Assert.Equal(0, f.Prefetch.TakeCounters().WaitMilliseconds); + Task waiter; + Monitor.Enter(f.QueueLock); + try + { + // The worker cannot start the job while this thread holds the queue lock, so the take waits for it. + var late = f.Prefetch.Submit(banks); + waiter = Task.Run(() => f.Take(banks, late)); + Assert.False(waiter.Wait(50)); + } + finally { Monitor.Exit(f.QueueLock); } + + Assert.True(waiter.Wait(TimeSpan.FromSeconds(10)) && waiter.Result); + Assert.True(f.Prefetch.TakeCounters().WaitMilliseconds > 0); + } } From 14aaf8165dc4a6699b26df524e8bc1c5013fa05f Mon Sep 17 00:00:00 2001 From: kuba Date: Wed, 28 Oct 2026 18:24:31 +0100 Subject: [PATCH 58/82] [GPU] Repeated storage-binding and command-read lookups skip the walks that found nothing last time. DeferredLabelCopiesTouch remembers, per guest range, how many held label copies a scan covered and resumes after them, since the copies are append-only until ClearHeldLabelCopies. MetalGpuWriteRanges.PendingSerial remembers 256-byte granules no live record or label overlaps, dropped when Note writes into them and when a fold merges records. Both are on by default; SHARPEMU_METAL_V2_LABEL_SCAN_MEMO=0 and SHARPEMU_METAL_V2_WRITE_CLEAN_MEMO=0 turn them off. --- .../V2/MetalCommandStream.cs | 59 +++++- .../V2/MetalGpuWriteRanges.cs | 112 +++++++++++ .../MetalEndOfPipeOrderTests.LabelScatter.cs | 96 ++++++++++ .../MetalGpuWriteRangesTests.cs | 181 ++++++++++++++++++ 4 files changed, 445 insertions(+), 3 deletions(-) diff --git a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalCommandStream.cs b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalCommandStream.cs index 0bed9595c..213c3cb6b 100644 --- a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalCommandStream.cs +++ b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalCommandStream.cs @@ -1352,6 +1352,26 @@ private readonly record struct DeferredLabelCopy( Environment.GetEnvironmentVariable("SHARPEMU_METAL_V2_LABEL_PAGE_FILTER") != "0"; private readonly MetalPageRangeFilter _deferredGuestPages = new(); private readonly MetalPageRangeFilter _deferredBufferPages = new(); + + // A guest range a scan found no held copy in, with how many copies it covered. Copies are only + // appended until ClearHeldLabelCopies and never change, so the same range asked again can only + // be touched by copies held since: its scan starts there. Draws bind the same storage ranges + // over and over while the held copies grow to the commit (Silent Hill: ~17000 held, a scan of + // thousands of them for one binding in fifteen). SHARPEMU_METAL_V2_LABEL_SCAN_MEMO=0 scans + // every held copy each time again. + private static readonly bool LabelScanMemoEnabled = + SharpEmu.ShaderCompiler.PerfSwitches.Enabled("SHARPEMU_METAL_V2_LABEL_SCAN_MEMO"); + private const int LabelScanMemoBits = 10; + private struct LabelScanMemo + { + public ulong Address; + public ulong Size; + public uint Epoch; + public int Scanned; + } + + private readonly LabelScanMemo[] _labelScanMemo = new LabelScanMemo[1 << LabelScanMemoBits]; + private uint _labelScanEpoch = 1; private long _labelLookupQueries, _labelHullRejects, _labelPageRejects, _labelExactScans, _labelEntriesInspected; private int _labelMaxHeldCopies; @@ -1407,6 +1427,7 @@ private void HoldLabelCopies(ReadOnlySpan copies) private void ClearHeldLabelCopies() { _deferredLabelCopies.Clear(); + _labelScanEpoch++; if (DeferredLabelFilterEnabled) { _deferredGuestPages.Clear(); @@ -1606,27 +1627,59 @@ private bool DeferredLabelCopiesTouch(ulong address, ulong size) if (SharpEmu.Libs.VideoOut.RenderPhaseProfile.Enabled) _labelHullRejects++; return false; } + + var held = _deferredLabelCopies.Count; + ref var memo = ref _labelScanMemo[ + LabelScanMemoEnabled + ? (int)(((address ^ (size * 0x9E3779B97F4A7C15UL)) * 0xFF51AFD7ED558CCDUL) >> (64 - LabelScanMemoBits)) + : 0]; + var from = 0; + if (LabelScanMemoEnabled && memo.Epoch == _labelScanEpoch && memo.Address == address && memo.Size == size) + { + from = memo.Scanned; + if (from == held) + { + return false; + } + } + if (DeferredLabelFilterEnabled && !_deferredGuestPages.MayOverlap(address, size)) { if (SharpEmu.Libs.VideoOut.RenderPhaseProfile.Enabled) _labelPageRejects++; + RememberLabelScan(ref memo, address, size, held); return false; } var copies = System.Runtime.InteropServices.CollectionsMarshal.AsSpan(_deferredLabelCopies); - for (var index = 0; index < copies.Length; index++) + for (var index = from; index < copies.Length; index++) { ref readonly var copy = ref copies[index]; if (copy.GuestAddress < address + size && address < copy.GuestAddress + (ulong)copy.Bytes) { - NoteLabelScan(index + 1); + NoteLabelScan(index + 1 - from); return true; } } - NoteLabelScan(copies.Length); + NoteLabelScan(copies.Length - from); + RememberLabelScan(ref memo, address, size, held); return false; } + // Only an answer of "no copy touches it" is kept: a touch lands the copies, which clears them. + private void RememberLabelScan(ref LabelScanMemo memo, ulong address, ulong size, int scanned) + { + if (!LabelScanMemoEnabled) + { + return; + } + + memo.Address = address; + memo.Size = size; + memo.Epoch = _labelScanEpoch; + memo.Scanned = scanned; + } + // Buffer identity is part of the page key: aliases with different guest addresses // must still land before a binding to the same native buffer bytes. private bool DeferredLabelCopiesTouch(in MetalGuestRange range) diff --git a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalGpuWriteRanges.cs b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalGpuWriteRanges.cs index 0a5993f08..a51273f48 100644 --- a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalGpuWriteRanges.cs +++ b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalGpuWriteRanges.cs @@ -155,6 +155,7 @@ public ulong Note( Version++; var sequence = ++_sequence; var end = address + size; + InvalidateClean(address, end); NoteNewest(address, end, serial); if (origin == MetalGpuWriteOrigin.EndOfPipeLabel) { @@ -550,6 +551,111 @@ internal ulong UnindexedPendingSerial(ulong address, ulong size, ulong completed return found; } + // 256-byte granules no live record or label overlaps, kept for the stream's reads: the + // command stream asks about every packet and register table it reads, thousands a frame, + // and the records' 2 MiB buckets and the label pages cannot tell a read next to a + // scattered label from one far from any, so the answer was a walk of them all (Silent Hill: + // the top self-time item of its GPU worker). Live records leave only by Retire and Clear, + // which can only make a granule cleaner, and arrive only through Note (its range's granules + // are dropped there) or merge in FoldOldest (everything is dropped), so a granule kept here + // has no overlapping record whatever serial the next query completes. The unbounded serial + // is not part of it: PendingSerial reads that for every query. Worker thread only, like Note. + // SHARPEMU_METAL_V2_WRITE_CLEAN_MEMO=0 walks the records for every query again. + private static readonly bool CleanMemoEnabled = + SharpEmu.ShaderCompiler.PerfSwitches.Enabled("SHARPEMU_METAL_V2_WRITE_CLEAN_MEMO"); + private const int CleanGranuleShift = 8; + private const int CleanSlotBits = 10; + // A query wider than this many granules is not looked up. + private const ulong MaxCleanGranules = 16; + private readonly ulong[] _cleanGranules = new ulong[1 << CleanSlotBits]; + + // The slot holds the granule plus one, so an empty one matches no granule. + private static int CleanSlot(ulong granule) => (int)((granule * 0x9E3779B97F4A7C15UL) >> (64 - CleanSlotBits)); + + private void InvalidateClean(ulong address, ulong end) + { + var first = address >> CleanGranuleShift; + var last = (end - 1) >> CleanGranuleShift; + if (last - first >= MaxCleanGranules) + { + Array.Clear(_cleanGranules); + return; + } + + for (var granule = first; granule <= last; granule++) + { + ref var slot = ref _cleanGranules[CleanSlot(granule)]; + if (slot == granule + 1) + { + slot = 0; + } + } + } + + // True when no live record or label overlaps [address, end): every granule it touches is + // known clean, or the one walk of the granule-aligned span finds nothing, which makes them so. + private bool TryAnswerCleanGranules(ulong address, ulong end) + { + var first = address >> CleanGranuleShift; + var last = (end - 1) >> CleanGranuleShift; + if (last - first >= MaxCleanGranules || last >= ulong.MaxValue >> CleanGranuleShift) + { + return false; + } + + var known = true; + for (var granule = first; granule <= last; granule++) + { + if (_cleanGranules[CleanSlot(granule)] != granule + 1) + { + known = false; + break; + } + } + + if (known) + { + return true; + } + + if (AnyLiveOverlap(first << CleanGranuleShift, (last + 1) << CleanGranuleShift)) + { + return false; + } + + for (var granule = first; granule <= last; granule++) + { + _cleanGranules[CleanSlot(granule)] = granule + 1; + } + + return true; + } + + private bool AnyLiveOverlap(ulong address, ulong end) + { + var records = RecordsMayCover(address, end) ? _count : 0; + for (var index = 0; index < records; index++) + { + ref var record = ref _records[index]; + if (record.End > address && record.Start < end) + { + return true; + } + } + + var labels = LabelPagesMayCover(address, end) ? _labelCount : 0; + for (var index = 0; index < labels; index++) + { + ref var label = ref _labels[index]; + if (label.End > address && label.Start < end) + { + return true; + } + } + + return false; + } + public bool MayCover(ulong address, ulong size) => _unboundedSerial != 0 || ((_count != 0 || _labelCount != 0) && size != 0 && address < _boundsEnd && _boundsStart < address + size); @@ -568,6 +674,10 @@ public ulong PendingSerial(ulong address, ulong size, ulong completed) { LastMatch = (0, ulong.MaxValue, pending, MetalGpuWriteOrigin.Unbounded); } + else if (CleanMemoEnabled && TryAnswerCleanGranules(address, end)) + { + return 0; + } var records = RecordsMayCover(address, end) ? _count : 0; @@ -1074,6 +1184,8 @@ private void FoldOldest() } _count = kept + tail; + // A merged record spans the gaps between the ones it took in. + Array.Clear(_cleanGranules); _recordBuckets.Clear(); _wideRecords.Clear(); Array.Clear(_bucketHashCounts); diff --git a/tests/SharpEmu.Gpu.Metal.Tests/MetalEndOfPipeOrderTests.LabelScatter.cs b/tests/SharpEmu.Gpu.Metal.Tests/MetalEndOfPipeOrderTests.LabelScatter.cs index ce8c6a88d..8977736ac 100644 --- a/tests/SharpEmu.Gpu.Metal.Tests/MetalEndOfPipeOrderTests.LabelScatter.cs +++ b/tests/SharpEmu.Gpu.Metal.Tests/MetalEndOfPipeOrderTests.LabelScatter.cs @@ -84,6 +84,54 @@ public void AHeldBatchWithAnUnalignedCopyLandsThroughBlits() Assert.Equal(0x332211UL, rig.Backing.Read64(LaterLabel + 0x400) & 0xFF_FFFF); } + // A binding that no held copy touched is not scanned again until another copy is held, and + // the copy held afterwards is found: the second answer starts where the first left off. + [Fact] + public void AnUntouchedRangeIsFoundTouchedOnceACopyOverItIsHeld() + { + if (!RequireDevice()) return; + using var rig = new Rig(Device.Value); + var host = rig.Host; + + _ = host.EnsureComputeEncoder(); + for (var index = 0; index < 64; index++) + { + host.RecordCommandProcessorWrite(LaterLabel + (ulong)index * 16, BitConverter.GetBytes(0x3000UL + (ulong)index), "WRITE_DATA"); + } + + // Between two held qwords, and inside the span and the pages the held copies cover. + var between = LaterLabel + 3 * 16 + 8; + host.OrderAfterDeferredCopies(between, 8); + host.OrderAfterDeferredCopies(between, 8); + Assert.Equal(64, host.DeferredLabelCopyCount); + + host.RecordCommandProcessorWrite(LaterLabel + 70 * 16, BitConverter.GetBytes(0x4000UL), "WRITE_DATA"); + host.OrderAfterDeferredCopies(between, 8); + Assert.Equal(65, host.DeferredLabelCopyCount); + + host.RecordCommandProcessorWrite(between, BitConverter.GetBytes(0x5000UL), "WRITE_DATA"); + host.OrderAfterDeferredCopies(between, 8); + Assert.Equal(0, host.DeferredLabelCopyCount); + + // The copies landed: a new batch starts from nothing, and the old answer is not reused. + _ = host.EnsureComputeEncoder(); + for (var index = 0; index < 8; index++) + { + host.RecordCommandProcessorWrite(LaterLabel + (ulong)index * 32, BitConverter.GetBytes(0x6000UL + (ulong)index), "WRITE_DATA"); + } + + host.OrderAfterDeferredCopies(between, 8); + Assert.Equal(8, host.DeferredLabelCopyCount); + host.RecordCommandProcessorWrite(between, BitConverter.GetBytes(0x7000UL), "WRITE_DATA"); + host.OrderAfterDeferredCopies(between, 8); + Assert.Equal(0, host.DeferredLabelCopyCount); + + host.FlushCommandBuffer(); + WaitForGpu(host, host.LastWorkSerial); + WaitForCompletions(host); + Assert.Equal(0x7000UL, rig.Backing.Read64(between)); + } + // SHARPEMU_LABEL_SCATTER_BENCH_OUTPUT=: the worker's cost of landing a held batch, and the // GPU's time to run it, through blits and through the scatter, alternating per round. [Fact] @@ -132,4 +180,52 @@ public void MeasureLabelScatterWhenRequested() architecture = System.Runtime.InteropServices.RuntimeInformation.ProcessArchitecture.ToString(), rounds }, new JsonSerializerOptions { WriteIndented = true })); } + // SHARPEMU_LABEL_SCAN_BENCH_OUTPUT=: nanoseconds per binding lookup against held copies, for + // ranges that sit between them (so no filter rejects them and each lookup scans), with a copy + // held now and then. Run once per setting of SHARPEMU_METAL_V2_LABEL_SCAN_MEMO. + [Fact] + public void MeasureLabelScanWhenRequested() + { + var path = Environment.GetEnvironmentVariable("SHARPEMU_LABEL_SCAN_BENCH_OUTPUT"); + if (path is null || !RequireDevice()) return; + using var rig = new Rig(Device.Value); + var host = rig.Host; + var rounds = new List(); + var value = new byte[8]; + const ulong benchBase = GuestBase + 0x10000; + foreach (var held in new[] { 256, 2048 }) + { + for (var round = 0; round < 14; round++) + { + _ = host.EnsureComputeEncoder(); + for (var index = 0; index < held; index++) + { + host.RecordCommandProcessorWrite(benchBase + (ulong)index * 16, value, "WRITE_DATA"); + } + + const int queries = 200_000; + var started = Stopwatch.GetTimestamp(); + for (var query = 0; query < queries; query++) + { + // Sixty-four distinct 8-byte gaps between held qwords, asked in turn. + host.OrderAfterDeferredCopies(benchBase + (ulong)(query % 64) * 16 * (ulong)(held / 64) + 8, 8); + if (query % 4096 == 4095) + { + host.RecordCommandProcessorWrite(benchBase + 0x8000 + (ulong)(query / 4096) * 16, value, "WRITE_DATA"); + } + } + + var elapsed = Stopwatch.GetTimestamp() - started; + Assert.True(host.DeferredLabelCopyCount >= held); + host.OrderAfterDeferredCopies(0, ulong.MaxValue); + if (round < 8) continue; + rounds.Add(new { held, round, nsPerQuery = elapsed * 1e9 / Stopwatch.Frequency / queries }); + } + } + + File.WriteAllText(path, JsonSerializer.Serialize(new { + architecture = System.Runtime.InteropServices.RuntimeInformation.ProcessArchitecture.ToString(), + scanMemo = Environment.GetEnvironmentVariable("SHARPEMU_METAL_V2_LABEL_SCAN_MEMO") != "0", rounds }, + new JsonSerializerOptions { WriteIndented = true })); + } } diff --git a/tests/SharpEmu.Gpu.Metal.Tests/MetalGpuWriteRangesTests.cs b/tests/SharpEmu.Gpu.Metal.Tests/MetalGpuWriteRangesTests.cs index e778553ef..8601645f9 100644 --- a/tests/SharpEmu.Gpu.Metal.Tests/MetalGpuWriteRangesTests.cs +++ b/tests/SharpEmu.Gpu.Metal.Tests/MetalGpuWriteRangesTests.cs @@ -1,5 +1,7 @@ // Copyright (C) 2026 SharpEmu Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later +using System.Diagnostics; +using System.Text.Json; using SharpEmu.Gpu.Metal.V2; using Xunit; @@ -493,4 +495,183 @@ public void IndexedQueriesAgreeWithTheUnindexedWalk(int seed) } } } + // A range a query found nothing live over answers the next query without a walk, until a write + // lands in the same 256-byte granule: the neighbours stay answered, the written one does not. + [Fact] + public void ACleanGranuleIsForgottenWhenAWriteLandsInIt() + { + var ranges = new MetalGpuWriteRanges(); + ranges.Note(0x10_0000, 0x40, serial: 4); + Assert.Equal(0UL, ranges.PendingSerial(0x10_0100, 0x10, completed: 0)); + Assert.Equal(0UL, ranges.PendingSerial(0x10_0100, 0x10, completed: 0)); + Assert.Equal(0UL, ranges.PendingSerial(0x10_0200, 0x10, completed: 0)); + + ranges.Note(0x10_0180, 8, serial: 6, MetalGpuWriteOrigin.EndOfPipeLabel); + Assert.Equal(6UL, ranges.PendingSerial(0x10_0100, 0x100, completed: 0)); + Assert.Equal(6UL, ranges.PendingSerial(0x10_0180, 4, completed: 0)); + Assert.Equal(0UL, ranges.PendingSerial(0x10_0200, 0x10, completed: 0)); + Assert.Equal(0UL, ranges.PendingSerial(0x10_0100, 0x10, completed: 0)); + + // A write wider than the lookups drops every granule, and a granule next to the record + // is not answered by the record that ends at its edge. + Assert.Equal(0UL, ranges.PendingSerial(0x10_0040, 0x10, completed: 0)); + ranges.Note(0x10_0000, 0x20_0000, serial: 9); + Assert.Equal(9UL, ranges.PendingSerial(0x10_0040, 0x10, completed: 0)); + Assert.Equal(9UL, ranges.PendingSerial(0x10_0200, 0x10, completed: 0)); + Assert.Equal(0UL, ranges.PendingSerial(0x30_0000, 0x10, completed: 0)); + } + + // The unbounded serial is read for every query, not remembered with the granules. + [Fact] + public void ACleanGranuleStillWaitsForAnUnboundedWrite() + { + var ranges = new MetalGpuWriteRanges(); + Assert.Equal(0UL, ranges.PendingSerial(0x10_0000, 0x10, completed: 0)); + Assert.Equal(0UL, ranges.PendingSerial(0x10_0000, 0x10, completed: 0)); + ranges.NoteUnbounded(5); + Assert.Equal(5UL, ranges.PendingSerial(0x10_0000, 0x10, completed: 0)); + Assert.Equal(0UL, ranges.PendingSerial(0x10_0000, 0x10, completed: 5)); + Assert.Equal(0UL, ranges.PendingSerial(0x10_0000, 0x10, completed: 5)); + } + + // Folding merges the oldest records over the gaps between them: granules answered clean + // before a fold may lie in such a gap. + [Fact] + public void AFoldDropsTheGranulesItsMergedRecordsNowCover() + { + var ranges = new MetalGpuWriteRanges(); + var gaps = new List(); + for (var index = 0; index < 511; index++) + { + var start = 0x100_0000UL + (ulong)index * 0x1_0000; + ranges.Note(start, 0x100, serial: 3 + (ulong)index / 64); + gaps.Add(start + 0x8000); + } + + foreach (var gap in gaps) + { + Assert.Equal(0UL, ranges.PendingSerial(gap, 0x10, completed: 0)); + } + + var folds = ranges.Folds; + for (var index = 0; index < 64 && ranges.Folds == folds; index++) + { + ranges.Note(0x900_0000UL + (ulong)index * 0x1_0000, 0x100, serial: 20); + } + + Assert.True(ranges.Folds > folds); + foreach (var gap in gaps) + { + Assert.Equal(ranges.UnindexedPendingSerial(gap, 0x10, 0), ranges.PendingSerial(gap, 0x10, completed: 0)); + } + } + + // Narrow, repeating queries over a small address space, so granules are answered, dropped by + // writes, retired and folded, and every answer is the walk of the records. + [Theory] + [InlineData(1)] + [InlineData(2)] + [InlineData(3)] + public void RepeatedNarrowQueriesAlwaysAgreeWithTheWalkOfEveryRecord(int seed) + { + var random = new Random(seed); + var ranges = new MetalGpuWriteRanges(); + ulong serial = 1; + ulong completed = 0; + for (var step = 0; step < 6000; step++) + { + switch (random.Next(12)) + { + case 0 when random.Next(8) == 0: + completed = Math.Min(serial, completed + (ulong)random.Next(1, 4)); + break; + case 1: + serial++; + break; + case 2 when random.Next(200) == 0: + ranges.NoteUnbounded(serial); + break; + case 3 when random.Next(100) == 0: + ranges.Clear(completed); + break; + case 4 or 5 or 6: + { + var origin = random.Next(3) == 0 ? MetalGpuWriteOrigin.EndOfPipeLabel : MetalGpuWriteOrigin.StorageBuffer; + var size = origin == MetalGpuWriteOrigin.EndOfPipeLabel ? 8UL : (ulong)random.Next(1, 0x3000); + ranges.Note(0x40_0000 + (ulong)random.Next(0, 0x80000), size, serial, origin); + break; + } + } + + for (var query = 0; query < 8; query++) + { + var address = 0x40_0000 + (ulong)random.Next(0, 0x80000); + var size = (ulong)(random.Next(6) == 0 ? random.Next(1, 0x1800) : random.Next(1, 0x20)); + var hinted = completed > 0 && random.Next(3) == 0 ? completed - 1 : completed; + var expected = ranges.UnindexedPendingSerial(address, size, hinted); + Assert.Equal(expected, ranges.PendingSerial(address, size, hinted)); + } + } + } + + // SHARPEMU_WRITE_RANGES_BENCH_OUTPUT=: nanoseconds per PendingSerial for the stream's + // sequential reads (eight 16-byte reads at the start of each page of command memory) over pages + // that hold labels and written storage ranges elsewhere on them, so the buckets and label + // pages cannot rule a read out; every tenth page has a record over its reads. A label is + // noted now and then. Run once per setting of SHARPEMU_METAL_V2_WRITE_CLEAN_MEMO. + [Fact] + public void MeasurePendingSerialWhenRequested() + { + var path = Environment.GetEnvironmentVariable("SHARPEMU_WRITE_RANGES_BENCH_OUTPUT"); + if (path is null) return; + var rounds = new List(); + foreach (var dirtyEvery in new[] { 0, 10 }) + { + foreach (var labels in new[] { 256, 2048 }) + { + for (var round = 0; round < 14; round++) + { + var random = new Random(round); + var ranges = new MetalGpuWriteRanges(); + const ulong area = 0x10_0000_0000; + for (var index = 0; index < 300; index++) + { + ranges.Note(area + ((ulong)random.Next(0, 0x4000) << 12) + 0x400, (ulong)random.Next(0x100, 0x800), serial: 5); + } + + for (var index = 0; index < labels; index++) + { + ranges.Note(area + ((ulong)random.Next(0, 0x4000) << 12) + 0x100 + (ulong)random.Next(0, 0xE00) / 8 * 8, 8, serial: 5, MetalGpuWriteOrigin.EndOfPipeLabel); + } + + for (var page = 0; dirtyEvery != 0 && page < 0x4000; page += dirtyEvery) + { + ranges.Note(area + ((ulong)page << 12), 0x80, serial: 5); + } + + const int queries = 400_000; + var found = 0UL; + var started = Stopwatch.GetTimestamp(); + for (var query = 0; query < queries; query++) + { + var address = area + ((ulong)(query / 8 % 0x4000) << 12) + (ulong)(query % 8) * 16; + found += ranges.PendingSerial(address, 16, completed: 1); + if (query % 512 == 0) + { + ranges.Note(area + ((ulong)random.Next(0, 0x4000) << 12) + 0x100 + (ulong)random.Next(0, 0xE00) / 8 * 8, 8, serial: 5, MetalGpuWriteOrigin.EndOfPipeLabel); + } + } + + var elapsed = Stopwatch.GetTimestamp() - started; + if (round < 8) continue; + rounds.Add(new { dirtyEvery, labels, round, nsPerQuery = elapsed * 1e9 / Stopwatch.Frequency / queries, nonzero = found }); + } + } + } + + File.WriteAllText(path, JsonSerializer.Serialize(new { + architecture = System.Runtime.InteropServices.RuntimeInformation.ProcessArchitecture.ToString(), + cleanMemo = Environment.GetEnvironmentVariable("SHARPEMU_METAL_V2_WRITE_CLEAN_MEMO") != "0", rounds }, + new JsonSerializerOptions { WriteIndented = true })); + } } From 7dd4296142fd9a3a5e958ce9528b855669d3c51f Mon Sep 17 00:00:00 2001 From: kuba Date: Wed, 28 Oct 2026 18:38:22 +0100 Subject: [PATCH 59/82] [GPU] An occlusion dump resolves, invalidates and notes its blocks as one run instead of once per block. A dump of 16 blocks resolved a window, forgot the overlays twice, invalidated images and noted a GPU write sixteen times, and left a write record per block. It now resolves one window over the run, forgets only each block's own bytes in the overlays (a pending store between the blocks still speaks for them), and invalidates and notes the run once. SHARPEMU_METAL_V2_COALESCE_OCCLUSION_DUMPS=0 restores the per-block path. --- .../V2/MetalCommandStream.cs | 44 +++++++++++++- .../V2/MetalRenderHost.Occlusion.cs | 58 ++++++++++++++----- .../SharpEmu.Gpu.Metal/V2/MetalRenderHost.cs | 9 +++ .../MetalEndOfPipeOrderTests.OcclusionDump.cs | 51 ++++++++++++++++ 4 files changed, 146 insertions(+), 16 deletions(-) create mode 100644 tests/SharpEmu.Gpu.Metal.Tests/MetalEndOfPipeOrderTests.OcclusionDump.cs diff --git a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalCommandStream.cs b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalCommandStream.cs index 213c3cb6b..8c12a2686 100644 --- a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalCommandStream.cs +++ b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalCommandStream.cs @@ -795,7 +795,8 @@ private ulong EncodeOrderedCopies( string? profileKind, ulong profileBytes, bool sourceIsFinal = true, - bool deferStableGpuResult = false) + bool deferStableGpuResult = false, + bool noteAsOneRange = false) { ulong serial; // A copy whose source later work in the same encoder may still change (the GDS a @@ -895,6 +896,47 @@ private ulong EncodeOrderedCopies( -1, guestAddress, bytes, landedValue, landedKind, deferred: true)); } + if (noteAsOneRange && copies.Length > 1) + { + // Copies the caller made into one small run (an occlusion dump's blocks): the image cache, + // the overlays and the write ranges each take the run once, which also keeps them from + // holding a record per block. The few bytes between the copies are noted with them: the + // note over them can only make a read wait or an image reload that did not need to. The + // two overlays are not: a pending store into the bytes between the copies still speaks for + // them, so each copy's own bytes are forgotten and nothing wider. + var low = ulong.MaxValue; + var high = 0UL; + using (SharpEmu.Libs.VideoOut.RenderPhaseProfile.MeasureDetail( + SharpEmu.Libs.VideoOut.RenderPhaseProfile.Phase.WriteNoteOverlays)) + { + foreach (ref readonly var copy in copies) + { + low = Math.Min(low, copy.GuestAddress); + high = Math.Max(high, copy.GuestAddress + (ulong)copy.Bytes); + _cpOverlay.Forget(copy.GuestAddress, (ulong)copy.Bytes); + _labelOverlay.Forget(copy.GuestAddress, (ulong)copy.Bytes); + } + } + + if (origin != MetalGpuWriteOrigin.EndOfPipeLabel) + { + _imageCache.InvalidateMemoryFromGpu(low, high - low); + } + else + { + _imageCache.ForgetStandingDccClaims(low, high - low); + } + + using (SharpEmu.Libs.VideoOut.RenderPhaseProfile.MeasureDetail( + SharpEmu.Libs.VideoOut.RenderPhaseProfile.Phase.CommandCpBookkeeping)) + { + NoteGuestMemoryWriteBeyondOverlays(low, high - low, origin); + } + + _pendingLabelWrites += (ulong)copies.Length; + return serial; + } + foreach (ref readonly var copy in copies) { if (origin != MetalGpuWriteOrigin.EndOfPipeLabel) diff --git a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.Occlusion.cs b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.Occlusion.cs index 1b66e89da..1bc1da6ec 100644 --- a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.Occlusion.cs +++ b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.Occlusion.cs @@ -13,6 +13,13 @@ internal sealed partial class MetalRenderHost internal bool NativeOcclusionEnabled { get; set; } = Environment.GetEnvironmentVariable("SHARPEMU_METAL_V2_NATIVE_OCCLUSION") != "0"; + // SHARPEMU_METAL_V2_COALESCE_OCCLUSION_DUMPS=0 resolves and notes every block of a dump on its own again. + internal bool CoalesceOcclusionDumps { get; set; } = + SharpEmu.ShaderCompiler.PerfSwitches.Enabled("SHARPEMU_METAL_V2_COALESCE_OCCLUSION_DUMPS"); + + // A dump covers at most 24 blocks of at most 32 bytes; past this the blocks are not one result buffer. + private const ulong MaxCoalescedDumpBytes = 1024; + private const int OcclusionIntervalCapacity = 4096; internal int NativeOcclusionBatchBudget { get; set; } = 256; private const ulong OcclusionReady = 1UL << 63; @@ -221,29 +228,49 @@ public bool DumpOcclusionCounter(int queueId, ulong address, uint instanceMask, if (instanceMask == 0) return true; Span copies = stackalloc DeferredLabelCopy[24]; var count = 0; - for (var block = 0; block < 24; block++) + var selected = instanceMask & 0xFF_FFFFu; + // The blocks sit in one run of guest memory, so one window usually covers them all: one + // resolve for the run instead of one per block. A run no single window covers falls back + // to resolving each block. + var run = default(MetalGuestRange); + var firstBlock = 0; + if (CoalesceOcclusionDumps && selected != 0) { - if ((instanceMask & (1u << block)) == 0) continue; - var destinationAddress = checked(address + (ulong)block * strideBytes); - var destination = _guestMemory.ResolveWritable(destinationAddress, sizeof(ulong)); - if (!destination.IsValid) throw Fatal($"Native occlusion has no writable result window at 0x{destinationAddress:X16}."); - copies[count++] = new DeferredLabelCopy(0, 0, destination.Buffer, destination.Offset, sizeof(ulong), destinationAddress); + firstBlock = System.Numerics.BitOperations.TrailingZeroCount(selected); + var lastBlock = System.Numerics.BitOperations.Log2(selected); + var runBytes = checked((ulong)(lastBlock - firstBlock) * strideBytes + sizeof(ulong)); + if (runBytes <= MaxCoalescedDumpBytes) + { + run = _guestMemory.ResolveWritable(checked(address + (ulong)firstBlock * strideBytes), runBytes); + if (run.Size < runBytes) run = default; + } } var batch = _oqOpen; var source = batch?.Results ?? _oqBank; var offset = batch is null ? (ulong)queueId * sizeof(ulong) : (ulong)batch.Slot * sizeof(ulong); - var index = 0; for (var block = 0; block < 24; block++) { if ((instanceMask & (1u << block)) == 0) continue; - copies[index] = copies[index] with + var destinationAddress = checked(address + (ulong)block * strideBytes); + var destinationBuffer = run.Buffer; + var destinationOffset = run.Offset + (ulong)(block - firstBlock) * strideBytes; + if (!run.IsValid) { - StagingBuffer = block == 0 ? source : _oqBank, - StagingOffset = block == 0 ? offset : (ulong)CommandStreamQueue.QueueCount * sizeof(ulong), - }; - _cpOverlay.Forget(copies[index].GuestAddress, sizeof(ulong)); - _labelOverlay.Forget(copies[index].GuestAddress, sizeof(ulong)); - index++; + var destination = _guestMemory.ResolveWritable(destinationAddress, sizeof(ulong)); + if (!destination.IsValid) throw Fatal($"Native occlusion has no writable result window at 0x{destinationAddress:X16}."); + destinationBuffer = destination.Buffer; + destinationOffset = destination.Offset; + } + copies[count++] = new DeferredLabelCopy( + block == 0 ? source : _oqBank, + block == 0 ? offset : (ulong)CommandStreamQueue.QueueCount * sizeof(ulong), + destinationBuffer, destinationOffset, sizeof(ulong), destinationAddress); + if (!CoalesceOcclusionDumps) + { + // The note below forgets them as well; this is the order before it did. + _cpOverlay.Forget(destinationAddress, sizeof(ulong)); + _labelOverlay.Forget(destinationAddress, sizeof(ulong)); + } } if (batch is not null) { @@ -254,7 +281,8 @@ public bool DumpOcclusionCounter(int queueId, ulong address, uint instanceMask, if (batch.Slot < OcclusionIntervalCapacity) InitializeOcclusionSlot(batch); } _ = EncodeOrderedCopies(copies[..count], address, null, 0, MetalGpuWriteOrigin.Copy, - "occlusion dump", null, (ulong)count * sizeof(ulong), sourceIsFinal: false, deferStableGpuResult: batch is not null); + "occlusion dump", null, (ulong)count * sizeof(ulong), sourceIsFinal: false, deferStableGpuResult: batch is not null, + noteAsOneRange: CoalesceOcclusionDumps); if (batch is { Slot: OcclusionIntervalCapacity }) EndRendering(); _oqDumps++; return true; diff --git a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.cs b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.cs index daae738c3..b614b9845 100644 --- a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.cs +++ b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.cs @@ -1240,6 +1240,15 @@ public void NoteGuestMemoryWrite(ulong address, ulong size, MetalGpuWriteOrigin _labelOverlay.Forget(address, size); } + NoteGuestMemoryWriteBeyondOverlays(address, size, origin); + } + + /// + /// without the two overlays, for a caller that has forgotten the + /// bytes it wrote itself and notes a run that also spans bytes it did not write. + /// + public void NoteGuestMemoryWriteBeyondOverlays(ulong address, ulong size, MetalGpuWriteOrigin origin) + { using (var stagedScope = SharpEmu.Libs.VideoOut.RenderPhaseProfile.MeasureDetail(SharpEmu.Libs.VideoOut.RenderPhaseProfile.Phase.WriteNoteStaged)) { _stagedStorageCopies?.Forget(address, size); diff --git a/tests/SharpEmu.Gpu.Metal.Tests/MetalEndOfPipeOrderTests.OcclusionDump.cs b/tests/SharpEmu.Gpu.Metal.Tests/MetalEndOfPipeOrderTests.OcclusionDump.cs new file mode 100644 index 000000000..fcef6df50 --- /dev/null +++ b/tests/SharpEmu.Gpu.Metal.Tests/MetalEndOfPipeOrderTests.OcclusionDump.cs @@ -0,0 +1,51 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using Xunit; + +namespace SharpEmu.Gpu.Metal.Tests; + +public sealed unsafe partial class MetalEndOfPipeOrderTests +{ + // A dump's blocks are noted as one run, but a pending store into the bytes between them still + // speaks for those bytes (UE4 zeroes a query's whole block with WRITE_DATA before the begin dump), + // and the dump itself lands the same words either way. + [Theory] + [InlineData(true)] + [InlineData(false)] + public void AnOcclusionDumpLeavesAPendingStoreBetweenItsBlocksAndLandsItsOwnWords(bool coalesce) + { + if (!RequireDevice()) return; + using var rig = new Rig(Device.Value); + var host = rig.Host; + host.NativeOcclusionEnabled = true; + host.CoalesceOcclusionDumps = coalesce; + const ulong query = GuestBase + 0x30000; + const ulong ready = 1UL << 63; + + _ = host.EnsureComputeEncoder(); + host.RecordCommandProcessorWrite(query, Enumerable.Repeat((byte)0xAB, 256).ToArray(), "WRITE_DATA"); + Assert.True(host.DumpOcclusionCounter(0, query, 0xFFFF, 16)); + + // Nothing has landed: the bytes between the blocks are still the store's, and the read is answered + // from the pending store instead of waiting for the GPU to land it. + _ = host.TakeOverlayReads(); + Span between = stackalloc byte[sizeof(ulong)]; + for (var block = 0; block < 16; block++) + { + Assert.True(host.TryReadGuest(query + (ulong)block * 16 + 8, between)); + Assert.Equal(0xABABABABABABABABUL, BitConverter.ToUInt64(between)); + } + + Assert.Equal(16, host.TakeOverlayReads()); + + host.FlushCommandBuffer(); + WaitForGpu(host, host.LastWorkSerial); + WaitForCompletions(host); + for (var block = 0; block < 16; block++) + { + Assert.Equal(ready, rig.Backing.Read64(query + (ulong)block * 16)); + Assert.Equal(0xABABABABABABABABUL, rig.Backing.Read64(query + (ulong)block * 16 + 8)); + } + } +} From e4b00171276c2708c6bec4eaf646365753dedf15 Mon Sep 17 00:00:00 2001 From: kuba Date: Wed, 28 Oct 2026 18:38:22 +0100 Subject: [PATCH 60/82] [GPU] Add env-gated benches for interpreter throughput, end-of-pipe and dump cost, and worker stream cost. SHARPEMU_INTERPRETER_BENCH_OUTPUT (Libs tests) times each packet type and a frame-shaped stream through a null host; SHARPEMU_NONDRAW_BENCH_OUTPUT (Metal tests) times labels and dumps through the render host rig; SHARPEMU_WORKER_FRAME_BENCH_OUTPUT times draws with dumps, labels or writes after each through the whole path. Each returns at once without its variable. --- .../MetalEndOfPipeOrderTests.NonDrawBench.cs | 119 ++++++++ .../Scenes/WorkerFrameBenchTests.cs | 136 ++++++++++ .../GpuCommandInterpreterBenchTests.cs | 255 ++++++++++++++++++ 3 files changed, 510 insertions(+) create mode 100644 tests/SharpEmu.Gpu.Metal.Tests/MetalEndOfPipeOrderTests.NonDrawBench.cs create mode 100644 tests/SharpEmu.Gpu.Metal.Tests/Scenes/WorkerFrameBenchTests.cs create mode 100644 tests/SharpEmu.Libs.Tests/Gpu/GpuCommands/GpuCommandInterpreterBenchTests.cs diff --git a/tests/SharpEmu.Gpu.Metal.Tests/MetalEndOfPipeOrderTests.NonDrawBench.cs b/tests/SharpEmu.Gpu.Metal.Tests/MetalEndOfPipeOrderTests.NonDrawBench.cs new file mode 100644 index 000000000..57e19a6a3 --- /dev/null +++ b/tests/SharpEmu.Gpu.Metal.Tests/MetalEndOfPipeOrderTests.NonDrawBench.cs @@ -0,0 +1,119 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using System.Diagnostics; +using System.Text.Json; +using SharpEmu.Libs.Gpu.GpuCommands; +using Xunit; + +namespace SharpEmu.Gpu.Metal.Tests; + +public sealed unsafe partial class MetalEndOfPipeOrderTests +{ + // SHARPEMU_NONDRAW_BENCH_OUTPUT=: the worker's cost of one end-of-pipe label and one + // occlusion dump, with no encoder open (the copy is its own transfer) and with a compute + // encoder open (the copy is held for the encoder's end, as it is behind an open render pass), + // and of the commit that lands what was held. Calls run in batches of 128, a commit between. + [Fact] + public void MeasureEndOfPipeAndOcclusionDumpWhenRequested() + { + var path = Environment.GetEnvironmentVariable("SHARPEMU_NONDRAW_BENCH_OUTPUT"); + if (path is null || !RequireDevice()) return; + using var rig = new Rig(Device.Value); + var host = rig.Host; + host.NativeOcclusionEnabled = true; + const ulong labels = GuestBase + 0x20000; + const ulong queries = GuestBase + 0x30000; + const int batch = 128; + var results = new List(); + // SHARPEMU_NONDRAW_BENCH_ROUNDS lengthens a run for a sampling profiler; ..._ONLY keeps the cases whose name contains it. + var rounds = int.TryParse(Environment.GetEnvironmentVariable("SHARPEMU_NONDRAW_BENCH_ROUNDS"), out var requested) ? requested : 14; + var only = Environment.GetEnvironmentVariable("SHARPEMU_NONDRAW_BENCH_ONLY"); + + void Measure(string name, bool computeOpen, Action call) + { + if (only is not null && !name.Contains(only, StringComparison.Ordinal)) return; + var callNs = new List(); + var commitNs = new List(); + long allocated = 0; + long calls = 0; + for (var round = 0; round < rounds; round++) + { + if (computeOpen) _ = host.EnsureComputeEncoder(); + var allocatedBefore = GC.GetAllocatedBytesForCurrentThread(); + var started = Stopwatch.GetTimestamp(); + for (var index = 0; index < batch; index++) call(index); + var called = Stopwatch.GetTimestamp(); + var allocatedAfter = GC.GetAllocatedBytesForCurrentThread(); + var serial = host.FlushCommandBuffer(); + var flushed = Stopwatch.GetTimestamp(); + WaitForGpu(host, serial); + WaitForCompletions(host); + if (round < 4) continue; + callNs.Add((called - started) * 1e9 / Stopwatch.Frequency / batch); + commitNs.Add((flushed - called) * 1e9 / Stopwatch.Frequency); + allocated += allocatedAfter - allocatedBefore; + calls += batch; + } + + callNs.Sort(); + commitNs.Sort(); + results.Add(new + { + name, + computeOpen, + nsPerCallMedian = callNs[callNs.Count / 2], + nsPerCallBest = callNs[0], + commitMicrosecondsMedian = commitNs[commitNs.Count / 2] / 1000, + allocatedBytesPerCall = (double)allocated / calls, + }); + } + + foreach (var computeOpen in new[] { false, true }) + { + var open = computeOpen ? "held" : "immediate"; + Measure($"eop_write32_{open}", computeOpen, index => + host.RecordEndOfPipe(new EndOfPipeWrite(EndOfPipeWriteKind.Write32, 1, labels + (ulong)index * 16, (uint)index))); + Measure($"eop_write64_{open}", computeOpen, index => + host.RecordEndOfPipe(new EndOfPipeWrite(EndOfPipeWriteKind.Write64, 1, labels + (ulong)index * 16, (ulong)index))); + Measure($"eop_interrupt_only_{open}", computeOpen, index => + host.RecordEndOfPipe(new EndOfPipeWrite(EndOfPipeWriteKind.InterruptOnly, 1, EventId: 0, ContextId: (uint)index))); + Measure($"occlusion_dump_16_blocks_{open}", computeOpen, index => + Assert.True(host.DumpOcclusionCounter(0, queries + (ulong)index * 256, 0xFFFF, 16))); + Measure($"occlusion_dump_1_block_{open}", computeOpen, index => + Assert.True(host.DumpOcclusionCounter(0, queries + (ulong)index * 256, 1, 8))); + Measure($"cp_write_8_bytes_{open}", computeOpen, index => + host.RecordCommandProcessorWrite(labels + (ulong)index * 16, BitConverter.GetBytes((ulong)index), "WRITE_DATA")); + } + + // The dump as it runs behind an open render pass: a visibility batch is open, so the copies are held for the + // encoder's end. The batch is made by hand (no pass is drawn here) and rewound each round. + { + var hostType = typeof(SharpEmu.Gpu.Metal.V2.MetalRenderHost); + const System.Reflection.BindingFlags any = System.Reflection.BindingFlags.Instance | System.Reflection.BindingFlags.NonPublic; + var batchType = hostType.GetNestedType("OcclusionBatch", System.Reflection.BindingFlags.NonPublic)!; + var newBuffer = hostType.GetMethod("NewOcclusionBuffer", any)!; + var counts = (nint)newBuffer.Invoke(host, [4096 * 8])!; + var resultBuffer = (nint)newBuffer.Invoke(host, [4096 * 8])!; + var openBatch = Activator.CreateInstance(batchType, any | System.Reflection.BindingFlags.Public, null, [counts, resultBuffer], null)!; + hostType.GetField("_oqOpen", any)!.SetValue(host, openBatch); + var slot = batchType.GetField("Slot", any | System.Reflection.BindingFlags.Public)!; + foreach (var coalesce in new[] { false, true, false, true }) + { + host.CoalesceOcclusionDumps = coalesce; + Measure($"occlusion_dump_16_blocks_batch_open_coalesce_{(coalesce ? "on" : "off")}", true, index => + { + if (index == 0) slot.SetValue(openBatch, 0); + Assert.True(host.DumpOcclusionCounter(0, queries + (ulong)index * 256, 0xFFFF, 16)); + }); + } + + hostType.GetField("_oqOpen", any)!.SetValue(host, null); + host.CoalesceOcclusionDumps = true; + } + + File.WriteAllText(path, JsonSerializer.Serialize(new { + architecture = System.Runtime.InteropServices.RuntimeInformation.ProcessArchitecture.ToString(), results }, + new JsonSerializerOptions { WriteIndented = true })); + } +} diff --git a/tests/SharpEmu.Gpu.Metal.Tests/Scenes/WorkerFrameBenchTests.cs b/tests/SharpEmu.Gpu.Metal.Tests/Scenes/WorkerFrameBenchTests.cs new file mode 100644 index 000000000..47065c65f --- /dev/null +++ b/tests/SharpEmu.Gpu.Metal.Tests/Scenes/WorkerFrameBenchTests.cs @@ -0,0 +1,136 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using System.Diagnostics; +using System.Text.Json; +using Xunit; +using Xunit.Abstractions; + +namespace SharpEmu.Gpu.Metal.Tests.Scenes; + +public sealed class WorkerFrameBenchTests(ITestOutputHelper output) : SceneTestBase(output) +{ + private const uint Size = 32; + private const int Draws = 150; + + // SHARPEMU_WORKER_FRAME_BENCH_OUTPUT=: a stream of 150 small draws with nothing else, and the + // same stream with an occlusion dump, an end-of-pipe label, a WRITE_DATA or a pair of dumps after every + // draw, each run through the whole path (interpreter, translation, Metal host, GPU). The difference + // over the draws-only stream, per added packet, is what the packet costs the worker with a render + // pass open. Variants alternate per round so a drift in the machine falls on all of them. + [Fact] + public void MeasureWorkerStreamCostsWhenRequested() + { + var path = Environment.GetEnvironmentVariable("SHARPEMU_WORKER_FRAME_BENCH_OUTPUT"); + if (path is null) return; + using var rig = CreateRig(); + if (rig is null) return; + rig.Host.NativeOcclusionEnabled = true; + var target = rig.ColorTarget(Size, Size); + var queries = rig.Memory.Allocate(Draws * 256UL, 256); + var labels = rig.Memory.Allocate(Draws * 16UL, 256); + var data = rig.Memory.Allocate(Draws * 16UL, 256); + + ScenePm4 Stream(Action? after, uint depthCount = 1u << 8) + { + var pm4 = new ScenePm4(); + var state = new SceneState(Size, Size).ColorTarget(0, target, Size, Size); + state.Context[Cx.DbCountControl] = depthCount; + for (var draw = 0; draw < Draws; draw++) + { + rig.DrawMesh(pm4, state, SceneDraws.Quad(0.1f, 1, 0, 0)); + after?.Invoke(pm4, draw); + } + + return pm4; + } + + var variants = new (string Name, int PerDraw, ScenePm4 Stream)[] + { + ("draws_only", 0, Stream(null)), + ("dump_16_blocks", 1, Stream((pm4, draw) => pm4.EventWithAddress(ScenePm4.EventPixelPipeStatDump, 1, queries + (ulong)draw * 256))), + ("dump_5x_16_blocks", 5, Stream((pm4, draw) => + { + for (var dump = 0; dump < 5; dump++) + pm4.EventWithAddress(ScenePm4.EventPixelPipeStatDump, 1, queries + (ulong)draw * 256 + (ulong)dump * 8); + })), + ("dump_pair_16_blocks", 2, Stream((pm4, draw) => + { + pm4.EventWithAddress(ScenePm4.EventPixelPipeStatDump, 1, queries + (ulong)draw * 256); + pm4.EventWithAddress(ScenePm4.EventPixelPipeStatDump, 1, queries + (ulong)draw * 256 + 8); + })), + ("release_mem_write32", 1, Stream((pm4, draw) => ReleaseMemory(pm4, labels + (ulong)draw * 16, 1u << 29, (uint)draw))), + ("release_mem_write32_interrupt", 1, Stream((pm4, draw) => ReleaseMemory(pm4, labels + (ulong)draw * 16, (1u << 29) | (2u << 24), (uint)draw))), + ("write_data_2_dwords", 1, Stream((pm4, draw) => + pm4.Packet(ScenePm4.OpWriteData, 0x100, (uint)(data + (ulong)draw * 16), (uint)((data + (ulong)draw * 16) >> 32), (uint)draw, 2))), + }; + + // SHARPEMU_WORKER_FRAME_BENCH_ONLY= runs that variant alone (a sampling profiler wants one), and + // ..._ROUNDS makes the run longer. + var only = Environment.GetEnvironmentVariable("SHARPEMU_WORKER_FRAME_BENCH_ONLY"); + if (only is not null) + { + var names = only.Split(','); + variants = variants.Where(variant => variant.Name == "draws_only" || names.Contains(variant.Name)).ToArray(); + } + var rounds = int.TryParse(Environment.GetEnvironmentVariable("SHARPEMU_WORKER_FRAME_BENCH_ROUNDS"), out var requested) ? requested : 12; + // Each stream is uploaded once and queued again every round, so a long run does not use up the scene's memory. + var words = variants.Select(variant => variant.Stream.ToArray()).ToArray(); + var uploaded = words.Select(stream => rig.Memory.Upload(stream, 256)).ToArray(); + ulong submission = 1000; + var times = variants.Select(_ => new List()).ToArray(); + var allocated = variants.Select(_ => new List()).ToArray(); + for (var round = 0; round < rounds + 2; round++) + { + for (var index = 0; index < variants.Length; index++) + { + var variant = variants[(index + round) % variants.Length]; + var slot = Array.IndexOf(variants, variant); + var before = GC.GetTotalAllocatedBytes(false); + var started = Stopwatch.GetTimestamp(); + rig.Host.CommandStream.EnqueueGraphics(uploaded[slot], (uint)words[slot].Length, submission++, null); + rig.Drain(); + var elapsed = (Stopwatch.GetTimestamp() - started) * 1000.0 / Stopwatch.Frequency; + var bytes = GC.GetTotalAllocatedBytes(false) - before; + if (round < 2) continue; + times[slot].Add(elapsed); + allocated[slot].Add(bytes); + } + } + + var baseline = Median(times[0]); + var baselineBytes = MedianLong(allocated[0]); + File.WriteAllText(path, JsonSerializer.Serialize(new + { + architecture = System.Runtime.InteropServices.RuntimeInformation.ProcessArchitecture.ToString(), + draws = Draws, + variants = variants.Select((variant, slot) => new + { + name = variant.Name, + submitMillisecondsMedian = Median(times[slot]), + submitMillisecondsBest = times[slot].Min(), + microsecondsPerExtraDumpOverOne = variant.Name == "dump_5x_16_blocks" + ? (Median(times[slot]) - Median(times[Array.FindIndex(variants, other => other.Name == "dump_16_blocks")])) * 1000.0 / (Draws * 4) + : 0, + microsecondsPerAddedPacket = variant.PerDraw == 0 ? 0 : (Median(times[slot]) - baseline) * 1000.0 / (Draws * variant.PerDraw), + allocatedKilobytesPerSubmit = MedianLong(allocated[slot]) / 1024.0, + allocatedBytesPerAddedPacket = variant.PerDraw == 0 ? 0 : (MedianLong(allocated[slot]) - baselineBytes) / (double)(Draws * variant.PerDraw), + }), + }, new JsonSerializerOptions { WriteIndented = true })); + } + + private static void ReleaseMemory(ScenePm4 pm4, ulong label, uint control, uint value) => + pm4.Packet(0x49, 0x28u | (5u << 8), control, (uint)label, (uint)(label >> 32), value, 0, 0); + + private static double Median(List values) + { + var sorted = values.Order().ToList(); + return sorted[sorted.Count / 2]; + } + + private static long MedianLong(List values) + { + var sorted = values.Order().ToList(); + return sorted[sorted.Count / 2]; + } +} diff --git a/tests/SharpEmu.Libs.Tests/Gpu/GpuCommands/GpuCommandInterpreterBenchTests.cs b/tests/SharpEmu.Libs.Tests/Gpu/GpuCommands/GpuCommandInterpreterBenchTests.cs new file mode 100644 index 000000000..c0704c033 --- /dev/null +++ b/tests/SharpEmu.Libs.Tests/Gpu/GpuCommands/GpuCommandInterpreterBenchTests.cs @@ -0,0 +1,255 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using System.Diagnostics; +using System.Text.Json; +using SharpEmu.Libs.Gpu.GpuCommands; +using SharpEmu.Libs.Gpu.GpuCommands.Registers; +using SharpEmu.Libs.Gpu.Scheduling; +using Xunit; + +namespace SharpEmu.Libs.Tests.Gpu.GpuCommands; + +public sealed class GpuCommandInterpreterBenchTests +{ + private const ulong Base = RecordingCommandStreamHost.MemoryBase; + private const ulong CommandAddress = Base + 0x1000; + private const ulong ContextTable = Base + 0x4_0000; + private const ulong ShaderTable = Base + 0x4_1000; + private const ulong LabelBase = Base + 0x5_0000; + private const ulong DataBase = Base + 0x6_0000; + private const ulong QueryBase = Base + 0x7_0000; + private const ulong IndexBase = Base + 0x8_0000; + + // A host with no GPU: every host call is counted and does no work, so the time is the + // interpreter's own. Dumps take the native path, as the Metal host's do. + private sealed class NullHost(FakeCpuMemory memory, CountingSink sink) : + ManagedCommandStreamHost(memory, sink, sink), ICommandStreamHost + { + public int Draws; + public int Dumps; + + // The Metal host hands over command blocks and says whether a block is still current at every packet. + public long BlockVersion; + + public bool SupportsCommandBlocks => true; + + public bool TryReadCommandBlock(ulong address, Span block, out CommandReadToken token) + { + token = new CommandReadToken(BlockVersion, 0, 0, 0); + return Memory.TryRead(address, block); + } + + public bool IsCommandBlockCurrent(in CommandReadToken token) => token.First == BlockVersion; + + public override void DrawIndexed(ulong submitId, in DrawIndexedArguments arguments) => Draws++; + + public bool DumpOcclusionCounter(int queueId, ulong address, uint instanceMask, uint strideBytes) + { + Dumps++; + return true; + } + } + + private sealed class CountingSink : IEndOfPipeSink, ICommandStreamFlipTarget + { + public int Interrupts; + + public void TriggerInterrupt(int eventId, uint contextId) => Interrupts++; + + public ulong Prepare(int handle, int index, int flipMode, long flipArgument) => 1; + + public void Complete(ulong requestId) { } + + public void Presented(ulong requestId) { } + + public bool IsDone(int handle, int index) => true; + } + + private static uint[] Packet(uint opcode, params uint[] payload) => StreamRunner.Packet(opcode, payload); + + private static uint[] Custom(uint code, params uint[] payload) => + StreamRunner.CustomPacket(PacketOpcode.Nop, code, payload); + + private static uint[] ReleaseMemory(ulong label, uint value) => Packet(PacketOpcode.ReleaseMemory, + 0x28u | (5u << 8), (2u << 24) | (1u << 29), StreamRunner.Low(label), StreamRunner.High(label), value, 0, 0); + + private static uint[] Dump(ulong address) => Packet(PacketOpcode.EventWrite, + 0x139, StreamRunner.Low(address), StreamRunner.High(address)); + + private static uint[] SelectQueries() => Packet(PacketOpcode.EventWrite, + 0x138, (2u << 9) | (0xFFFFu << 11), 0); + + // Context registers a table or packet can write with a zero value, found by writing each. + private static uint[] WritableContextRegisters() + { + var found = new List(); + for (var offset = 0u; offset < RegisterBankLayout.ContextRegisterCount && found.Count < 32; offset++) + { + if (RegisterWriteTable.ContextIndirect[offset] is null || offset == RegisterBankLayout.ContextNop) continue; + var scratch = new RegisterBanks(message => new InvalidOperationException(message)); + try + { + RegisterWriteTable.WriteContextEntry(scratch, offset, 0, 0); + found.Add(offset); + } + catch (Exception) + { + } + } + + return found.ToArray(); + } + + private static void Table(FakeCpuMemory memory, ulong address, uint[] registers, int count) + { + var words = new uint[count * 2]; + for (var index = 0; index < count; index++) + { + words[index * 2] = registers[index % registers.Length]; + words[index * 2 + 1] = 0; + } + + Assert.True(memory.TryWrite(address, System.Runtime.InteropServices.MemoryMarshal.AsBytes(words.AsSpan()))); + } + + private static uint[] ShaderRegisters(uint first, int count) + { + var registers = new uint[count]; + for (var index = 0; index < count; index++) registers[index] = first + (uint)index; + return registers; + } + + private sealed record Workload(string Name, int PacketCount, uint[] Words); + + private static Workload Repeat(string name, int times, Func body) + { + var all = new List(); + var packets = 0; + for (var index = 0; index < times; index++) + { + foreach (var packet in body(index)) + { + all.Add(packet); + packets++; + } + } + + return new Workload(name, packets, StreamRunner.Concat(all.ToArray())); + } + + // SHARPEMU_INTERPRETER_BENCH_OUTPUT=: ns per packet for each packet type and for a stream + // shaped like a frame, and the bytes the worker thread allocates per frame. + [Fact] + public void MeasureInterpreterWhenRequested() + { + var path = Environment.GetEnvironmentVariable("SHARPEMU_INTERPRETER_BENCH_OUTPUT"); + if (path is null) return; + + var memory = new FakeCpuMemory(Base, RecordingCommandStreamHost.MemorySize); + var sink = new CountingSink(); + var host = new NullHost(memory, sink); + var interpreter = new GpuCommandInterpreter(host, 0, 0); + var contextRegisters = WritableContextRegisters(); + Assert.NotEmpty(contextRegisters); + Table(memory, ContextTable, contextRegisters, 24); + Table(memory, ShaderTable, ShaderRegisters(ShaderRegisterOffset.SpiShaderUserDataPs0, 16), 16); + + const int Reps = 400; + var workloads = new List + { + Repeat("set_context_reg_1", Reps, i => [Packet(PacketOpcode.SetContextRegister, contextRegisters[i % contextRegisters.Length], 0)]), + Repeat("set_context_reg_run4", Reps, i => [Packet(PacketOpcode.SetContextRegister, ContextRegisterOffset.PaScViewportScissor0Tl, 0, 0, 0, 0)]), + Repeat("set_sh_reg_run6", Reps, i => [Packet(PacketOpcode.SetShaderRegister, ShaderRegisterOffset.SpiShaderUserDataPs0, 1, 2, 3, 4, 5, 6)]), + Repeat("set_sh_reg_run16", Reps, i => [Packet(PacketOpcode.SetShaderRegister, [ShaderRegisterOffset.SpiShaderUserDataVs0, .. Enumerable.Range(0, 16).Select(v => (uint)v)])]), + Repeat("context_table_24", Reps, i => [Custom(PacketCustomCode.ContextRegisterTable, 24, StreamRunner.Low(ContextTable), StreamRunner.High(ContextTable))]), + Repeat("shader_table_16", Reps, i => [Custom(PacketCustomCode.ShaderRegisterTable, 16, StreamRunner.Low(ShaderTable), StreamRunner.High(ShaderTable))]), + Repeat("draw_index_2", Reps, i => [Packet(PacketOpcode.DrawIndex2, 100, StreamRunner.Low(IndexBase), StreamRunner.High(IndexBase), 60, 0x20)]), + Repeat("write_data_2", Reps, i => [Packet(PacketOpcode.WriteData, 0x100, StreamRunner.Low(DataBase), StreamRunner.High(DataBase), 1, 2)]), + Repeat("release_mem_write32_interrupt", Reps, i => [ReleaseMemory(LabelBase + (ulong)(i % 64) * 16, (uint)i)]), + Repeat("zpass_dump_16_blocks", Reps, i => [i == 0 ? SelectQueries() : [], Dump(QueryBase)]), + Repeat("marker_push_pop", Reps, i => [Custom(PacketCustomCode.PushMarker, 0, 0), Custom(PacketCustomCode.PopMarker, 0, 0)]), + Repeat("nop_filler", Reps, i => [Packet(PacketOpcode.Nop, 0, 0)]), + }; + + // A frame: per draw, a few context packets, shader packets and tables, then the draw; + // a release or write every few draws, and a pair of occlusion dumps around some. + workloads.Add(Repeat("frame_1000_draws", 1000, i => + { + var packets = new List + { + Packet(PacketOpcode.SetContextRegister, contextRegisters[i % contextRegisters.Length], 0), + Packet(PacketOpcode.SetContextRegister, ContextRegisterOffset.PaScViewportScissor0Tl, 0, 0), + Packet(PacketOpcode.SetShaderRegister, ShaderRegisterOffset.SpiShaderUserDataPs0, 1, 2, 3, 4, 5, 6), + Packet(PacketOpcode.SetShaderRegister, ShaderRegisterOffset.SpiShaderUserDataVs0, 1, 2, 3, 4), + Custom(PacketCustomCode.ShaderRegisterTable, 16, StreamRunner.Low(ShaderTable), StreamRunner.High(ShaderTable)), + Packet(PacketOpcode.DrawIndex2, 100, StreamRunner.Low(IndexBase), StreamRunner.High(IndexBase), 60, 0x20), + }; + if (i % 4 == 0) + packets.Insert(0, Custom(PacketCustomCode.ContextRegisterTable, 24, StreamRunner.Low(ContextTable), StreamRunner.High(ContextTable))); + if (i % 10 == 0) + { + packets.Add(Packet(PacketOpcode.WriteData, 0x100, StreamRunner.Low(DataBase), StreamRunner.High(DataBase), 1, 2)); + packets.Add(ReleaseMemory(LabelBase + (ulong)(i % 64) * 16, (uint)i)); + packets.Add(Custom(PacketCustomCode.PushMarker, 0, 0)); + packets.Add(Custom(PacketCustomCode.PopMarker, 0, 0)); + } + + if (i % 25 == 0) + { + packets.Add(SelectQueries()); + packets.Add(Dump(QueryBase)); + packets.Add(Dump(QueryBase + 8)); + } + + return packets.ToArray(); + })); + + var results = new List(); + foreach (var workload in workloads) + { + Assert.True(memory.TryWrite(CommandAddress, System.Runtime.InteropServices.MemoryMarshal.AsBytes(workload.Words.AsSpan()))); + var dwords = (uint)workload.Words.Length; + Assert.True(CommandAddress + dwords * 4UL < ContextTable); + double Pass() + { + var commands = new PacketCursorStack(); + var started = Stopwatch.GetTimestamp(); + interpreter.Process(commands, CommandAddress, dwords); + return (Stopwatch.GetTimestamp() - started) * 1e9 / Stopwatch.Frequency; + } + + // Long enough for the tiered JIT to promote what the stream runs through. + var warmUntil = Stopwatch.GetTimestamp() + Stopwatch.Frequency * (workload.PacketCount > 3000 ? 3 : 1) / 2; + while (Stopwatch.GetTimestamp() < warmUntil) Pass(); + var samples = new double[21]; + long allocated = 0; + for (var pass = 0; pass < samples.Length; pass++) + { + var before = GC.GetAllocatedBytesForCurrentThread(); + samples[pass] = Pass(); + allocated = GC.GetAllocatedBytesForCurrentThread() - before; + } + + Array.Sort(samples); + results.Add(new + { + name = workload.Name, + packets = workload.PacketCount, + dwords, + nsPerPacketMedian = samples[samples.Length / 2] / workload.PacketCount, + nsPerPacketBest = samples[0] / workload.PacketCount, + allocatedBytesPerPass = allocated, + allocatedBytesPerPacket = (double)allocated / workload.PacketCount, + }); + } + + File.WriteAllText(path, JsonSerializer.Serialize(new + { + architecture = System.Runtime.InteropServices.RuntimeInformation.ProcessArchitecture.ToString(), + draws = host.Draws, + dumps = host.Dumps, + workloads = results, + }, new JsonSerializerOptions { WriteIndented = true })); + } +} From ac594d14e8ae473a0c3b4b6df603199c65db58dd Mon Sep 17 00:00:00 2001 From: kuba Date: Wed, 28 Oct 2026 18:43:07 +0100 Subject: [PATCH 61/82] [GPU] A register table zeroes a stack block sized to its entries, and the unsynchronized copy only when tracing. ApplyRegisterTable zeroed two 1 KB stack blocks per table packet. The block is now sized to min(count, 128) entries and the second exists only for SHARPEMU_GPU_TRACE_REGISTER_TABLES. Null-host bench, ns per packet (best of runs): 24-entry context table 180 -> 170, 16-entry shader table 137 -> 124. --- .../Gpu/GpuCommands/GpuCommandInterpreter.Registers.cs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/SharpEmu.Libs/Gpu/GpuCommands/GpuCommandInterpreter.Registers.cs b/src/SharpEmu.Libs/Gpu/GpuCommands/GpuCommandInterpreter.Registers.cs index 85fa6747e..604b0391d 100644 --- a/src/SharpEmu.Libs/Gpu/GpuCommands/GpuCommandInterpreter.Registers.cs +++ b/src/SharpEmu.Libs/Gpu/GpuCommands/GpuCommandInterpreter.Registers.cs @@ -259,8 +259,10 @@ private void ApplyRegisterTable(RegisterBank bank, ulong tableAddress, uint coun if (bank == RegisterBank.Context && ScissorTrace.Instance is { Active: true } scissorTrace) scissorTrace.RegisterTable(TypedRegisters, SubmitId, QueueId, in packet, tableAddress, count); - Span tableBlock = stackalloc byte[entriesPerBlock * entryBytes]; - Span beforeSynchronization = stackalloc byte[entriesPerBlock * entryBytes]; + // Both are zeroed when allocated: a table is usually a few dozen entries, so the block is sized + // to it, and the unsynchronized copy exists only for the table trace. + Span tableBlock = stackalloc byte[(int)Math.Min(count, (uint)entriesPerBlock) * entryBytes]; + Span beforeSynchronization = TraceRegisterTables ? stackalloc byte[entriesPerBlock * entryBytes] : default; for (var index = 0u; index < count; index++) { var blockEntry = (int)(index % entriesPerBlock); From 29f064c8116389422baf82c1d2152ba5b8e9ec4a Mon Sep 17 00:00:00 2001 From: kuba Date: Wed, 28 Oct 2026 18:43:07 +0100 Subject: [PATCH 62/82] [GPU] The worker stream bench adds a five-dump, a WRITE_DATA and a zero-fill-then-dump variant, and an open-batch dump case. The open-batch case in the Metal rig opens a visibility batch by hand so dump copies are held as they are behind a render pass; the stream variants time the added packets per draw. --- .../Scenes/WorkerFrameBenchTests.cs | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/SharpEmu.Gpu.Metal.Tests/Scenes/WorkerFrameBenchTests.cs b/tests/SharpEmu.Gpu.Metal.Tests/Scenes/WorkerFrameBenchTests.cs index 47065c65f..503c96317 100644 --- a/tests/SharpEmu.Gpu.Metal.Tests/Scenes/WorkerFrameBenchTests.cs +++ b/tests/SharpEmu.Gpu.Metal.Tests/Scenes/WorkerFrameBenchTests.cs @@ -54,6 +54,19 @@ ScenePm4 Stream(Action? after, uint depthCount = 1u << 8) for (var dump = 0; dump < 5; dump++) pm4.EventWithAddress(ScenePm4.EventPixelPipeStatDump, 1, queries + (ulong)draw * 256 + (ulong)dump * 8); })), + ("write_data_256_bytes", 1, Stream((pm4, draw) => + { + var query = queries + (ulong)draw * 256; + pm4.Packet(ScenePm4.OpWriteData, [0x100u, (uint)query, (uint)(query >> 32), .. new uint[64]]); + })), + // UE4 and UE5 zero a query's 256 bytes with WRITE_DATA, then dump its begin and end into it. + ("zero_fill_then_dump_pair", 3, Stream((pm4, draw) => + { + var query = queries + (ulong)draw * 256; + pm4.Packet(ScenePm4.OpWriteData, [0x100u, (uint)query, (uint)(query >> 32), .. new uint[64]]); + pm4.EventWithAddress(ScenePm4.EventPixelPipeStatDump, 1, query); + pm4.EventWithAddress(ScenePm4.EventPixelPipeStatDump, 1, query + 8); + })), ("dump_pair_16_blocks", 2, Stream((pm4, draw) => { pm4.EventWithAddress(ScenePm4.EventPixelPipeStatDump, 1, queries + (ulong)draw * 256); From 1f56f55ce01525289d6feedbf5498d7bb00c84df Mon Sep 17 00:00:00 2001 From: kuba Date: Wed, 28 Oct 2026 18:38:01 +0100 Subject: [PATCH 63/82] [GPU] An env-gated bench records repeated Metal draws and reports microseconds per draw per variation. SHARPEMU_DRAW_RECORD_BENCH_OUTPUT= runs one 256-draw PM4 stream (three sampled textures, depth, two vertex buffers) in five variations: the same state, an alternating texture, sixteen alternating pixel programs, a render pass every four draws, and all three mixed. It reports wall and recording-thread CPU time per draw, and the SHARPEMU_PROFILE_RENDER self time per phase when that is on. Returns immediately without the variable. --- .../Scenes/DrawRecordBenchTests.cs | 235 ++++++++++++++++++ 1 file changed, 235 insertions(+) create mode 100644 tests/SharpEmu.Gpu.Metal.Tests/Scenes/DrawRecordBenchTests.cs diff --git a/tests/SharpEmu.Gpu.Metal.Tests/Scenes/DrawRecordBenchTests.cs b/tests/SharpEmu.Gpu.Metal.Tests/Scenes/DrawRecordBenchTests.cs new file mode 100644 index 000000000..dd2b428db --- /dev/null +++ b/tests/SharpEmu.Gpu.Metal.Tests/Scenes/DrawRecordBenchTests.cs @@ -0,0 +1,235 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using System.Diagnostics; +using System.Reflection; +using System.Runtime.InteropServices; +using System.Text.Json; +using SharpEmu.Gpu.Metal.V2; +using SharpEmu.Libs.VideoOut; +using Xunit; +using Xunit.Abstractions; + +namespace SharpEmu.Gpu.Metal.Tests.Scenes; + +/// +/// The worker's per-draw recording cost (RenderExecutor.RecordDraw and everything below it): the same +/// three-texture, depth-tested draw repeated, then with one binding or the pixel program alternating. +/// +public sealed class DrawRecordBenchTests(ITestOutputHelper output) : SceneTestBase(output) +{ + private const uint Rgba8 = 56; + private const uint TextureSize = 8; + private const uint TargetSize = 64; + private const int DrawsPerSubmit = 256; + + // Three images sampled through one sampler and summed (T# s[0:7], s[8:15], s[16:23], S# s[24:27]; the + // primitive mask is s28), the way a material pass reads several textures. + private static readonly uint[] SampledThree = + [ + 0xBEFC031C, // s_mov_b32 m0, s28 + 0xC8080000, // v_interp_p1_f32_e32 v2, v0, attr0.x + 0xC8090001, // v_interp_p2_f32_e32 v2, v1, attr0.x + 0xC80C0100, // v_interp_p1_f32_e32 v3, v0, attr0.y + 0xC80D0101, // v_interp_p2_f32_e32 v3, v1, attr0.y + 0xF0800F08, 0x00C00402, // image_sample v[4:7], v[2:3], s[0:7], s[24:27] dmask:0xf dim:SQ_RSRC_IMG_2D + 0xF0800F08, 0x00C20802, // image_sample v[8:11], v[2:3], s[8:15], s[24:27] dmask:0xf dim:SQ_RSRC_IMG_2D + 0xF0800F08, 0x00C40C02, // image_sample v[12:15], v[2:3], s[16:23], s[24:27] dmask:0xf dim:SQ_RSRC_IMG_2D + 0xBF8C3F70, // s_waitcnt vmcnt(0) + 0x06080908, 0x060A0B09, 0x060C0D0A, 0x060E0F0B, // v_add_f32 v4..v7, v8..v11, v4..v7 + 0x06080D08, 0x060A0F09, 0x060C110A, 0x060E130B, // v_add_f32 v4..v7, v12..v15, v4..v7 + 0xF800180F, 0x07060504, // exp mrt0 v4, v5, v6, v7 done vm + 0xBF810000, // s_endpgm + ]; + + // The same program under another hash: k s_nop instructions in front of it. + private static uint[] SampledThreeNops(int count) => [.. Enumerable.Repeat(0xBF800000u, count), .. SampledThree]; + + private static readonly Func PhaseTicks = + typeof(RenderPhaseProfile).GetMethod("PhaseTicks", BindingFlags.Static | BindingFlags.NonPublic)! + .CreateDelegate>(); + + // CLOCK_THREAD_CPUTIME_ID: the recording thread's own CPU time, which a busy machine does not stretch the way it does wall time. + [DllImport("libSystem.dylib")] + private static extern ulong clock_gettime_nsec_np(int clockId); + + private static ulong ThreadCpuNanoseconds() => clock_gettime_nsec_np(16); + + // ReportCounters zeroes the host's render-pass count every two seconds, which is what keeps its first-pass diagnostics + // (strings built per pass while the count is under 4096) running in the emulator; the bench does the same per round. + private static readonly System.Reflection.FieldInfo RenderPasses = + typeof(MetalRenderHost).GetField("_renderPasses", BindingFlags.Instance | BindingFlags.NonPublic)!; + + private static ulong SolidTexture(SceneRig rig, uint abgr) + { + var texels = new uint[TextureSize * 64]; + Array.Fill(texels, abgr); + var address = rig.Memory.Allocate(TextureSize * 64 * 4, 0x10000); + rig.Memory.Write(address, MemoryMarshal.AsBytes(texels.AsSpan())); + return address; + } + + // Same: one state, 256 draws. AlternatingTexture: texture 0 flips every draw. AlternatingProgram: sixteen + // pixel programs in turn. PassChurn: two colour targets, a new render pass every four draws. Mixed: all three. + private enum Variation { Same, AlternatingTexture, AlternatingProgram, PassChurn, Mixed } + + // SHARPEMU_DRAW_RECORD_BENCH_OUTPUT=: microseconds per recorded draw for each variation, and + // with SHARPEMU_PROFILE_RENDER=1 the self time per render phase. Draws record on this thread + // (no command thread), so the number is the worker's CPU for packets plus RecordDraw. _ROUNDS and _WARM set + // the measured and warm-up submissions, _VARIATION keeps one case (a native sampler wants a long one). + [Fact] + public void MeasureDrawRecordWhenRequested() + { + var path = Environment.GetEnvironmentVariable("SHARPEMU_DRAW_RECORD_BENCH_OUTPUT"); + if (path is null) return; + Environment.SetEnvironmentVariable("SHARPEMU_TEST_REQUIRE_DEVICE", "1"); + using var rig = CreateRig(); + Assert.NotNull(rig); + var colors = new[] { rig.ColorTarget(TargetSize, TargetSize, 16, 0), rig.ColorTarget(TargetSize, TargetSize, 16, 0) }; + var depth = rig.Memory.Allocate(TargetSize * TargetSize * 4 + 0x10000, 0x10000); + var textures = Enumerable.Range(0, 10).Select(index => SolidTexture(rig, 0x80000040u + (uint)index * 0x00101000u)).ToArray(); + var meshData = rig.UploadMesh(SceneDraws.TexturedQuad()); + var vertexProgram = rig.Program(ScenePrograms.FetchVertex); + var programs = Enumerable.Range(0, 16).Select(count => rig.Program(SampledThreeNops(count))).ToArray(); + var rounds = int.TryParse(Environment.GetEnvironmentVariable("SHARPEMU_DRAW_RECORD_BENCH_ROUNDS"), out var configured) ? configured : 60; + var warm = int.TryParse(Environment.GetEnvironmentVariable("SHARPEMU_DRAW_RECORD_BENCH_WARM"), out var warmed) ? warmed : 400; + var only = Environment.GetEnvironmentVariable("SHARPEMU_DRAW_RECORD_BENCH_VARIATION"); + var phases = Enum.GetValues().Where(phase => phase != RenderPhaseProfile.Phase.Count).ToArray(); + + uint[] Stream(Variation variation) + { + var textureChanges = variation is Variation.AlternatingTexture or Variation.Mixed; + var programChanges = variation is Variation.AlternatingProgram or Variation.Mixed; + var passChanges = variation is Variation.PassChurn or Variation.Mixed; + var passLength = variation == Variation.Mixed ? 8 : 4; + var pm4 = new ScenePm4(); + for (var draw = 0; draw < DrawsPerSubmit; draw++) + { + var texture = textures[textureChanges ? (variation == Variation.Mixed ? draw % 7 : draw & 1) : 0]; + var program = programs[programChanges ? (variation == Variation.Mixed ? draw % 5 : draw) % programs.Length : 0]; + var color = colors[passChanges ? draw / passLength & 1 : 0]; + uint[] pixelData = + [ + .. Descriptors.Texture(texture, Rgba8, TextureSize, TextureSize), + .. Descriptors.Texture(textures[8], Rgba8, TextureSize, TextureSize), + .. Descriptors.Texture(textures[9], Rgba8, TextureSize, TextureSize), + .. Descriptors.Sampler(), + ]; + if (draw == 0 || (passChanges && draw % passLength == 0)) + { + new SceneState(TargetSize, TargetSize) + .ColorTarget(0, color, TargetSize, TargetSize, format: 14, numberType: 7) + .DepthTarget(depth, TargetSize, TargetSize, zFormat: 3) + .DepthControl(test: true, write: true, CompareFunc.Always) + .VertexShader(vertexProgram, meshData) + .PixelShader(program, pixelData) + .Emit(pm4); + } + else if (textureChanges || programChanges) + { + // A guest rewrites the pixel stage's registers when its material changes, nothing else. + var changed = new SceneState(TargetSize, TargetSize); + changed.Context.Clear(); + changed.Uconfig.Clear(); + changed.Shader.Clear(); + changed.PixelShader(program, pixelData).Emit(pm4); + } + + pm4.NumInstances(1); + pm4.DrawAuto(6); + } + + return pm4.ToArray(); + } + + var results = new List(); + ulong submissionId = 1000; + var profiling = RenderPhaseProfile.Enabled; + foreach (var variation in Enum.GetValues().Where(variation => only is null || variation.ToString() == only)) + { + var words = Stream(variation); + // The stream is placed once: every round submits the same packets. + var streamAddress = rig.Memory.Upload(words, 256); + var microseconds = new List(); + var cpuMicroseconds = new List(); + var phaseTotals = new Dictionary(); + var hostDrawMilliseconds = 0.0; + long allocatedBytes = 0; + var gen0Collections = 0; + // The first variation also pays for the JIT's tier-up of the shared recording path. + var warmRounds = results.Count == 0 ? warm * 2 : warm; + for (var round = 0; round < warmRounds + rounds; round++) + { + RenderPasses.SetValue(rig.Host, 0L); + var before = profiling ? phases.Select(phase => PhaseTicks(phase)).ToArray() : null; + rig.Host.CommandStream.EnqueueGraphics(streamAddress, (uint)words.Length, ++submissionId, null); + long started; + ulong cpuStarted; + var allocatedBefore = GC.GetAllocatedBytesForCurrentThread(); + var collectionsBefore = GC.CollectionCount(0); + using (profiling ? RenderPhaseProfile.Measure(RenderPhaseProfile.Phase.CommandStream) : default) + { + cpuStarted = ThreadCpuNanoseconds(); + started = Stopwatch.GetTimestamp(); + rig.RecordPending(); + } + + var elapsed = Stopwatch.GetTimestamp() - started; + var cpuElapsed = ThreadCpuNanoseconds() - cpuStarted; + var allocated = GC.GetAllocatedBytesForCurrentThread() - allocatedBefore; + var collections = GC.CollectionCount(0) - collectionsBefore; + var recordedBeforeDrain = rig.Host.ReadAndResetPerfCounters().Draws; + var drainStarted = Stopwatch.GetTimestamp(); + rig.Drain(); + var drained = Stopwatch.GetTimestamp() - drainStarted; + if (round == warmRounds + 3) Output.WriteLine($"{variation}: {recordedBeforeDrain} draws recorded by RecordPending, drain {drained * 1e6 / Stopwatch.Frequency:F0} us"); + // The warm-up rounds fill the pipeline and image caches and let the JIT reach its optimized code. + var counters = rig.Host.ReadAndResetPerfCounters(); + if (round < warmRounds) continue; + Assert.Equal(DrawsPerSubmit, recordedBeforeDrain + counters.Draws); + microseconds.Add(elapsed * 1e6 / Stopwatch.Frequency / DrawsPerSubmit); + cpuMicroseconds.Add(cpuElapsed / 1e3 / DrawsPerSubmit); + allocatedBytes += allocated; + gen0Collections += collections; + hostDrawMilliseconds += counters.DrawMs; + if (before is null) continue; + for (var index = 0; index < phases.Length; index++) + { + var delta = PhaseTicks(phases[index]) - before[index]; + if (delta == 0) continue; + var key = phases[index].ToString(); + phaseTotals[key] = phaseTotals.GetValueOrDefault(key) + delta * 1e6 / Stopwatch.Frequency / DrawsPerSubmit; + } + } + + microseconds.Sort(); + cpuMicroseconds.Sort(); + var median = microseconds[microseconds.Count / 2]; + Output.WriteLine($"{variation}: wall median {median:F2} us/draw, p25 {microseconds[microseconds.Count / 4]:F2}, min {microseconds[0]:F2}; " + + $"thread cpu median {cpuMicroseconds[cpuMicroseconds.Count / 2]:F2}, p25 {cpuMicroseconds[cpuMicroseconds.Count / 4]:F2}; " + + $"{allocatedBytes / (double)(rounds * DrawsPerSubmit):F0} B/draw"); + results.Add(new + { + variation = variation.ToString(), + draws = DrawsPerSubmit, + rounds, + medianMicrosecondsPerDraw = median, + minMicrosecondsPerDraw = microseconds[0], + p25MicrosecondsPerDraw = microseconds[microseconds.Count / 4], + cpuMedianMicrosecondsPerDraw = cpuMicroseconds[cpuMicroseconds.Count / 2], + cpuP25MicrosecondsPerDraw = cpuMicroseconds[cpuMicroseconds.Count / 4], + maxMicrosecondsPerDraw = microseconds[^1], + allocatedBytesPerDraw = allocatedBytes / (double)(rounds * DrawsPerSubmit), + gen0CollectionsPerThousandDraws = gen0Collections * 1000.0 / (rounds * DrawsPerSubmit), + hostDrawMicrosecondsPerDraw = hostDrawMilliseconds * 1000.0 / (rounds * DrawsPerSubmit), + phaseMicrosecondsPerDraw = phaseTotals.Where(pair => pair.Value / rounds >= 0.05) + .OrderByDescending(pair => pair.Value).ToDictionary(pair => pair.Key, pair => Math.Round(pair.Value / rounds, 2)), + }); + } + + Assert.Empty(rig.Fatals); + File.WriteAllText(path, JsonSerializer.Serialize( + new { arch = RuntimeInformation.ProcessArchitecture.ToString(), profiling, results }, + new JsonSerializerOptions { WriteIndented = true })); + } +} From 7d2dc991a9c373e83e10fc1cdc76b5245e65706b Mon Sep 17 00:00:00 2001 From: kuba Date: Wed, 28 Oct 2026 18:47:29 +0100 Subject: [PATCH 64/82] [GPU] A pipeline key compares only the vertex bindings and attributes it counts. PipelineVertexInputState.Equals compared the whole 32-entry arrays; their records are padded structs, so every per-draw pipeline map hit compared 64 mostly-empty records one by one (32 of the 87 samples in ShaderPipelineCache.CreateGraphicsPipeline on the Silent Hill worker). It now compares the counted prefix, as GetHashCode already hashes it. The per-draw lookup of a 300-pipeline map drops from about 120 ns to about 106 ns (GraphicsPipelineKeyTests.MeasurePipelineKeyLookupWhenRequested, arm64, thread CPU time). --- .../Gpu/Pipelines/PipelineKeys.cs | 6 +- .../Gpu/Pipelines/GraphicsPipelineKeyTests.cs | 124 ++++++++++++++++++ 2 files changed, 129 insertions(+), 1 deletion(-) create mode 100644 tests/SharpEmu.Libs.Tests/Gpu/Pipelines/GraphicsPipelineKeyTests.cs diff --git a/src/SharpEmu.Libs/Gpu/Pipelines/PipelineKeys.cs b/src/SharpEmu.Libs/Gpu/Pipelines/PipelineKeys.cs index 36108bd9e..bef76b863 100644 --- a/src/SharpEmu.Libs/Gpu/Pipelines/PipelineKeys.cs +++ b/src/SharpEmu.Libs/Gpu/Pipelines/PipelineKeys.cs @@ -206,9 +206,13 @@ public void Reset() AttributeCount = 0; } + // Only the counted entries are the state, as in GetHashCode: the arrays are MaxBuffers long and their records + // are not bitwise comparable, so comparing whole arrays made every pipeline lookup compare the empty tail + // one record at a time. public bool Equals(PipelineVertexInputState? other) => other is not null && BindingCount == other.BindingCount && AttributeCount == other.AttributeCount && - Bindings.AsSpan().SequenceEqual(other.Bindings) && Attributes.AsSpan().SequenceEqual(other.Attributes); + Bindings.AsSpan(0, BindingCount).SequenceEqual(other.Bindings.AsSpan(0, BindingCount)) && + Attributes.AsSpan(0, AttributeCount).SequenceEqual(other.Attributes.AsSpan(0, AttributeCount)); public override bool Equals(object? obj) => Equals(obj as PipelineVertexInputState); diff --git a/tests/SharpEmu.Libs.Tests/Gpu/Pipelines/GraphicsPipelineKeyTests.cs b/tests/SharpEmu.Libs.Tests/Gpu/Pipelines/GraphicsPipelineKeyTests.cs new file mode 100644 index 000000000..a7730c874 --- /dev/null +++ b/tests/SharpEmu.Libs.Tests/Gpu/Pipelines/GraphicsPipelineKeyTests.cs @@ -0,0 +1,124 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using System.Collections.Concurrent; +using System.Diagnostics; +using System.Text.Json; +using SharpEmu.Libs.Gpu.Abstractions; +using SharpEmu.Libs.Gpu.Pipelines; +using SharpEmu.PerfAudit; +using Xunit; + +namespace SharpEmu.Libs.Tests.Gpu.Pipelines; + +public sealed class GraphicsPipelineKeyTests +{ + private static GraphicsPipelineKey Key(ulong vertexProgram, uint bindings, uint attributes, uint strideSeed = 16) + { + var key = new GraphicsPipelineKey + { + Rendering = new PipelineRenderingState { ColorCount = 1, DepthFormat = GpuFormat.D32Sfloat }, + VertexInput = new PipelineVertexInputState { BindingCount = (byte)bindings, AttributeCount = (byte)attributes }, + StaticParameters = new PipelineStaticParameters { ColorCount = 1, WithDepth = true }, + VertexProgramId = vertexProgram, + PixelProgramId = vertexProgram * 3, + }; + key.Rendering.ColorFormats[0] = GpuFormat.B8G8R8A8Unorm; + for (var index = 0; index < bindings; index++) + { + key.VertexInput.Bindings[index] = new PipelineVertexBinding(strideSeed * (uint)(index + 1), index == 2); + } + + for (var index = 0; index < attributes; index++) + { + key.VertexInput.Attributes[index] = new PipelineVertexAttribute((uint)index * 4, (byte)(index % Math.Max(1, (int)bindings))); + } + + return key; + } + + [Fact] + public void KeysAreEqualWhenProgramsAndTheUsedVertexInputMatch() + { + Assert.Equal(Key(7, 3, 5), Key(7, 3, 5)); + Assert.Equal(Key(7, 3, 5).GetHashCode(), Key(7, 3, 5).GetHashCode()); + Assert.NotEqual(Key(7, 3, 5), Key(8, 3, 5)); + Assert.NotEqual(Key(7, 3, 5), Key(7, 3, 5, strideSeed: 32)); + Assert.NotEqual(Key(7, 3, 5), Key(7, 2, 5)); + Assert.NotEqual(Key(7, 3, 5), Key(7, 3, 4)); + } + + [Fact] + public void AChangeInTheLastUsedBindingOrAttributeMakesKeysDiffer() + { + var changedBinding = Key(7, 4, 6); + changedBinding.VertexInput.Bindings[3] = new PipelineVertexBinding(999, true); + Assert.NotEqual(Key(7, 4, 6), changedBinding); + var changedAttribute = Key(7, 4, 6); + changedAttribute.VertexInput.Attributes[5] = new PipelineVertexAttribute(999, 3); + Assert.NotEqual(Key(7, 4, 6), changedAttribute); + } + + [Fact] + public void AFilledLookupKeyThatWasReusedMatchesAFreshOne() + { + // The draw path fills one thread-local key for every draw; the entries a larger draw left past the count + // are cleared by Reset and never part of the comparison. + var lookup = Key(7, 6, 6); + lookup.VertexInput.Reset(); + lookup.VertexInput.BindingCount = 2; + lookup.VertexInput.AttributeCount = 3; + var fresh = Key(7, 2, 3); + for (var index = 0; index < 2; index++) lookup.VertexInput.Bindings[index] = fresh.VertexInput.Bindings[index]; + for (var index = 0; index < 3; index++) lookup.VertexInput.Attributes[index] = fresh.VertexInput.Attributes[index]; + Assert.Equal(fresh.VertexInput, lookup.VertexInput); + Assert.Equal(fresh.VertexInput.GetHashCode(), lookup.VertexInput.GetHashCode()); + } + + // SHARPEMU_PIPELINE_KEY_BENCH_OUTPUT=: the cost of the per-draw pipeline map lookup - a reused lookup key + // against a map of a few hundred pipelines, the way ShaderPipelineCache.CreateGraphicsPipeline probes it. + [Fact] + public void MeasurePipelineKeyLookupWhenRequested() + { + var path = Environment.GetEnvironmentVariable("SHARPEMU_PIPELINE_KEY_BENCH_OUTPUT"); + if (path is null) return; + var map = new ConcurrentDictionary(); + for (var index = 0; index < 300; index++) map[Key((ulong)index + 1, (uint)(index % 4) + 1, (uint)(index % 5) + 2)] = index; + var results = new List(); + foreach (var (bindings, attributes) in new[] { (2u, 3u), (4u, 6u) }) + { + var lookup = Key(150, bindings, attributes); + const int iterations = 2_000_000; + var samples = new List(); + long sum = 0; + for (var round = 0; round < 12; round++) + { + var started = OperatingSystem.IsMacOS() ? DarwinThreadCpuClock.Nanoseconds() : 0; + var wall = Stopwatch.GetTimestamp(); + for (var iteration = 0; iteration < iterations; iteration++) + { + lookup.VertexInput.Reset(); + lookup.VertexProgramId = (ulong)(iteration % 300) + 1; + lookup.PixelProgramId = lookup.VertexProgramId * 3; + lookup.VertexInput.BindingCount = (byte)((lookup.VertexProgramId - 1) % 4 + 1); + lookup.VertexInput.AttributeCount = (byte)((lookup.VertexProgramId - 1) % 5 + 2); + for (var index = 0; index < lookup.VertexInput.BindingCount; index++) + lookup.VertexInput.Bindings[index] = new PipelineVertexBinding(16u * (uint)(index + 1), index == 2); + for (var index = 0; index < lookup.VertexInput.AttributeCount; index++) + lookup.VertexInput.Attributes[index] = new PipelineVertexAttribute((uint)index * 4, (byte)(index % lookup.VertexInput.BindingCount)); + sum += map.TryGetValue(lookup, out var found) ? found : -1; + } + + var elapsed = OperatingSystem.IsMacOS() + ? (DarwinThreadCpuClock.Nanoseconds() - started) / (double)iterations + : Stopwatch.GetElapsedTime(wall).TotalNanoseconds / iterations; + if (round >= 2) samples.Add(elapsed); + } + + samples.Sort(); + results.Add(new { bindings, attributes, medianNanosecondsPerLookup = samples[samples.Count / 2], minNanosecondsPerLookup = samples[0], sum }); + } + + File.WriteAllText(path, JsonSerializer.Serialize(results, new JsonSerializerOptions { WriteIndented = true })); + } +} From d5314cfdc066aa403fca388e7b70a8818f2f8a7e Mon Sep 17 00:00:00 2001 From: kuba Date: Wed, 28 Oct 2026 19:00:50 +0100 Subject: [PATCH 65/82] [GPU] The draw record bench can sample larger linear images. SHARPEMU_DRAW_RECORD_BENCH_TEXTURE= sets the edge of the three sampled images (8 by default). Edges of 512 and 1024 change the recording cost by less than the run-to-run noise, so the draw path does not scale with image size. --- .../Scenes/DrawRecordBenchTests.cs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/SharpEmu.Gpu.Metal.Tests/Scenes/DrawRecordBenchTests.cs b/tests/SharpEmu.Gpu.Metal.Tests/Scenes/DrawRecordBenchTests.cs index dd2b428db..d4957ad80 100644 --- a/tests/SharpEmu.Gpu.Metal.Tests/Scenes/DrawRecordBenchTests.cs +++ b/tests/SharpEmu.Gpu.Metal.Tests/Scenes/DrawRecordBenchTests.cs @@ -19,7 +19,9 @@ namespace SharpEmu.Gpu.Metal.Tests.Scenes; public sealed class DrawRecordBenchTests(ITestOutputHelper output) : SceneTestBase(output) { private const uint Rgba8 = 56; - private const uint TextureSize = 8; + // SHARPEMU_DRAW_RECORD_BENCH_TEXTURE= samples edge x edge linear images (8 by default; a title's are megabytes). + private static readonly uint TextureSize = + uint.TryParse(Environment.GetEnvironmentVariable("SHARPEMU_DRAW_RECORD_BENCH_TEXTURE"), out var edge) ? edge : 8; private const uint TargetSize = 64; private const int DrawsPerSubmit = 256; @@ -62,9 +64,10 @@ public sealed class DrawRecordBenchTests(ITestOutputHelper output) : SceneTestBa private static ulong SolidTexture(SceneRig rig, uint abgr) { - var texels = new uint[TextureSize * 64]; + var pitchTexels = SceneDraws.LinearPitchBytes(TextureSize, 4) / 4; + var texels = new uint[pitchTexels * TextureSize]; Array.Fill(texels, abgr); - var address = rig.Memory.Allocate(TextureSize * 64 * 4, 0x10000); + var address = rig.Memory.Allocate((ulong)texels.Length * 4, 0x10000); rig.Memory.Write(address, MemoryMarshal.AsBytes(texels.AsSpan())); return address; } From 550395ce8e716108a9f4038edab6125545cc32ac Mon Sep 17 00:00:00 2001 From: kuba Date: Wed, 28 Oct 2026 19:48:55 +0100 Subject: [PATCH 66/82] [GPU] The draw record bench pre-fills a title's state sizes and records an occlusion dump after every draw. SHARPEMU_DRAW_RECORD_BENCH_FILL=images=,windows=,overlay=,writes=,dccfills= fills the cached images, the guest windows the draws fetch from, the pending command-processor writes, the GPU write ranges and the recorded DCC fills before the measured rounds. SHARPEMU_SCENE_MEMORY_MB sizes the scene's block (mapped anonymously above 64 MB) so a few hundred windows fit, and ..._DRAWS sets the draws per submission. The new Production variation is Mixed plus a dump per draw and a label every eighth. --- .../Scenes/DrawRecordBenchTests.cs | 132 ++++++++++++++++-- .../Scenes/SceneMemory.cs | 32 ++++- 2 files changed, 145 insertions(+), 19 deletions(-) diff --git a/tests/SharpEmu.Gpu.Metal.Tests/Scenes/DrawRecordBenchTests.cs b/tests/SharpEmu.Gpu.Metal.Tests/Scenes/DrawRecordBenchTests.cs index d4957ad80..8b9219aaf 100644 --- a/tests/SharpEmu.Gpu.Metal.Tests/Scenes/DrawRecordBenchTests.cs +++ b/tests/SharpEmu.Gpu.Metal.Tests/Scenes/DrawRecordBenchTests.cs @@ -23,7 +23,20 @@ public sealed class DrawRecordBenchTests(ITestOutputHelper output) : SceneTestBa private static readonly uint TextureSize = uint.TryParse(Environment.GetEnvironmentVariable("SHARPEMU_DRAW_RECORD_BENCH_TEXTURE"), out var edge) ? edge : 8; private const uint TargetSize = 64; - private const int DrawsPerSubmit = 256; + // SHARPEMU_DRAW_RECORD_BENCH_DRAWS= draws per submission (256 by default). + private static readonly int DrawsPerSubmit = + int.TryParse(Environment.GetEnvironmentVariable("SHARPEMU_DRAW_RECORD_BENCH_DRAWS"), out var drawCount) && drawCount > 0 ? drawCount : 256; + + // SHARPEMU_DRAW_RECORD_BENCH_FILL=images=760,windows=300,overlay=400,writes=400,dccfills=500 pre-fills the host's + // state to a title's size before the draws are measured: cached images over distinct 1 MiB-spaced ranges, guest + // windows the draws' vertex buffers are spread over (needs SHARPEMU_SCENE_MEMORY_MB of about 33 per window), + // pending command-processor writes, GPU write ranges, and recorded DCC fills. + private static readonly Dictionary Fill = + (Environment.GetEnvironmentVariable("SHARPEMU_DRAW_RECORD_BENCH_FILL") ?? "") + .Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries) + .Select(item => item.Split('=')) + .Where(parts => parts.Length == 2 && int.TryParse(parts[1], out _)) + .ToDictionary(parts => parts[0], parts => int.Parse(parts[1])); // Three images sampled through one sampler and summed (T# s[0:7], s[8:15], s[16:23], S# s[24:27]; the // primitive mask is s28), the way a material pass reads several textures. @@ -74,7 +87,16 @@ private static ulong SolidTexture(SceneRig rig, uint abgr) // Same: one state, 256 draws. AlternatingTexture: texture 0 flips every draw. AlternatingProgram: sixteen // pixel programs in turn. PassChurn: two colour targets, a new render pass every four draws. Mixed: all three. - private enum Variation { Same, AlternatingTexture, AlternatingProgram, PassChurn, Mixed } + // Production: Mixed with an occlusion dump after every draw and an end-of-pipe label after every eighth, as a + // title's frame has them. + private enum Variation { Same, AlternatingTexture, AlternatingProgram, PassChurn, Mixed, Production } + + private static readonly FieldInfo CpOverlayField = + typeof(MetalRenderHost).GetField("_cpOverlay", BindingFlags.Instance | BindingFlags.NonPublic)!; + private static readonly FieldInfo GpuWritesField = + typeof(MetalRenderHost).GetField("_gpuWrites", BindingFlags.Instance | BindingFlags.NonPublic)!; + private static readonly MethodInfo RecordDccFill = + typeof(MetalImageCache).GetMethod("RecordDccFillLocked", BindingFlags.Instance | BindingFlags.NonPublic)!; // SHARPEMU_DRAW_RECORD_BENCH_OUTPUT=: microseconds per recorded draw for each variation, and // with SHARPEMU_PROFILE_RENDER=1 the self time per render phase. Draws record on this thread @@ -92,6 +114,14 @@ public void MeasureDrawRecordWhenRequested() var depth = rig.Memory.Allocate(TargetSize * TargetSize * 4 + 0x10000, 0x10000); var textures = Enumerable.Range(0, 10).Select(index => SolidTexture(rig, 0x80000040u + (uint)index * 0x00101000u)).ToArray(); var meshData = rig.UploadMesh(SceneDraws.TexturedQuad()); + // One mesh per guest window (32 MiB granules): draw n fetches from window n modulo the count. + var windowMeshes = Enumerable.Range(0, Fill.GetValueOrDefault("windows")).Select(_ => + { + rig.Memory.Allocate(1, 32UL * 1024 * 1024); + return rig.UploadMesh(SceneDraws.TexturedQuad()); + }).ToArray(); + var queries = rig.Memory.Allocate((ulong)DrawsPerSubmit * 256, 256); + var labels = rig.Memory.Allocate((ulong)DrawsPerSubmit * 16, 256); var vertexProgram = rig.Program(ScenePrograms.FetchVertex); var programs = Enumerable.Range(0, 16).Select(count => rig.Program(SampledThreeNops(count))).ToArray(); var rounds = int.TryParse(Environment.GetEnvironmentVariable("SHARPEMU_DRAW_RECORD_BENCH_ROUNDS"), out var configured) ? configured : 60; @@ -101,16 +131,18 @@ public void MeasureDrawRecordWhenRequested() uint[] Stream(Variation variation) { - var textureChanges = variation is Variation.AlternatingTexture or Variation.Mixed; - var programChanges = variation is Variation.AlternatingProgram or Variation.Mixed; - var passChanges = variation is Variation.PassChurn or Variation.Mixed; - var passLength = variation == Variation.Mixed ? 8 : 4; + var mixed = variation is Variation.Mixed or Variation.Production; + var textureChanges = variation is Variation.AlternatingTexture || mixed; + var programChanges = variation is Variation.AlternatingProgram || mixed; + var passChanges = variation is Variation.PassChurn || mixed; + var passLength = mixed ? 8 : 4; var pm4 = new ScenePm4(); for (var draw = 0; draw < DrawsPerSubmit; draw++) { - var texture = textures[textureChanges ? (variation == Variation.Mixed ? draw % 7 : draw & 1) : 0]; - var program = programs[programChanges ? (variation == Variation.Mixed ? draw % 5 : draw) % programs.Length : 0]; + var texture = textures[textureChanges ? (mixed ? draw % 7 : draw & 1) : 0]; + var program = programs[programChanges ? (mixed ? draw % 5 : draw) % programs.Length : 0]; var color = colors[passChanges ? draw / passLength & 1 : 0]; + var mesh = windowMeshes.Length != 0 ? windowMeshes[draw % windowMeshes.Length] : meshData; uint[] pixelData = [ .. Descriptors.Texture(texture, Rgba8, TextureSize, TextureSize), @@ -120,36 +152,106 @@ .. Descriptors.Sampler(), ]; if (draw == 0 || (passChanges && draw % passLength == 0)) { - new SceneState(TargetSize, TargetSize) + var full = new SceneState(TargetSize, TargetSize) .ColorTarget(0, color, TargetSize, TargetSize, format: 14, numberType: 7) .DepthTarget(depth, TargetSize, TargetSize, zFormat: 3) .DepthControl(test: true, write: true, CompareFunc.Always) - .VertexShader(vertexProgram, meshData) - .PixelShader(program, pixelData) - .Emit(pm4); + .VertexShader(vertexProgram, mesh) + .PixelShader(program, pixelData); + if (variation == Variation.Production) full.Context[Cx.DbCountControl] = 1u << 8; + full.Emit(pm4); } - else if (textureChanges || programChanges) + else if (textureChanges || programChanges || windowMeshes.Length != 0) { // A guest rewrites the pixel stage's registers when its material changes, nothing else. var changed = new SceneState(TargetSize, TargetSize); changed.Context.Clear(); changed.Uconfig.Clear(); changed.Shader.Clear(); + if (windowMeshes.Length != 0) changed.VertexShader(vertexProgram, mesh); changed.PixelShader(program, pixelData).Emit(pm4); } pm4.NumInstances(1); pm4.DrawAuto(6); + if (variation != Variation.Production) continue; + pm4.EventWithAddress(ScenePm4.EventPixelPipeStatDump, 1, queries + (ulong)draw * 256); + if (draw % 8 == 7) pm4.Packet(0x49, 0x28u | (5u << 8), 1u << 29, (uint)(labels + (ulong)draw * 16), (uint)((labels + (ulong)draw * 16) >> 32), (uint)draw, 0, 0); } return pm4.ToArray(); } - var results = new List(); ulong submissionId = 1000; + if (Fill.GetValueOrDefault("images") is > 0 and var imageCount) + { + // Sampled once so each becomes a cached image; 1 MiB apart, which is how a title's few-megabyte targets sit. + var fillTextures = Enumerable.Range(0, imageCount).Select(index => + { + rig.Memory.Allocate(1, 1UL << 20); + return SolidTexture(rig, 0x80000040u + (uint)index * 0x1010u); + }).ToArray(); + var fillStream = new ScenePm4(); + for (var index = 0; index < imageCount; index += 3) + { + uint[] pixelData = + [ + .. Descriptors.Texture(fillTextures[index], Rgba8, TextureSize, TextureSize), + .. Descriptors.Texture(fillTextures[(index + 1) % imageCount], Rgba8, TextureSize, TextureSize), + .. Descriptors.Texture(fillTextures[(index + 2) % imageCount], Rgba8, TextureSize, TextureSize), + .. Descriptors.Sampler(), + ]; + var state = new SceneState(TargetSize, TargetSize); + if (index == 0) + { + state.ColorTarget(0, colors[0], TargetSize, TargetSize, format: 14, numberType: 7) + .DepthTarget(depth, TargetSize, TargetSize, zFormat: 3) + .DepthControl(test: true, write: true, CompareFunc.Always) + .VertexShader(vertexProgram, meshData); + } + else + { + state.Context.Clear(); + state.Uconfig.Clear(); + state.Shader.Clear(); + } + + state.PixelShader(programs[0], pixelData).Emit(fillStream); + fillStream.NumInstances(1); + fillStream.DrawAuto(6); + } + + for (var pass = 0; pass < 2; pass++) rig.Submit(fillStream); + Output.WriteLine($"fill: {imageCount} images, host reports {rig.Host.ImageCache.ImageCount} cached"); + } + + // 16 MiB each for the overlay entries, the write ranges and the DCC fills, away from every address a draw uses. + var fillRegion = rig.Memory.Allocate(3UL << 24, 1UL << 20); + const ulong NeverCompleted = ulong.MaxValue / 2; + var overlayCount = Fill.GetValueOrDefault("overlay"); + if (overlayCount > 0) + { + var overlay = (MetalPendingWriteOverlay)CpOverlayField.GetValue(rig.Host)!; + Span value = stackalloc byte[8]; + for (var index = 0; index < overlayCount; index++) overlay.Add(fillRegion + (ulong)index * 256, value, NeverCompleted); + } + + var writeCount = Fill.GetValueOrDefault("writes"); + if (writeCount > 0) + { + var writes = (MetalGpuWriteRanges)GpuWritesField.GetValue(rig.Host)!; + for (var index = 0; index < writeCount; index++) + writes.Note(fillRegion + (1UL << 24) + (ulong)index * 0x2000, 0x1000, NeverCompleted + (ulong)index); + } + + for (var index = 0; index < Fill.GetValueOrDefault("dccfills"); index++) + RecordDccFill.Invoke(rig.Host.ImageCache, [fillRegion + (2UL << 24) + (ulong)index * 0x2000, 0x1000UL, (byte)0, true]); + + var results = new List(); var profiling = RenderPhaseProfile.Enabled; foreach (var variation in Enum.GetValues().Where(variation => only is null || variation.ToString() == only)) { + rig.Host.NativeOcclusionEnabled = variation == Variation.Production; var words = Stream(variation); // The stream is placed once: every round submits the same packets. var streamAddress = rig.Memory.Upload(words, 256); @@ -232,7 +334,7 @@ .. Descriptors.Sampler(), Assert.Empty(rig.Fatals); File.WriteAllText(path, JsonSerializer.Serialize( - new { arch = RuntimeInformation.ProcessArchitecture.ToString(), profiling, results }, + new { arch = RuntimeInformation.ProcessArchitecture.ToString(), profiling, fill = Fill, results }, new JsonSerializerOptions { WriteIndented = true })); } } diff --git a/tests/SharpEmu.Gpu.Metal.Tests/Scenes/SceneMemory.cs b/tests/SharpEmu.Gpu.Metal.Tests/Scenes/SceneMemory.cs index 6b5fbec9f..581ec1f6e 100644 --- a/tests/SharpEmu.Gpu.Metal.Tests/Scenes/SceneMemory.cs +++ b/tests/SharpEmu.Gpu.Metal.Tests/Scenes/SceneMemory.cs @@ -29,13 +29,37 @@ internal sealed class SceneAddressSpace : IGuestAddressSpace internal sealed unsafe class SceneMemory : IGuestBackedSpace, ICpuMemory, IDisposable { public const ulong Base = 0x2_0000_0000; - public const ulong Size = 64 * 1024 * 1024; - private readonly void* _block = NativeMemory.AlignedAlloc((nuint)Size, 16384); + // SHARPEMU_SCENE_MEMORY_MB= sizes the block (64 by default). A larger one is mapped anonymously, so the + // pages a scene never touches cost nothing, and it spans as many guest windows as a title's address table does. + private const ulong DefaultSize = 64 * 1024 * 1024; + public static readonly ulong Size = + (ulong.TryParse(Environment.GetEnvironmentVariable("SHARPEMU_SCENE_MEMORY_MB"), out var megabytes) && megabytes > 64 ? megabytes * 1024 * 1024 : DefaultSize); + private readonly void* _block = Size > DefaultSize ? Map(Size) : NativeMemory.AlignedAlloc((nuint)Size, 16384); private ulong _next = Base; - public SceneMemory() => NativeMemory.Clear(_block, (nuint)Size); + [DllImport("libSystem.dylib", EntryPoint = "mmap")] + private static extern nint mmap(nint address, nuint length, int protection, int flags, int descriptor, long offset); - public void Dispose() => NativeMemory.AlignedFree(_block); + [DllImport("libSystem.dylib", EntryPoint = "munmap")] + private static extern int munmap(nint address, nuint length); + + private static void* Map(ulong size) + { + // PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON. + var block = mmap(0, (nuint)size, 3, 0x1002, -1, 0); + return block == -1 ? throw new InvalidOperationException($"Scene memory of 0x{size:X} bytes could not be mapped.") : (void*)block; + } + + public SceneMemory() + { + if (Size <= DefaultSize) NativeMemory.Clear(_block, (nuint)Size); + } + + public void Dispose() + { + if (Size > DefaultSize) munmap((nint)_block, (nuint)Size); + else NativeMemory.AlignedFree(_block); + } /// Hands out guest memory from a bump allocator. public ulong Allocate(ulong size, ulong alignment = 256) From 438d7e4cb32000a1725b0c7b8de374f2634b9595 Mon Sep 17 00:00:00 2001 From: kuba Date: Wed, 28 Oct 2026 19:48:55 +0100 Subject: [PATCH 67/82] [GPU] A recorded GPU write forgets pending overlay bytes without allocating and moves the overlay version only when an entry changed. The Dead array every partly covered entry allocated is a bit mask for entries up to 64 bytes. The three forgets that a GPU write range note follows (a store, a copy run, an occlusion dump) bump the overlay version only when they removed or narrowed an entry; host writes keep the unconditional bump, since nothing else tells a cache of memory a host store happened. SHARPEMU_METAL_V2_OVERLAY_FORGET_VERSION=0 bumps on every forget again. --- .../V2/MetalCommandStream.cs | 4 +- .../V2/MetalPendingWriteOverlay.cs | 56 +++++++++++++++++-- .../V2/MetalRenderHost.Occlusion.cs | 4 +- .../SharpEmu.Gpu.Metal/V2/MetalRenderHost.cs | 4 +- .../MetalPendingWriteOverlayTests.cs | 50 +++++++++++++++++ 5 files changed, 107 insertions(+), 11 deletions(-) diff --git a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalCommandStream.cs b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalCommandStream.cs index 8c12a2686..1155c4f24 100644 --- a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalCommandStream.cs +++ b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalCommandStream.cs @@ -913,8 +913,8 @@ private ulong EncodeOrderedCopies( { low = Math.Min(low, copy.GuestAddress); high = Math.Max(high, copy.GuestAddress + (ulong)copy.Bytes); - _cpOverlay.Forget(copy.GuestAddress, (ulong)copy.Bytes); - _labelOverlay.Forget(copy.GuestAddress, (ulong)copy.Bytes); + _cpOverlay.ForgetRecordedWrite(copy.GuestAddress, (ulong)copy.Bytes); + _labelOverlay.ForgetRecordedWrite(copy.GuestAddress, (ulong)copy.Bytes); } } diff --git a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalPendingWriteOverlay.cs b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalPendingWriteOverlay.cs index 7944ca990..604c36933 100644 --- a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalPendingWriteOverlay.cs +++ b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalPendingWriteOverlay.cs @@ -54,12 +54,33 @@ private sealed class Entry public ulong Serial; public ulong Sequence; public byte[]? Baseline; + // The bytes a later write covers: a bit per byte for an entry of up to 64 bytes (a dword, a qword, a + // label: all of them in practice), so a partial forget allocates nothing; a flag array for a longer one. + public ulong DeadMask; public bool[]? Dead; public bool Removed; public ulong End => Address + (ulong)Bytes.Length; + + public bool IsDead(ulong offset) => + Dead is null ? offset < 64 && ((DeadMask >> (int)offset) & 1) != 0 : Dead[offset]; + + public void MarkDead(int start, int count) + { + if (Bytes.Length <= 64) + { + DeadMask |= (count == 64 ? ulong.MaxValue : (1UL << count) - 1) << start; + return; + } + + (Dead ??= new bool[Bytes.Length]).AsSpan(start, count).Fill(true); + } } + // SHARPEMU_METAL_V2_OVERLAY_FORGET_VERSION=0 moves the version on every forget again, as before. + private static readonly bool ForgetBumpsOnlyOnChange = + SharpEmu.ShaderCompiler.PerfSwitches.Enabled("SHARPEMU_METAL_V2_OVERLAY_FORGET_VERSION"); + // A command-processor write is almost always a dword or a qword, and one entry with its // byte array per write was garbage the worker made at every draw's occlusion dump. Entries // of up to MaxPooledBytes are kept, by size, once they have left the page lists and the @@ -100,6 +121,7 @@ private void RecycleEntry(Entry entry) entry.Baseline = null; entry.Dead = null; + entry.DeadMask = 0; entry.Removed = false; _spareEntries[length >> 2].Push(entry); _spareEntryCount++; @@ -207,6 +229,27 @@ public void Retire(ulong completedSerial) public void Forget(ulong address, ulong size) { Interlocked.Increment(ref _version); + ForgetCore(address, size); + } + + /// + /// for a store the GPU write ranges are told of right after (so their own version + /// moves for it): this overlay's version moves only when an entry stopped speaking for some of the + /// range. A forget that finds nothing there leaves what a read through the overlay answers as it was, + /// and a cache keyed on the version need not drop what it holds on every dump. + /// + public void ForgetRecordedWrite(ulong address, ulong size) + { + if (!ForgetBumpsOnlyOnChange) + { + Interlocked.Increment(ref _version); + } + + ForgetCore(address, size); + } + + private void ForgetCore(ulong address, ulong size) + { if (size == 0 || Count == 0 || !MayOverlap(address, size)) { return; @@ -328,7 +371,7 @@ public void Apply(ulong address, Span destination) var end = Math.Min(address + size, entry.End); for (var at = start; at < end; at++) { - if (entry.Dead is null || !entry.Dead[at - entry.Address]) + if (!entry.IsDead(at - entry.Address)) { destination[(int)(at - address)] = entry.Bytes[at - entry.Address]; } @@ -517,7 +560,7 @@ public void Visit(Entry entry) var end = Math.Min(address + size, entry.End); for (var at = start; at < end; at++) { - if (entry.Dead is null || !entry.Dead[at - entry.Address]) + if (!entry.IsDead(at - entry.Address)) { covered[at - address] = true; Any = true; @@ -549,14 +592,17 @@ public void Visit(Entry entry) { var start = Math.Max(address, entry.Address); var end = Math.Min(address + size, entry.End); + // After the change, under the gate: a version a reader sees moved came with the entry's new state. if (start <= entry.Address && end >= entry.End) { owner.RemoveLocked(entry); - return; + } + else + { + entry.MarkDead((int)(start - entry.Address), (int)(end - start)); } - entry.Dead ??= new bool[entry.Bytes.Length]; - entry.Dead.AsSpan((int)(start - entry.Address), (int)(end - start)).Fill(true); + Interlocked.Increment(ref owner._version); } } diff --git a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.Occlusion.cs b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.Occlusion.cs index 1bc1da6ec..d38b8ba51 100644 --- a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.Occlusion.cs +++ b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.Occlusion.cs @@ -268,8 +268,8 @@ public bool DumpOcclusionCounter(int queueId, ulong address, uint instanceMask, if (!CoalesceOcclusionDumps) { // The note below forgets them as well; this is the order before it did. - _cpOverlay.Forget(destinationAddress, sizeof(ulong)); - _labelOverlay.Forget(destinationAddress, sizeof(ulong)); + _cpOverlay.ForgetRecordedWrite(destinationAddress, sizeof(ulong)); + _labelOverlay.ForgetRecordedWrite(destinationAddress, sizeof(ulong)); } } if (batch is not null) diff --git a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.cs b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.cs index b614b9845..e8a2c92b7 100644 --- a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.cs +++ b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.cs @@ -1236,8 +1236,8 @@ public void NoteGuestMemoryWrite(ulong address, ulong size, MetalGpuWriteOrigin // of them has to see from now on, whenever it lands. using (var overlayScope = SharpEmu.Libs.VideoOut.RenderPhaseProfile.MeasureDetail(SharpEmu.Libs.VideoOut.RenderPhaseProfile.Phase.WriteNoteOverlays)) { - _cpOverlay.Forget(address, size); - _labelOverlay.Forget(address, size); + _cpOverlay.ForgetRecordedWrite(address, size); + _labelOverlay.ForgetRecordedWrite(address, size); } NoteGuestMemoryWriteBeyondOverlays(address, size, origin); diff --git a/tests/SharpEmu.Gpu.Metal.Tests/MetalPendingWriteOverlayTests.cs b/tests/SharpEmu.Gpu.Metal.Tests/MetalPendingWriteOverlayTests.cs index fdba7587c..f59e1f0d2 100644 --- a/tests/SharpEmu.Gpu.Metal.Tests/MetalPendingWriteOverlayTests.cs +++ b/tests/SharpEmu.Gpu.Metal.Tests/MetalPendingWriteOverlayTests.cs @@ -95,6 +95,56 @@ public void ALaterWriteOverTheRangeIsForgotten(bool pageFilter) Assert.Equal(MetalOverlayCoverage.None, overlay.Coverage(0x7000_0010, 4, () => 0)); } + [Theory] + [InlineData(true)] + [InlineData(false)] + public void AForgetOfARecordedWriteMovesTheVersionOnlyWhenAnEntryChanged(bool pageFilter) + { + var overlay = new MetalPendingWriteOverlay(pageFilter); + overlay.Add(0x7000_0010, [1, 2, 3, 4], serial: 5); + + var version = overlay.Version; + overlay.ForgetRecordedWrite(0x7000_0010 + Page, 4); + overlay.ForgetRecordedWrite(0x7000_0000, 8); + Assert.Equal(version, overlay.Version); + + overlay.ForgetRecordedWrite(0x7000_0012, 2); + Assert.NotEqual(version, overlay.Version); + Assert.Equal(MetalOverlayCoverage.Partial, overlay.Coverage(0x7000_0010, 4, () => 0)); + Span bytes = [9, 9, 9, 9]; + overlay.Apply(0x7000_0010, bytes); + Assert.Equal([1, 2, 9, 9], bytes.ToArray()); + + version = overlay.Version; + overlay.ForgetRecordedWrite(0x7000_0000, 0x100); + Assert.NotEqual(version, overlay.Version); + Assert.Equal(0, overlay.Count); + + // The unconditional forget still moves it for a range that holds nothing. + version = overlay.Version; + overlay.Forget(0x7000_0000, 8); + Assert.NotEqual(version, overlay.Version); + } + + [Fact] + public void ALongEntryForgetsOnlyTheBytesALaterWriteCovers() + { + var overlay = new MetalPendingWriteOverlay(); + overlay.Add(0x7000_0000, new byte[100].Select((_, index) => (byte)(index + 1)).ToArray(), serial: 5); + + overlay.ForgetRecordedWrite(0x7000_0000 + 10, 4); + overlay.ForgetRecordedWrite(0x7000_0000 + 70, 4); + var bytes = new byte[100]; + overlay.Apply(0x7000_0000, bytes); + for (var index = 0; index < 100; index++) + { + var forgotten = index is >= 10 and < 14 or >= 70 and < 74; + Assert.Equal(forgotten ? 0 : index + 1, bytes[index]); + } + + Assert.Equal(MetalOverlayCoverage.Partial, overlay.Coverage(0x7000_0000, 100, () => 0)); + } + [Fact] public void PagesThatShareACountStillAnswerOnlyForTheirOwnWrites() { From bddded97cf00bfe45163f6d14ed333b62bf7e8af Mon Sep 17 00:00:00 2001 From: kuba Date: Wed, 28 Oct 2026 19:49:00 +0100 Subject: [PATCH 68/82] [GPU] Recorded DCC fills are found by the page they start on instead of by a walk over all of them. Every GPU store and every slice lookup enumerated the whole recorded-fill dictionary; a fill-start page count plus the largest fill size now decides that a store reaches none. SHARPEMU_METAL_V2_DCC_CLAIM_INDEX=0 walks them all again. --- .../V2/MetalImageCache.Tracking.cs | 93 ++++++++++++++++++- 1 file changed, 88 insertions(+), 5 deletions(-) diff --git a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalImageCache.Tracking.cs b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalImageCache.Tracking.cs index 8194a88e0..94a571842 100644 --- a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalImageCache.Tracking.cs +++ b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalImageCache.Tracking.cs @@ -868,6 +868,11 @@ private bool GpuModifiedImageOverlapsLocked(ulong address, ulong size) // Whether any recorded fill reaches the slice. Caller holds the lock. private bool RecordedDccFillOverlapsLocked(ulong address, ulong size) { + if (!MayHoldRecordedDccFill(address, size)) + { + return false; + } + foreach (var (start, fill) in _recordedDccFills) { if (address < start + fill.Size && start < address + size) @@ -1320,6 +1325,79 @@ private struct RecordedDccFill public bool AwaitingOwnWrite; } + // The recorded fills by the 4 KiB page their start is on, and the largest one ever recorded: a store + // reaches a fill only if the fill starts less than that size below the store's first byte, or inside + // the store, so a store over pages none of those starts is on reaches none, without the walk over every + // fill that each GPU store and each slice lookup made (a frame leaves hundreds). The sizes only grow, + // so the bound only loosens. SHARPEMU_METAL_V2_DCC_CLAIM_INDEX=0 walks them all, as before. + private readonly Dictionary _recordedDccFillPages = new(); + private ulong _recordedDccFillMaxSize; + + // Caller holds the lock. + private void SetRecordedDccFillLocked(ulong address, RecordedDccFill fill) + { + if (_recordedDccFills.ContainsKey(address)) + { + UnindexRecordedDccFillLocked(address); + } + + _recordedDccFills[address] = fill; + System.Runtime.InteropServices.CollectionsMarshal.GetValueRefOrAddDefault( + _recordedDccFillPages, address >> DccClaimPageShift, out _)++; + _recordedDccFillMaxSize = Math.Max(_recordedDccFillMaxSize, fill.Size); + } + + // Caller holds the lock. + private void RemoveRecordedDccFillLocked(ulong address) + { + if (_recordedDccFills.Remove(address)) + { + UnindexRecordedDccFillLocked(address); + } + } + + private void UnindexRecordedDccFillLocked(ulong address) + { + var page = address >> DccClaimPageShift; + ref var count = ref System.Runtime.InteropServices.CollectionsMarshal.GetValueRefOrNullRef(_recordedDccFillPages, page); + if (!System.Runtime.CompilerServices.Unsafe.IsNullRef(ref count) && --count <= 0) + { + _recordedDccFillPages.Remove(page); + } + } + + // Whether a recorded fill may reach the range; false only when none does. Caller holds the lock. + private bool MayHoldRecordedDccFill(ulong address, ulong size) + { + if (_recordedDccFills.Count == 0) + { + return false; + } + + if (!DccClaimIndexEnabled || size == 0 || address + (size - 1) < address) + { + return true; + } + + var lowestStart = address >= _recordedDccFillMaxSize ? address - (_recordedDccFillMaxSize - 1) : 0; + var first = lowestStart >> DccClaimPageShift; + var last = (address + (size - 1)) >> DccClaimPageShift; + if (last - first >= DccClaimIndexMaxPages) + { + return true; + } + + for (var page = first; page <= last; page++) + { + if (_recordedDccFillPages.ContainsKey(page)) + { + return true; + } + } + + return false; + } + /// Caller holds the lock. private void RecordDccFillLocked(ulong address, ulong size, byte code, bool record) { @@ -1327,7 +1405,7 @@ private void RecordDccFillLocked(ulong address, ulong size, byte code, bool reco if (record) { ForgetDccRunMarkersLocked(null, address, size); - _recordedDccFills[address] = new RecordedDccFill { Size = size, Code = code, AwaitingOwnWrite = true }; + SetRecordedDccFillLocked(address, new RecordedDccFill { Size = size, Code = code, AwaitingOwnWrite = true }); Epochs.NoteDccChanged(); } } @@ -1340,7 +1418,7 @@ private void RecordDccFillLocked(ulong address, ulong size, byte code, bool reco /// private void NoteRecordedDccStoreLocked(ulong address, ulong size) { - if (_recordedDccFills.Count == 0) + if (!MayHoldRecordedDccFill(address, size)) { return; } @@ -1361,7 +1439,7 @@ private void NoteRecordedDccStoreLocked(ulong address, ulong size) continue; } - _recordedDccFills.Remove(start); + RemoveRecordedDccFillLocked(start); Epochs.NoteDccChanged(); } } @@ -1369,7 +1447,7 @@ private void NoteRecordedDccStoreLocked(ulong address, ulong size) /// Caller holds the lock. private void ForgetRecordedDccFillsLocked(ulong address, ulong size) { - if (_recordedDccFills.Count == 0) + if (!MayHoldRecordedDccFill(address, size)) { return; } @@ -1379,7 +1457,7 @@ private void ForgetRecordedDccFillsLocked(ulong address, ulong size) { if (address < start + fill.Size && start < address + size) { - _recordedDccFills.Remove(start); + RemoveRecordedDccFillLocked(start); Epochs.NoteDccChanged(); } } @@ -1394,6 +1472,11 @@ private void ForgetRecordedDccFillsLocked(ulong address, ulong size) private bool TryFindRecordedDccFillLocked(ulong address, ulong size, out byte code) { code = 0; + if (!MayHoldRecordedDccFill(address, size)) + { + return false; + } + foreach (var (start, fill) in _recordedDccFills) { if (address >= start && address + size <= start + fill.Size) From 0131fb32b63369b9de10f746036bae82c619bb8f Mon Sep 17 00:00:00 2001 From: kuba Date: Wed, 28 Oct 2026 19:49:00 +0100 Subject: [PATCH 69/82] [GPU] A window first touched mid-submission is appended to the resident list instead of rebuilding it. The list was rebuilt, and every buffer in it named again on the encoder, at each of the few hundred first touches of a submission. The list now only grows while its version holds and an encoder is told only the buffers added since its last draw. SHARPEMU_METAL_V2_RESIDENT_APPEND=0 rebuilds on every touch again. --- .../SharpEmu.Gpu.Metal/V2/MetalGuestMemory.cs | 21 ++++++++++++++++--- .../V2/MetalRenderHost.Descriptors.cs | 20 ++++++++++++------ 2 files changed, 32 insertions(+), 9 deletions(-) diff --git a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalGuestMemory.cs b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalGuestMemory.cs index 4fe3392a9..6d69b745a 100644 --- a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalGuestMemory.cs +++ b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalGuestMemory.cs @@ -1239,7 +1239,17 @@ private bool TryResolveWindowCore(ulong gpuVa, ulong size, out MetalGuestRange r if (_touchedWindows.Add(window.Id)) { - _residentDirty = true; + if (ResidentAppendEnabled && !_residentDirty && _residencySet == 0 && window.Buffer != 0) + { + // A first touch adds one buffer to a list that is otherwise current: put it on the end + // (ResidentVersion holds, ResidentBuffers grows) instead of rebuilding every window's + // entry, which a submission touching a few hundred windows did a few hundred times. + Append(ref _submissionResidentBuffers, ref _submissionResidentCount, window.Buffer); + } + else + { + _residentDirty = true; + } } var windowOffset = offset - window.Start; @@ -1828,9 +1838,14 @@ public ReadOnlySpan ResidentBuffers } } + // SHARPEMU_METAL_V2_RESIDENT_APPEND=0 rebuilds the list on every first touch of a window, as before. + private static readonly bool ResidentAppendEnabled = + SharpEmu.ShaderCompiler.PerfSwitches.Enabled("SHARPEMU_METAL_V2_RESIDENT_APPEND"); + /// - /// Changes whenever is rebuilt: the list is the same set while - /// it holds. Read after the list, which is what rebuilds it. + /// Changes whenever is rebuilt: while it holds, the list at any later + /// read starts with the list of an earlier one (a window touched since only adds to its end). + /// Read after the list, which is what rebuilds it. /// public long ResidentVersion => _residentVersion; diff --git a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.Descriptors.cs b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.Descriptors.cs index 345a94470..af5570f8a 100644 --- a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.Descriptors.cs +++ b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.Descriptors.cs @@ -1812,7 +1812,8 @@ private void ResolveDeviceAddressBuffers() private const int ResidentMarkCompute = 1; private const int ResidentMarkDeviceAddress = 2; - private readonly record struct ResidentMark(long Epoch, long Version, nuint Stages) + // Count: how many of the list's buffers the encoder has been told, for a list that only grows while its version holds. + private readonly record struct ResidentMark(long Epoch, long Version, nuint Stages, int Count = 0) { public bool Matches(long epoch, long version, nuint stages) => Epoch == epoch && Version == version && Stages == stages; @@ -1830,14 +1831,21 @@ private void NameSubmissionResidents(nint encoder, nuint stages, int markIndex) var buffers = _guestMemory.ResidentBuffers; var version = _guestMemory.ResidentVersion; ref var mark = ref _residentMarks[markIndex]; - if (ResidentWatermark && mark.Matches(_encoderEpoch, version, stages)) + var named = 0; + if (ResidentWatermark && mark.Matches(_encoderEpoch, version, stages) && mark.Count <= buffers.Length) { - _residentWalksSkipped++; - return; + if (mark.Count == buffers.Length) + { + _residentWalksSkipped++; + return; + } + + // Windows touched since the last draw were added to the end: name those, not every one again. + named = mark.Count; } - NameResidentBuffers(encoder, buffers, stages); - mark = new ResidentMark(_encoderEpoch, version, stages); + NameResidentBuffers(encoder, buffers[named..], stages); + mark = new ResidentMark(_encoderEpoch, version, stages, buffers.Length); } private nint[] _useResourcesScratch = new nint[64]; From 7627a3c1bec95c55e4d7de26fb92ce047b17f8f4 Mon Sep 17 00:00:00 2001 From: kuba Date: Wed, 28 Oct 2026 19:59:27 +0100 Subject: [PATCH 70/82] [GPU] An env-gated bench times translator code shapes against cheaper forms of the same work. Standalone compute micro-kernels (exec guards, mul+add contraction, V_MIN/V_MAX sequences, f16 packing, LDS indexing, bounds-checked buffer loads, device-address resolves) run with the translated-guest compile options; GPU time is the median of interleaved command-buffer GPUStart/End samples. SHARPEMU_MSL_PATTERN_BENCH_OUTPUT=. --- .../MetalMslPatternBenchTests.Kernels.cs | 7037 +++++++++++++++++ .../MetalMslPatternBenchTests.cs | 262 + 2 files changed, 7299 insertions(+) create mode 100644 tests/SharpEmu.Gpu.Metal.Tests/MetalMslPatternBenchTests.Kernels.cs create mode 100644 tests/SharpEmu.Gpu.Metal.Tests/MetalMslPatternBenchTests.cs diff --git a/tests/SharpEmu.Gpu.Metal.Tests/MetalMslPatternBenchTests.Kernels.cs b/tests/SharpEmu.Gpu.Metal.Tests/MetalMslPatternBenchTests.Kernels.cs new file mode 100644 index 000000000..4334d330e --- /dev/null +++ b/tests/SharpEmu.Gpu.Metal.Tests/MetalMslPatternBenchTests.Kernels.cs @@ -0,0 +1,7037 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +namespace SharpEmu.Gpu.Metal.Tests; + +// The micro-kernels MetalMslPatternBenchTests runs. The helpers behind the //@DEV_PRELUDE and +// //@BUF_PRELUDE markers are what the translator emits for a Silent Hill mesh shader +// (Templates/resource_prelude.msl with this title's address-space constants filled in). +public sealed partial class MetalMslPatternBenchTests +{ + private const string DevPrelude = """ +struct Gen5AddressRange +{ + ulong first_page; + ulong page_count; + device uint* base; +}; + +// The range this invocation resolved last. Guest accesses cluster: a loop-invariant +// base read every trip, a stream walking one buffer, a read-modify-write of the word it +// just read. A hit skips the binary search; the ranges cannot change while a dispatch +// runs, and the cache lives only as long as the invocation, so it is never stale. A +// zero page_count matches nothing, which is the empty cache. +struct Gen5RangeCache +{ + ulong first_page; + ulong page_count; + device uint* base; +}; + +// Finds the mapped range of a guest address; a miss records the page in the fault bitmap. +static inline device uint* sharpemu_resolve_device_address(device const Gen5AddressRange* ranges, uint range_count, device uint* faults, uint fault_words, thread Gen5RangeCache& cache, ulong address, thread bool& valid) +{ + ulong masked = address & 0xFFFFFFFFFFFFul; + ulong page = masked >> 14u; + if (page - cache.first_page < cache.page_count) + { + valid = true; + ulong byte_offset = ((page - cache.first_page) << 14u) + (masked & 0x3FFFul); + return (device uint*)((device uchar*)cache.base + byte_offset); + } + + // range_count carries the table's entry count in its low 31 bits and, in bit 31, whether + // the host appended a granule index after the entries. The index narrows the search to + // the entries that can matter for this page; the result is the same upper bound the + // full search finds, and a page outside the index's coverage takes the full search. + uint entry_count = range_count & 0x7FFFFFFFu; + uint low = 0u; + uint high = entry_count; + if ((range_count & 0x80000000u) != 0u) + { + device const uint* index = (device const uint*)(ranges + entry_count); + ulong granule = (page >> index[0]) - (ulong(index[3]) << 32 | ulong(index[2])); + if (granule < ulong(index[1])) + { + uint2 bucket = *(device const uint2*)(index + 4u + (uint)granule * 2u); + low = bucket.x; + high = bucket.y; + } + } + for (uint step = 0u; step < 17u && low < high; step++) + { + uint middle = low + ((high - low) >> 1u); + bool move_up = ranges[middle].first_page <= page; + low = move_up ? middle + 1u : low; + high = move_up ? high : middle; + } + valid = false; + if (low != 0u) + { + Gen5AddressRange range = ranges[low - 1u]; + // The low 32 bits of page_count are the page count. Bit 63 marks a range in a guest + // memory window this submission did not name resident, bits 32..61 its granule. + // The access still goes through - the window's pages are mapped for the life of + // the buffer - and the granule is reported in the feedback words (word 0 counts such + // lookups, then from word 8 one bit per granule) so the host names it from then on. + ulong page_count = range.page_count & 0xFFFFFFFFul; + if (page - range.first_page < page_count) + { + ulong flags = range.page_count >> 62u; + if (flags == 1ul) + { + // Bit 62: the range's backing chunks are single-copy released (their GPU alias + // pages are a hole), bits 32..61 its first backing page. The hit is counted in + // word 1 and its backing page set in the bitmap words 2 and 3 locate, so the + // host gives the chunk back and keeps it. Word 4 set is the shadow mode, where + // nothing was emptied and the access goes through; otherwise it reads zero and + // a store is dropped. Never cached: every access to the range is seen. + uint backing_page = ((uint)(range.page_count >> 32u) & 0x3FFFFFFFu) + (uint)(page - range.first_page); + uint bitmap_word = backing_page >> 5u; + if (bitmap_word < faults[3]) + { + atomic_fetch_or_explicit((device atomic_uint*)(faults + faults[2] + bitmap_word), 1u << (backing_page & 31u), memory_order_relaxed); + } + atomic_fetch_add_explicit((device atomic_uint*)(faults + 1u), 1u, memory_order_relaxed); + if (faults[4] == 0u) + { + return nullptr; + } + valid = true; + ulong trap_offset = ((page - range.first_page) << 14u) + (masked & 0x3FFFul); + return (device uint*)((device uchar*)range.base + trap_offset); + } + if (flags != 0ul) + { + uint granule = (uint)(range.page_count >> 32u) & 0x3FFFFFFFu; + atomic_fetch_or_explicit((device atomic_uint*)(faults + 8u + (granule >> 5u)), 1u << (granule & 31u), memory_order_relaxed); + atomic_fetch_add_explicit((device atomic_uint*)faults, 1u, memory_order_relaxed); + } + valid = true; + cache.first_page = range.first_page; + cache.page_count = page_count; + cache.base = range.base; + ulong byte_offset = ((page - range.first_page) << 14u) + (masked & 0x3FFFul); + return (device uint*)((device uchar*)range.base + byte_offset); + } + } + uint page_index = (uint)page; + uint word = page_index >> 5u; + if (word < fault_words) + { + atomic_fetch_or_explicit((device atomic_uint*)(faults + word), 1u << (page_index & 31u), memory_order_relaxed); + } + return nullptr; +} + +static inline uint sharpemu_load_device_dword(device const Gen5AddressRange* ranges, uint range_count, device uint* faults, uint fault_words, thread Gen5RangeCache& cache, ulong address) +{ + bool valid = false; + device uint* word = sharpemu_resolve_device_address(ranges, range_count, faults, fault_words, cache, address, valid); + return valid ? *word : 0u; +} + +// Resolves the first dword of a contiguous multi-dword read once for the whole span. +// A span that stays inside one 16 KB page is contiguous host memory, so its dwords +// index off the one pointer; one that crosses a page reports invalid and the caller +// reads it dword by dword, which is what every dword would have cost before. +static inline device uint* sharpemu_resolve_device_span(device const Gen5AddressRange* ranges, uint range_count, device uint* faults, uint fault_words, thread Gen5RangeCache& cache, ulong address, uint bytes, thread bool& valid) +{ + device uint* first = sharpemu_resolve_device_address(ranges, range_count, faults, fault_words, cache, address, valid); + valid = valid && (address & 0x3FFFul) + bytes <= 0x4000ul; + return first; +} + +// An unaligned value assembled from the two aligned dwords it spans. +static inline uint sharpemu_load_device_unaligned(device const Gen5AddressRange* ranges, uint range_count, device uint* faults, uint fault_words, thread Gen5RangeCache& cache, ulong address, uint count) +{ + uint alignment = (uint)(address & 3ul); + ulong aligned = address & ~3ul; + uint shift = alignment * 8u; + uint low = sharpemu_load_device_dword(ranges, range_count, faults, fault_words, cache, aligned); + uint high = (alignment + count > 4u) ? sharpemu_load_device_dword(ranges, range_count, faults, fault_words, cache, aligned + 4ul) : 0u; + return (low >> shift) | (shift == 0u ? 0u : (high << (32u - shift))); +} + +"""; + + private const string BufPrelude = """ +static inline uint sharpemu_load_dword(device uint* b, uint bytes, uint addr) +{ + uint index = addr >> 2; + return index < (bytes >> 2) ? b[index] : 0u; +} + +// Up to four consecutive raw dwords of an S_BUFFER_LOAD/BUFFER_LOAD_DWORDXn, each what +// sharpemu_load_dword reads at addr + 4 * i. A run that lies inside the binding is one +// bounds check and adjacent loads; one that crosses the end reads dword by dword. +template +static inline uint4 sharpemu_load_dwords(device uint* b, uint bytes, uint addr) +{ + uint4 result = uint4(0u); + uint index = addr >> 2; + uint words = bytes >> 2; + if (index < words && (uint)N <= words - index) + { + for (int i = 0; i < N; i++) + { + result[i] = b[index + (uint)i]; + } + } + else + { + for (int i = 0; i < N; i++) + { + result[i] = sharpemu_load_dword(b, bytes, addr + (uint)(4 * i)); + } + } + return result; +} + +"""; + + private static readonly Dictionary RawKernels = new() + { + ["buf1_perlane_constant_checked"] = """ +// bench group=buf_one_perlane_constant variant=checked threads=1048576 tg=32 iters=1 check=1 +#include +using namespace metal; +#pragma clang fp contract(off) +//@BUF_PRELUDE_CONSTANT + +static inline uint load1_select(constant uint* b, uint bytes, uint addr) +{ + uint index = addr >> 2; + uint words = bytes >> 2; + uint last = max(words, 1u) - 1u; + uint v = b[index > last ? last : index]; + return index < words ? v : 0u; +} + +kernel void bench(device uint* data [[buffer(0)]], device uint* out [[buffer(1)]], constant uint4& prm [[buffer(2)]], + constant uint* cdata [[buffer(5)]], + uint tid [[thread_position_in_grid]], uint lane [[thread_index_in_simdgroup]]) +{ + constant uint* b = cdata; + uint bytes = prm.y; + uint acc = tid; + acc = acc * 3u + sharpemu_load_dword(b, bytes, (0u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (16384u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (32768u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (49152u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (65536u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (81920u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (98304u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (114688u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (131072u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (147456u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (163840u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (180224u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (196608u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (212992u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (229376u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (245760u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (262144u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (278528u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (294912u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (311296u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (327680u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (344064u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (360448u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (376832u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (393216u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (409600u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (425984u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (442368u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (458752u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (475136u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (491520u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (507904u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (524288u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (540672u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (557056u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (573440u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (589824u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (606208u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (622592u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (638976u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (655360u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (671744u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (688128u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (704512u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (720896u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (737280u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (753664u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (770048u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (786432u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (802816u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (819200u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (835584u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (851968u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (868352u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (884736u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (901120u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (917504u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (933888u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (950272u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (966656u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (983040u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (999424u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (1015808u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (1032192u + ((tid * 4u) & 0x3FFFu))); + out[tid] = acc; +} +""", + ["buf1_perlane_constant_raw"] = """ +// bench group=buf_one_perlane_constant variant=raw threads=1048576 tg=32 iters=1 check=1 +#include +using namespace metal; +#pragma clang fp contract(off) +//@BUF_PRELUDE_CONSTANT + +static inline uint load1_select(constant uint* b, uint bytes, uint addr) +{ + uint index = addr >> 2; + uint words = bytes >> 2; + uint last = max(words, 1u) - 1u; + uint v = b[index > last ? last : index]; + return index < words ? v : 0u; +} + +kernel void bench(device uint* data [[buffer(0)]], device uint* out [[buffer(1)]], constant uint4& prm [[buffer(2)]], + constant uint* cdata [[buffer(5)]], + uint tid [[thread_position_in_grid]], uint lane [[thread_index_in_simdgroup]]) +{ + constant uint* b = cdata; + uint bytes = prm.y; + uint acc = tid; + acc = acc * 3u + b[(0u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(16384u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(32768u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(49152u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(65536u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(81920u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(98304u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(114688u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(131072u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(147456u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(163840u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(180224u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(196608u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(212992u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(229376u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(245760u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(262144u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(278528u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(294912u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(311296u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(327680u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(344064u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(360448u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(376832u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(393216u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(409600u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(425984u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(442368u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(458752u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(475136u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(491520u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(507904u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(524288u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(540672u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(557056u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(573440u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(589824u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(606208u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(622592u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(638976u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(655360u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(671744u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(688128u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(704512u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(720896u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(737280u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(753664u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(770048u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(786432u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(802816u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(819200u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(835584u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(851968u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(868352u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(884736u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(901120u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(917504u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(933888u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(950272u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(966656u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(983040u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(999424u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(1015808u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(1032192u + ((tid * 4u) & 0x3FFFu)) >> 2]; + out[tid] = acc; +} +""", + ["buf1_perlane_constant_select"] = """ +// bench group=buf_one_perlane_constant variant=select threads=1048576 tg=32 iters=1 check=1 +#include +using namespace metal; +#pragma clang fp contract(off) +//@BUF_PRELUDE_CONSTANT + +static inline uint load1_select(constant uint* b, uint bytes, uint addr) +{ + uint index = addr >> 2; + uint words = bytes >> 2; + uint last = max(words, 1u) - 1u; + uint v = b[index > last ? last : index]; + return index < words ? v : 0u; +} + +kernel void bench(device uint* data [[buffer(0)]], device uint* out [[buffer(1)]], constant uint4& prm [[buffer(2)]], + constant uint* cdata [[buffer(5)]], + uint tid [[thread_position_in_grid]], uint lane [[thread_index_in_simdgroup]]) +{ + constant uint* b = cdata; + uint bytes = prm.y; + uint acc = tid; + acc = acc * 3u + load1_select(b, bytes, (0u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (16384u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (32768u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (49152u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (65536u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (81920u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (98304u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (114688u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (131072u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (147456u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (163840u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (180224u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (196608u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (212992u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (229376u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (245760u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (262144u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (278528u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (294912u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (311296u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (327680u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (344064u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (360448u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (376832u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (393216u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (409600u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (425984u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (442368u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (458752u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (475136u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (491520u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (507904u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (524288u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (540672u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (557056u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (573440u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (589824u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (606208u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (622592u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (638976u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (655360u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (671744u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (688128u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (704512u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (720896u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (737280u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (753664u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (770048u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (786432u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (802816u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (819200u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (835584u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (851968u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (868352u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (884736u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (901120u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (917504u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (933888u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (950272u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (966656u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (983040u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (999424u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (1015808u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (1032192u + ((tid * 4u) & 0x3FFFu))); + out[tid] = acc; +} +""", + ["buf1_perlane_device_checked"] = """ +// bench group=buf_one_perlane_device variant=checked threads=1048576 tg=32 iters=1 check=1 +#include +using namespace metal; +#pragma clang fp contract(off) +//@BUF_PRELUDE + +static inline uint load1_select(device uint* b, uint bytes, uint addr) +{ + uint index = addr >> 2; + uint words = bytes >> 2; + uint last = max(words, 1u) - 1u; + uint v = b[index > last ? last : index]; + return index < words ? v : 0u; +} + +kernel void bench(device uint* data [[buffer(0)]], device uint* out [[buffer(1)]], constant uint4& prm [[buffer(2)]], + constant uint* cdata [[buffer(5)]], + uint tid [[thread_position_in_grid]], uint lane [[thread_index_in_simdgroup]]) +{ + device uint* b = data; + uint bytes = prm.y; + uint acc = tid; + acc = acc * 3u + sharpemu_load_dword(b, bytes, (0u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (16384u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (32768u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (49152u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (65536u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (81920u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (98304u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (114688u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (131072u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (147456u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (163840u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (180224u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (196608u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (212992u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (229376u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (245760u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (262144u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (278528u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (294912u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (311296u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (327680u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (344064u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (360448u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (376832u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (393216u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (409600u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (425984u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (442368u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (458752u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (475136u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (491520u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (507904u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (524288u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (540672u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (557056u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (573440u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (589824u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (606208u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (622592u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (638976u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (655360u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (671744u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (688128u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (704512u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (720896u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (737280u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (753664u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (770048u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (786432u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (802816u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (819200u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (835584u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (851968u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (868352u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (884736u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (901120u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (917504u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (933888u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (950272u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (966656u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (983040u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (999424u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (1015808u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + sharpemu_load_dword(b, bytes, (1032192u + ((tid * 4u) & 0x3FFFu))); + out[tid] = acc; +} +""", + ["buf1_perlane_device_raw"] = """ +// bench group=buf_one_perlane_device variant=raw threads=1048576 tg=32 iters=1 check=1 +#include +using namespace metal; +#pragma clang fp contract(off) +//@BUF_PRELUDE + +static inline uint load1_select(device uint* b, uint bytes, uint addr) +{ + uint index = addr >> 2; + uint words = bytes >> 2; + uint last = max(words, 1u) - 1u; + uint v = b[index > last ? last : index]; + return index < words ? v : 0u; +} + +kernel void bench(device uint* data [[buffer(0)]], device uint* out [[buffer(1)]], constant uint4& prm [[buffer(2)]], + constant uint* cdata [[buffer(5)]], + uint tid [[thread_position_in_grid]], uint lane [[thread_index_in_simdgroup]]) +{ + device uint* b = data; + uint bytes = prm.y; + uint acc = tid; + acc = acc * 3u + b[(0u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(16384u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(32768u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(49152u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(65536u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(81920u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(98304u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(114688u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(131072u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(147456u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(163840u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(180224u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(196608u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(212992u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(229376u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(245760u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(262144u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(278528u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(294912u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(311296u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(327680u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(344064u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(360448u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(376832u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(393216u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(409600u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(425984u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(442368u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(458752u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(475136u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(491520u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(507904u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(524288u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(540672u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(557056u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(573440u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(589824u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(606208u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(622592u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(638976u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(655360u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(671744u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(688128u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(704512u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(720896u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(737280u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(753664u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(770048u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(786432u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(802816u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(819200u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(835584u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(851968u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(868352u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(884736u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(901120u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(917504u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(933888u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(950272u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(966656u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(983040u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(999424u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(1015808u + ((tid * 4u) & 0x3FFFu)) >> 2]; + acc = acc * 3u + b[(1032192u + ((tid * 4u) & 0x3FFFu)) >> 2]; + out[tid] = acc; +} +""", + ["buf1_perlane_device_select"] = """ +// bench group=buf_one_perlane_device variant=select threads=1048576 tg=32 iters=1 check=1 +#include +using namespace metal; +#pragma clang fp contract(off) +//@BUF_PRELUDE + +static inline uint load1_select(device uint* b, uint bytes, uint addr) +{ + uint index = addr >> 2; + uint words = bytes >> 2; + uint last = max(words, 1u) - 1u; + uint v = b[index > last ? last : index]; + return index < words ? v : 0u; +} + +kernel void bench(device uint* data [[buffer(0)]], device uint* out [[buffer(1)]], constant uint4& prm [[buffer(2)]], + constant uint* cdata [[buffer(5)]], + uint tid [[thread_position_in_grid]], uint lane [[thread_index_in_simdgroup]]) +{ + device uint* b = data; + uint bytes = prm.y; + uint acc = tid; + acc = acc * 3u + load1_select(b, bytes, (0u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (16384u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (32768u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (49152u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (65536u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (81920u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (98304u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (114688u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (131072u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (147456u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (163840u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (180224u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (196608u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (212992u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (229376u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (245760u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (262144u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (278528u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (294912u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (311296u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (327680u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (344064u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (360448u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (376832u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (393216u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (409600u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (425984u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (442368u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (458752u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (475136u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (491520u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (507904u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (524288u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (540672u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (557056u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (573440u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (589824u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (606208u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (622592u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (638976u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (655360u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (671744u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (688128u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (704512u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (720896u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (737280u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (753664u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (770048u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (786432u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (802816u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (819200u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (835584u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (851968u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (868352u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (884736u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (901120u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (917504u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (933888u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (950272u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (966656u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (983040u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (999424u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (1015808u + ((tid * 4u) & 0x3FFFu))); + acc = acc * 3u + load1_select(b, bytes, (1032192u + ((tid * 4u) & 0x3FFFu))); + out[tid] = acc; +} +""", + ["buf1_uniform_constant_checked"] = """ +// bench group=buf_one_uniform_constant variant=checked threads=1048576 tg=32 iters=1 check=1 +#include +using namespace metal; +#pragma clang fp contract(off) +//@BUF_PRELUDE_CONSTANT + +static inline uint load1_select(constant uint* b, uint bytes, uint addr) +{ + uint index = addr >> 2; + uint words = bytes >> 2; + uint last = max(words, 1u) - 1u; + uint v = b[index > last ? last : index]; + return index < words ? v : 0u; +} + +kernel void bench(device uint* data [[buffer(0)]], device uint* out [[buffer(1)]], constant uint4& prm [[buffer(2)]], + constant uint* cdata [[buffer(5)]], + uint tid [[thread_position_in_grid]], uint lane [[thread_index_in_simdgroup]]) +{ + constant uint* b = cdata; + uint bytes = prm.y; + uint acc = tid; + acc = acc * 3u + sharpemu_load_dword(b, bytes, 0u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 4u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 8u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 12u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 16u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 20u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 24u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 28u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 32u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 36u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 40u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 44u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 48u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 52u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 56u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 60u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 64u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 68u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 72u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 76u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 80u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 84u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 88u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 92u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 96u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 100u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 104u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 108u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 112u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 116u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 120u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 124u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 128u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 132u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 136u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 140u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 144u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 148u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 152u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 156u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 160u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 164u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 168u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 172u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 176u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 180u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 184u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 188u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 192u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 196u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 200u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 204u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 208u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 212u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 216u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 220u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 224u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 228u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 232u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 236u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 240u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 244u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 248u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 252u); + out[tid] = acc; +} +""", + ["buf1_uniform_constant_raw"] = """ +// bench group=buf_one_uniform_constant variant=raw threads=1048576 tg=32 iters=1 check=1 +#include +using namespace metal; +#pragma clang fp contract(off) +//@BUF_PRELUDE_CONSTANT + +static inline uint load1_select(constant uint* b, uint bytes, uint addr) +{ + uint index = addr >> 2; + uint words = bytes >> 2; + uint last = max(words, 1u) - 1u; + uint v = b[index > last ? last : index]; + return index < words ? v : 0u; +} + +kernel void bench(device uint* data [[buffer(0)]], device uint* out [[buffer(1)]], constant uint4& prm [[buffer(2)]], + constant uint* cdata [[buffer(5)]], + uint tid [[thread_position_in_grid]], uint lane [[thread_index_in_simdgroup]]) +{ + constant uint* b = cdata; + uint bytes = prm.y; + uint acc = tid; + acc = acc * 3u + b[0u >> 2]; + acc = acc * 3u + b[4u >> 2]; + acc = acc * 3u + b[8u >> 2]; + acc = acc * 3u + b[12u >> 2]; + acc = acc * 3u + b[16u >> 2]; + acc = acc * 3u + b[20u >> 2]; + acc = acc * 3u + b[24u >> 2]; + acc = acc * 3u + b[28u >> 2]; + acc = acc * 3u + b[32u >> 2]; + acc = acc * 3u + b[36u >> 2]; + acc = acc * 3u + b[40u >> 2]; + acc = acc * 3u + b[44u >> 2]; + acc = acc * 3u + b[48u >> 2]; + acc = acc * 3u + b[52u >> 2]; + acc = acc * 3u + b[56u >> 2]; + acc = acc * 3u + b[60u >> 2]; + acc = acc * 3u + b[64u >> 2]; + acc = acc * 3u + b[68u >> 2]; + acc = acc * 3u + b[72u >> 2]; + acc = acc * 3u + b[76u >> 2]; + acc = acc * 3u + b[80u >> 2]; + acc = acc * 3u + b[84u >> 2]; + acc = acc * 3u + b[88u >> 2]; + acc = acc * 3u + b[92u >> 2]; + acc = acc * 3u + b[96u >> 2]; + acc = acc * 3u + b[100u >> 2]; + acc = acc * 3u + b[104u >> 2]; + acc = acc * 3u + b[108u >> 2]; + acc = acc * 3u + b[112u >> 2]; + acc = acc * 3u + b[116u >> 2]; + acc = acc * 3u + b[120u >> 2]; + acc = acc * 3u + b[124u >> 2]; + acc = acc * 3u + b[128u >> 2]; + acc = acc * 3u + b[132u >> 2]; + acc = acc * 3u + b[136u >> 2]; + acc = acc * 3u + b[140u >> 2]; + acc = acc * 3u + b[144u >> 2]; + acc = acc * 3u + b[148u >> 2]; + acc = acc * 3u + b[152u >> 2]; + acc = acc * 3u + b[156u >> 2]; + acc = acc * 3u + b[160u >> 2]; + acc = acc * 3u + b[164u >> 2]; + acc = acc * 3u + b[168u >> 2]; + acc = acc * 3u + b[172u >> 2]; + acc = acc * 3u + b[176u >> 2]; + acc = acc * 3u + b[180u >> 2]; + acc = acc * 3u + b[184u >> 2]; + acc = acc * 3u + b[188u >> 2]; + acc = acc * 3u + b[192u >> 2]; + acc = acc * 3u + b[196u >> 2]; + acc = acc * 3u + b[200u >> 2]; + acc = acc * 3u + b[204u >> 2]; + acc = acc * 3u + b[208u >> 2]; + acc = acc * 3u + b[212u >> 2]; + acc = acc * 3u + b[216u >> 2]; + acc = acc * 3u + b[220u >> 2]; + acc = acc * 3u + b[224u >> 2]; + acc = acc * 3u + b[228u >> 2]; + acc = acc * 3u + b[232u >> 2]; + acc = acc * 3u + b[236u >> 2]; + acc = acc * 3u + b[240u >> 2]; + acc = acc * 3u + b[244u >> 2]; + acc = acc * 3u + b[248u >> 2]; + acc = acc * 3u + b[252u >> 2]; + out[tid] = acc; +} +""", + ["buf1_uniform_constant_select"] = """ +// bench group=buf_one_uniform_constant variant=select threads=1048576 tg=32 iters=1 check=1 +#include +using namespace metal; +#pragma clang fp contract(off) +//@BUF_PRELUDE_CONSTANT + +static inline uint load1_select(constant uint* b, uint bytes, uint addr) +{ + uint index = addr >> 2; + uint words = bytes >> 2; + uint last = max(words, 1u) - 1u; + uint v = b[index > last ? last : index]; + return index < words ? v : 0u; +} + +kernel void bench(device uint* data [[buffer(0)]], device uint* out [[buffer(1)]], constant uint4& prm [[buffer(2)]], + constant uint* cdata [[buffer(5)]], + uint tid [[thread_position_in_grid]], uint lane [[thread_index_in_simdgroup]]) +{ + constant uint* b = cdata; + uint bytes = prm.y; + uint acc = tid; + acc = acc * 3u + load1_select(b, bytes, 0u); + acc = acc * 3u + load1_select(b, bytes, 4u); + acc = acc * 3u + load1_select(b, bytes, 8u); + acc = acc * 3u + load1_select(b, bytes, 12u); + acc = acc * 3u + load1_select(b, bytes, 16u); + acc = acc * 3u + load1_select(b, bytes, 20u); + acc = acc * 3u + load1_select(b, bytes, 24u); + acc = acc * 3u + load1_select(b, bytes, 28u); + acc = acc * 3u + load1_select(b, bytes, 32u); + acc = acc * 3u + load1_select(b, bytes, 36u); + acc = acc * 3u + load1_select(b, bytes, 40u); + acc = acc * 3u + load1_select(b, bytes, 44u); + acc = acc * 3u + load1_select(b, bytes, 48u); + acc = acc * 3u + load1_select(b, bytes, 52u); + acc = acc * 3u + load1_select(b, bytes, 56u); + acc = acc * 3u + load1_select(b, bytes, 60u); + acc = acc * 3u + load1_select(b, bytes, 64u); + acc = acc * 3u + load1_select(b, bytes, 68u); + acc = acc * 3u + load1_select(b, bytes, 72u); + acc = acc * 3u + load1_select(b, bytes, 76u); + acc = acc * 3u + load1_select(b, bytes, 80u); + acc = acc * 3u + load1_select(b, bytes, 84u); + acc = acc * 3u + load1_select(b, bytes, 88u); + acc = acc * 3u + load1_select(b, bytes, 92u); + acc = acc * 3u + load1_select(b, bytes, 96u); + acc = acc * 3u + load1_select(b, bytes, 100u); + acc = acc * 3u + load1_select(b, bytes, 104u); + acc = acc * 3u + load1_select(b, bytes, 108u); + acc = acc * 3u + load1_select(b, bytes, 112u); + acc = acc * 3u + load1_select(b, bytes, 116u); + acc = acc * 3u + load1_select(b, bytes, 120u); + acc = acc * 3u + load1_select(b, bytes, 124u); + acc = acc * 3u + load1_select(b, bytes, 128u); + acc = acc * 3u + load1_select(b, bytes, 132u); + acc = acc * 3u + load1_select(b, bytes, 136u); + acc = acc * 3u + load1_select(b, bytes, 140u); + acc = acc * 3u + load1_select(b, bytes, 144u); + acc = acc * 3u + load1_select(b, bytes, 148u); + acc = acc * 3u + load1_select(b, bytes, 152u); + acc = acc * 3u + load1_select(b, bytes, 156u); + acc = acc * 3u + load1_select(b, bytes, 160u); + acc = acc * 3u + load1_select(b, bytes, 164u); + acc = acc * 3u + load1_select(b, bytes, 168u); + acc = acc * 3u + load1_select(b, bytes, 172u); + acc = acc * 3u + load1_select(b, bytes, 176u); + acc = acc * 3u + load1_select(b, bytes, 180u); + acc = acc * 3u + load1_select(b, bytes, 184u); + acc = acc * 3u + load1_select(b, bytes, 188u); + acc = acc * 3u + load1_select(b, bytes, 192u); + acc = acc * 3u + load1_select(b, bytes, 196u); + acc = acc * 3u + load1_select(b, bytes, 200u); + acc = acc * 3u + load1_select(b, bytes, 204u); + acc = acc * 3u + load1_select(b, bytes, 208u); + acc = acc * 3u + load1_select(b, bytes, 212u); + acc = acc * 3u + load1_select(b, bytes, 216u); + acc = acc * 3u + load1_select(b, bytes, 220u); + acc = acc * 3u + load1_select(b, bytes, 224u); + acc = acc * 3u + load1_select(b, bytes, 228u); + acc = acc * 3u + load1_select(b, bytes, 232u); + acc = acc * 3u + load1_select(b, bytes, 236u); + acc = acc * 3u + load1_select(b, bytes, 240u); + acc = acc * 3u + load1_select(b, bytes, 244u); + acc = acc * 3u + load1_select(b, bytes, 248u); + acc = acc * 3u + load1_select(b, bytes, 252u); + out[tid] = acc; +} +""", + ["buf1_uniform_device_checked"] = """ +// bench group=buf_one_uniform_device variant=checked threads=1048576 tg=32 iters=1 check=1 +#include +using namespace metal; +#pragma clang fp contract(off) +//@BUF_PRELUDE + +static inline uint load1_select(device uint* b, uint bytes, uint addr) +{ + uint index = addr >> 2; + uint words = bytes >> 2; + uint last = max(words, 1u) - 1u; + uint v = b[index > last ? last : index]; + return index < words ? v : 0u; +} + +kernel void bench(device uint* data [[buffer(0)]], device uint* out [[buffer(1)]], constant uint4& prm [[buffer(2)]], + constant uint* cdata [[buffer(5)]], + uint tid [[thread_position_in_grid]], uint lane [[thread_index_in_simdgroup]]) +{ + device uint* b = data; + uint bytes = prm.y; + uint acc = tid; + acc = acc * 3u + sharpemu_load_dword(b, bytes, 0u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 4u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 8u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 12u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 16u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 20u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 24u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 28u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 32u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 36u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 40u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 44u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 48u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 52u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 56u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 60u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 64u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 68u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 72u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 76u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 80u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 84u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 88u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 92u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 96u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 100u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 104u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 108u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 112u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 116u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 120u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 124u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 128u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 132u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 136u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 140u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 144u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 148u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 152u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 156u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 160u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 164u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 168u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 172u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 176u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 180u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 184u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 188u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 192u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 196u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 200u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 204u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 208u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 212u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 216u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 220u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 224u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 228u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 232u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 236u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 240u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 244u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 248u); + acc = acc * 3u + sharpemu_load_dword(b, bytes, 252u); + out[tid] = acc; +} +""", + ["buf1_uniform_device_raw"] = """ +// bench group=buf_one_uniform_device variant=raw threads=1048576 tg=32 iters=1 check=1 +#include +using namespace metal; +#pragma clang fp contract(off) +//@BUF_PRELUDE + +static inline uint load1_select(device uint* b, uint bytes, uint addr) +{ + uint index = addr >> 2; + uint words = bytes >> 2; + uint last = max(words, 1u) - 1u; + uint v = b[index > last ? last : index]; + return index < words ? v : 0u; +} + +kernel void bench(device uint* data [[buffer(0)]], device uint* out [[buffer(1)]], constant uint4& prm [[buffer(2)]], + constant uint* cdata [[buffer(5)]], + uint tid [[thread_position_in_grid]], uint lane [[thread_index_in_simdgroup]]) +{ + device uint* b = data; + uint bytes = prm.y; + uint acc = tid; + acc = acc * 3u + b[0u >> 2]; + acc = acc * 3u + b[4u >> 2]; + acc = acc * 3u + b[8u >> 2]; + acc = acc * 3u + b[12u >> 2]; + acc = acc * 3u + b[16u >> 2]; + acc = acc * 3u + b[20u >> 2]; + acc = acc * 3u + b[24u >> 2]; + acc = acc * 3u + b[28u >> 2]; + acc = acc * 3u + b[32u >> 2]; + acc = acc * 3u + b[36u >> 2]; + acc = acc * 3u + b[40u >> 2]; + acc = acc * 3u + b[44u >> 2]; + acc = acc * 3u + b[48u >> 2]; + acc = acc * 3u + b[52u >> 2]; + acc = acc * 3u + b[56u >> 2]; + acc = acc * 3u + b[60u >> 2]; + acc = acc * 3u + b[64u >> 2]; + acc = acc * 3u + b[68u >> 2]; + acc = acc * 3u + b[72u >> 2]; + acc = acc * 3u + b[76u >> 2]; + acc = acc * 3u + b[80u >> 2]; + acc = acc * 3u + b[84u >> 2]; + acc = acc * 3u + b[88u >> 2]; + acc = acc * 3u + b[92u >> 2]; + acc = acc * 3u + b[96u >> 2]; + acc = acc * 3u + b[100u >> 2]; + acc = acc * 3u + b[104u >> 2]; + acc = acc * 3u + b[108u >> 2]; + acc = acc * 3u + b[112u >> 2]; + acc = acc * 3u + b[116u >> 2]; + acc = acc * 3u + b[120u >> 2]; + acc = acc * 3u + b[124u >> 2]; + acc = acc * 3u + b[128u >> 2]; + acc = acc * 3u + b[132u >> 2]; + acc = acc * 3u + b[136u >> 2]; + acc = acc * 3u + b[140u >> 2]; + acc = acc * 3u + b[144u >> 2]; + acc = acc * 3u + b[148u >> 2]; + acc = acc * 3u + b[152u >> 2]; + acc = acc * 3u + b[156u >> 2]; + acc = acc * 3u + b[160u >> 2]; + acc = acc * 3u + b[164u >> 2]; + acc = acc * 3u + b[168u >> 2]; + acc = acc * 3u + b[172u >> 2]; + acc = acc * 3u + b[176u >> 2]; + acc = acc * 3u + b[180u >> 2]; + acc = acc * 3u + b[184u >> 2]; + acc = acc * 3u + b[188u >> 2]; + acc = acc * 3u + b[192u >> 2]; + acc = acc * 3u + b[196u >> 2]; + acc = acc * 3u + b[200u >> 2]; + acc = acc * 3u + b[204u >> 2]; + acc = acc * 3u + b[208u >> 2]; + acc = acc * 3u + b[212u >> 2]; + acc = acc * 3u + b[216u >> 2]; + acc = acc * 3u + b[220u >> 2]; + acc = acc * 3u + b[224u >> 2]; + acc = acc * 3u + b[228u >> 2]; + acc = acc * 3u + b[232u >> 2]; + acc = acc * 3u + b[236u >> 2]; + acc = acc * 3u + b[240u >> 2]; + acc = acc * 3u + b[244u >> 2]; + acc = acc * 3u + b[248u >> 2]; + acc = acc * 3u + b[252u >> 2]; + out[tid] = acc; +} +""", + ["buf1_uniform_device_select"] = """ +// bench group=buf_one_uniform_device variant=select threads=1048576 tg=32 iters=1 check=1 +#include +using namespace metal; +#pragma clang fp contract(off) +//@BUF_PRELUDE + +static inline uint load1_select(device uint* b, uint bytes, uint addr) +{ + uint index = addr >> 2; + uint words = bytes >> 2; + uint last = max(words, 1u) - 1u; + uint v = b[index > last ? last : index]; + return index < words ? v : 0u; +} + +kernel void bench(device uint* data [[buffer(0)]], device uint* out [[buffer(1)]], constant uint4& prm [[buffer(2)]], + constant uint* cdata [[buffer(5)]], + uint tid [[thread_position_in_grid]], uint lane [[thread_index_in_simdgroup]]) +{ + device uint* b = data; + uint bytes = prm.y; + uint acc = tid; + acc = acc * 3u + load1_select(b, bytes, 0u); + acc = acc * 3u + load1_select(b, bytes, 4u); + acc = acc * 3u + load1_select(b, bytes, 8u); + acc = acc * 3u + load1_select(b, bytes, 12u); + acc = acc * 3u + load1_select(b, bytes, 16u); + acc = acc * 3u + load1_select(b, bytes, 20u); + acc = acc * 3u + load1_select(b, bytes, 24u); + acc = acc * 3u + load1_select(b, bytes, 28u); + acc = acc * 3u + load1_select(b, bytes, 32u); + acc = acc * 3u + load1_select(b, bytes, 36u); + acc = acc * 3u + load1_select(b, bytes, 40u); + acc = acc * 3u + load1_select(b, bytes, 44u); + acc = acc * 3u + load1_select(b, bytes, 48u); + acc = acc * 3u + load1_select(b, bytes, 52u); + acc = acc * 3u + load1_select(b, bytes, 56u); + acc = acc * 3u + load1_select(b, bytes, 60u); + acc = acc * 3u + load1_select(b, bytes, 64u); + acc = acc * 3u + load1_select(b, bytes, 68u); + acc = acc * 3u + load1_select(b, bytes, 72u); + acc = acc * 3u + load1_select(b, bytes, 76u); + acc = acc * 3u + load1_select(b, bytes, 80u); + acc = acc * 3u + load1_select(b, bytes, 84u); + acc = acc * 3u + load1_select(b, bytes, 88u); + acc = acc * 3u + load1_select(b, bytes, 92u); + acc = acc * 3u + load1_select(b, bytes, 96u); + acc = acc * 3u + load1_select(b, bytes, 100u); + acc = acc * 3u + load1_select(b, bytes, 104u); + acc = acc * 3u + load1_select(b, bytes, 108u); + acc = acc * 3u + load1_select(b, bytes, 112u); + acc = acc * 3u + load1_select(b, bytes, 116u); + acc = acc * 3u + load1_select(b, bytes, 120u); + acc = acc * 3u + load1_select(b, bytes, 124u); + acc = acc * 3u + load1_select(b, bytes, 128u); + acc = acc * 3u + load1_select(b, bytes, 132u); + acc = acc * 3u + load1_select(b, bytes, 136u); + acc = acc * 3u + load1_select(b, bytes, 140u); + acc = acc * 3u + load1_select(b, bytes, 144u); + acc = acc * 3u + load1_select(b, bytes, 148u); + acc = acc * 3u + load1_select(b, bytes, 152u); + acc = acc * 3u + load1_select(b, bytes, 156u); + acc = acc * 3u + load1_select(b, bytes, 160u); + acc = acc * 3u + load1_select(b, bytes, 164u); + acc = acc * 3u + load1_select(b, bytes, 168u); + acc = acc * 3u + load1_select(b, bytes, 172u); + acc = acc * 3u + load1_select(b, bytes, 176u); + acc = acc * 3u + load1_select(b, bytes, 180u); + acc = acc * 3u + load1_select(b, bytes, 184u); + acc = acc * 3u + load1_select(b, bytes, 188u); + acc = acc * 3u + load1_select(b, bytes, 192u); + acc = acc * 3u + load1_select(b, bytes, 196u); + acc = acc * 3u + load1_select(b, bytes, 200u); + acc = acc * 3u + load1_select(b, bytes, 204u); + acc = acc * 3u + load1_select(b, bytes, 208u); + acc = acc * 3u + load1_select(b, bytes, 212u); + acc = acc * 3u + load1_select(b, bytes, 216u); + acc = acc * 3u + load1_select(b, bytes, 220u); + acc = acc * 3u + load1_select(b, bytes, 224u); + acc = acc * 3u + load1_select(b, bytes, 228u); + acc = acc * 3u + load1_select(b, bytes, 232u); + acc = acc * 3u + load1_select(b, bytes, 236u); + acc = acc * 3u + load1_select(b, bytes, 240u); + acc = acc * 3u + load1_select(b, bytes, 244u); + acc = acc * 3u + load1_select(b, bytes, 248u); + acc = acc * 3u + load1_select(b, bytes, 252u); + out[tid] = acc; +} +""", + ["buf2_perlane_constant_checked"] = """ +// bench group=buf_two_perlane_constant variant=checked threads=1048576 tg=32 iters=1 check=1 +#include +using namespace metal; +#pragma clang fp contract(off) +//@BUF_PRELUDE_CONSTANT + +// Fast path inline, the dword-by-dword path out of line. +__attribute__((noinline)) static uint2 slow2(constant uint* b, uint bytes, uint addr) +{ + return uint2(sharpemu_load_dword(b, bytes, addr), sharpemu_load_dword(b, bytes, addr + 4u)); +} +static inline uint2 load2_outlined(constant uint* b, uint bytes, uint addr) +{ + uint index = addr >> 2; + uint words = bytes >> 2; + if (index < words && 2u <= words - index) + { + return uint2(b[index], b[index + 1u]); + } + return slow2(b, bytes, addr); +} +// Branch-free: every dword is read at an index clamped into the binding and selected against +// its own bounds test, which is what sharpemu_load_dword answers per dword. Needs a binding of +// at least one dword. +static inline uint2 load2_select(constant uint* b, uint bytes, uint addr) +{ + uint index = addr >> 2; + uint words = bytes >> 2; + uint last = max(words, 1u) - 1u; + uint2 r = uint2(0u); + for (int i = 0; i < 2; i++) + { + uint ix = ((addr >> 2) + (uint)i) & 0x3FFFFFFFu; + uint v = b[ix > last ? last : ix]; + r[i] = ix < words ? v : 0u; + } + return r; +} + +kernel void bench(device uint* data [[buffer(0)]], device uint* out [[buffer(1)]], constant uint4& prm [[buffer(2)]], + constant uint* cdata [[buffer(5)]], + uint tid [[thread_position_in_grid]], uint lane [[thread_index_in_simdgroup]]) +{ + constant uint* b = cdata; + uint bytes = prm.y; + uint acc = tid; + { uint4 w = sharpemu_load_dwords<2>(b, bytes, (0u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, (65536u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, (131072u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, (196608u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, (262144u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, (327680u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, (393216u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, (458752u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, (524288u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, (589824u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, (655360u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, (720896u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, (786432u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, (851968u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, (917504u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, (983040u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, (1048576u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, (1114112u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, (1179648u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, (1245184u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, (1310720u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, (1376256u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, (1441792u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, (1507328u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, (1572864u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, (1638400u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, (1703936u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, (1769472u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, (1835008u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, (1900544u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, (1966080u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, (2031616u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + out[tid] = acc; +} +""", + ["buf2_perlane_constant_outlined"] = """ +// bench group=buf_two_perlane_constant variant=outlined threads=1048576 tg=32 iters=1 check=1 +#include +using namespace metal; +#pragma clang fp contract(off) +//@BUF_PRELUDE_CONSTANT + +// Fast path inline, the dword-by-dword path out of line. +__attribute__((noinline)) static uint2 slow2(constant uint* b, uint bytes, uint addr) +{ + return uint2(sharpemu_load_dword(b, bytes, addr), sharpemu_load_dword(b, bytes, addr + 4u)); +} +static inline uint2 load2_outlined(constant uint* b, uint bytes, uint addr) +{ + uint index = addr >> 2; + uint words = bytes >> 2; + if (index < words && 2u <= words - index) + { + return uint2(b[index], b[index + 1u]); + } + return slow2(b, bytes, addr); +} +// Branch-free: every dword is read at an index clamped into the binding and selected against +// its own bounds test, which is what sharpemu_load_dword answers per dword. Needs a binding of +// at least one dword. +static inline uint2 load2_select(constant uint* b, uint bytes, uint addr) +{ + uint index = addr >> 2; + uint words = bytes >> 2; + uint last = max(words, 1u) - 1u; + uint2 r = uint2(0u); + for (int i = 0; i < 2; i++) + { + uint ix = ((addr >> 2) + (uint)i) & 0x3FFFFFFFu; + uint v = b[ix > last ? last : ix]; + r[i] = ix < words ? v : 0u; + } + return r; +} + +kernel void bench(device uint* data [[buffer(0)]], device uint* out [[buffer(1)]], constant uint4& prm [[buffer(2)]], + constant uint* cdata [[buffer(5)]], + uint tid [[thread_position_in_grid]], uint lane [[thread_index_in_simdgroup]]) +{ + constant uint* b = cdata; + uint bytes = prm.y; + uint acc = tid; + { uint2 w = load2_outlined(b, bytes, (0u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, (65536u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, (131072u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, (196608u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, (262144u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, (327680u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, (393216u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, (458752u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, (524288u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, (589824u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, (655360u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, (720896u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, (786432u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, (851968u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, (917504u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, (983040u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, (1048576u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, (1114112u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, (1179648u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, (1245184u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, (1310720u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, (1376256u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, (1441792u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, (1507328u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, (1572864u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, (1638400u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, (1703936u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, (1769472u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, (1835008u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, (1900544u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, (1966080u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, (2031616u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + out[tid] = acc; +} +""", + ["buf2_perlane_constant_raw"] = """ +// bench group=buf_two_perlane_constant variant=raw threads=1048576 tg=32 iters=1 check=1 +#include +using namespace metal; +#pragma clang fp contract(off) +//@BUF_PRELUDE_CONSTANT + +// Fast path inline, the dword-by-dword path out of line. +__attribute__((noinline)) static uint2 slow2(constant uint* b, uint bytes, uint addr) +{ + return uint2(sharpemu_load_dword(b, bytes, addr), sharpemu_load_dword(b, bytes, addr + 4u)); +} +static inline uint2 load2_outlined(constant uint* b, uint bytes, uint addr) +{ + uint index = addr >> 2; + uint words = bytes >> 2; + if (index < words && 2u <= words - index) + { + return uint2(b[index], b[index + 1u]); + } + return slow2(b, bytes, addr); +} +// Branch-free: every dword is read at an index clamped into the binding and selected against +// its own bounds test, which is what sharpemu_load_dword answers per dword. Needs a binding of +// at least one dword. +static inline uint2 load2_select(constant uint* b, uint bytes, uint addr) +{ + uint index = addr >> 2; + uint words = bytes >> 2; + uint last = max(words, 1u) - 1u; + uint2 r = uint2(0u); + for (int i = 0; i < 2; i++) + { + uint ix = ((addr >> 2) + (uint)i) & 0x3FFFFFFFu; + uint v = b[ix > last ? last : ix]; + r[i] = ix < words ? v : 0u; + } + return r; +} + +kernel void bench(device uint* data [[buffer(0)]], device uint* out [[buffer(1)]], constant uint4& prm [[buffer(2)]], + constant uint* cdata [[buffer(5)]], + uint tid [[thread_position_in_grid]], uint lane [[thread_index_in_simdgroup]]) +{ + constant uint* b = cdata; + uint bytes = prm.y; + uint acc = tid; + { uint i = (0u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = (65536u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = (131072u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = (196608u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = (262144u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = (327680u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = (393216u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = (458752u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = (524288u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = (589824u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = (655360u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = (720896u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = (786432u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = (851968u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = (917504u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = (983040u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = (1048576u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = (1114112u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = (1179648u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = (1245184u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = (1310720u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = (1376256u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = (1441792u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = (1507328u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = (1572864u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = (1638400u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = (1703936u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = (1769472u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = (1835008u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = (1900544u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = (1966080u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = (2031616u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; } + out[tid] = acc; +} +""", + ["buf2_perlane_constant_select"] = """ +// bench group=buf_two_perlane_constant variant=select threads=1048576 tg=32 iters=1 check=1 +#include +using namespace metal; +#pragma clang fp contract(off) +//@BUF_PRELUDE_CONSTANT + +// Fast path inline, the dword-by-dword path out of line. +__attribute__((noinline)) static uint2 slow2(constant uint* b, uint bytes, uint addr) +{ + return uint2(sharpemu_load_dword(b, bytes, addr), sharpemu_load_dword(b, bytes, addr + 4u)); +} +static inline uint2 load2_outlined(constant uint* b, uint bytes, uint addr) +{ + uint index = addr >> 2; + uint words = bytes >> 2; + if (index < words && 2u <= words - index) + { + return uint2(b[index], b[index + 1u]); + } + return slow2(b, bytes, addr); +} +// Branch-free: every dword is read at an index clamped into the binding and selected against +// its own bounds test, which is what sharpemu_load_dword answers per dword. Needs a binding of +// at least one dword. +static inline uint2 load2_select(constant uint* b, uint bytes, uint addr) +{ + uint index = addr >> 2; + uint words = bytes >> 2; + uint last = max(words, 1u) - 1u; + uint2 r = uint2(0u); + for (int i = 0; i < 2; i++) + { + uint ix = ((addr >> 2) + (uint)i) & 0x3FFFFFFFu; + uint v = b[ix > last ? last : ix]; + r[i] = ix < words ? v : 0u; + } + return r; +} + +kernel void bench(device uint* data [[buffer(0)]], device uint* out [[buffer(1)]], constant uint4& prm [[buffer(2)]], + constant uint* cdata [[buffer(5)]], + uint tid [[thread_position_in_grid]], uint lane [[thread_index_in_simdgroup]]) +{ + constant uint* b = cdata; + uint bytes = prm.y; + uint acc = tid; + { uint2 w = load2_select(b, bytes, (0u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, (65536u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, (131072u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, (196608u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, (262144u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, (327680u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, (393216u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, (458752u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, (524288u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, (589824u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, (655360u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, (720896u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, (786432u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, (851968u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, (917504u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, (983040u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, (1048576u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, (1114112u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, (1179648u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, (1245184u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, (1310720u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, (1376256u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, (1441792u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, (1507328u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, (1572864u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, (1638400u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, (1703936u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, (1769472u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, (1835008u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, (1900544u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, (1966080u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, (2031616u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + out[tid] = acc; +} +""", + ["buf2_perlane_device_checked"] = """ +// bench group=buf_two_perlane_device variant=checked threads=1048576 tg=32 iters=1 check=1 +#include +using namespace metal; +#pragma clang fp contract(off) +//@BUF_PRELUDE + +// Fast path inline, the dword-by-dword path out of line. +__attribute__((noinline)) static uint2 slow2(device uint* b, uint bytes, uint addr) +{ + return uint2(sharpemu_load_dword(b, bytes, addr), sharpemu_load_dword(b, bytes, addr + 4u)); +} +static inline uint2 load2_outlined(device uint* b, uint bytes, uint addr) +{ + uint index = addr >> 2; + uint words = bytes >> 2; + if (index < words && 2u <= words - index) + { + return uint2(b[index], b[index + 1u]); + } + return slow2(b, bytes, addr); +} +// Branch-free: every dword is read at an index clamped into the binding and selected against +// its own bounds test, which is what sharpemu_load_dword answers per dword. Needs a binding of +// at least one dword. +static inline uint2 load2_select(device uint* b, uint bytes, uint addr) +{ + uint index = addr >> 2; + uint words = bytes >> 2; + uint last = max(words, 1u) - 1u; + uint2 r = uint2(0u); + for (int i = 0; i < 2; i++) + { + uint ix = ((addr >> 2) + (uint)i) & 0x3FFFFFFFu; + uint v = b[ix > last ? last : ix]; + r[i] = ix < words ? v : 0u; + } + return r; +} + +kernel void bench(device uint* data [[buffer(0)]], device uint* out [[buffer(1)]], constant uint4& prm [[buffer(2)]], + constant uint* cdata [[buffer(5)]], + uint tid [[thread_position_in_grid]], uint lane [[thread_index_in_simdgroup]]) +{ + device uint* b = data; + uint bytes = prm.y; + uint acc = tid; + { uint4 w = sharpemu_load_dwords<2>(b, bytes, (0u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, (65536u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, (131072u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, (196608u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, (262144u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, (327680u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, (393216u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, (458752u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, (524288u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, (589824u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, (655360u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, (720896u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, (786432u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, (851968u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, (917504u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, (983040u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, (1048576u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, (1114112u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, (1179648u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, (1245184u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, (1310720u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, (1376256u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, (1441792u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, (1507328u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, (1572864u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, (1638400u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, (1703936u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, (1769472u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, (1835008u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, (1900544u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, (1966080u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, (2031616u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + out[tid] = acc; +} +""", + ["buf2_perlane_device_outlined"] = """ +// bench group=buf_two_perlane_device variant=outlined threads=1048576 tg=32 iters=1 check=1 +#include +using namespace metal; +#pragma clang fp contract(off) +//@BUF_PRELUDE + +// Fast path inline, the dword-by-dword path out of line. +__attribute__((noinline)) static uint2 slow2(device uint* b, uint bytes, uint addr) +{ + return uint2(sharpemu_load_dword(b, bytes, addr), sharpemu_load_dword(b, bytes, addr + 4u)); +} +static inline uint2 load2_outlined(device uint* b, uint bytes, uint addr) +{ + uint index = addr >> 2; + uint words = bytes >> 2; + if (index < words && 2u <= words - index) + { + return uint2(b[index], b[index + 1u]); + } + return slow2(b, bytes, addr); +} +// Branch-free: every dword is read at an index clamped into the binding and selected against +// its own bounds test, which is what sharpemu_load_dword answers per dword. Needs a binding of +// at least one dword. +static inline uint2 load2_select(device uint* b, uint bytes, uint addr) +{ + uint index = addr >> 2; + uint words = bytes >> 2; + uint last = max(words, 1u) - 1u; + uint2 r = uint2(0u); + for (int i = 0; i < 2; i++) + { + uint ix = ((addr >> 2) + (uint)i) & 0x3FFFFFFFu; + uint v = b[ix > last ? last : ix]; + r[i] = ix < words ? v : 0u; + } + return r; +} + +kernel void bench(device uint* data [[buffer(0)]], device uint* out [[buffer(1)]], constant uint4& prm [[buffer(2)]], + constant uint* cdata [[buffer(5)]], + uint tid [[thread_position_in_grid]], uint lane [[thread_index_in_simdgroup]]) +{ + device uint* b = data; + uint bytes = prm.y; + uint acc = tid; + { uint2 w = load2_outlined(b, bytes, (0u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, (65536u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, (131072u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, (196608u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, (262144u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, (327680u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, (393216u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, (458752u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, (524288u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, (589824u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, (655360u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, (720896u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, (786432u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, (851968u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, (917504u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, (983040u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, (1048576u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, (1114112u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, (1179648u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, (1245184u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, (1310720u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, (1376256u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, (1441792u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, (1507328u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, (1572864u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, (1638400u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, (1703936u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, (1769472u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, (1835008u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, (1900544u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, (1966080u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, (2031616u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + out[tid] = acc; +} +""", + ["buf2_perlane_device_raw"] = """ +// bench group=buf_two_perlane_device variant=raw threads=1048576 tg=32 iters=1 check=1 +#include +using namespace metal; +#pragma clang fp contract(off) +//@BUF_PRELUDE + +// Fast path inline, the dword-by-dword path out of line. +__attribute__((noinline)) static uint2 slow2(device uint* b, uint bytes, uint addr) +{ + return uint2(sharpemu_load_dword(b, bytes, addr), sharpemu_load_dword(b, bytes, addr + 4u)); +} +static inline uint2 load2_outlined(device uint* b, uint bytes, uint addr) +{ + uint index = addr >> 2; + uint words = bytes >> 2; + if (index < words && 2u <= words - index) + { + return uint2(b[index], b[index + 1u]); + } + return slow2(b, bytes, addr); +} +// Branch-free: every dword is read at an index clamped into the binding and selected against +// its own bounds test, which is what sharpemu_load_dword answers per dword. Needs a binding of +// at least one dword. +static inline uint2 load2_select(device uint* b, uint bytes, uint addr) +{ + uint index = addr >> 2; + uint words = bytes >> 2; + uint last = max(words, 1u) - 1u; + uint2 r = uint2(0u); + for (int i = 0; i < 2; i++) + { + uint ix = ((addr >> 2) + (uint)i) & 0x3FFFFFFFu; + uint v = b[ix > last ? last : ix]; + r[i] = ix < words ? v : 0u; + } + return r; +} + +kernel void bench(device uint* data [[buffer(0)]], device uint* out [[buffer(1)]], constant uint4& prm [[buffer(2)]], + constant uint* cdata [[buffer(5)]], + uint tid [[thread_position_in_grid]], uint lane [[thread_index_in_simdgroup]]) +{ + device uint* b = data; + uint bytes = prm.y; + uint acc = tid; + { uint i = (0u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = (65536u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = (131072u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = (196608u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = (262144u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = (327680u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = (393216u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = (458752u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = (524288u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = (589824u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = (655360u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = (720896u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = (786432u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = (851968u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = (917504u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = (983040u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = (1048576u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = (1114112u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = (1179648u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = (1245184u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = (1310720u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = (1376256u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = (1441792u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = (1507328u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = (1572864u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = (1638400u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = (1703936u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = (1769472u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = (1835008u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = (1900544u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = (1966080u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = (2031616u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; } + out[tid] = acc; +} +""", + ["buf2_perlane_device_select"] = """ +// bench group=buf_two_perlane_device variant=select threads=1048576 tg=32 iters=1 check=1 +#include +using namespace metal; +#pragma clang fp contract(off) +//@BUF_PRELUDE + +// Fast path inline, the dword-by-dword path out of line. +__attribute__((noinline)) static uint2 slow2(device uint* b, uint bytes, uint addr) +{ + return uint2(sharpemu_load_dword(b, bytes, addr), sharpemu_load_dword(b, bytes, addr + 4u)); +} +static inline uint2 load2_outlined(device uint* b, uint bytes, uint addr) +{ + uint index = addr >> 2; + uint words = bytes >> 2; + if (index < words && 2u <= words - index) + { + return uint2(b[index], b[index + 1u]); + } + return slow2(b, bytes, addr); +} +// Branch-free: every dword is read at an index clamped into the binding and selected against +// its own bounds test, which is what sharpemu_load_dword answers per dword. Needs a binding of +// at least one dword. +static inline uint2 load2_select(device uint* b, uint bytes, uint addr) +{ + uint index = addr >> 2; + uint words = bytes >> 2; + uint last = max(words, 1u) - 1u; + uint2 r = uint2(0u); + for (int i = 0; i < 2; i++) + { + uint ix = ((addr >> 2) + (uint)i) & 0x3FFFFFFFu; + uint v = b[ix > last ? last : ix]; + r[i] = ix < words ? v : 0u; + } + return r; +} + +kernel void bench(device uint* data [[buffer(0)]], device uint* out [[buffer(1)]], constant uint4& prm [[buffer(2)]], + constant uint* cdata [[buffer(5)]], + uint tid [[thread_position_in_grid]], uint lane [[thread_index_in_simdgroup]]) +{ + device uint* b = data; + uint bytes = prm.y; + uint acc = tid; + { uint2 w = load2_select(b, bytes, (0u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, (65536u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, (131072u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, (196608u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, (262144u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, (327680u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, (393216u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, (458752u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, (524288u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, (589824u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, (655360u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, (720896u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, (786432u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, (851968u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, (917504u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, (983040u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, (1048576u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, (1114112u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, (1179648u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, (1245184u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, (1310720u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, (1376256u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, (1441792u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, (1507328u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, (1572864u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, (1638400u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, (1703936u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, (1769472u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, (1835008u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, (1900544u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, (1966080u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, (2031616u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; } + out[tid] = acc; +} +""", + ["buf2_uniform_constant_checked"] = """ +// bench group=buf_two_uniform_constant variant=checked threads=1048576 tg=32 iters=1 check=1 +#include +using namespace metal; +#pragma clang fp contract(off) +//@BUF_PRELUDE_CONSTANT + +// Fast path inline, the dword-by-dword path out of line. +__attribute__((noinline)) static uint2 slow2(constant uint* b, uint bytes, uint addr) +{ + return uint2(sharpemu_load_dword(b, bytes, addr), sharpemu_load_dword(b, bytes, addr + 4u)); +} +static inline uint2 load2_outlined(constant uint* b, uint bytes, uint addr) +{ + uint index = addr >> 2; + uint words = bytes >> 2; + if (index < words && 2u <= words - index) + { + return uint2(b[index], b[index + 1u]); + } + return slow2(b, bytes, addr); +} +// Branch-free: every dword is read at an index clamped into the binding and selected against +// its own bounds test, which is what sharpemu_load_dword answers per dword. Needs a binding of +// at least one dword. +static inline uint2 load2_select(constant uint* b, uint bytes, uint addr) +{ + uint index = addr >> 2; + uint words = bytes >> 2; + uint last = max(words, 1u) - 1u; + uint2 r = uint2(0u); + for (int i = 0; i < 2; i++) + { + uint ix = ((addr >> 2) + (uint)i) & 0x3FFFFFFFu; + uint v = b[ix > last ? last : ix]; + r[i] = ix < words ? v : 0u; + } + return r; +} + +kernel void bench(device uint* data [[buffer(0)]], device uint* out [[buffer(1)]], constant uint4& prm [[buffer(2)]], + constant uint* cdata [[buffer(5)]], + uint tid [[thread_position_in_grid]], uint lane [[thread_index_in_simdgroup]]) +{ + constant uint* b = cdata; + uint bytes = prm.y; + uint acc = tid; + { uint4 w = sharpemu_load_dwords<2>(b, bytes, 0u); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, 48u); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, 96u); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, 144u); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, 192u); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, 240u); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, 288u); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, 336u); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, 384u); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, 432u); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, 480u); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, 528u); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, 576u); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, 624u); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, 672u); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, 720u); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, 768u); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, 816u); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, 864u); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, 912u); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, 960u); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, 1008u); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, 1056u); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, 1104u); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, 1152u); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, 1200u); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, 1248u); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, 1296u); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, 1344u); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, 1392u); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, 1440u); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, 1488u); acc ^= w.x; acc += w.y; } + out[tid] = acc; +} +""", + ["buf2_uniform_constant_outlined"] = """ +// bench group=buf_two_uniform_constant variant=outlined threads=1048576 tg=32 iters=1 check=1 +#include +using namespace metal; +#pragma clang fp contract(off) +//@BUF_PRELUDE_CONSTANT + +// Fast path inline, the dword-by-dword path out of line. +__attribute__((noinline)) static uint2 slow2(constant uint* b, uint bytes, uint addr) +{ + return uint2(sharpemu_load_dword(b, bytes, addr), sharpemu_load_dword(b, bytes, addr + 4u)); +} +static inline uint2 load2_outlined(constant uint* b, uint bytes, uint addr) +{ + uint index = addr >> 2; + uint words = bytes >> 2; + if (index < words && 2u <= words - index) + { + return uint2(b[index], b[index + 1u]); + } + return slow2(b, bytes, addr); +} +// Branch-free: every dword is read at an index clamped into the binding and selected against +// its own bounds test, which is what sharpemu_load_dword answers per dword. Needs a binding of +// at least one dword. +static inline uint2 load2_select(constant uint* b, uint bytes, uint addr) +{ + uint index = addr >> 2; + uint words = bytes >> 2; + uint last = max(words, 1u) - 1u; + uint2 r = uint2(0u); + for (int i = 0; i < 2; i++) + { + uint ix = ((addr >> 2) + (uint)i) & 0x3FFFFFFFu; + uint v = b[ix > last ? last : ix]; + r[i] = ix < words ? v : 0u; + } + return r; +} + +kernel void bench(device uint* data [[buffer(0)]], device uint* out [[buffer(1)]], constant uint4& prm [[buffer(2)]], + constant uint* cdata [[buffer(5)]], + uint tid [[thread_position_in_grid]], uint lane [[thread_index_in_simdgroup]]) +{ + constant uint* b = cdata; + uint bytes = prm.y; + uint acc = tid; + { uint2 w = load2_outlined(b, bytes, 0u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, 48u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, 96u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, 144u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, 192u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, 240u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, 288u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, 336u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, 384u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, 432u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, 480u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, 528u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, 576u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, 624u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, 672u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, 720u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, 768u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, 816u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, 864u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, 912u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, 960u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, 1008u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, 1056u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, 1104u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, 1152u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, 1200u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, 1248u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, 1296u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, 1344u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, 1392u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, 1440u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, 1488u); acc ^= w.x; acc += w.y; } + out[tid] = acc; +} +""", + ["buf2_uniform_constant_raw"] = """ +// bench group=buf_two_uniform_constant variant=raw threads=1048576 tg=32 iters=1 check=1 +#include +using namespace metal; +#pragma clang fp contract(off) +//@BUF_PRELUDE_CONSTANT + +// Fast path inline, the dword-by-dword path out of line. +__attribute__((noinline)) static uint2 slow2(constant uint* b, uint bytes, uint addr) +{ + return uint2(sharpemu_load_dword(b, bytes, addr), sharpemu_load_dword(b, bytes, addr + 4u)); +} +static inline uint2 load2_outlined(constant uint* b, uint bytes, uint addr) +{ + uint index = addr >> 2; + uint words = bytes >> 2; + if (index < words && 2u <= words - index) + { + return uint2(b[index], b[index + 1u]); + } + return slow2(b, bytes, addr); +} +// Branch-free: every dword is read at an index clamped into the binding and selected against +// its own bounds test, which is what sharpemu_load_dword answers per dword. Needs a binding of +// at least one dword. +static inline uint2 load2_select(constant uint* b, uint bytes, uint addr) +{ + uint index = addr >> 2; + uint words = bytes >> 2; + uint last = max(words, 1u) - 1u; + uint2 r = uint2(0u); + for (int i = 0; i < 2; i++) + { + uint ix = ((addr >> 2) + (uint)i) & 0x3FFFFFFFu; + uint v = b[ix > last ? last : ix]; + r[i] = ix < words ? v : 0u; + } + return r; +} + +kernel void bench(device uint* data [[buffer(0)]], device uint* out [[buffer(1)]], constant uint4& prm [[buffer(2)]], + constant uint* cdata [[buffer(5)]], + uint tid [[thread_position_in_grid]], uint lane [[thread_index_in_simdgroup]]) +{ + constant uint* b = cdata; + uint bytes = prm.y; + uint acc = tid; + { uint i = 0u >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = 48u >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = 96u >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = 144u >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = 192u >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = 240u >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = 288u >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = 336u >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = 384u >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = 432u >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = 480u >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = 528u >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = 576u >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = 624u >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = 672u >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = 720u >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = 768u >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = 816u >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = 864u >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = 912u >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = 960u >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = 1008u >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = 1056u >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = 1104u >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = 1152u >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = 1200u >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = 1248u >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = 1296u >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = 1344u >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = 1392u >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = 1440u >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = 1488u >> 2; acc ^= b[i]; acc += b[i + 1u]; } + out[tid] = acc; +} +""", + ["buf2_uniform_constant_select"] = """ +// bench group=buf_two_uniform_constant variant=select threads=1048576 tg=32 iters=1 check=1 +#include +using namespace metal; +#pragma clang fp contract(off) +//@BUF_PRELUDE_CONSTANT + +// Fast path inline, the dword-by-dword path out of line. +__attribute__((noinline)) static uint2 slow2(constant uint* b, uint bytes, uint addr) +{ + return uint2(sharpemu_load_dword(b, bytes, addr), sharpemu_load_dword(b, bytes, addr + 4u)); +} +static inline uint2 load2_outlined(constant uint* b, uint bytes, uint addr) +{ + uint index = addr >> 2; + uint words = bytes >> 2; + if (index < words && 2u <= words - index) + { + return uint2(b[index], b[index + 1u]); + } + return slow2(b, bytes, addr); +} +// Branch-free: every dword is read at an index clamped into the binding and selected against +// its own bounds test, which is what sharpemu_load_dword answers per dword. Needs a binding of +// at least one dword. +static inline uint2 load2_select(constant uint* b, uint bytes, uint addr) +{ + uint index = addr >> 2; + uint words = bytes >> 2; + uint last = max(words, 1u) - 1u; + uint2 r = uint2(0u); + for (int i = 0; i < 2; i++) + { + uint ix = ((addr >> 2) + (uint)i) & 0x3FFFFFFFu; + uint v = b[ix > last ? last : ix]; + r[i] = ix < words ? v : 0u; + } + return r; +} + +kernel void bench(device uint* data [[buffer(0)]], device uint* out [[buffer(1)]], constant uint4& prm [[buffer(2)]], + constant uint* cdata [[buffer(5)]], + uint tid [[thread_position_in_grid]], uint lane [[thread_index_in_simdgroup]]) +{ + constant uint* b = cdata; + uint bytes = prm.y; + uint acc = tid; + { uint2 w = load2_select(b, bytes, 0u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, 48u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, 96u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, 144u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, 192u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, 240u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, 288u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, 336u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, 384u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, 432u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, 480u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, 528u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, 576u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, 624u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, 672u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, 720u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, 768u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, 816u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, 864u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, 912u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, 960u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, 1008u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, 1056u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, 1104u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, 1152u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, 1200u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, 1248u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, 1296u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, 1344u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, 1392u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, 1440u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, 1488u); acc ^= w.x; acc += w.y; } + out[tid] = acc; +} +""", + ["buf2_uniform_device_checked"] = """ +// bench group=buf_two_uniform_device variant=checked threads=1048576 tg=32 iters=1 check=1 +#include +using namespace metal; +#pragma clang fp contract(off) +//@BUF_PRELUDE + +// Fast path inline, the dword-by-dword path out of line. +__attribute__((noinline)) static uint2 slow2(device uint* b, uint bytes, uint addr) +{ + return uint2(sharpemu_load_dword(b, bytes, addr), sharpemu_load_dword(b, bytes, addr + 4u)); +} +static inline uint2 load2_outlined(device uint* b, uint bytes, uint addr) +{ + uint index = addr >> 2; + uint words = bytes >> 2; + if (index < words && 2u <= words - index) + { + return uint2(b[index], b[index + 1u]); + } + return slow2(b, bytes, addr); +} +// Branch-free: every dword is read at an index clamped into the binding and selected against +// its own bounds test, which is what sharpemu_load_dword answers per dword. Needs a binding of +// at least one dword. +static inline uint2 load2_select(device uint* b, uint bytes, uint addr) +{ + uint index = addr >> 2; + uint words = bytes >> 2; + uint last = max(words, 1u) - 1u; + uint2 r = uint2(0u); + for (int i = 0; i < 2; i++) + { + uint ix = ((addr >> 2) + (uint)i) & 0x3FFFFFFFu; + uint v = b[ix > last ? last : ix]; + r[i] = ix < words ? v : 0u; + } + return r; +} + +kernel void bench(device uint* data [[buffer(0)]], device uint* out [[buffer(1)]], constant uint4& prm [[buffer(2)]], + constant uint* cdata [[buffer(5)]], + uint tid [[thread_position_in_grid]], uint lane [[thread_index_in_simdgroup]]) +{ + device uint* b = data; + uint bytes = prm.y; + uint acc = tid; + { uint4 w = sharpemu_load_dwords<2>(b, bytes, 0u); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, 48u); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, 96u); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, 144u); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, 192u); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, 240u); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, 288u); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, 336u); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, 384u); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, 432u); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, 480u); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, 528u); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, 576u); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, 624u); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, 672u); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, 720u); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, 768u); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, 816u); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, 864u); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, 912u); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, 960u); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, 1008u); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, 1056u); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, 1104u); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, 1152u); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, 1200u); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, 1248u); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, 1296u); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, 1344u); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, 1392u); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, 1440u); acc ^= w.x; acc += w.y; } + { uint4 w = sharpemu_load_dwords<2>(b, bytes, 1488u); acc ^= w.x; acc += w.y; } + out[tid] = acc; +} +""", + ["buf2_uniform_device_outlined"] = """ +// bench group=buf_two_uniform_device variant=outlined threads=1048576 tg=32 iters=1 check=1 +#include +using namespace metal; +#pragma clang fp contract(off) +//@BUF_PRELUDE + +// Fast path inline, the dword-by-dword path out of line. +__attribute__((noinline)) static uint2 slow2(device uint* b, uint bytes, uint addr) +{ + return uint2(sharpemu_load_dword(b, bytes, addr), sharpemu_load_dword(b, bytes, addr + 4u)); +} +static inline uint2 load2_outlined(device uint* b, uint bytes, uint addr) +{ + uint index = addr >> 2; + uint words = bytes >> 2; + if (index < words && 2u <= words - index) + { + return uint2(b[index], b[index + 1u]); + } + return slow2(b, bytes, addr); +} +// Branch-free: every dword is read at an index clamped into the binding and selected against +// its own bounds test, which is what sharpemu_load_dword answers per dword. Needs a binding of +// at least one dword. +static inline uint2 load2_select(device uint* b, uint bytes, uint addr) +{ + uint index = addr >> 2; + uint words = bytes >> 2; + uint last = max(words, 1u) - 1u; + uint2 r = uint2(0u); + for (int i = 0; i < 2; i++) + { + uint ix = ((addr >> 2) + (uint)i) & 0x3FFFFFFFu; + uint v = b[ix > last ? last : ix]; + r[i] = ix < words ? v : 0u; + } + return r; +} + +kernel void bench(device uint* data [[buffer(0)]], device uint* out [[buffer(1)]], constant uint4& prm [[buffer(2)]], + constant uint* cdata [[buffer(5)]], + uint tid [[thread_position_in_grid]], uint lane [[thread_index_in_simdgroup]]) +{ + device uint* b = data; + uint bytes = prm.y; + uint acc = tid; + { uint2 w = load2_outlined(b, bytes, 0u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, 48u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, 96u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, 144u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, 192u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, 240u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, 288u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, 336u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, 384u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, 432u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, 480u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, 528u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, 576u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, 624u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, 672u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, 720u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, 768u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, 816u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, 864u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, 912u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, 960u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, 1008u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, 1056u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, 1104u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, 1152u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, 1200u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, 1248u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, 1296u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, 1344u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, 1392u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, 1440u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_outlined(b, bytes, 1488u); acc ^= w.x; acc += w.y; } + out[tid] = acc; +} +""", + ["buf2_uniform_device_raw"] = """ +// bench group=buf_two_uniform_device variant=raw threads=1048576 tg=32 iters=1 check=1 +#include +using namespace metal; +#pragma clang fp contract(off) +//@BUF_PRELUDE + +// Fast path inline, the dword-by-dword path out of line. +__attribute__((noinline)) static uint2 slow2(device uint* b, uint bytes, uint addr) +{ + return uint2(sharpemu_load_dword(b, bytes, addr), sharpemu_load_dword(b, bytes, addr + 4u)); +} +static inline uint2 load2_outlined(device uint* b, uint bytes, uint addr) +{ + uint index = addr >> 2; + uint words = bytes >> 2; + if (index < words && 2u <= words - index) + { + return uint2(b[index], b[index + 1u]); + } + return slow2(b, bytes, addr); +} +// Branch-free: every dword is read at an index clamped into the binding and selected against +// its own bounds test, which is what sharpemu_load_dword answers per dword. Needs a binding of +// at least one dword. +static inline uint2 load2_select(device uint* b, uint bytes, uint addr) +{ + uint index = addr >> 2; + uint words = bytes >> 2; + uint last = max(words, 1u) - 1u; + uint2 r = uint2(0u); + for (int i = 0; i < 2; i++) + { + uint ix = ((addr >> 2) + (uint)i) & 0x3FFFFFFFu; + uint v = b[ix > last ? last : ix]; + r[i] = ix < words ? v : 0u; + } + return r; +} + +kernel void bench(device uint* data [[buffer(0)]], device uint* out [[buffer(1)]], constant uint4& prm [[buffer(2)]], + constant uint* cdata [[buffer(5)]], + uint tid [[thread_position_in_grid]], uint lane [[thread_index_in_simdgroup]]) +{ + device uint* b = data; + uint bytes = prm.y; + uint acc = tid; + { uint i = 0u >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = 48u >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = 96u >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = 144u >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = 192u >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = 240u >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = 288u >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = 336u >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = 384u >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = 432u >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = 480u >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = 528u >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = 576u >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = 624u >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = 672u >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = 720u >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = 768u >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = 816u >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = 864u >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = 912u >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = 960u >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = 1008u >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = 1056u >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = 1104u >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = 1152u >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = 1200u >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = 1248u >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = 1296u >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = 1344u >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = 1392u >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = 1440u >> 2; acc ^= b[i]; acc += b[i + 1u]; } + { uint i = 1488u >> 2; acc ^= b[i]; acc += b[i + 1u]; } + out[tid] = acc; +} +""", + ["buf2_uniform_device_select"] = """ +// bench group=buf_two_uniform_device variant=select threads=1048576 tg=32 iters=1 check=1 +#include +using namespace metal; +#pragma clang fp contract(off) +//@BUF_PRELUDE + +// Fast path inline, the dword-by-dword path out of line. +__attribute__((noinline)) static uint2 slow2(device uint* b, uint bytes, uint addr) +{ + return uint2(sharpemu_load_dword(b, bytes, addr), sharpemu_load_dword(b, bytes, addr + 4u)); +} +static inline uint2 load2_outlined(device uint* b, uint bytes, uint addr) +{ + uint index = addr >> 2; + uint words = bytes >> 2; + if (index < words && 2u <= words - index) + { + return uint2(b[index], b[index + 1u]); + } + return slow2(b, bytes, addr); +} +// Branch-free: every dword is read at an index clamped into the binding and selected against +// its own bounds test, which is what sharpemu_load_dword answers per dword. Needs a binding of +// at least one dword. +static inline uint2 load2_select(device uint* b, uint bytes, uint addr) +{ + uint index = addr >> 2; + uint words = bytes >> 2; + uint last = max(words, 1u) - 1u; + uint2 r = uint2(0u); + for (int i = 0; i < 2; i++) + { + uint ix = ((addr >> 2) + (uint)i) & 0x3FFFFFFFu; + uint v = b[ix > last ? last : ix]; + r[i] = ix < words ? v : 0u; + } + return r; +} + +kernel void bench(device uint* data [[buffer(0)]], device uint* out [[buffer(1)]], constant uint4& prm [[buffer(2)]], + constant uint* cdata [[buffer(5)]], + uint tid [[thread_position_in_grid]], uint lane [[thread_index_in_simdgroup]]) +{ + device uint* b = data; + uint bytes = prm.y; + uint acc = tid; + { uint2 w = load2_select(b, bytes, 0u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, 48u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, 96u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, 144u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, 192u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, 240u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, 288u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, 336u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, 384u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, 432u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, 480u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, 528u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, 576u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, 624u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, 672u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, 720u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, 768u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, 816u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, 864u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, 912u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, 960u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, 1008u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, 1056u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, 1104u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, 1152u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, 1200u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, 1248u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, 1296u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, 1344u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, 1392u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, 1440u); acc ^= w.x; acc += w.y; } + { uint2 w = load2_select(b, bytes, 1488u); acc ^= w.x; acc += w.y; } + out[tid] = acc; +} +""", + ["buf_perlane_constant_checked"] = """ +// bench group=buf_perlane_constant variant=checked threads=1048576 tg=32 iters=1 check=1 +#include +using namespace metal; +#pragma clang fp contract(off) +//@BUF_PRELUDE_CONSTANT + +// Fast path inline, the dword-by-dword path out of line. +__attribute__((noinline)) static uint4 slow4(constant uint* b, uint bytes, uint addr) +{ + return uint4(sharpemu_load_dword(b, bytes, addr), sharpemu_load_dword(b, bytes, addr + 4u), + sharpemu_load_dword(b, bytes, addr + 8u), sharpemu_load_dword(b, bytes, addr + 12u)); +} +static inline uint4 load4_outlined(constant uint* b, uint bytes, uint addr) +{ + uint index = addr >> 2; + uint words = bytes >> 2; + if (index < words && 4u <= words - index) + { + return uint4(b[index], b[index + 1u], b[index + 2u], b[index + 3u]); + } + return slow4(b, bytes, addr); +} +// Branch-free: every dword is read at an index clamped into the binding and selected against +// its own bounds test, which is what sharpemu_load_dword answers per dword. Needs a binding of +// at least one dword. +static inline uint4 load4_select(constant uint* b, uint bytes, uint addr) +{ + uint index = addr >> 2; + uint words = bytes >> 2; + uint last = max(words, 1u) - 1u; + uint4 r = uint4(0u); + for (int i = 0; i < 4; i++) + { + uint ix = ((addr >> 2) + (uint)i) & 0x3FFFFFFFu; + uint v = b[ix > last ? last : ix]; + r[i] = ix < words ? v : 0u; + } + return r; +} + +kernel void bench(device uint* data [[buffer(0)]], device uint* out [[buffer(1)]], constant uint4& prm [[buffer(2)]], + constant uint* cdata [[buffer(5)]], + uint tid [[thread_position_in_grid]], uint lane [[thread_index_in_simdgroup]]) +{ + constant uint* b = cdata; + uint bytes = prm.y; + uint acc = tid; + { uint4 w = sharpemu_load_dwords<4>(b, bytes, (0u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, (65536u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, (131072u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, (196608u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, (262144u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, (327680u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, (393216u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, (458752u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, (524288u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, (589824u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, (655360u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, (720896u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, (786432u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, (851968u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, (917504u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, (983040u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, (1048576u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, (1114112u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, (1179648u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, (1245184u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, (1310720u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, (1376256u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, (1441792u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, (1507328u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, (1572864u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, (1638400u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, (1703936u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, (1769472u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, (1835008u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, (1900544u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, (1966080u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, (2031616u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + out[tid] = acc; +} +""", + ["buf_perlane_constant_outlined"] = """ +// bench group=buf_perlane_constant variant=outlined threads=1048576 tg=32 iters=1 check=1 +#include +using namespace metal; +#pragma clang fp contract(off) +//@BUF_PRELUDE_CONSTANT + +// Fast path inline, the dword-by-dword path out of line. +__attribute__((noinline)) static uint4 slow4(constant uint* b, uint bytes, uint addr) +{ + return uint4(sharpemu_load_dword(b, bytes, addr), sharpemu_load_dword(b, bytes, addr + 4u), + sharpemu_load_dword(b, bytes, addr + 8u), sharpemu_load_dword(b, bytes, addr + 12u)); +} +static inline uint4 load4_outlined(constant uint* b, uint bytes, uint addr) +{ + uint index = addr >> 2; + uint words = bytes >> 2; + if (index < words && 4u <= words - index) + { + return uint4(b[index], b[index + 1u], b[index + 2u], b[index + 3u]); + } + return slow4(b, bytes, addr); +} +// Branch-free: every dword is read at an index clamped into the binding and selected against +// its own bounds test, which is what sharpemu_load_dword answers per dword. Needs a binding of +// at least one dword. +static inline uint4 load4_select(constant uint* b, uint bytes, uint addr) +{ + uint index = addr >> 2; + uint words = bytes >> 2; + uint last = max(words, 1u) - 1u; + uint4 r = uint4(0u); + for (int i = 0; i < 4; i++) + { + uint ix = ((addr >> 2) + (uint)i) & 0x3FFFFFFFu; + uint v = b[ix > last ? last : ix]; + r[i] = ix < words ? v : 0u; + } + return r; +} + +kernel void bench(device uint* data [[buffer(0)]], device uint* out [[buffer(1)]], constant uint4& prm [[buffer(2)]], + constant uint* cdata [[buffer(5)]], + uint tid [[thread_position_in_grid]], uint lane [[thread_index_in_simdgroup]]) +{ + constant uint* b = cdata; + uint bytes = prm.y; + uint acc = tid; + { uint4 w = load4_outlined(b, bytes, (0u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, (65536u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, (131072u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, (196608u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, (262144u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, (327680u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, (393216u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, (458752u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, (524288u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, (589824u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, (655360u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, (720896u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, (786432u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, (851968u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, (917504u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, (983040u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, (1048576u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, (1114112u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, (1179648u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, (1245184u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, (1310720u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, (1376256u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, (1441792u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, (1507328u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, (1572864u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, (1638400u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, (1703936u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, (1769472u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, (1835008u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, (1900544u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, (1966080u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, (2031616u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + out[tid] = acc; +} +""", + ["buf_perlane_constant_raw"] = """ +// bench group=buf_perlane_constant variant=raw threads=1048576 tg=32 iters=1 check=1 +#include +using namespace metal; +#pragma clang fp contract(off) +//@BUF_PRELUDE_CONSTANT + +// Fast path inline, the dword-by-dword path out of line. +__attribute__((noinline)) static uint4 slow4(constant uint* b, uint bytes, uint addr) +{ + return uint4(sharpemu_load_dword(b, bytes, addr), sharpemu_load_dword(b, bytes, addr + 4u), + sharpemu_load_dword(b, bytes, addr + 8u), sharpemu_load_dword(b, bytes, addr + 12u)); +} +static inline uint4 load4_outlined(constant uint* b, uint bytes, uint addr) +{ + uint index = addr >> 2; + uint words = bytes >> 2; + if (index < words && 4u <= words - index) + { + return uint4(b[index], b[index + 1u], b[index + 2u], b[index + 3u]); + } + return slow4(b, bytes, addr); +} +// Branch-free: every dword is read at an index clamped into the binding and selected against +// its own bounds test, which is what sharpemu_load_dword answers per dword. Needs a binding of +// at least one dword. +static inline uint4 load4_select(constant uint* b, uint bytes, uint addr) +{ + uint index = addr >> 2; + uint words = bytes >> 2; + uint last = max(words, 1u) - 1u; + uint4 r = uint4(0u); + for (int i = 0; i < 4; i++) + { + uint ix = ((addr >> 2) + (uint)i) & 0x3FFFFFFFu; + uint v = b[ix > last ? last : ix]; + r[i] = ix < words ? v : 0u; + } + return r; +} + +kernel void bench(device uint* data [[buffer(0)]], device uint* out [[buffer(1)]], constant uint4& prm [[buffer(2)]], + constant uint* cdata [[buffer(5)]], + uint tid [[thread_position_in_grid]], uint lane [[thread_index_in_simdgroup]]) +{ + constant uint* b = cdata; + uint bytes = prm.y; + uint acc = tid; + { uint i = (0u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = (65536u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = (131072u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = (196608u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = (262144u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = (327680u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = (393216u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = (458752u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = (524288u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = (589824u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = (655360u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = (720896u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = (786432u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = (851968u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = (917504u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = (983040u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = (1048576u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = (1114112u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = (1179648u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = (1245184u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = (1310720u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = (1376256u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = (1441792u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = (1507328u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = (1572864u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = (1638400u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = (1703936u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = (1769472u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = (1835008u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = (1900544u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = (1966080u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = (2031616u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + out[tid] = acc; +} +""", + ["buf_perlane_constant_select"] = """ +// bench group=buf_perlane_constant variant=select threads=1048576 tg=32 iters=1 check=1 +#include +using namespace metal; +#pragma clang fp contract(off) +//@BUF_PRELUDE_CONSTANT + +// Fast path inline, the dword-by-dword path out of line. +__attribute__((noinline)) static uint4 slow4(constant uint* b, uint bytes, uint addr) +{ + return uint4(sharpemu_load_dword(b, bytes, addr), sharpemu_load_dword(b, bytes, addr + 4u), + sharpemu_load_dword(b, bytes, addr + 8u), sharpemu_load_dword(b, bytes, addr + 12u)); +} +static inline uint4 load4_outlined(constant uint* b, uint bytes, uint addr) +{ + uint index = addr >> 2; + uint words = bytes >> 2; + if (index < words && 4u <= words - index) + { + return uint4(b[index], b[index + 1u], b[index + 2u], b[index + 3u]); + } + return slow4(b, bytes, addr); +} +// Branch-free: every dword is read at an index clamped into the binding and selected against +// its own bounds test, which is what sharpemu_load_dword answers per dword. Needs a binding of +// at least one dword. +static inline uint4 load4_select(constant uint* b, uint bytes, uint addr) +{ + uint index = addr >> 2; + uint words = bytes >> 2; + uint last = max(words, 1u) - 1u; + uint4 r = uint4(0u); + for (int i = 0; i < 4; i++) + { + uint ix = ((addr >> 2) + (uint)i) & 0x3FFFFFFFu; + uint v = b[ix > last ? last : ix]; + r[i] = ix < words ? v : 0u; + } + return r; +} + +kernel void bench(device uint* data [[buffer(0)]], device uint* out [[buffer(1)]], constant uint4& prm [[buffer(2)]], + constant uint* cdata [[buffer(5)]], + uint tid [[thread_position_in_grid]], uint lane [[thread_index_in_simdgroup]]) +{ + constant uint* b = cdata; + uint bytes = prm.y; + uint acc = tid; + { uint4 w = load4_select(b, bytes, (0u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, (65536u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, (131072u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, (196608u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, (262144u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, (327680u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, (393216u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, (458752u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, (524288u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, (589824u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, (655360u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, (720896u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, (786432u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, (851968u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, (917504u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, (983040u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, (1048576u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, (1114112u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, (1179648u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, (1245184u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, (1310720u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, (1376256u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, (1441792u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, (1507328u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, (1572864u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, (1638400u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, (1703936u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, (1769472u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, (1835008u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, (1900544u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, (1966080u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, (2031616u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + out[tid] = acc; +} +""", + ["buf_perlane_device_checked"] = """ +// bench group=buf_perlane_device variant=checked threads=1048576 tg=32 iters=1 check=1 +#include +using namespace metal; +#pragma clang fp contract(off) +//@BUF_PRELUDE + +// Fast path inline, the dword-by-dword path out of line. +__attribute__((noinline)) static uint4 slow4(device uint* b, uint bytes, uint addr) +{ + return uint4(sharpemu_load_dword(b, bytes, addr), sharpemu_load_dword(b, bytes, addr + 4u), + sharpemu_load_dword(b, bytes, addr + 8u), sharpemu_load_dword(b, bytes, addr + 12u)); +} +static inline uint4 load4_outlined(device uint* b, uint bytes, uint addr) +{ + uint index = addr >> 2; + uint words = bytes >> 2; + if (index < words && 4u <= words - index) + { + return uint4(b[index], b[index + 1u], b[index + 2u], b[index + 3u]); + } + return slow4(b, bytes, addr); +} +// Branch-free: every dword is read at an index clamped into the binding and selected against +// its own bounds test, which is what sharpemu_load_dword answers per dword. Needs a binding of +// at least one dword. +static inline uint4 load4_select(device uint* b, uint bytes, uint addr) +{ + uint index = addr >> 2; + uint words = bytes >> 2; + uint last = max(words, 1u) - 1u; + uint4 r = uint4(0u); + for (int i = 0; i < 4; i++) + { + uint ix = ((addr >> 2) + (uint)i) & 0x3FFFFFFFu; + uint v = b[ix > last ? last : ix]; + r[i] = ix < words ? v : 0u; + } + return r; +} + +kernel void bench(device uint* data [[buffer(0)]], device uint* out [[buffer(1)]], constant uint4& prm [[buffer(2)]], + constant uint* cdata [[buffer(5)]], + uint tid [[thread_position_in_grid]], uint lane [[thread_index_in_simdgroup]]) +{ + device uint* b = data; + uint bytes = prm.y; + uint acc = tid; + { uint4 w = sharpemu_load_dwords<4>(b, bytes, (0u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, (65536u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, (131072u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, (196608u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, (262144u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, (327680u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, (393216u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, (458752u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, (524288u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, (589824u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, (655360u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, (720896u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, (786432u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, (851968u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, (917504u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, (983040u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, (1048576u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, (1114112u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, (1179648u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, (1245184u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, (1310720u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, (1376256u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, (1441792u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, (1507328u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, (1572864u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, (1638400u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, (1703936u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, (1769472u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, (1835008u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, (1900544u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, (1966080u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, (2031616u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + out[tid] = acc; +} +""", + ["buf_perlane_device_outlined"] = """ +// bench group=buf_perlane_device variant=outlined threads=1048576 tg=32 iters=1 check=1 +#include +using namespace metal; +#pragma clang fp contract(off) +//@BUF_PRELUDE + +// Fast path inline, the dword-by-dword path out of line. +__attribute__((noinline)) static uint4 slow4(device uint* b, uint bytes, uint addr) +{ + return uint4(sharpemu_load_dword(b, bytes, addr), sharpemu_load_dword(b, bytes, addr + 4u), + sharpemu_load_dword(b, bytes, addr + 8u), sharpemu_load_dword(b, bytes, addr + 12u)); +} +static inline uint4 load4_outlined(device uint* b, uint bytes, uint addr) +{ + uint index = addr >> 2; + uint words = bytes >> 2; + if (index < words && 4u <= words - index) + { + return uint4(b[index], b[index + 1u], b[index + 2u], b[index + 3u]); + } + return slow4(b, bytes, addr); +} +// Branch-free: every dword is read at an index clamped into the binding and selected against +// its own bounds test, which is what sharpemu_load_dword answers per dword. Needs a binding of +// at least one dword. +static inline uint4 load4_select(device uint* b, uint bytes, uint addr) +{ + uint index = addr >> 2; + uint words = bytes >> 2; + uint last = max(words, 1u) - 1u; + uint4 r = uint4(0u); + for (int i = 0; i < 4; i++) + { + uint ix = ((addr >> 2) + (uint)i) & 0x3FFFFFFFu; + uint v = b[ix > last ? last : ix]; + r[i] = ix < words ? v : 0u; + } + return r; +} + +kernel void bench(device uint* data [[buffer(0)]], device uint* out [[buffer(1)]], constant uint4& prm [[buffer(2)]], + constant uint* cdata [[buffer(5)]], + uint tid [[thread_position_in_grid]], uint lane [[thread_index_in_simdgroup]]) +{ + device uint* b = data; + uint bytes = prm.y; + uint acc = tid; + { uint4 w = load4_outlined(b, bytes, (0u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, (65536u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, (131072u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, (196608u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, (262144u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, (327680u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, (393216u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, (458752u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, (524288u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, (589824u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, (655360u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, (720896u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, (786432u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, (851968u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, (917504u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, (983040u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, (1048576u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, (1114112u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, (1179648u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, (1245184u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, (1310720u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, (1376256u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, (1441792u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, (1507328u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, (1572864u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, (1638400u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, (1703936u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, (1769472u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, (1835008u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, (1900544u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, (1966080u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, (2031616u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + out[tid] = acc; +} +""", + ["buf_perlane_device_raw"] = """ +// bench group=buf_perlane_device variant=raw threads=1048576 tg=32 iters=1 check=1 +#include +using namespace metal; +#pragma clang fp contract(off) +//@BUF_PRELUDE + +// Fast path inline, the dword-by-dword path out of line. +__attribute__((noinline)) static uint4 slow4(device uint* b, uint bytes, uint addr) +{ + return uint4(sharpemu_load_dword(b, bytes, addr), sharpemu_load_dword(b, bytes, addr + 4u), + sharpemu_load_dword(b, bytes, addr + 8u), sharpemu_load_dword(b, bytes, addr + 12u)); +} +static inline uint4 load4_outlined(device uint* b, uint bytes, uint addr) +{ + uint index = addr >> 2; + uint words = bytes >> 2; + if (index < words && 4u <= words - index) + { + return uint4(b[index], b[index + 1u], b[index + 2u], b[index + 3u]); + } + return slow4(b, bytes, addr); +} +// Branch-free: every dword is read at an index clamped into the binding and selected against +// its own bounds test, which is what sharpemu_load_dword answers per dword. Needs a binding of +// at least one dword. +static inline uint4 load4_select(device uint* b, uint bytes, uint addr) +{ + uint index = addr >> 2; + uint words = bytes >> 2; + uint last = max(words, 1u) - 1u; + uint4 r = uint4(0u); + for (int i = 0; i < 4; i++) + { + uint ix = ((addr >> 2) + (uint)i) & 0x3FFFFFFFu; + uint v = b[ix > last ? last : ix]; + r[i] = ix < words ? v : 0u; + } + return r; +} + +kernel void bench(device uint* data [[buffer(0)]], device uint* out [[buffer(1)]], constant uint4& prm [[buffer(2)]], + constant uint* cdata [[buffer(5)]], + uint tid [[thread_position_in_grid]], uint lane [[thread_index_in_simdgroup]]) +{ + device uint* b = data; + uint bytes = prm.y; + uint acc = tid; + { uint i = (0u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = (65536u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = (131072u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = (196608u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = (262144u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = (327680u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = (393216u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = (458752u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = (524288u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = (589824u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = (655360u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = (720896u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = (786432u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = (851968u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = (917504u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = (983040u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = (1048576u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = (1114112u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = (1179648u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = (1245184u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = (1310720u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = (1376256u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = (1441792u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = (1507328u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = (1572864u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = (1638400u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = (1703936u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = (1769472u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = (1835008u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = (1900544u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = (1966080u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = (2031616u + ((tid * 16u) & 0xFFFFu)) >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + out[tid] = acc; +} +""", + ["buf_perlane_device_select"] = """ +// bench group=buf_perlane_device variant=select threads=1048576 tg=32 iters=1 check=1 +#include +using namespace metal; +#pragma clang fp contract(off) +//@BUF_PRELUDE + +// Fast path inline, the dword-by-dword path out of line. +__attribute__((noinline)) static uint4 slow4(device uint* b, uint bytes, uint addr) +{ + return uint4(sharpemu_load_dword(b, bytes, addr), sharpemu_load_dword(b, bytes, addr + 4u), + sharpemu_load_dword(b, bytes, addr + 8u), sharpemu_load_dword(b, bytes, addr + 12u)); +} +static inline uint4 load4_outlined(device uint* b, uint bytes, uint addr) +{ + uint index = addr >> 2; + uint words = bytes >> 2; + if (index < words && 4u <= words - index) + { + return uint4(b[index], b[index + 1u], b[index + 2u], b[index + 3u]); + } + return slow4(b, bytes, addr); +} +// Branch-free: every dword is read at an index clamped into the binding and selected against +// its own bounds test, which is what sharpemu_load_dword answers per dword. Needs a binding of +// at least one dword. +static inline uint4 load4_select(device uint* b, uint bytes, uint addr) +{ + uint index = addr >> 2; + uint words = bytes >> 2; + uint last = max(words, 1u) - 1u; + uint4 r = uint4(0u); + for (int i = 0; i < 4; i++) + { + uint ix = ((addr >> 2) + (uint)i) & 0x3FFFFFFFu; + uint v = b[ix > last ? last : ix]; + r[i] = ix < words ? v : 0u; + } + return r; +} + +kernel void bench(device uint* data [[buffer(0)]], device uint* out [[buffer(1)]], constant uint4& prm [[buffer(2)]], + constant uint* cdata [[buffer(5)]], + uint tid [[thread_position_in_grid]], uint lane [[thread_index_in_simdgroup]]) +{ + device uint* b = data; + uint bytes = prm.y; + uint acc = tid; + { uint4 w = load4_select(b, bytes, (0u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, (65536u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, (131072u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, (196608u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, (262144u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, (327680u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, (393216u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, (458752u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, (524288u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, (589824u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, (655360u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, (720896u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, (786432u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, (851968u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, (917504u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, (983040u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, (1048576u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, (1114112u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, (1179648u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, (1245184u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, (1310720u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, (1376256u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, (1441792u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, (1507328u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, (1572864u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, (1638400u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, (1703936u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, (1769472u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, (1835008u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, (1900544u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, (1966080u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, (2031616u + ((tid * 16u) & 0xFFFFu))); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + out[tid] = acc; +} +""", + ["buf_uniform_constant_checked"] = """ +// bench group=buf_uniform_constant variant=checked threads=1048576 tg=32 iters=1 check=1 +#include +using namespace metal; +#pragma clang fp contract(off) +//@BUF_PRELUDE_CONSTANT + +// Fast path inline, the dword-by-dword path out of line. +__attribute__((noinline)) static uint4 slow4(constant uint* b, uint bytes, uint addr) +{ + return uint4(sharpemu_load_dword(b, bytes, addr), sharpemu_load_dword(b, bytes, addr + 4u), + sharpemu_load_dword(b, bytes, addr + 8u), sharpemu_load_dword(b, bytes, addr + 12u)); +} +static inline uint4 load4_outlined(constant uint* b, uint bytes, uint addr) +{ + uint index = addr >> 2; + uint words = bytes >> 2; + if (index < words && 4u <= words - index) + { + return uint4(b[index], b[index + 1u], b[index + 2u], b[index + 3u]); + } + return slow4(b, bytes, addr); +} +// Branch-free: every dword is read at an index clamped into the binding and selected against +// its own bounds test, which is what sharpemu_load_dword answers per dword. Needs a binding of +// at least one dword. +static inline uint4 load4_select(constant uint* b, uint bytes, uint addr) +{ + uint index = addr >> 2; + uint words = bytes >> 2; + uint last = max(words, 1u) - 1u; + uint4 r = uint4(0u); + for (int i = 0; i < 4; i++) + { + uint ix = ((addr >> 2) + (uint)i) & 0x3FFFFFFFu; + uint v = b[ix > last ? last : ix]; + r[i] = ix < words ? v : 0u; + } + return r; +} + +kernel void bench(device uint* data [[buffer(0)]], device uint* out [[buffer(1)]], constant uint4& prm [[buffer(2)]], + constant uint* cdata [[buffer(5)]], + uint tid [[thread_position_in_grid]], uint lane [[thread_index_in_simdgroup]]) +{ + constant uint* b = cdata; + uint bytes = prm.y; + uint acc = tid; + { uint4 w = sharpemu_load_dwords<4>(b, bytes, 0u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, 48u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, 96u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, 144u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, 192u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, 240u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, 288u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, 336u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, 384u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, 432u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, 480u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, 528u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, 576u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, 624u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, 672u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, 720u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, 768u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, 816u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, 864u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, 912u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, 960u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, 1008u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, 1056u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, 1104u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, 1152u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, 1200u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, 1248u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, 1296u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, 1344u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, 1392u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, 1440u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, 1488u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + out[tid] = acc; +} +""", + ["buf_uniform_constant_outlined"] = """ +// bench group=buf_uniform_constant variant=outlined threads=1048576 tg=32 iters=1 check=1 +#include +using namespace metal; +#pragma clang fp contract(off) +//@BUF_PRELUDE_CONSTANT + +// Fast path inline, the dword-by-dword path out of line. +__attribute__((noinline)) static uint4 slow4(constant uint* b, uint bytes, uint addr) +{ + return uint4(sharpemu_load_dword(b, bytes, addr), sharpemu_load_dword(b, bytes, addr + 4u), + sharpemu_load_dword(b, bytes, addr + 8u), sharpemu_load_dword(b, bytes, addr + 12u)); +} +static inline uint4 load4_outlined(constant uint* b, uint bytes, uint addr) +{ + uint index = addr >> 2; + uint words = bytes >> 2; + if (index < words && 4u <= words - index) + { + return uint4(b[index], b[index + 1u], b[index + 2u], b[index + 3u]); + } + return slow4(b, bytes, addr); +} +// Branch-free: every dword is read at an index clamped into the binding and selected against +// its own bounds test, which is what sharpemu_load_dword answers per dword. Needs a binding of +// at least one dword. +static inline uint4 load4_select(constant uint* b, uint bytes, uint addr) +{ + uint index = addr >> 2; + uint words = bytes >> 2; + uint last = max(words, 1u) - 1u; + uint4 r = uint4(0u); + for (int i = 0; i < 4; i++) + { + uint ix = ((addr >> 2) + (uint)i) & 0x3FFFFFFFu; + uint v = b[ix > last ? last : ix]; + r[i] = ix < words ? v : 0u; + } + return r; +} + +kernel void bench(device uint* data [[buffer(0)]], device uint* out [[buffer(1)]], constant uint4& prm [[buffer(2)]], + constant uint* cdata [[buffer(5)]], + uint tid [[thread_position_in_grid]], uint lane [[thread_index_in_simdgroup]]) +{ + constant uint* b = cdata; + uint bytes = prm.y; + uint acc = tid; + { uint4 w = load4_outlined(b, bytes, 0u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, 48u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, 96u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, 144u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, 192u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, 240u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, 288u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, 336u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, 384u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, 432u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, 480u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, 528u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, 576u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, 624u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, 672u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, 720u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, 768u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, 816u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, 864u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, 912u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, 960u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, 1008u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, 1056u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, 1104u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, 1152u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, 1200u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, 1248u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, 1296u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, 1344u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, 1392u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, 1440u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, 1488u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + out[tid] = acc; +} +""", + ["buf_uniform_constant_raw"] = """ +// bench group=buf_uniform_constant variant=raw threads=1048576 tg=32 iters=1 check=1 +#include +using namespace metal; +#pragma clang fp contract(off) +//@BUF_PRELUDE_CONSTANT + +// Fast path inline, the dword-by-dword path out of line. +__attribute__((noinline)) static uint4 slow4(constant uint* b, uint bytes, uint addr) +{ + return uint4(sharpemu_load_dword(b, bytes, addr), sharpemu_load_dword(b, bytes, addr + 4u), + sharpemu_load_dword(b, bytes, addr + 8u), sharpemu_load_dword(b, bytes, addr + 12u)); +} +static inline uint4 load4_outlined(constant uint* b, uint bytes, uint addr) +{ + uint index = addr >> 2; + uint words = bytes >> 2; + if (index < words && 4u <= words - index) + { + return uint4(b[index], b[index + 1u], b[index + 2u], b[index + 3u]); + } + return slow4(b, bytes, addr); +} +// Branch-free: every dword is read at an index clamped into the binding and selected against +// its own bounds test, which is what sharpemu_load_dword answers per dword. Needs a binding of +// at least one dword. +static inline uint4 load4_select(constant uint* b, uint bytes, uint addr) +{ + uint index = addr >> 2; + uint words = bytes >> 2; + uint last = max(words, 1u) - 1u; + uint4 r = uint4(0u); + for (int i = 0; i < 4; i++) + { + uint ix = ((addr >> 2) + (uint)i) & 0x3FFFFFFFu; + uint v = b[ix > last ? last : ix]; + r[i] = ix < words ? v : 0u; + } + return r; +} + +kernel void bench(device uint* data [[buffer(0)]], device uint* out [[buffer(1)]], constant uint4& prm [[buffer(2)]], + constant uint* cdata [[buffer(5)]], + uint tid [[thread_position_in_grid]], uint lane [[thread_index_in_simdgroup]]) +{ + constant uint* b = cdata; + uint bytes = prm.y; + uint acc = tid; + { uint i = 0u >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = 48u >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = 96u >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = 144u >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = 192u >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = 240u >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = 288u >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = 336u >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = 384u >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = 432u >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = 480u >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = 528u >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = 576u >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = 624u >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = 672u >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = 720u >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = 768u >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = 816u >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = 864u >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = 912u >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = 960u >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = 1008u >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = 1056u >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = 1104u >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = 1152u >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = 1200u >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = 1248u >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = 1296u >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = 1344u >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = 1392u >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = 1440u >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = 1488u >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + out[tid] = acc; +} +""", + ["buf_uniform_constant_select"] = """ +// bench group=buf_uniform_constant variant=select threads=1048576 tg=32 iters=1 check=1 +#include +using namespace metal; +#pragma clang fp contract(off) +//@BUF_PRELUDE_CONSTANT + +// Fast path inline, the dword-by-dword path out of line. +__attribute__((noinline)) static uint4 slow4(constant uint* b, uint bytes, uint addr) +{ + return uint4(sharpemu_load_dword(b, bytes, addr), sharpemu_load_dword(b, bytes, addr + 4u), + sharpemu_load_dword(b, bytes, addr + 8u), sharpemu_load_dword(b, bytes, addr + 12u)); +} +static inline uint4 load4_outlined(constant uint* b, uint bytes, uint addr) +{ + uint index = addr >> 2; + uint words = bytes >> 2; + if (index < words && 4u <= words - index) + { + return uint4(b[index], b[index + 1u], b[index + 2u], b[index + 3u]); + } + return slow4(b, bytes, addr); +} +// Branch-free: every dword is read at an index clamped into the binding and selected against +// its own bounds test, which is what sharpemu_load_dword answers per dword. Needs a binding of +// at least one dword. +static inline uint4 load4_select(constant uint* b, uint bytes, uint addr) +{ + uint index = addr >> 2; + uint words = bytes >> 2; + uint last = max(words, 1u) - 1u; + uint4 r = uint4(0u); + for (int i = 0; i < 4; i++) + { + uint ix = ((addr >> 2) + (uint)i) & 0x3FFFFFFFu; + uint v = b[ix > last ? last : ix]; + r[i] = ix < words ? v : 0u; + } + return r; +} + +kernel void bench(device uint* data [[buffer(0)]], device uint* out [[buffer(1)]], constant uint4& prm [[buffer(2)]], + constant uint* cdata [[buffer(5)]], + uint tid [[thread_position_in_grid]], uint lane [[thread_index_in_simdgroup]]) +{ + constant uint* b = cdata; + uint bytes = prm.y; + uint acc = tid; + { uint4 w = load4_select(b, bytes, 0u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, 48u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, 96u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, 144u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, 192u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, 240u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, 288u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, 336u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, 384u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, 432u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, 480u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, 528u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, 576u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, 624u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, 672u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, 720u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, 768u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, 816u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, 864u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, 912u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, 960u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, 1008u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, 1056u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, 1104u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, 1152u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, 1200u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, 1248u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, 1296u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, 1344u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, 1392u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, 1440u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, 1488u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + out[tid] = acc; +} +""", + ["buf_uniform_device_checked"] = """ +// bench group=buf_uniform_device variant=checked threads=1048576 tg=32 iters=1 check=1 +#include +using namespace metal; +#pragma clang fp contract(off) +//@BUF_PRELUDE + +// Fast path inline, the dword-by-dword path out of line. +__attribute__((noinline)) static uint4 slow4(device uint* b, uint bytes, uint addr) +{ + return uint4(sharpemu_load_dword(b, bytes, addr), sharpemu_load_dword(b, bytes, addr + 4u), + sharpemu_load_dword(b, bytes, addr + 8u), sharpemu_load_dword(b, bytes, addr + 12u)); +} +static inline uint4 load4_outlined(device uint* b, uint bytes, uint addr) +{ + uint index = addr >> 2; + uint words = bytes >> 2; + if (index < words && 4u <= words - index) + { + return uint4(b[index], b[index + 1u], b[index + 2u], b[index + 3u]); + } + return slow4(b, bytes, addr); +} +// Branch-free: every dword is read at an index clamped into the binding and selected against +// its own bounds test, which is what sharpemu_load_dword answers per dword. Needs a binding of +// at least one dword. +static inline uint4 load4_select(device uint* b, uint bytes, uint addr) +{ + uint index = addr >> 2; + uint words = bytes >> 2; + uint last = max(words, 1u) - 1u; + uint4 r = uint4(0u); + for (int i = 0; i < 4; i++) + { + uint ix = ((addr >> 2) + (uint)i) & 0x3FFFFFFFu; + uint v = b[ix > last ? last : ix]; + r[i] = ix < words ? v : 0u; + } + return r; +} + +kernel void bench(device uint* data [[buffer(0)]], device uint* out [[buffer(1)]], constant uint4& prm [[buffer(2)]], + constant uint* cdata [[buffer(5)]], + uint tid [[thread_position_in_grid]], uint lane [[thread_index_in_simdgroup]]) +{ + device uint* b = data; + uint bytes = prm.y; + uint acc = tid; + { uint4 w = sharpemu_load_dwords<4>(b, bytes, 0u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, 48u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, 96u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, 144u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, 192u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, 240u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, 288u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, 336u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, 384u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, 432u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, 480u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, 528u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, 576u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, 624u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, 672u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, 720u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, 768u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, 816u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, 864u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, 912u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, 960u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, 1008u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, 1056u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, 1104u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, 1152u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, 1200u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, 1248u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, 1296u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, 1344u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, 1392u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, 1440u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = sharpemu_load_dwords<4>(b, bytes, 1488u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + out[tid] = acc; +} +""", + ["buf_uniform_device_outlined"] = """ +// bench group=buf_uniform_device variant=outlined threads=1048576 tg=32 iters=1 check=1 +#include +using namespace metal; +#pragma clang fp contract(off) +//@BUF_PRELUDE + +// Fast path inline, the dword-by-dword path out of line. +__attribute__((noinline)) static uint4 slow4(device uint* b, uint bytes, uint addr) +{ + return uint4(sharpemu_load_dword(b, bytes, addr), sharpemu_load_dword(b, bytes, addr + 4u), + sharpemu_load_dword(b, bytes, addr + 8u), sharpemu_load_dword(b, bytes, addr + 12u)); +} +static inline uint4 load4_outlined(device uint* b, uint bytes, uint addr) +{ + uint index = addr >> 2; + uint words = bytes >> 2; + if (index < words && 4u <= words - index) + { + return uint4(b[index], b[index + 1u], b[index + 2u], b[index + 3u]); + } + return slow4(b, bytes, addr); +} +// Branch-free: every dword is read at an index clamped into the binding and selected against +// its own bounds test, which is what sharpemu_load_dword answers per dword. Needs a binding of +// at least one dword. +static inline uint4 load4_select(device uint* b, uint bytes, uint addr) +{ + uint index = addr >> 2; + uint words = bytes >> 2; + uint last = max(words, 1u) - 1u; + uint4 r = uint4(0u); + for (int i = 0; i < 4; i++) + { + uint ix = ((addr >> 2) + (uint)i) & 0x3FFFFFFFu; + uint v = b[ix > last ? last : ix]; + r[i] = ix < words ? v : 0u; + } + return r; +} + +kernel void bench(device uint* data [[buffer(0)]], device uint* out [[buffer(1)]], constant uint4& prm [[buffer(2)]], + constant uint* cdata [[buffer(5)]], + uint tid [[thread_position_in_grid]], uint lane [[thread_index_in_simdgroup]]) +{ + device uint* b = data; + uint bytes = prm.y; + uint acc = tid; + { uint4 w = load4_outlined(b, bytes, 0u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, 48u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, 96u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, 144u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, 192u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, 240u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, 288u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, 336u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, 384u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, 432u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, 480u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, 528u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, 576u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, 624u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, 672u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, 720u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, 768u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, 816u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, 864u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, 912u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, 960u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, 1008u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, 1056u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, 1104u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, 1152u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, 1200u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, 1248u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, 1296u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, 1344u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, 1392u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, 1440u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_outlined(b, bytes, 1488u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + out[tid] = acc; +} +""", + ["buf_uniform_device_raw"] = """ +// bench group=buf_uniform_device variant=raw threads=1048576 tg=32 iters=1 check=1 +#include +using namespace metal; +#pragma clang fp contract(off) +//@BUF_PRELUDE + +// Fast path inline, the dword-by-dword path out of line. +__attribute__((noinline)) static uint4 slow4(device uint* b, uint bytes, uint addr) +{ + return uint4(sharpemu_load_dword(b, bytes, addr), sharpemu_load_dword(b, bytes, addr + 4u), + sharpemu_load_dword(b, bytes, addr + 8u), sharpemu_load_dword(b, bytes, addr + 12u)); +} +static inline uint4 load4_outlined(device uint* b, uint bytes, uint addr) +{ + uint index = addr >> 2; + uint words = bytes >> 2; + if (index < words && 4u <= words - index) + { + return uint4(b[index], b[index + 1u], b[index + 2u], b[index + 3u]); + } + return slow4(b, bytes, addr); +} +// Branch-free: every dword is read at an index clamped into the binding and selected against +// its own bounds test, which is what sharpemu_load_dword answers per dword. Needs a binding of +// at least one dword. +static inline uint4 load4_select(device uint* b, uint bytes, uint addr) +{ + uint index = addr >> 2; + uint words = bytes >> 2; + uint last = max(words, 1u) - 1u; + uint4 r = uint4(0u); + for (int i = 0; i < 4; i++) + { + uint ix = ((addr >> 2) + (uint)i) & 0x3FFFFFFFu; + uint v = b[ix > last ? last : ix]; + r[i] = ix < words ? v : 0u; + } + return r; +} + +kernel void bench(device uint* data [[buffer(0)]], device uint* out [[buffer(1)]], constant uint4& prm [[buffer(2)]], + constant uint* cdata [[buffer(5)]], + uint tid [[thread_position_in_grid]], uint lane [[thread_index_in_simdgroup]]) +{ + device uint* b = data; + uint bytes = prm.y; + uint acc = tid; + { uint i = 0u >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = 48u >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = 96u >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = 144u >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = 192u >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = 240u >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = 288u >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = 336u >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = 384u >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = 432u >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = 480u >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = 528u >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = 576u >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = 624u >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = 672u >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = 720u >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = 768u >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = 816u >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = 864u >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = 912u >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = 960u >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = 1008u >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = 1056u >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = 1104u >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = 1152u >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = 1200u >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = 1248u >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = 1296u >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = 1344u >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = 1392u >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = 1440u >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + { uint i = 1488u >> 2; acc ^= b[i]; acc += b[i + 1u]; acc ^= b[i + 2u]; acc += b[i + 3u]; } + out[tid] = acc; +} +""", + ["buf_uniform_device_select"] = """ +// bench group=buf_uniform_device variant=select threads=1048576 tg=32 iters=1 check=1 +#include +using namespace metal; +#pragma clang fp contract(off) +//@BUF_PRELUDE + +// Fast path inline, the dword-by-dword path out of line. +__attribute__((noinline)) static uint4 slow4(device uint* b, uint bytes, uint addr) +{ + return uint4(sharpemu_load_dword(b, bytes, addr), sharpemu_load_dword(b, bytes, addr + 4u), + sharpemu_load_dword(b, bytes, addr + 8u), sharpemu_load_dword(b, bytes, addr + 12u)); +} +static inline uint4 load4_outlined(device uint* b, uint bytes, uint addr) +{ + uint index = addr >> 2; + uint words = bytes >> 2; + if (index < words && 4u <= words - index) + { + return uint4(b[index], b[index + 1u], b[index + 2u], b[index + 3u]); + } + return slow4(b, bytes, addr); +} +// Branch-free: every dword is read at an index clamped into the binding and selected against +// its own bounds test, which is what sharpemu_load_dword answers per dword. Needs a binding of +// at least one dword. +static inline uint4 load4_select(device uint* b, uint bytes, uint addr) +{ + uint index = addr >> 2; + uint words = bytes >> 2; + uint last = max(words, 1u) - 1u; + uint4 r = uint4(0u); + for (int i = 0; i < 4; i++) + { + uint ix = ((addr >> 2) + (uint)i) & 0x3FFFFFFFu; + uint v = b[ix > last ? last : ix]; + r[i] = ix < words ? v : 0u; + } + return r; +} + +kernel void bench(device uint* data [[buffer(0)]], device uint* out [[buffer(1)]], constant uint4& prm [[buffer(2)]], + constant uint* cdata [[buffer(5)]], + uint tid [[thread_position_in_grid]], uint lane [[thread_index_in_simdgroup]]) +{ + device uint* b = data; + uint bytes = prm.y; + uint acc = tid; + { uint4 w = load4_select(b, bytes, 0u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, 48u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, 96u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, 144u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, 192u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, 240u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, 288u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, 336u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, 384u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, 432u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, 480u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, 528u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, 576u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, 624u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, 672u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, 720u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, 768u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, 816u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, 864u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, 912u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, 960u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, 1008u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, 1056u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, 1104u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, 1152u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, 1200u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, 1248u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, 1296u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, 1344u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, 1392u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, 1440u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { uint4 w = load4_select(b, bytes, 1488u); acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + out[tid] = acc; +} +""", + ["dev_altranges_direct"] = """ +// bench group=dev_altranges variant=direct threads=262144 tg=32 iters=1 check=1 +#include +using namespace metal; +#pragma clang fp contract(off) +//@DEV_PRELUDE + +kernel void bench( + device uint* data [[buffer(0)]], + device uint* out [[buffer(1)]], + constant uint4& prm [[buffer(2)]], + device const Gen5AddressRange* sharpemu_ranges [[buffer(3)]], + device uint* sharpemu_faults [[buffer(4)]], + uint tid [[thread_position_in_grid]], + uint lane [[thread_index_in_simdgroup]], + uint tgi [[thread_index_in_threadgroup]]) + +{ + uint range_count = prm.w; + Gen5RangeCache cache = { 0ul, 0ul, nullptr }; + ulong gA = ((ulong)prm.z << 32) | (ulong)prm.y; + ulong gB = gA + 0x800000ul; + uint acc = tid; + { ulong a = (gA + 0ul + (ulong)((tid * 16u) & 0x1FFFFu)); + device uint* p = (device uint*)((device uchar*)(data + 0) + 0ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= p[0]; acc += p[1]; acc ^= p[2]; acc += p[3]; } + { ulong a = (gB + 0ul + (ulong)((tid * 16u) & 0x1FFFFu)); + device uint* p = (device uint*)((device uchar*)(data + 0x200000) + 0ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= p[0]; acc += p[1]; acc ^= p[2]; acc += p[3]; } + { ulong a = (gA + 131072ul + (ulong)((tid * 16u) & 0x1FFFFu)); + device uint* p = (device uint*)((device uchar*)(data + 0) + 131072ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= p[0]; acc += p[1]; acc ^= p[2]; acc += p[3]; } + { ulong a = (gB + 131072ul + (ulong)((tid * 16u) & 0x1FFFFu)); + device uint* p = (device uint*)((device uchar*)(data + 0x200000) + 131072ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= p[0]; acc += p[1]; acc ^= p[2]; acc += p[3]; } + { ulong a = (gA + 262144ul + (ulong)((tid * 16u) & 0x1FFFFu)); + device uint* p = (device uint*)((device uchar*)(data + 0) + 262144ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= p[0]; acc += p[1]; acc ^= p[2]; acc += p[3]; } + { ulong a = (gB + 262144ul + (ulong)((tid * 16u) & 0x1FFFFu)); + device uint* p = (device uint*)((device uchar*)(data + 0x200000) + 262144ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= p[0]; acc += p[1]; acc ^= p[2]; acc += p[3]; } + { ulong a = (gA + 393216ul + (ulong)((tid * 16u) & 0x1FFFFu)); + device uint* p = (device uint*)((device uchar*)(data + 0) + 393216ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= p[0]; acc += p[1]; acc ^= p[2]; acc += p[3]; } + { ulong a = (gB + 393216ul + (ulong)((tid * 16u) & 0x1FFFFu)); + device uint* p = (device uint*)((device uchar*)(data + 0x200000) + 393216ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= p[0]; acc += p[1]; acc ^= p[2]; acc += p[3]; } + { ulong a = (gA + 524288ul + (ulong)((tid * 16u) & 0x1FFFFu)); + device uint* p = (device uint*)((device uchar*)(data + 0) + 524288ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= p[0]; acc += p[1]; acc ^= p[2]; acc += p[3]; } + { ulong a = (gB + 524288ul + (ulong)((tid * 16u) & 0x1FFFFu)); + device uint* p = (device uint*)((device uchar*)(data + 0x200000) + 524288ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= p[0]; acc += p[1]; acc ^= p[2]; acc += p[3]; } + { ulong a = (gA + 655360ul + (ulong)((tid * 16u) & 0x1FFFFu)); + device uint* p = (device uint*)((device uchar*)(data + 0) + 655360ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= p[0]; acc += p[1]; acc ^= p[2]; acc += p[3]; } + { ulong a = (gB + 655360ul + (ulong)((tid * 16u) & 0x1FFFFu)); + device uint* p = (device uint*)((device uchar*)(data + 0x200000) + 655360ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= p[0]; acc += p[1]; acc ^= p[2]; acc += p[3]; } + { ulong a = (gA + 786432ul + (ulong)((tid * 16u) & 0x1FFFFu)); + device uint* p = (device uint*)((device uchar*)(data + 0) + 786432ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= p[0]; acc += p[1]; acc ^= p[2]; acc += p[3]; } + { ulong a = (gB + 786432ul + (ulong)((tid * 16u) & 0x1FFFFu)); + device uint* p = (device uint*)((device uchar*)(data + 0x200000) + 786432ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= p[0]; acc += p[1]; acc ^= p[2]; acc += p[3]; } + { ulong a = (gA + 917504ul + (ulong)((tid * 16u) & 0x1FFFFu)); + device uint* p = (device uint*)((device uchar*)(data + 0) + 917504ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= p[0]; acc += p[1]; acc ^= p[2]; acc += p[3]; } + { ulong a = (gB + 917504ul + (ulong)((tid * 16u) & 0x1FFFFu)); + device uint* p = (device uint*)((device uchar*)(data + 0x200000) + 917504ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= p[0]; acc += p[1]; acc ^= p[2]; acc += p[3]; } + { ulong a = (gA + 1048576ul + (ulong)((tid * 16u) & 0x1FFFFu)); + device uint* p = (device uint*)((device uchar*)(data + 0) + 1048576ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= p[0]; acc += p[1]; acc ^= p[2]; acc += p[3]; } + { ulong a = (gB + 1048576ul + (ulong)((tid * 16u) & 0x1FFFFu)); + device uint* p = (device uint*)((device uchar*)(data + 0x200000) + 1048576ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= p[0]; acc += p[1]; acc ^= p[2]; acc += p[3]; } + { ulong a = (gA + 1179648ul + (ulong)((tid * 16u) & 0x1FFFFu)); + device uint* p = (device uint*)((device uchar*)(data + 0) + 1179648ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= p[0]; acc += p[1]; acc ^= p[2]; acc += p[3]; } + { ulong a = (gB + 1179648ul + (ulong)((tid * 16u) & 0x1FFFFu)); + device uint* p = (device uint*)((device uchar*)(data + 0x200000) + 1179648ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= p[0]; acc += p[1]; acc ^= p[2]; acc += p[3]; } + { ulong a = (gA + 1310720ul + (ulong)((tid * 16u) & 0x1FFFFu)); + device uint* p = (device uint*)((device uchar*)(data + 0) + 1310720ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= p[0]; acc += p[1]; acc ^= p[2]; acc += p[3]; } + { ulong a = (gB + 1310720ul + (ulong)((tid * 16u) & 0x1FFFFu)); + device uint* p = (device uint*)((device uchar*)(data + 0x200000) + 1310720ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= p[0]; acc += p[1]; acc ^= p[2]; acc += p[3]; } + { ulong a = (gA + 1441792ul + (ulong)((tid * 16u) & 0x1FFFFu)); + device uint* p = (device uint*)((device uchar*)(data + 0) + 1441792ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= p[0]; acc += p[1]; acc ^= p[2]; acc += p[3]; } + { ulong a = (gB + 1441792ul + (ulong)((tid * 16u) & 0x1FFFFu)); + device uint* p = (device uint*)((device uchar*)(data + 0x200000) + 1441792ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= p[0]; acc += p[1]; acc ^= p[2]; acc += p[3]; } + { ulong a = (gA + 1572864ul + (ulong)((tid * 16u) & 0x1FFFFu)); + device uint* p = (device uint*)((device uchar*)(data + 0) + 1572864ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= p[0]; acc += p[1]; acc ^= p[2]; acc += p[3]; } + { ulong a = (gB + 1572864ul + (ulong)((tid * 16u) & 0x1FFFFu)); + device uint* p = (device uint*)((device uchar*)(data + 0x200000) + 1572864ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= p[0]; acc += p[1]; acc ^= p[2]; acc += p[3]; } + { ulong a = (gA + 1703936ul + (ulong)((tid * 16u) & 0x1FFFFu)); + device uint* p = (device uint*)((device uchar*)(data + 0) + 1703936ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= p[0]; acc += p[1]; acc ^= p[2]; acc += p[3]; } + { ulong a = (gB + 1703936ul + (ulong)((tid * 16u) & 0x1FFFFu)); + device uint* p = (device uint*)((device uchar*)(data + 0x200000) + 1703936ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= p[0]; acc += p[1]; acc ^= p[2]; acc += p[3]; } + { ulong a = (gA + 1835008ul + (ulong)((tid * 16u) & 0x1FFFFu)); + device uint* p = (device uint*)((device uchar*)(data + 0) + 1835008ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= p[0]; acc += p[1]; acc ^= p[2]; acc += p[3]; } + { ulong a = (gB + 1835008ul + (ulong)((tid * 16u) & 0x1FFFFu)); + device uint* p = (device uint*)((device uchar*)(data + 0x200000) + 1835008ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= p[0]; acc += p[1]; acc ^= p[2]; acc += p[3]; } + { ulong a = (gA + 1966080ul + (ulong)((tid * 16u) & 0x1FFFFu)); + device uint* p = (device uint*)((device uchar*)(data + 0) + 1966080ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= p[0]; acc += p[1]; acc ^= p[2]; acc += p[3]; } + { ulong a = (gB + 1966080ul + (ulong)((tid * 16u) & 0x1FFFFu)); + device uint* p = (device uint*)((device uchar*)(data + 0x200000) + 1966080ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= p[0]; acc += p[1]; acc ^= p[2]; acc += p[3]; } + out[tid] = acc; +} +""", + ["dev_altranges_perdword"] = """ +// bench group=dev_altranges variant=perdword threads=262144 tg=32 iters=1 check=1 +#include +using namespace metal; +#pragma clang fp contract(off) +//@DEV_PRELUDE + +kernel void bench( + device uint* data [[buffer(0)]], + device uint* out [[buffer(1)]], + constant uint4& prm [[buffer(2)]], + device const Gen5AddressRange* sharpemu_ranges [[buffer(3)]], + device uint* sharpemu_faults [[buffer(4)]], + uint tid [[thread_position_in_grid]], + uint lane [[thread_index_in_simdgroup]], + uint tgi [[thread_index_in_threadgroup]]) + +{ + uint range_count = prm.w; + Gen5RangeCache cache = { 0ul, 0ul, nullptr }; + ulong gA = ((ulong)prm.z << 32) | (ulong)prm.y; + ulong gB = gA + 0x800000ul; + uint acc = tid; + { ulong a = (gA + 0ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); } + { ulong a = (gB + 0ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); } + { ulong a = (gA + 131072ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); } + { ulong a = (gB + 131072ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); } + { ulong a = (gA + 262144ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); } + { ulong a = (gB + 262144ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); } + { ulong a = (gA + 393216ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); } + { ulong a = (gB + 393216ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); } + { ulong a = (gA + 524288ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); } + { ulong a = (gB + 524288ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); } + { ulong a = (gA + 655360ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); } + { ulong a = (gB + 655360ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); } + { ulong a = (gA + 786432ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); } + { ulong a = (gB + 786432ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); } + { ulong a = (gA + 917504ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); } + { ulong a = (gB + 917504ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); } + { ulong a = (gA + 1048576ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); } + { ulong a = (gB + 1048576ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); } + { ulong a = (gA + 1179648ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); } + { ulong a = (gB + 1179648ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); } + { ulong a = (gA + 1310720ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); } + { ulong a = (gB + 1310720ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); } + { ulong a = (gA + 1441792ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); } + { ulong a = (gB + 1441792ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); } + { ulong a = (gA + 1572864ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); } + { ulong a = (gB + 1572864ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); } + { ulong a = (gA + 1703936ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); } + { ulong a = (gB + 1703936ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); } + { ulong a = (gA + 1835008ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); } + { ulong a = (gB + 1835008ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); } + { ulong a = (gA + 1966080ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); } + { ulong a = (gB + 1966080ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); } + out[tid] = acc; +} +""", + ["dev_altranges_span_cache2"] = """ +// bench group=dev_altranges variant=span_cache2 threads=262144 tg=32 iters=1 check=1 +#include +using namespace metal; +#pragma clang fp contract(off) +//@DEV_PRELUDE +struct Cache2 { Gen5RangeCache c0; Gen5RangeCache c1; }; +static inline device uint* resolve2(device const Gen5AddressRange* ranges, uint range_count, device uint* faults, thread Cache2& c, ulong address, thread bool& valid) +{ + ulong masked = address & 0xFFFFFFFFFFFFul; + ulong page = masked >> 14u; + if (page - c.c0.first_page < c.c0.page_count) + { + valid = true; + return (device uint*)((device uchar*)c.c0.base + ((page - c.c0.first_page) << 14u) + (masked & 0x3FFFul)); + } + if (page - c.c1.first_page < c.c1.page_count) + { + valid = true; + Gen5RangeCache t = c.c0; c.c0 = c.c1; c.c1 = t; + return (device uint*)((device uchar*)c.c0.base + ((page - c.c0.first_page) << 14u) + (masked & 0x3FFFul)); + } + c.c1 = c.c0; + return sharpemu_resolve_device_address(ranges, range_count, faults, 1024u, c.c0, address, valid); +} +kernel void bench( + device uint* data [[buffer(0)]], + device uint* out [[buffer(1)]], + constant uint4& prm [[buffer(2)]], + device const Gen5AddressRange* sharpemu_ranges [[buffer(3)]], + device uint* sharpemu_faults [[buffer(4)]], + uint tid [[thread_position_in_grid]], + uint lane [[thread_index_in_simdgroup]], + uint tgi [[thread_index_in_threadgroup]]) + +{ + uint range_count = prm.w; + Cache2 c2 = { { 0ul, 0ul, nullptr }, { 0ul, 0ul, nullptr } }; + ulong gA = ((ulong)prm.z << 32) | (ulong)prm.y; + ulong gB = gA + 0x800000ul; + uint acc = tid; + { ulong a = (gA + 0ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = resolve2(sharpemu_ranges, range_count, sharpemu_faults, c2, a, ok); + ok = ok && (a & 0x3FFFul) + 16ul <= 0x4000ul; + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 0ul); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gB + 0ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = resolve2(sharpemu_ranges, range_count, sharpemu_faults, c2, a, ok); + ok = ok && (a & 0x3FFFul) + 16ul <= 0x4000ul; + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 0ul); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gA + 131072ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = resolve2(sharpemu_ranges, range_count, sharpemu_faults, c2, a, ok); + ok = ok && (a & 0x3FFFul) + 16ul <= 0x4000ul; + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 0ul); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gB + 131072ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = resolve2(sharpemu_ranges, range_count, sharpemu_faults, c2, a, ok); + ok = ok && (a & 0x3FFFul) + 16ul <= 0x4000ul; + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 0ul); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gA + 262144ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = resolve2(sharpemu_ranges, range_count, sharpemu_faults, c2, a, ok); + ok = ok && (a & 0x3FFFul) + 16ul <= 0x4000ul; + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 0ul); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gB + 262144ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = resolve2(sharpemu_ranges, range_count, sharpemu_faults, c2, a, ok); + ok = ok && (a & 0x3FFFul) + 16ul <= 0x4000ul; + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 0ul); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gA + 393216ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = resolve2(sharpemu_ranges, range_count, sharpemu_faults, c2, a, ok); + ok = ok && (a & 0x3FFFul) + 16ul <= 0x4000ul; + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 0ul); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gB + 393216ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = resolve2(sharpemu_ranges, range_count, sharpemu_faults, c2, a, ok); + ok = ok && (a & 0x3FFFul) + 16ul <= 0x4000ul; + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 0ul); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gA + 524288ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = resolve2(sharpemu_ranges, range_count, sharpemu_faults, c2, a, ok); + ok = ok && (a & 0x3FFFul) + 16ul <= 0x4000ul; + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 0ul); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gB + 524288ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = resolve2(sharpemu_ranges, range_count, sharpemu_faults, c2, a, ok); + ok = ok && (a & 0x3FFFul) + 16ul <= 0x4000ul; + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 0ul); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gA + 655360ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = resolve2(sharpemu_ranges, range_count, sharpemu_faults, c2, a, ok); + ok = ok && (a & 0x3FFFul) + 16ul <= 0x4000ul; + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 0ul); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gB + 655360ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = resolve2(sharpemu_ranges, range_count, sharpemu_faults, c2, a, ok); + ok = ok && (a & 0x3FFFul) + 16ul <= 0x4000ul; + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 0ul); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gA + 786432ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = resolve2(sharpemu_ranges, range_count, sharpemu_faults, c2, a, ok); + ok = ok && (a & 0x3FFFul) + 16ul <= 0x4000ul; + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 0ul); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gB + 786432ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = resolve2(sharpemu_ranges, range_count, sharpemu_faults, c2, a, ok); + ok = ok && (a & 0x3FFFul) + 16ul <= 0x4000ul; + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 0ul); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gA + 917504ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = resolve2(sharpemu_ranges, range_count, sharpemu_faults, c2, a, ok); + ok = ok && (a & 0x3FFFul) + 16ul <= 0x4000ul; + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 0ul); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gB + 917504ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = resolve2(sharpemu_ranges, range_count, sharpemu_faults, c2, a, ok); + ok = ok && (a & 0x3FFFul) + 16ul <= 0x4000ul; + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 0ul); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gA + 1048576ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = resolve2(sharpemu_ranges, range_count, sharpemu_faults, c2, a, ok); + ok = ok && (a & 0x3FFFul) + 16ul <= 0x4000ul; + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 0ul); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gB + 1048576ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = resolve2(sharpemu_ranges, range_count, sharpemu_faults, c2, a, ok); + ok = ok && (a & 0x3FFFul) + 16ul <= 0x4000ul; + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 0ul); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gA + 1179648ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = resolve2(sharpemu_ranges, range_count, sharpemu_faults, c2, a, ok); + ok = ok && (a & 0x3FFFul) + 16ul <= 0x4000ul; + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 0ul); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gB + 1179648ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = resolve2(sharpemu_ranges, range_count, sharpemu_faults, c2, a, ok); + ok = ok && (a & 0x3FFFul) + 16ul <= 0x4000ul; + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 0ul); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gA + 1310720ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = resolve2(sharpemu_ranges, range_count, sharpemu_faults, c2, a, ok); + ok = ok && (a & 0x3FFFul) + 16ul <= 0x4000ul; + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 0ul); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gB + 1310720ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = resolve2(sharpemu_ranges, range_count, sharpemu_faults, c2, a, ok); + ok = ok && (a & 0x3FFFul) + 16ul <= 0x4000ul; + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 0ul); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gA + 1441792ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = resolve2(sharpemu_ranges, range_count, sharpemu_faults, c2, a, ok); + ok = ok && (a & 0x3FFFul) + 16ul <= 0x4000ul; + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 0ul); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gB + 1441792ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = resolve2(sharpemu_ranges, range_count, sharpemu_faults, c2, a, ok); + ok = ok && (a & 0x3FFFul) + 16ul <= 0x4000ul; + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 0ul); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gA + 1572864ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = resolve2(sharpemu_ranges, range_count, sharpemu_faults, c2, a, ok); + ok = ok && (a & 0x3FFFul) + 16ul <= 0x4000ul; + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 0ul); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gB + 1572864ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = resolve2(sharpemu_ranges, range_count, sharpemu_faults, c2, a, ok); + ok = ok && (a & 0x3FFFul) + 16ul <= 0x4000ul; + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 0ul); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gA + 1703936ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = resolve2(sharpemu_ranges, range_count, sharpemu_faults, c2, a, ok); + ok = ok && (a & 0x3FFFul) + 16ul <= 0x4000ul; + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 0ul); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gB + 1703936ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = resolve2(sharpemu_ranges, range_count, sharpemu_faults, c2, a, ok); + ok = ok && (a & 0x3FFFul) + 16ul <= 0x4000ul; + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 0ul); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gA + 1835008ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = resolve2(sharpemu_ranges, range_count, sharpemu_faults, c2, a, ok); + ok = ok && (a & 0x3FFFul) + 16ul <= 0x4000ul; + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 0ul); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gB + 1835008ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = resolve2(sharpemu_ranges, range_count, sharpemu_faults, c2, a, ok); + ok = ok && (a & 0x3FFFul) + 16ul <= 0x4000ul; + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 0ul); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gA + 1966080ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = resolve2(sharpemu_ranges, range_count, sharpemu_faults, c2, a, ok); + ok = ok && (a & 0x3FFFul) + 16ul <= 0x4000ul; + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 0ul); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gB + 1966080ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = resolve2(sharpemu_ranges, range_count, sharpemu_faults, c2, a, ok); + ok = ok && (a & 0x3FFFul) + 16ul <= 0x4000ul; + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 0ul); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + out[tid] = acc; +} +""", + ["dev_altranges_span_inline"] = """ +// bench group=dev_altranges variant=span_inline threads=262144 tg=32 iters=1 check=1 +#include +using namespace metal; +#pragma clang fp contract(off) +//@DEV_PRELUDE + +kernel void bench( + device uint* data [[buffer(0)]], + device uint* out [[buffer(1)]], + constant uint4& prm [[buffer(2)]], + device const Gen5AddressRange* sharpemu_ranges [[buffer(3)]], + device uint* sharpemu_faults [[buffer(4)]], + uint tid [[thread_position_in_grid]], + uint lane [[thread_index_in_simdgroup]], + uint tgi [[thread_index_in_threadgroup]]) + +{ + uint range_count = prm.w; + Gen5RangeCache cache = { 0ul, 0ul, nullptr }; + ulong gA = ((ulong)prm.z << 32) | (ulong)prm.y; + ulong gB = gA + 0x800000ul; + uint acc = tid; + { ulong a = (gA + 0ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gB + 0ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gA + 131072ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gB + 131072ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gA + 262144ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gB + 262144ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gA + 393216ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gB + 393216ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gA + 524288ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gB + 524288ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gA + 655360ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gB + 655360ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gA + 786432ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gB + 786432ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gA + 917504ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gB + 917504ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gA + 1048576ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gB + 1048576ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gA + 1179648ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gB + 1179648ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gA + 1310720ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gB + 1310720ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gA + 1441792ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gB + 1441792ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gA + 1572864ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gB + 1572864ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gA + 1703936ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gB + 1703936ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gA + 1835008ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gB + 1835008ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gA + 1966080ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gB + 1966080ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + out[tid] = acc; +} +""", + ["dev_altranges_span_outlined"] = """ +// bench group=dev_altranges variant=span_outlined threads=262144 tg=32 iters=1 check=1 +#include +using namespace metal; +#pragma clang fp contract(off) +//@DEV_PRELUDE + +__attribute__((noinline)) static uint4 slow4(device const Gen5AddressRange* r, uint rc, device uint* f, thread Gen5RangeCache& cache, ulong a) +{ + return uint4(sharpemu_load_device_dword(r, rc, f, 1024u, cache, a), sharpemu_load_device_dword(r, rc, f, 1024u, cache, a + 4ul), + sharpemu_load_device_dword(r, rc, f, 1024u, cache, a + 8ul), sharpemu_load_device_dword(r, rc, f, 1024u, cache, a + 12ul)); +} + +kernel void bench( + device uint* data [[buffer(0)]], + device uint* out [[buffer(1)]], + constant uint4& prm [[buffer(2)]], + device const Gen5AddressRange* sharpemu_ranges [[buffer(3)]], + device uint* sharpemu_faults [[buffer(4)]], + uint tid [[thread_position_in_grid]], + uint lane [[thread_index_in_simdgroup]], + uint tgi [[thread_index_in_threadgroup]]) + +{ + uint range_count = prm.w; + Gen5RangeCache cache = { 0ul, 0ul, nullptr }; + ulong gA = ((ulong)prm.z << 32) | (ulong)prm.y; + ulong gB = gA + 0x800000ul; + uint acc = tid; + { ulong a = (gA + 0ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint4 w = ok ? uint4(p[0], p[1], p[2], p[3]) : slow4(sharpemu_ranges, range_count, sharpemu_faults, cache, a); + acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { ulong a = (gB + 0ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint4 w = ok ? uint4(p[0], p[1], p[2], p[3]) : slow4(sharpemu_ranges, range_count, sharpemu_faults, cache, a); + acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { ulong a = (gA + 131072ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint4 w = ok ? uint4(p[0], p[1], p[2], p[3]) : slow4(sharpemu_ranges, range_count, sharpemu_faults, cache, a); + acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { ulong a = (gB + 131072ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint4 w = ok ? uint4(p[0], p[1], p[2], p[3]) : slow4(sharpemu_ranges, range_count, sharpemu_faults, cache, a); + acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { ulong a = (gA + 262144ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint4 w = ok ? uint4(p[0], p[1], p[2], p[3]) : slow4(sharpemu_ranges, range_count, sharpemu_faults, cache, a); + acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { ulong a = (gB + 262144ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint4 w = ok ? uint4(p[0], p[1], p[2], p[3]) : slow4(sharpemu_ranges, range_count, sharpemu_faults, cache, a); + acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { ulong a = (gA + 393216ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint4 w = ok ? uint4(p[0], p[1], p[2], p[3]) : slow4(sharpemu_ranges, range_count, sharpemu_faults, cache, a); + acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { ulong a = (gB + 393216ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint4 w = ok ? uint4(p[0], p[1], p[2], p[3]) : slow4(sharpemu_ranges, range_count, sharpemu_faults, cache, a); + acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { ulong a = (gA + 524288ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint4 w = ok ? uint4(p[0], p[1], p[2], p[3]) : slow4(sharpemu_ranges, range_count, sharpemu_faults, cache, a); + acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { ulong a = (gB + 524288ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint4 w = ok ? uint4(p[0], p[1], p[2], p[3]) : slow4(sharpemu_ranges, range_count, sharpemu_faults, cache, a); + acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { ulong a = (gA + 655360ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint4 w = ok ? uint4(p[0], p[1], p[2], p[3]) : slow4(sharpemu_ranges, range_count, sharpemu_faults, cache, a); + acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { ulong a = (gB + 655360ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint4 w = ok ? uint4(p[0], p[1], p[2], p[3]) : slow4(sharpemu_ranges, range_count, sharpemu_faults, cache, a); + acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { ulong a = (gA + 786432ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint4 w = ok ? uint4(p[0], p[1], p[2], p[3]) : slow4(sharpemu_ranges, range_count, sharpemu_faults, cache, a); + acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { ulong a = (gB + 786432ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint4 w = ok ? uint4(p[0], p[1], p[2], p[3]) : slow4(sharpemu_ranges, range_count, sharpemu_faults, cache, a); + acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { ulong a = (gA + 917504ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint4 w = ok ? uint4(p[0], p[1], p[2], p[3]) : slow4(sharpemu_ranges, range_count, sharpemu_faults, cache, a); + acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { ulong a = (gB + 917504ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint4 w = ok ? uint4(p[0], p[1], p[2], p[3]) : slow4(sharpemu_ranges, range_count, sharpemu_faults, cache, a); + acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { ulong a = (gA + 1048576ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint4 w = ok ? uint4(p[0], p[1], p[2], p[3]) : slow4(sharpemu_ranges, range_count, sharpemu_faults, cache, a); + acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { ulong a = (gB + 1048576ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint4 w = ok ? uint4(p[0], p[1], p[2], p[3]) : slow4(sharpemu_ranges, range_count, sharpemu_faults, cache, a); + acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { ulong a = (gA + 1179648ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint4 w = ok ? uint4(p[0], p[1], p[2], p[3]) : slow4(sharpemu_ranges, range_count, sharpemu_faults, cache, a); + acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { ulong a = (gB + 1179648ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint4 w = ok ? uint4(p[0], p[1], p[2], p[3]) : slow4(sharpemu_ranges, range_count, sharpemu_faults, cache, a); + acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { ulong a = (gA + 1310720ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint4 w = ok ? uint4(p[0], p[1], p[2], p[3]) : slow4(sharpemu_ranges, range_count, sharpemu_faults, cache, a); + acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { ulong a = (gB + 1310720ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint4 w = ok ? uint4(p[0], p[1], p[2], p[3]) : slow4(sharpemu_ranges, range_count, sharpemu_faults, cache, a); + acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { ulong a = (gA + 1441792ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint4 w = ok ? uint4(p[0], p[1], p[2], p[3]) : slow4(sharpemu_ranges, range_count, sharpemu_faults, cache, a); + acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { ulong a = (gB + 1441792ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint4 w = ok ? uint4(p[0], p[1], p[2], p[3]) : slow4(sharpemu_ranges, range_count, sharpemu_faults, cache, a); + acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { ulong a = (gA + 1572864ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint4 w = ok ? uint4(p[0], p[1], p[2], p[3]) : slow4(sharpemu_ranges, range_count, sharpemu_faults, cache, a); + acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { ulong a = (gB + 1572864ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint4 w = ok ? uint4(p[0], p[1], p[2], p[3]) : slow4(sharpemu_ranges, range_count, sharpemu_faults, cache, a); + acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { ulong a = (gA + 1703936ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint4 w = ok ? uint4(p[0], p[1], p[2], p[3]) : slow4(sharpemu_ranges, range_count, sharpemu_faults, cache, a); + acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { ulong a = (gB + 1703936ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint4 w = ok ? uint4(p[0], p[1], p[2], p[3]) : slow4(sharpemu_ranges, range_count, sharpemu_faults, cache, a); + acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { ulong a = (gA + 1835008ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint4 w = ok ? uint4(p[0], p[1], p[2], p[3]) : slow4(sharpemu_ranges, range_count, sharpemu_faults, cache, a); + acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { ulong a = (gB + 1835008ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint4 w = ok ? uint4(p[0], p[1], p[2], p[3]) : slow4(sharpemu_ranges, range_count, sharpemu_faults, cache, a); + acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { ulong a = (gA + 1966080ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint4 w = ok ? uint4(p[0], p[1], p[2], p[3]) : slow4(sharpemu_ranges, range_count, sharpemu_faults, cache, a); + acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { ulong a = (gB + 1966080ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint4 w = ok ? uint4(p[0], p[1], p[2], p[3]) : slow4(sharpemu_ranges, range_count, sharpemu_faults, cache, a); + acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + out[tid] = acc; +} +""", + ["dev_onerange_direct"] = """ +// bench group=dev_onerange variant=direct threads=262144 tg=32 iters=1 check=1 +#include +using namespace metal; +#pragma clang fp contract(off) +//@DEV_PRELUDE + +kernel void bench( + device uint* data [[buffer(0)]], + device uint* out [[buffer(1)]], + constant uint4& prm [[buffer(2)]], + device const Gen5AddressRange* sharpemu_ranges [[buffer(3)]], + device uint* sharpemu_faults [[buffer(4)]], + uint tid [[thread_position_in_grid]], + uint lane [[thread_index_in_simdgroup]], + uint tgi [[thread_index_in_threadgroup]]) + +{ + uint range_count = prm.w; + Gen5RangeCache cache = { 0ul, 0ul, nullptr }; + ulong gA = ((ulong)prm.z << 32) | (ulong)prm.y; + ulong gB = gA + 0x800000ul; + uint acc = tid; + { ulong a = (gA + 0ul + (ulong)((tid * 16u) & 0x1FFFFu)); + device uint* p = (device uint*)((device uchar*)(data + 0) + 0ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= p[0]; acc += p[1]; acc ^= p[2]; acc += p[3]; } + { ulong a = (gA + 131072ul + (ulong)((tid * 16u) & 0x1FFFFu)); + device uint* p = (device uint*)((device uchar*)(data + 0) + 131072ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= p[0]; acc += p[1]; acc ^= p[2]; acc += p[3]; } + { ulong a = (gA + 262144ul + (ulong)((tid * 16u) & 0x1FFFFu)); + device uint* p = (device uint*)((device uchar*)(data + 0) + 262144ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= p[0]; acc += p[1]; acc ^= p[2]; acc += p[3]; } + { ulong a = (gA + 393216ul + (ulong)((tid * 16u) & 0x1FFFFu)); + device uint* p = (device uint*)((device uchar*)(data + 0) + 393216ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= p[0]; acc += p[1]; acc ^= p[2]; acc += p[3]; } + { ulong a = (gA + 524288ul + (ulong)((tid * 16u) & 0x1FFFFu)); + device uint* p = (device uint*)((device uchar*)(data + 0) + 524288ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= p[0]; acc += p[1]; acc ^= p[2]; acc += p[3]; } + { ulong a = (gA + 655360ul + (ulong)((tid * 16u) & 0x1FFFFu)); + device uint* p = (device uint*)((device uchar*)(data + 0) + 655360ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= p[0]; acc += p[1]; acc ^= p[2]; acc += p[3]; } + { ulong a = (gA + 786432ul + (ulong)((tid * 16u) & 0x1FFFFu)); + device uint* p = (device uint*)((device uchar*)(data + 0) + 786432ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= p[0]; acc += p[1]; acc ^= p[2]; acc += p[3]; } + { ulong a = (gA + 917504ul + (ulong)((tid * 16u) & 0x1FFFFu)); + device uint* p = (device uint*)((device uchar*)(data + 0) + 917504ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= p[0]; acc += p[1]; acc ^= p[2]; acc += p[3]; } + { ulong a = (gA + 1048576ul + (ulong)((tid * 16u) & 0x1FFFFu)); + device uint* p = (device uint*)((device uchar*)(data + 0) + 1048576ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= p[0]; acc += p[1]; acc ^= p[2]; acc += p[3]; } + { ulong a = (gA + 1179648ul + (ulong)((tid * 16u) & 0x1FFFFu)); + device uint* p = (device uint*)((device uchar*)(data + 0) + 1179648ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= p[0]; acc += p[1]; acc ^= p[2]; acc += p[3]; } + { ulong a = (gA + 1310720ul + (ulong)((tid * 16u) & 0x1FFFFu)); + device uint* p = (device uint*)((device uchar*)(data + 0) + 1310720ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= p[0]; acc += p[1]; acc ^= p[2]; acc += p[3]; } + { ulong a = (gA + 1441792ul + (ulong)((tid * 16u) & 0x1FFFFu)); + device uint* p = (device uint*)((device uchar*)(data + 0) + 1441792ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= p[0]; acc += p[1]; acc ^= p[2]; acc += p[3]; } + { ulong a = (gA + 1572864ul + (ulong)((tid * 16u) & 0x1FFFFu)); + device uint* p = (device uint*)((device uchar*)(data + 0) + 1572864ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= p[0]; acc += p[1]; acc ^= p[2]; acc += p[3]; } + { ulong a = (gA + 1703936ul + (ulong)((tid * 16u) & 0x1FFFFu)); + device uint* p = (device uint*)((device uchar*)(data + 0) + 1703936ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= p[0]; acc += p[1]; acc ^= p[2]; acc += p[3]; } + { ulong a = (gA + 1835008ul + (ulong)((tid * 16u) & 0x1FFFFu)); + device uint* p = (device uint*)((device uchar*)(data + 0) + 1835008ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= p[0]; acc += p[1]; acc ^= p[2]; acc += p[3]; } + { ulong a = (gA + 1966080ul + (ulong)((tid * 16u) & 0x1FFFFu)); + device uint* p = (device uint*)((device uchar*)(data + 0) + 1966080ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= p[0]; acc += p[1]; acc ^= p[2]; acc += p[3]; } + { ulong a = (gA + 2097152ul + (ulong)((tid * 16u) & 0x1FFFFu)); + device uint* p = (device uint*)((device uchar*)(data + 0) + 2097152ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= p[0]; acc += p[1]; acc ^= p[2]; acc += p[3]; } + { ulong a = (gA + 2228224ul + (ulong)((tid * 16u) & 0x1FFFFu)); + device uint* p = (device uint*)((device uchar*)(data + 0) + 2228224ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= p[0]; acc += p[1]; acc ^= p[2]; acc += p[3]; } + { ulong a = (gA + 2359296ul + (ulong)((tid * 16u) & 0x1FFFFu)); + device uint* p = (device uint*)((device uchar*)(data + 0) + 2359296ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= p[0]; acc += p[1]; acc ^= p[2]; acc += p[3]; } + { ulong a = (gA + 2490368ul + (ulong)((tid * 16u) & 0x1FFFFu)); + device uint* p = (device uint*)((device uchar*)(data + 0) + 2490368ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= p[0]; acc += p[1]; acc ^= p[2]; acc += p[3]; } + { ulong a = (gA + 2621440ul + (ulong)((tid * 16u) & 0x1FFFFu)); + device uint* p = (device uint*)((device uchar*)(data + 0) + 2621440ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= p[0]; acc += p[1]; acc ^= p[2]; acc += p[3]; } + { ulong a = (gA + 2752512ul + (ulong)((tid * 16u) & 0x1FFFFu)); + device uint* p = (device uint*)((device uchar*)(data + 0) + 2752512ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= p[0]; acc += p[1]; acc ^= p[2]; acc += p[3]; } + { ulong a = (gA + 2883584ul + (ulong)((tid * 16u) & 0x1FFFFu)); + device uint* p = (device uint*)((device uchar*)(data + 0) + 2883584ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= p[0]; acc += p[1]; acc ^= p[2]; acc += p[3]; } + { ulong a = (gA + 3014656ul + (ulong)((tid * 16u) & 0x1FFFFu)); + device uint* p = (device uint*)((device uchar*)(data + 0) + 3014656ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= p[0]; acc += p[1]; acc ^= p[2]; acc += p[3]; } + { ulong a = (gA + 3145728ul + (ulong)((tid * 16u) & 0x1FFFFu)); + device uint* p = (device uint*)((device uchar*)(data + 0) + 3145728ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= p[0]; acc += p[1]; acc ^= p[2]; acc += p[3]; } + { ulong a = (gA + 3276800ul + (ulong)((tid * 16u) & 0x1FFFFu)); + device uint* p = (device uint*)((device uchar*)(data + 0) + 3276800ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= p[0]; acc += p[1]; acc ^= p[2]; acc += p[3]; } + { ulong a = (gA + 3407872ul + (ulong)((tid * 16u) & 0x1FFFFu)); + device uint* p = (device uint*)((device uchar*)(data + 0) + 3407872ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= p[0]; acc += p[1]; acc ^= p[2]; acc += p[3]; } + { ulong a = (gA + 3538944ul + (ulong)((tid * 16u) & 0x1FFFFu)); + device uint* p = (device uint*)((device uchar*)(data + 0) + 3538944ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= p[0]; acc += p[1]; acc ^= p[2]; acc += p[3]; } + { ulong a = (gA + 3670016ul + (ulong)((tid * 16u) & 0x1FFFFu)); + device uint* p = (device uint*)((device uchar*)(data + 0) + 3670016ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= p[0]; acc += p[1]; acc ^= p[2]; acc += p[3]; } + { ulong a = (gA + 3801088ul + (ulong)((tid * 16u) & 0x1FFFFu)); + device uint* p = (device uint*)((device uchar*)(data + 0) + 3801088ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= p[0]; acc += p[1]; acc ^= p[2]; acc += p[3]; } + { ulong a = (gA + 3932160ul + (ulong)((tid * 16u) & 0x1FFFFu)); + device uint* p = (device uint*)((device uchar*)(data + 0) + 3932160ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= p[0]; acc += p[1]; acc ^= p[2]; acc += p[3]; } + { ulong a = (gA + 4063232ul + (ulong)((tid * 16u) & 0x1FFFFu)); + device uint* p = (device uint*)((device uchar*)(data + 0) + 4063232ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= p[0]; acc += p[1]; acc ^= p[2]; acc += p[3]; } + out[tid] = acc; +} +""", + ["dev_onerange_perdword"] = """ +// bench group=dev_onerange variant=perdword threads=262144 tg=32 iters=1 check=1 +#include +using namespace metal; +#pragma clang fp contract(off) +//@DEV_PRELUDE + +kernel void bench( + device uint* data [[buffer(0)]], + device uint* out [[buffer(1)]], + constant uint4& prm [[buffer(2)]], + device const Gen5AddressRange* sharpemu_ranges [[buffer(3)]], + device uint* sharpemu_faults [[buffer(4)]], + uint tid [[thread_position_in_grid]], + uint lane [[thread_index_in_simdgroup]], + uint tgi [[thread_index_in_threadgroup]]) + +{ + uint range_count = prm.w; + Gen5RangeCache cache = { 0ul, 0ul, nullptr }; + ulong gA = ((ulong)prm.z << 32) | (ulong)prm.y; + ulong gB = gA + 0x800000ul; + uint acc = tid; + { ulong a = (gA + 0ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); } + { ulong a = (gA + 131072ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); } + { ulong a = (gA + 262144ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); } + { ulong a = (gA + 393216ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); } + { ulong a = (gA + 524288ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); } + { ulong a = (gA + 655360ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); } + { ulong a = (gA + 786432ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); } + { ulong a = (gA + 917504ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); } + { ulong a = (gA + 1048576ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); } + { ulong a = (gA + 1179648ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); } + { ulong a = (gA + 1310720ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); } + { ulong a = (gA + 1441792ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); } + { ulong a = (gA + 1572864ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); } + { ulong a = (gA + 1703936ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); } + { ulong a = (gA + 1835008ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); } + { ulong a = (gA + 1966080ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); } + { ulong a = (gA + 2097152ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); } + { ulong a = (gA + 2228224ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); } + { ulong a = (gA + 2359296ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); } + { ulong a = (gA + 2490368ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); } + { ulong a = (gA + 2621440ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); } + { ulong a = (gA + 2752512ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); } + { ulong a = (gA + 2883584ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); } + { ulong a = (gA + 3014656ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); } + { ulong a = (gA + 3145728ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); } + { ulong a = (gA + 3276800ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); } + { ulong a = (gA + 3407872ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); } + { ulong a = (gA + 3538944ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); } + { ulong a = (gA + 3670016ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); } + { ulong a = (gA + 3801088ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); } + { ulong a = (gA + 3932160ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); } + { ulong a = (gA + 4063232ul + (ulong)((tid * 16u) & 0x1FFFFu)); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + acc ^= sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); acc += sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); } + out[tid] = acc; +} +""", + ["dev_onerange_span_cache2"] = """ +// bench group=dev_onerange variant=span_cache2 threads=262144 tg=32 iters=1 check=1 +#include +using namespace metal; +#pragma clang fp contract(off) +//@DEV_PRELUDE +struct Cache2 { Gen5RangeCache c0; Gen5RangeCache c1; }; +static inline device uint* resolve2(device const Gen5AddressRange* ranges, uint range_count, device uint* faults, thread Cache2& c, ulong address, thread bool& valid) +{ + ulong masked = address & 0xFFFFFFFFFFFFul; + ulong page = masked >> 14u; + if (page - c.c0.first_page < c.c0.page_count) + { + valid = true; + return (device uint*)((device uchar*)c.c0.base + ((page - c.c0.first_page) << 14u) + (masked & 0x3FFFul)); + } + if (page - c.c1.first_page < c.c1.page_count) + { + valid = true; + Gen5RangeCache t = c.c0; c.c0 = c.c1; c.c1 = t; + return (device uint*)((device uchar*)c.c0.base + ((page - c.c0.first_page) << 14u) + (masked & 0x3FFFul)); + } + c.c1 = c.c0; + return sharpemu_resolve_device_address(ranges, range_count, faults, 1024u, c.c0, address, valid); +} +kernel void bench( + device uint* data [[buffer(0)]], + device uint* out [[buffer(1)]], + constant uint4& prm [[buffer(2)]], + device const Gen5AddressRange* sharpemu_ranges [[buffer(3)]], + device uint* sharpemu_faults [[buffer(4)]], + uint tid [[thread_position_in_grid]], + uint lane [[thread_index_in_simdgroup]], + uint tgi [[thread_index_in_threadgroup]]) + +{ + uint range_count = prm.w; + Cache2 c2 = { { 0ul, 0ul, nullptr }, { 0ul, 0ul, nullptr } }; + ulong gA = ((ulong)prm.z << 32) | (ulong)prm.y; + ulong gB = gA + 0x800000ul; + uint acc = tid; + { ulong a = (gA + 0ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = resolve2(sharpemu_ranges, range_count, sharpemu_faults, c2, a, ok); + ok = ok && (a & 0x3FFFul) + 16ul <= 0x4000ul; + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 0ul); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gA + 131072ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = resolve2(sharpemu_ranges, range_count, sharpemu_faults, c2, a, ok); + ok = ok && (a & 0x3FFFul) + 16ul <= 0x4000ul; + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 0ul); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gA + 262144ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = resolve2(sharpemu_ranges, range_count, sharpemu_faults, c2, a, ok); + ok = ok && (a & 0x3FFFul) + 16ul <= 0x4000ul; + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 0ul); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gA + 393216ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = resolve2(sharpemu_ranges, range_count, sharpemu_faults, c2, a, ok); + ok = ok && (a & 0x3FFFul) + 16ul <= 0x4000ul; + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 0ul); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gA + 524288ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = resolve2(sharpemu_ranges, range_count, sharpemu_faults, c2, a, ok); + ok = ok && (a & 0x3FFFul) + 16ul <= 0x4000ul; + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 0ul); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gA + 655360ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = resolve2(sharpemu_ranges, range_count, sharpemu_faults, c2, a, ok); + ok = ok && (a & 0x3FFFul) + 16ul <= 0x4000ul; + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 0ul); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gA + 786432ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = resolve2(sharpemu_ranges, range_count, sharpemu_faults, c2, a, ok); + ok = ok && (a & 0x3FFFul) + 16ul <= 0x4000ul; + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 0ul); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gA + 917504ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = resolve2(sharpemu_ranges, range_count, sharpemu_faults, c2, a, ok); + ok = ok && (a & 0x3FFFul) + 16ul <= 0x4000ul; + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 0ul); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gA + 1048576ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = resolve2(sharpemu_ranges, range_count, sharpemu_faults, c2, a, ok); + ok = ok && (a & 0x3FFFul) + 16ul <= 0x4000ul; + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 0ul); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gA + 1179648ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = resolve2(sharpemu_ranges, range_count, sharpemu_faults, c2, a, ok); + ok = ok && (a & 0x3FFFul) + 16ul <= 0x4000ul; + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 0ul); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gA + 1310720ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = resolve2(sharpemu_ranges, range_count, sharpemu_faults, c2, a, ok); + ok = ok && (a & 0x3FFFul) + 16ul <= 0x4000ul; + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 0ul); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gA + 1441792ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = resolve2(sharpemu_ranges, range_count, sharpemu_faults, c2, a, ok); + ok = ok && (a & 0x3FFFul) + 16ul <= 0x4000ul; + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 0ul); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gA + 1572864ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = resolve2(sharpemu_ranges, range_count, sharpemu_faults, c2, a, ok); + ok = ok && (a & 0x3FFFul) + 16ul <= 0x4000ul; + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 0ul); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gA + 1703936ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = resolve2(sharpemu_ranges, range_count, sharpemu_faults, c2, a, ok); + ok = ok && (a & 0x3FFFul) + 16ul <= 0x4000ul; + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 0ul); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gA + 1835008ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = resolve2(sharpemu_ranges, range_count, sharpemu_faults, c2, a, ok); + ok = ok && (a & 0x3FFFul) + 16ul <= 0x4000ul; + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 0ul); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gA + 1966080ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = resolve2(sharpemu_ranges, range_count, sharpemu_faults, c2, a, ok); + ok = ok && (a & 0x3FFFul) + 16ul <= 0x4000ul; + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 0ul); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gA + 2097152ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = resolve2(sharpemu_ranges, range_count, sharpemu_faults, c2, a, ok); + ok = ok && (a & 0x3FFFul) + 16ul <= 0x4000ul; + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 0ul); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gA + 2228224ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = resolve2(sharpemu_ranges, range_count, sharpemu_faults, c2, a, ok); + ok = ok && (a & 0x3FFFul) + 16ul <= 0x4000ul; + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 0ul); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gA + 2359296ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = resolve2(sharpemu_ranges, range_count, sharpemu_faults, c2, a, ok); + ok = ok && (a & 0x3FFFul) + 16ul <= 0x4000ul; + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 0ul); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gA + 2490368ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = resolve2(sharpemu_ranges, range_count, sharpemu_faults, c2, a, ok); + ok = ok && (a & 0x3FFFul) + 16ul <= 0x4000ul; + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 0ul); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gA + 2621440ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = resolve2(sharpemu_ranges, range_count, sharpemu_faults, c2, a, ok); + ok = ok && (a & 0x3FFFul) + 16ul <= 0x4000ul; + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 0ul); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gA + 2752512ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = resolve2(sharpemu_ranges, range_count, sharpemu_faults, c2, a, ok); + ok = ok && (a & 0x3FFFul) + 16ul <= 0x4000ul; + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 0ul); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gA + 2883584ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = resolve2(sharpemu_ranges, range_count, sharpemu_faults, c2, a, ok); + ok = ok && (a & 0x3FFFul) + 16ul <= 0x4000ul; + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 0ul); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gA + 3014656ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = resolve2(sharpemu_ranges, range_count, sharpemu_faults, c2, a, ok); + ok = ok && (a & 0x3FFFul) + 16ul <= 0x4000ul; + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 0ul); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gA + 3145728ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = resolve2(sharpemu_ranges, range_count, sharpemu_faults, c2, a, ok); + ok = ok && (a & 0x3FFFul) + 16ul <= 0x4000ul; + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 0ul); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gA + 3276800ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = resolve2(sharpemu_ranges, range_count, sharpemu_faults, c2, a, ok); + ok = ok && (a & 0x3FFFul) + 16ul <= 0x4000ul; + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 0ul); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gA + 3407872ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = resolve2(sharpemu_ranges, range_count, sharpemu_faults, c2, a, ok); + ok = ok && (a & 0x3FFFul) + 16ul <= 0x4000ul; + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 0ul); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gA + 3538944ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = resolve2(sharpemu_ranges, range_count, sharpemu_faults, c2, a, ok); + ok = ok && (a & 0x3FFFul) + 16ul <= 0x4000ul; + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 0ul); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gA + 3670016ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = resolve2(sharpemu_ranges, range_count, sharpemu_faults, c2, a, ok); + ok = ok && (a & 0x3FFFul) + 16ul <= 0x4000ul; + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 0ul); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gA + 3801088ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = resolve2(sharpemu_ranges, range_count, sharpemu_faults, c2, a, ok); + ok = ok && (a & 0x3FFFul) + 16ul <= 0x4000ul; + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 0ul); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gA + 3932160ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = resolve2(sharpemu_ranges, range_count, sharpemu_faults, c2, a, ok); + ok = ok && (a & 0x3FFFul) + 16ul <= 0x4000ul; + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 0ul); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gA + 4063232ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = resolve2(sharpemu_ranges, range_count, sharpemu_faults, c2, a, ok); + ok = ok && (a & 0x3FFFul) + 16ul <= 0x4000ul; + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 0ul); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, c2.c0, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + out[tid] = acc; +} +""", + ["dev_onerange_span_inline"] = """ +// bench group=dev_onerange variant=span_inline threads=262144 tg=32 iters=1 check=1 +#include +using namespace metal; +#pragma clang fp contract(off) +//@DEV_PRELUDE + +kernel void bench( + device uint* data [[buffer(0)]], + device uint* out [[buffer(1)]], + constant uint4& prm [[buffer(2)]], + device const Gen5AddressRange* sharpemu_ranges [[buffer(3)]], + device uint* sharpemu_faults [[buffer(4)]], + uint tid [[thread_position_in_grid]], + uint lane [[thread_index_in_simdgroup]], + uint tgi [[thread_index_in_threadgroup]]) + +{ + uint range_count = prm.w; + Gen5RangeCache cache = { 0ul, 0ul, nullptr }; + ulong gA = ((ulong)prm.z << 32) | (ulong)prm.y; + ulong gB = gA + 0x800000ul; + uint acc = tid; + { ulong a = (gA + 0ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gA + 131072ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gA + 262144ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gA + 393216ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gA + 524288ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gA + 655360ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gA + 786432ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gA + 917504ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gA + 1048576ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gA + 1179648ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gA + 1310720ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gA + 1441792ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gA + 1572864ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gA + 1703936ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gA + 1835008ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gA + 1966080ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gA + 2097152ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gA + 2228224ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gA + 2359296ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gA + 2490368ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gA + 2621440ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gA + 2752512ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gA + 2883584ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gA + 3014656ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gA + 3145728ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gA + 3276800ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gA + 3407872ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gA + 3538944ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gA + 3670016ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gA + 3801088ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gA + 3932160ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + { ulong a = (gA + 4063232ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint w0 = ok ? p[0] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a); + uint w1 = ok ? p[1] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 4ul); + uint w2 = ok ? p[2] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 8ul); + uint w3 = ok ? p[3] : sharpemu_load_device_dword(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a + 12ul); + acc ^= w0; acc += w1; acc ^= w2; acc += w3; } + out[tid] = acc; +} +""", + ["dev_onerange_span_outlined"] = """ +// bench group=dev_onerange variant=span_outlined threads=262144 tg=32 iters=1 check=1 +#include +using namespace metal; +#pragma clang fp contract(off) +//@DEV_PRELUDE + +__attribute__((noinline)) static uint4 slow4(device const Gen5AddressRange* r, uint rc, device uint* f, thread Gen5RangeCache& cache, ulong a) +{ + return uint4(sharpemu_load_device_dword(r, rc, f, 1024u, cache, a), sharpemu_load_device_dword(r, rc, f, 1024u, cache, a + 4ul), + sharpemu_load_device_dword(r, rc, f, 1024u, cache, a + 8ul), sharpemu_load_device_dword(r, rc, f, 1024u, cache, a + 12ul)); +} + +kernel void bench( + device uint* data [[buffer(0)]], + device uint* out [[buffer(1)]], + constant uint4& prm [[buffer(2)]], + device const Gen5AddressRange* sharpemu_ranges [[buffer(3)]], + device uint* sharpemu_faults [[buffer(4)]], + uint tid [[thread_position_in_grid]], + uint lane [[thread_index_in_simdgroup]], + uint tgi [[thread_index_in_threadgroup]]) + +{ + uint range_count = prm.w; + Gen5RangeCache cache = { 0ul, 0ul, nullptr }; + ulong gA = ((ulong)prm.z << 32) | (ulong)prm.y; + ulong gB = gA + 0x800000ul; + uint acc = tid; + { ulong a = (gA + 0ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint4 w = ok ? uint4(p[0], p[1], p[2], p[3]) : slow4(sharpemu_ranges, range_count, sharpemu_faults, cache, a); + acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { ulong a = (gA + 131072ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint4 w = ok ? uint4(p[0], p[1], p[2], p[3]) : slow4(sharpemu_ranges, range_count, sharpemu_faults, cache, a); + acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { ulong a = (gA + 262144ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint4 w = ok ? uint4(p[0], p[1], p[2], p[3]) : slow4(sharpemu_ranges, range_count, sharpemu_faults, cache, a); + acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { ulong a = (gA + 393216ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint4 w = ok ? uint4(p[0], p[1], p[2], p[3]) : slow4(sharpemu_ranges, range_count, sharpemu_faults, cache, a); + acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { ulong a = (gA + 524288ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint4 w = ok ? uint4(p[0], p[1], p[2], p[3]) : slow4(sharpemu_ranges, range_count, sharpemu_faults, cache, a); + acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { ulong a = (gA + 655360ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint4 w = ok ? uint4(p[0], p[1], p[2], p[3]) : slow4(sharpemu_ranges, range_count, sharpemu_faults, cache, a); + acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { ulong a = (gA + 786432ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint4 w = ok ? uint4(p[0], p[1], p[2], p[3]) : slow4(sharpemu_ranges, range_count, sharpemu_faults, cache, a); + acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { ulong a = (gA + 917504ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint4 w = ok ? uint4(p[0], p[1], p[2], p[3]) : slow4(sharpemu_ranges, range_count, sharpemu_faults, cache, a); + acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { ulong a = (gA + 1048576ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint4 w = ok ? uint4(p[0], p[1], p[2], p[3]) : slow4(sharpemu_ranges, range_count, sharpemu_faults, cache, a); + acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { ulong a = (gA + 1179648ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint4 w = ok ? uint4(p[0], p[1], p[2], p[3]) : slow4(sharpemu_ranges, range_count, sharpemu_faults, cache, a); + acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { ulong a = (gA + 1310720ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint4 w = ok ? uint4(p[0], p[1], p[2], p[3]) : slow4(sharpemu_ranges, range_count, sharpemu_faults, cache, a); + acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { ulong a = (gA + 1441792ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint4 w = ok ? uint4(p[0], p[1], p[2], p[3]) : slow4(sharpemu_ranges, range_count, sharpemu_faults, cache, a); + acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { ulong a = (gA + 1572864ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint4 w = ok ? uint4(p[0], p[1], p[2], p[3]) : slow4(sharpemu_ranges, range_count, sharpemu_faults, cache, a); + acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { ulong a = (gA + 1703936ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint4 w = ok ? uint4(p[0], p[1], p[2], p[3]) : slow4(sharpemu_ranges, range_count, sharpemu_faults, cache, a); + acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { ulong a = (gA + 1835008ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint4 w = ok ? uint4(p[0], p[1], p[2], p[3]) : slow4(sharpemu_ranges, range_count, sharpemu_faults, cache, a); + acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { ulong a = (gA + 1966080ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint4 w = ok ? uint4(p[0], p[1], p[2], p[3]) : slow4(sharpemu_ranges, range_count, sharpemu_faults, cache, a); + acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { ulong a = (gA + 2097152ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint4 w = ok ? uint4(p[0], p[1], p[2], p[3]) : slow4(sharpemu_ranges, range_count, sharpemu_faults, cache, a); + acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { ulong a = (gA + 2228224ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint4 w = ok ? uint4(p[0], p[1], p[2], p[3]) : slow4(sharpemu_ranges, range_count, sharpemu_faults, cache, a); + acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { ulong a = (gA + 2359296ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint4 w = ok ? uint4(p[0], p[1], p[2], p[3]) : slow4(sharpemu_ranges, range_count, sharpemu_faults, cache, a); + acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { ulong a = (gA + 2490368ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint4 w = ok ? uint4(p[0], p[1], p[2], p[3]) : slow4(sharpemu_ranges, range_count, sharpemu_faults, cache, a); + acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { ulong a = (gA + 2621440ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint4 w = ok ? uint4(p[0], p[1], p[2], p[3]) : slow4(sharpemu_ranges, range_count, sharpemu_faults, cache, a); + acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { ulong a = (gA + 2752512ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint4 w = ok ? uint4(p[0], p[1], p[2], p[3]) : slow4(sharpemu_ranges, range_count, sharpemu_faults, cache, a); + acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { ulong a = (gA + 2883584ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint4 w = ok ? uint4(p[0], p[1], p[2], p[3]) : slow4(sharpemu_ranges, range_count, sharpemu_faults, cache, a); + acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { ulong a = (gA + 3014656ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint4 w = ok ? uint4(p[0], p[1], p[2], p[3]) : slow4(sharpemu_ranges, range_count, sharpemu_faults, cache, a); + acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { ulong a = (gA + 3145728ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint4 w = ok ? uint4(p[0], p[1], p[2], p[3]) : slow4(sharpemu_ranges, range_count, sharpemu_faults, cache, a); + acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { ulong a = (gA + 3276800ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint4 w = ok ? uint4(p[0], p[1], p[2], p[3]) : slow4(sharpemu_ranges, range_count, sharpemu_faults, cache, a); + acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { ulong a = (gA + 3407872ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint4 w = ok ? uint4(p[0], p[1], p[2], p[3]) : slow4(sharpemu_ranges, range_count, sharpemu_faults, cache, a); + acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { ulong a = (gA + 3538944ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint4 w = ok ? uint4(p[0], p[1], p[2], p[3]) : slow4(sharpemu_ranges, range_count, sharpemu_faults, cache, a); + acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { ulong a = (gA + 3670016ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint4 w = ok ? uint4(p[0], p[1], p[2], p[3]) : slow4(sharpemu_ranges, range_count, sharpemu_faults, cache, a); + acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { ulong a = (gA + 3801088ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint4 w = ok ? uint4(p[0], p[1], p[2], p[3]) : slow4(sharpemu_ranges, range_count, sharpemu_faults, cache, a); + acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { ulong a = (gA + 3932160ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint4 w = ok ? uint4(p[0], p[1], p[2], p[3]) : slow4(sharpemu_ranges, range_count, sharpemu_faults, cache, a); + acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + { ulong a = (gA + 4063232ul + (ulong)((tid * 16u) & 0x1FFFFu)); + bool ok; + device uint* p = sharpemu_resolve_device_span(sharpemu_ranges, range_count, sharpemu_faults, 1024u, cache, a, 16u, ok); + uint4 w = ok ? uint4(p[0], p[1], p[2], p[3]) : slow4(sharpemu_ranges, range_count, sharpemu_faults, cache, a); + acc ^= w.x; acc += w.y; acc ^= w.z; acc += w.w; } + out[tid] = acc; +} +""", + ["exec_guarded"] = """ +// bench group=exec_guard variant=current threads=1048576 tg=32 iters=64 check=1 +#include +using namespace metal; +#pragma clang fp contract(off) + +kernel void bench(device uint* data [[buffer(0)]], device uint* out [[buffer(1)]], constant uint4& prm [[buffer(2)]], + uint tid [[thread_position_in_grid]], uint lane [[thread_index_in_simdgroup]]) +{ + uint m = data[0]; + bool exec = ((m >> lane) & 1u) != 0u; + uint v0 = (data[(tid * 3u) & 0x3FFFFFu] >> 9) | 0x3F800000u, v1 = (data[(tid * 5u + 1u) & 0x3FFFFFu] >> 9) | 0x3F800000u; + uint v2 = (data[(tid * 7u + 2u) & 0x3FFFFFu] >> 9) | 0x3F800000u, v3 = (data[(tid * 11u + 3u) & 0x3FFFFFu] >> 9) | 0x3F800000u; + uint v4 = v0 ^ 0x1234u, v5 = v1 ^ 0x4321u, v6 = v2 ^ 0x777u, v7 = v3 ^ 0x999u; + uint s0 = 0x3F000000u | (data[1] & 0xFFFFu), s1 = 0x3F000000u | (data[2] & 0xFFFFu), s2 = 0x3E800000u | (data[3] & 0xFFFFu), s3 = 0x3F7F0000u; + for (uint it = 0u; it < prm.x; it++) + { + { float t = as_type(s0) * as_type(v0); float u = as_type(v1); uint r = as_type(t + u); if (exec) { v2 = r; } } + { float t = as_type(s1) * as_type(v3); float u = as_type(v6); uint r = as_type(t + u); if (exec) { v1 = r; } } + { float t = as_type(s2) * as_type(v6); float u = as_type(v3); uint r = as_type(t + u); if (exec) { v0 = r; } } + { float t = as_type(s3) * as_type(v1); float u = as_type(v0); uint r = as_type(t + u); if (exec) { v7 = r; } } + { float t = as_type(s0) * as_type(v4); float u = as_type(v5); uint r = as_type(t + u); if (exec) { v6 = r; } } + { float t = as_type(s1) * as_type(v7); float u = as_type(v2); uint r = as_type(t + u); if (exec) { v5 = r; } } + { float t = as_type(s2) * as_type(v2); float u = as_type(v7); uint r = as_type(t + u); if (exec) { v4 = r; } } + { float t = as_type(s3) * as_type(v5); float u = as_type(v4); uint r = as_type(t + u); if (exec) { v3 = r; } } + { float t = as_type(s0) * as_type(v0); float u = as_type(v1); uint r = as_type(t + u); if (exec) { v2 = r; } } + { float t = as_type(s1) * as_type(v3); float u = as_type(v6); uint r = as_type(t + u); if (exec) { v1 = r; } } + { float t = as_type(s2) * as_type(v6); float u = as_type(v3); uint r = as_type(t + u); if (exec) { v0 = r; } } + { float t = as_type(s3) * as_type(v1); float u = as_type(v0); uint r = as_type(t + u); if (exec) { v7 = r; } } + { float t = as_type(s0) * as_type(v4); float u = as_type(v5); uint r = as_type(t + u); if (exec) { v6 = r; } } + { float t = as_type(s1) * as_type(v7); float u = as_type(v2); uint r = as_type(t + u); if (exec) { v5 = r; } } + { float t = as_type(s2) * as_type(v2); float u = as_type(v7); uint r = as_type(t + u); if (exec) { v4 = r; } } + { float t = as_type(s3) * as_type(v5); float u = as_type(v4); uint r = as_type(t + u); if (exec) { v3 = r; } } + { float t = as_type(s0) * as_type(v0); float u = as_type(v1); uint r = as_type(t + u); if (exec) { v2 = r; } } + { float t = as_type(s1) * as_type(v3); float u = as_type(v6); uint r = as_type(t + u); if (exec) { v1 = r; } } + { float t = as_type(s2) * as_type(v6); float u = as_type(v3); uint r = as_type(t + u); if (exec) { v0 = r; } } + { float t = as_type(s3) * as_type(v1); float u = as_type(v0); uint r = as_type(t + u); if (exec) { v7 = r; } } + { float t = as_type(s0) * as_type(v4); float u = as_type(v5); uint r = as_type(t + u); if (exec) { v6 = r; } } + { float t = as_type(s1) * as_type(v7); float u = as_type(v2); uint r = as_type(t + u); if (exec) { v5 = r; } } + { float t = as_type(s2) * as_type(v2); float u = as_type(v7); uint r = as_type(t + u); if (exec) { v4 = r; } } + { float t = as_type(s3) * as_type(v5); float u = as_type(v4); uint r = as_type(t + u); if (exec) { v3 = r; } } + } + out[tid] = v0 ^ v1 ^ v2 ^ v3 ^ v4 ^ v5 ^ v6 ^ v7; +} +""", + ["exec_unguarded"] = """ +// bench group=exec_guard variant=unguarded threads=1048576 tg=32 iters=64 check=1 +#include +using namespace metal; +#pragma clang fp contract(off) + +kernel void bench(device uint* data [[buffer(0)]], device uint* out [[buffer(1)]], constant uint4& prm [[buffer(2)]], + uint tid [[thread_position_in_grid]], uint lane [[thread_index_in_simdgroup]]) +{ + uint m = data[0]; + bool exec = ((m >> lane) & 1u) != 0u; + uint v0 = (data[(tid * 3u) & 0x3FFFFFu] >> 9) | 0x3F800000u, v1 = (data[(tid * 5u + 1u) & 0x3FFFFFu] >> 9) | 0x3F800000u; + uint v2 = (data[(tid * 7u + 2u) & 0x3FFFFFu] >> 9) | 0x3F800000u, v3 = (data[(tid * 11u + 3u) & 0x3FFFFFu] >> 9) | 0x3F800000u; + uint v4 = v0 ^ 0x1234u, v5 = v1 ^ 0x4321u, v6 = v2 ^ 0x777u, v7 = v3 ^ 0x999u; + uint s0 = 0x3F000000u | (data[1] & 0xFFFFu), s1 = 0x3F000000u | (data[2] & 0xFFFFu), s2 = 0x3E800000u | (data[3] & 0xFFFFu), s3 = 0x3F7F0000u; + for (uint it = 0u; it < prm.x; it++) + { + { float t = as_type(s0) * as_type(v0); float u = as_type(v1); uint r = as_type(t + u); v2 = r; } + { float t = as_type(s1) * as_type(v3); float u = as_type(v6); uint r = as_type(t + u); v1 = r; } + { float t = as_type(s2) * as_type(v6); float u = as_type(v3); uint r = as_type(t + u); v0 = r; } + { float t = as_type(s3) * as_type(v1); float u = as_type(v0); uint r = as_type(t + u); v7 = r; } + { float t = as_type(s0) * as_type(v4); float u = as_type(v5); uint r = as_type(t + u); v6 = r; } + { float t = as_type(s1) * as_type(v7); float u = as_type(v2); uint r = as_type(t + u); v5 = r; } + { float t = as_type(s2) * as_type(v2); float u = as_type(v7); uint r = as_type(t + u); v4 = r; } + { float t = as_type(s3) * as_type(v5); float u = as_type(v4); uint r = as_type(t + u); v3 = r; } + { float t = as_type(s0) * as_type(v0); float u = as_type(v1); uint r = as_type(t + u); v2 = r; } + { float t = as_type(s1) * as_type(v3); float u = as_type(v6); uint r = as_type(t + u); v1 = r; } + { float t = as_type(s2) * as_type(v6); float u = as_type(v3); uint r = as_type(t + u); v0 = r; } + { float t = as_type(s3) * as_type(v1); float u = as_type(v0); uint r = as_type(t + u); v7 = r; } + { float t = as_type(s0) * as_type(v4); float u = as_type(v5); uint r = as_type(t + u); v6 = r; } + { float t = as_type(s1) * as_type(v7); float u = as_type(v2); uint r = as_type(t + u); v5 = r; } + { float t = as_type(s2) * as_type(v2); float u = as_type(v7); uint r = as_type(t + u); v4 = r; } + { float t = as_type(s3) * as_type(v5); float u = as_type(v4); uint r = as_type(t + u); v3 = r; } + { float t = as_type(s0) * as_type(v0); float u = as_type(v1); uint r = as_type(t + u); v2 = r; } + { float t = as_type(s1) * as_type(v3); float u = as_type(v6); uint r = as_type(t + u); v1 = r; } + { float t = as_type(s2) * as_type(v6); float u = as_type(v3); uint r = as_type(t + u); v0 = r; } + { float t = as_type(s3) * as_type(v1); float u = as_type(v0); uint r = as_type(t + u); v7 = r; } + { float t = as_type(s0) * as_type(v4); float u = as_type(v5); uint r = as_type(t + u); v6 = r; } + { float t = as_type(s1) * as_type(v7); float u = as_type(v2); uint r = as_type(t + u); v5 = r; } + { float t = as_type(s2) * as_type(v2); float u = as_type(v7); uint r = as_type(t + u); v4 = r; } + { float t = as_type(s3) * as_type(v5); float u = as_type(v4); uint r = as_type(t + u); v3 = r; } + } + out[tid] = v0 ^ v1 ^ v2 ^ v3 ^ v4 ^ v5 ^ v6 ^ v7; +} +""", + ["f16_current"] = """ +// bench group=f16_pack variant=current threads=1048576 tg=64 iters=32 check=1 +#include +using namespace metal; +#pragma clang fp contract(off) + +static inline float rtz_cur(float t) +{ + return (isnan(t) || isinf(t)) ? t : (fabs(t) < 6.103515625e-05f ? (trunc(t * 16777216.0f) * 5.9604644775390625e-08f) : as_type(as_type(clamp(t, -65504.0f, 65504.0f)) & 0xFFFFE000u)); +} +static inline uint pack_cur(float x, float y) +{ + return ((uint)as_type(half(rtz_cur(x)))) | (((uint)as_type(half(rtz_cur(y)))) << 16); +} +// Integer round-toward-zero f32 -> f16 bits, saturating finite overflow to 65504. +static inline uint rtz_bits(uint f) +{ + uint sign = (f >> 16) & 0x8000u; + uint a = f & 0x7FFFFFFFu; + uint normal = min((a - 0x38000000u) >> 13, 0x7BFFu); + uint e = a >> 23; + uint m = (a & 0x7FFFFFu) | 0x800000u; + uint denorm = e >= 102u ? (m >> (126u - e)) : 0u; + uint special = a > 0x7F800000u ? (0x7E00u | ((a >> 13) & 0x1FFu)) : 0x7C00u; + uint r = a >= 0x7F800000u ? special : (a >= 0x38800000u ? normal : denorm); + return sign | r; +} +static inline uint pack_int(float x, float y) +{ + return rtz_bits(as_type(x)) | (rtz_bits(as_type(y)) << 16); +} + +kernel void bench(device uint* data [[buffer(0)]], device uint* out [[buffer(1)]], constant uint4& prm [[buffer(2)]], + uint tid [[thread_position_in_grid]], uint lane [[thread_index_in_simdgroup]]) +{ + uint acc = 0u; + for (uint i = 0u; i < prm.x; i++) + { + uint a = data[(tid * 2u + i * 977u) & 0x3FFFFFu]; + uint b = data[(tid * 2u + i * 977u + 1u) & 0x3FFFFFu]; + // exponents spread over denormal half, normal half and overflow; no NaN or infinity + uint ea = 90u + (a >> 27); uint eb = 90u + (b >> 27); + float x = as_type((a & 0x807FFFFFu) | (ea << 23)); + float y = as_type((b & 0x807FFFFFu) | (eb << 23)); + acc ^= pack_cur(x, y); + acc = acc * 3u + pack_cur(y, x); + } + out[tid] = acc; +} +""", + ["f16_int_rtz"] = """ +// bench group=f16_pack variant=int_rtz threads=1048576 tg=64 iters=32 check=1 +#include +using namespace metal; +#pragma clang fp contract(off) + +static inline float rtz_cur(float t) +{ + return (isnan(t) || isinf(t)) ? t : (fabs(t) < 6.103515625e-05f ? (trunc(t * 16777216.0f) * 5.9604644775390625e-08f) : as_type(as_type(clamp(t, -65504.0f, 65504.0f)) & 0xFFFFE000u)); +} +static inline uint pack_cur(float x, float y) +{ + return ((uint)as_type(half(rtz_cur(x)))) | (((uint)as_type(half(rtz_cur(y)))) << 16); +} +// Integer round-toward-zero f32 -> f16 bits, saturating finite overflow to 65504. +static inline uint rtz_bits(uint f) +{ + uint sign = (f >> 16) & 0x8000u; + uint a = f & 0x7FFFFFFFu; + uint normal = min((a - 0x38000000u) >> 13, 0x7BFFu); + uint e = a >> 23; + uint m = (a & 0x7FFFFFu) | 0x800000u; + uint denorm = e >= 102u ? (m >> (126u - e)) : 0u; + uint special = a > 0x7F800000u ? (0x7E00u | ((a >> 13) & 0x1FFu)) : 0x7C00u; + uint r = a >= 0x7F800000u ? special : (a >= 0x38800000u ? normal : denorm); + return sign | r; +} +static inline uint pack_int(float x, float y) +{ + return rtz_bits(as_type(x)) | (rtz_bits(as_type(y)) << 16); +} + +kernel void bench(device uint* data [[buffer(0)]], device uint* out [[buffer(1)]], constant uint4& prm [[buffer(2)]], + uint tid [[thread_position_in_grid]], uint lane [[thread_index_in_simdgroup]]) +{ + uint acc = 0u; + for (uint i = 0u; i < prm.x; i++) + { + uint a = data[(tid * 2u + i * 977u) & 0x3FFFFFu]; + uint b = data[(tid * 2u + i * 977u + 1u) & 0x3FFFFFu]; + // exponents spread over denormal half, normal half and overflow; no NaN or infinity + uint ea = 90u + (a >> 27); uint eb = 90u + (b >> 27); + float x = as_type((a & 0x807FFFFFu) | (ea << 23)); + float y = as_type((b & 0x807FFFFFu) | (eb << 23)); + acc ^= pack_int(x, y); + acc = acc * 3u + pack_int(y, x); + } + out[tid] = acc; +} +""", + ["lds_clampmask_noguard"] = """ +// bench group=lds_index variant=clampmask_noguard threads=262144 tg=32 iters=32 check=1 +#include +using namespace metal; +kernel void bench(device uint* data [[buffer(0)]], device uint* out [[buffer(1)]], constant uint4& prm [[buffer(2)]], + uint tid [[thread_position_in_grid]], uint lane [[thread_index_in_simdgroup]], uint tgi [[thread_index_in_threadgroup]]) +{ + threadgroup uint lds[1281]; + uint m = data[0]; + bool exec = ((m >> lane) & 1u) != 0u; + uint acc = tid; + uint base = (tgi & 31u) * 4u; + for (uint it = 0u; it < prm.x; it++) + { + uint a = base + (it & 7u) * 128u; + lds[min(((((a) + 1360u) >> 2) & 2047u), 1279u)] = acc + 0u; + lds[min(((((a) + 1364u) >> 2) & 2047u), 1279u)] = acc + 1u; + lds[min(((((a) + 1368u) >> 2) & 2047u), 1279u)] = acc + 2u; + lds[min(((((a) + 1372u) >> 2) & 2047u), 1279u)] = acc + 3u; + lds[min(((((a) + 1376u) >> 2) & 2047u), 1279u)] = acc + 4u; + lds[min(((((a) + 1380u) >> 2) & 2047u), 1279u)] = acc + 5u; + lds[min(((((a) + 1384u) >> 2) & 2047u), 1279u)] = acc + 6u; + lds[min(((((a) + 1388u) >> 2) & 2047u), 1279u)] = acc + 7u; + lds[min(((((a) + 1392u) >> 2) & 2047u), 1279u)] = acc + 8u; + lds[min(((((a) + 1396u) >> 2) & 2047u), 1279u)] = acc + 9u; + lds[min(((((a) + 1400u) >> 2) & 2047u), 1279u)] = acc + 10u; + lds[min(((((a) + 1404u) >> 2) & 2047u), 1279u)] = acc + 11u; + lds[min(((((a) + 1408u) >> 2) & 2047u), 1279u)] = acc + 12u; + lds[min(((((a) + 1412u) >> 2) & 2047u), 1279u)] = acc + 13u; + lds[min(((((a) + 1416u) >> 2) & 2047u), 1279u)] = acc + 14u; + lds[min(((((a) + 1420u) >> 2) & 2047u), 1279u)] = acc + 15u; + threadgroup_barrier(mem_flags::mem_threadgroup); + acc ^= lds[min(((((a) + 1360u) >> 2) & 2047u), 1279u)]; + acc ^= lds[min(((((a) + 1364u) >> 2) & 2047u), 1279u)]; + acc ^= lds[min(((((a) + 1368u) >> 2) & 2047u), 1279u)]; + acc ^= lds[min(((((a) + 1372u) >> 2) & 2047u), 1279u)]; + acc ^= lds[min(((((a) + 1376u) >> 2) & 2047u), 1279u)]; + acc ^= lds[min(((((a) + 1380u) >> 2) & 2047u), 1279u)]; + acc ^= lds[min(((((a) + 1384u) >> 2) & 2047u), 1279u)]; + acc ^= lds[min(((((a) + 1388u) >> 2) & 2047u), 1279u)]; + acc ^= lds[min(((((a) + 1392u) >> 2) & 2047u), 1279u)]; + acc ^= lds[min(((((a) + 1396u) >> 2) & 2047u), 1279u)]; + acc ^= lds[min(((((a) + 1400u) >> 2) & 2047u), 1279u)]; + acc ^= lds[min(((((a) + 1404u) >> 2) & 2047u), 1279u)]; + acc ^= lds[min(((((a) + 1408u) >> 2) & 2047u), 1279u)]; + acc ^= lds[min(((((a) + 1412u) >> 2) & 2047u), 1279u)]; + acc ^= lds[min(((((a) + 1416u) >> 2) & 2047u), 1279u)]; + acc ^= lds[min(((((a) + 1420u) >> 2) & 2047u), 1279u)]; + threadgroup_barrier(mem_flags::mem_threadgroup); + } + out[tid] = acc; +} +""", + ["lds_current"] = """ +// bench group=lds_index variant=current threads=262144 tg=32 iters=32 check=1 +#include +using namespace metal; +kernel void bench(device uint* data [[buffer(0)]], device uint* out [[buffer(1)]], constant uint4& prm [[buffer(2)]], + uint tid [[thread_position_in_grid]], uint lane [[thread_index_in_simdgroup]], uint tgi [[thread_index_in_threadgroup]]) +{ + threadgroup uint lds[1281]; + uint m = data[0]; + bool exec = ((m >> lane) & 1u) != 0u; + uint acc = tid; + uint base = (tgi & 31u) * 4u; + for (uint it = 0u; it < prm.x; it++) + { + uint a = base + (it & 7u) * 128u; + if (exec) { lds[min(((((a) + 1360u) >> 2) & 2047u), 1279u)] = acc + 0u; } + if (exec) { lds[min(((((a) + 1364u) >> 2) & 2047u), 1279u)] = acc + 1u; } + if (exec) { lds[min(((((a) + 1368u) >> 2) & 2047u), 1279u)] = acc + 2u; } + if (exec) { lds[min(((((a) + 1372u) >> 2) & 2047u), 1279u)] = acc + 3u; } + if (exec) { lds[min(((((a) + 1376u) >> 2) & 2047u), 1279u)] = acc + 4u; } + if (exec) { lds[min(((((a) + 1380u) >> 2) & 2047u), 1279u)] = acc + 5u; } + if (exec) { lds[min(((((a) + 1384u) >> 2) & 2047u), 1279u)] = acc + 6u; } + if (exec) { lds[min(((((a) + 1388u) >> 2) & 2047u), 1279u)] = acc + 7u; } + if (exec) { lds[min(((((a) + 1392u) >> 2) & 2047u), 1279u)] = acc + 8u; } + if (exec) { lds[min(((((a) + 1396u) >> 2) & 2047u), 1279u)] = acc + 9u; } + if (exec) { lds[min(((((a) + 1400u) >> 2) & 2047u), 1279u)] = acc + 10u; } + if (exec) { lds[min(((((a) + 1404u) >> 2) & 2047u), 1279u)] = acc + 11u; } + if (exec) { lds[min(((((a) + 1408u) >> 2) & 2047u), 1279u)] = acc + 12u; } + if (exec) { lds[min(((((a) + 1412u) >> 2) & 2047u), 1279u)] = acc + 13u; } + if (exec) { lds[min(((((a) + 1416u) >> 2) & 2047u), 1279u)] = acc + 14u; } + if (exec) { lds[min(((((a) + 1420u) >> 2) & 2047u), 1279u)] = acc + 15u; } + threadgroup_barrier(mem_flags::mem_threadgroup); + if (exec) { acc ^= lds[min(((((a) + 1360u) >> 2) & 2047u), 1279u)]; } + if (exec) { acc ^= lds[min(((((a) + 1364u) >> 2) & 2047u), 1279u)]; } + if (exec) { acc ^= lds[min(((((a) + 1368u) >> 2) & 2047u), 1279u)]; } + if (exec) { acc ^= lds[min(((((a) + 1372u) >> 2) & 2047u), 1279u)]; } + if (exec) { acc ^= lds[min(((((a) + 1376u) >> 2) & 2047u), 1279u)]; } + if (exec) { acc ^= lds[min(((((a) + 1380u) >> 2) & 2047u), 1279u)]; } + if (exec) { acc ^= lds[min(((((a) + 1384u) >> 2) & 2047u), 1279u)]; } + if (exec) { acc ^= lds[min(((((a) + 1388u) >> 2) & 2047u), 1279u)]; } + if (exec) { acc ^= lds[min(((((a) + 1392u) >> 2) & 2047u), 1279u)]; } + if (exec) { acc ^= lds[min(((((a) + 1396u) >> 2) & 2047u), 1279u)]; } + if (exec) { acc ^= lds[min(((((a) + 1400u) >> 2) & 2047u), 1279u)]; } + if (exec) { acc ^= lds[min(((((a) + 1404u) >> 2) & 2047u), 1279u)]; } + if (exec) { acc ^= lds[min(((((a) + 1408u) >> 2) & 2047u), 1279u)]; } + if (exec) { acc ^= lds[min(((((a) + 1412u) >> 2) & 2047u), 1279u)]; } + if (exec) { acc ^= lds[min(((((a) + 1416u) >> 2) & 2047u), 1279u)]; } + if (exec) { acc ^= lds[min(((((a) + 1420u) >> 2) & 2047u), 1279u)]; } + threadgroup_barrier(mem_flags::mem_threadgroup); + } + out[tid] = acc; +} +""", + ["lds_guard_only"] = """ +// bench group=lds_index variant=guard_only threads=262144 tg=32 iters=32 check=1 +#include +using namespace metal; +kernel void bench(device uint* data [[buffer(0)]], device uint* out [[buffer(1)]], constant uint4& prm [[buffer(2)]], + uint tid [[thread_position_in_grid]], uint lane [[thread_index_in_simdgroup]], uint tgi [[thread_index_in_threadgroup]]) +{ + threadgroup uint lds[1281]; + uint m = data[0]; + bool exec = ((m >> lane) & 1u) != 0u; + uint acc = tid; + uint base = (tgi & 31u) * 4u; + for (uint it = 0u; it < prm.x; it++) + { + uint a = base + (it & 7u) * 128u; + if (exec) { lds[(((a) + 1360u) >> 2)] = acc + 0u; } + if (exec) { lds[(((a) + 1364u) >> 2)] = acc + 1u; } + if (exec) { lds[(((a) + 1368u) >> 2)] = acc + 2u; } + if (exec) { lds[(((a) + 1372u) >> 2)] = acc + 3u; } + if (exec) { lds[(((a) + 1376u) >> 2)] = acc + 4u; } + if (exec) { lds[(((a) + 1380u) >> 2)] = acc + 5u; } + if (exec) { lds[(((a) + 1384u) >> 2)] = acc + 6u; } + if (exec) { lds[(((a) + 1388u) >> 2)] = acc + 7u; } + if (exec) { lds[(((a) + 1392u) >> 2)] = acc + 8u; } + if (exec) { lds[(((a) + 1396u) >> 2)] = acc + 9u; } + if (exec) { lds[(((a) + 1400u) >> 2)] = acc + 10u; } + if (exec) { lds[(((a) + 1404u) >> 2)] = acc + 11u; } + if (exec) { lds[(((a) + 1408u) >> 2)] = acc + 12u; } + if (exec) { lds[(((a) + 1412u) >> 2)] = acc + 13u; } + if (exec) { lds[(((a) + 1416u) >> 2)] = acc + 14u; } + if (exec) { lds[(((a) + 1420u) >> 2)] = acc + 15u; } + threadgroup_barrier(mem_flags::mem_threadgroup); + if (exec) { acc ^= lds[(((a) + 1360u) >> 2)]; } + if (exec) { acc ^= lds[(((a) + 1364u) >> 2)]; } + if (exec) { acc ^= lds[(((a) + 1368u) >> 2)]; } + if (exec) { acc ^= lds[(((a) + 1372u) >> 2)]; } + if (exec) { acc ^= lds[(((a) + 1376u) >> 2)]; } + if (exec) { acc ^= lds[(((a) + 1380u) >> 2)]; } + if (exec) { acc ^= lds[(((a) + 1384u) >> 2)]; } + if (exec) { acc ^= lds[(((a) + 1388u) >> 2)]; } + if (exec) { acc ^= lds[(((a) + 1392u) >> 2)]; } + if (exec) { acc ^= lds[(((a) + 1396u) >> 2)]; } + if (exec) { acc ^= lds[(((a) + 1400u) >> 2)]; } + if (exec) { acc ^= lds[(((a) + 1404u) >> 2)]; } + if (exec) { acc ^= lds[(((a) + 1408u) >> 2)]; } + if (exec) { acc ^= lds[(((a) + 1412u) >> 2)]; } + if (exec) { acc ^= lds[(((a) + 1416u) >> 2)]; } + if (exec) { acc ^= lds[(((a) + 1420u) >> 2)]; } + threadgroup_barrier(mem_flags::mem_threadgroup); + } + out[tid] = acc; +} +""", + ["lds_loadsel"] = """ +// bench group=lds_index variant=loadsel threads=262144 tg=32 iters=32 check=1 +#include +using namespace metal; +kernel void bench(device uint* data [[buffer(0)]], device uint* out [[buffer(1)]], constant uint4& prm [[buffer(2)]], + uint tid [[thread_position_in_grid]], uint lane [[thread_index_in_simdgroup]], uint tgi [[thread_index_in_threadgroup]]) +{ + threadgroup uint lds[1281]; + uint m = data[0]; + bool exec = ((m >> lane) & 1u) != 0u; + uint acc = tid; + uint base = (tgi & 31u) * 4u; + for (uint it = 0u; it < prm.x; it++) + { + uint a = base + (it & 7u) * 128u; + if (exec) { lds[min(((((a) + 1360u) >> 2) & 2047u), 1279u)] = acc + 0u; } + if (exec) { lds[min(((((a) + 1364u) >> 2) & 2047u), 1279u)] = acc + 1u; } + if (exec) { lds[min(((((a) + 1368u) >> 2) & 2047u), 1279u)] = acc + 2u; } + if (exec) { lds[min(((((a) + 1372u) >> 2) & 2047u), 1279u)] = acc + 3u; } + if (exec) { lds[min(((((a) + 1376u) >> 2) & 2047u), 1279u)] = acc + 4u; } + if (exec) { lds[min(((((a) + 1380u) >> 2) & 2047u), 1279u)] = acc + 5u; } + if (exec) { lds[min(((((a) + 1384u) >> 2) & 2047u), 1279u)] = acc + 6u; } + if (exec) { lds[min(((((a) + 1388u) >> 2) & 2047u), 1279u)] = acc + 7u; } + if (exec) { lds[min(((((a) + 1392u) >> 2) & 2047u), 1279u)] = acc + 8u; } + if (exec) { lds[min(((((a) + 1396u) >> 2) & 2047u), 1279u)] = acc + 9u; } + if (exec) { lds[min(((((a) + 1400u) >> 2) & 2047u), 1279u)] = acc + 10u; } + if (exec) { lds[min(((((a) + 1404u) >> 2) & 2047u), 1279u)] = acc + 11u; } + if (exec) { lds[min(((((a) + 1408u) >> 2) & 2047u), 1279u)] = acc + 12u; } + if (exec) { lds[min(((((a) + 1412u) >> 2) & 2047u), 1279u)] = acc + 13u; } + if (exec) { lds[min(((((a) + 1416u) >> 2) & 2047u), 1279u)] = acc + 14u; } + if (exec) { lds[min(((((a) + 1420u) >> 2) & 2047u), 1279u)] = acc + 15u; } + threadgroup_barrier(mem_flags::mem_threadgroup); + { uint r = lds[min(((((a) + 1360u) >> 2) & 2047u), 1279u)]; if (exec) { acc ^= r; } } + { uint r = lds[min(((((a) + 1364u) >> 2) & 2047u), 1279u)]; if (exec) { acc ^= r; } } + { uint r = lds[min(((((a) + 1368u) >> 2) & 2047u), 1279u)]; if (exec) { acc ^= r; } } + { uint r = lds[min(((((a) + 1372u) >> 2) & 2047u), 1279u)]; if (exec) { acc ^= r; } } + { uint r = lds[min(((((a) + 1376u) >> 2) & 2047u), 1279u)]; if (exec) { acc ^= r; } } + { uint r = lds[min(((((a) + 1380u) >> 2) & 2047u), 1279u)]; if (exec) { acc ^= r; } } + { uint r = lds[min(((((a) + 1384u) >> 2) & 2047u), 1279u)]; if (exec) { acc ^= r; } } + { uint r = lds[min(((((a) + 1388u) >> 2) & 2047u), 1279u)]; if (exec) { acc ^= r; } } + { uint r = lds[min(((((a) + 1392u) >> 2) & 2047u), 1279u)]; if (exec) { acc ^= r; } } + { uint r = lds[min(((((a) + 1396u) >> 2) & 2047u), 1279u)]; if (exec) { acc ^= r; } } + { uint r = lds[min(((((a) + 1400u) >> 2) & 2047u), 1279u)]; if (exec) { acc ^= r; } } + { uint r = lds[min(((((a) + 1404u) >> 2) & 2047u), 1279u)]; if (exec) { acc ^= r; } } + { uint r = lds[min(((((a) + 1408u) >> 2) & 2047u), 1279u)]; if (exec) { acc ^= r; } } + { uint r = lds[min(((((a) + 1412u) >> 2) & 2047u), 1279u)]; if (exec) { acc ^= r; } } + { uint r = lds[min(((((a) + 1416u) >> 2) & 2047u), 1279u)]; if (exec) { acc ^= r; } } + { uint r = lds[min(((((a) + 1420u) >> 2) & 2047u), 1279u)]; if (exec) { acc ^= r; } } + threadgroup_barrier(mem_flags::mem_threadgroup); + } + out[tid] = acc; +} +""", + ["lds_loadsel_storeslot"] = """ +// bench group=lds_index variant=loadsel_storeslot threads=262144 tg=32 iters=32 check=1 +#include +using namespace metal; +kernel void bench(device uint* data [[buffer(0)]], device uint* out [[buffer(1)]], constant uint4& prm [[buffer(2)]], + uint tid [[thread_position_in_grid]], uint lane [[thread_index_in_simdgroup]], uint tgi [[thread_index_in_threadgroup]]) +{ + threadgroup uint lds[1281]; + uint m = data[0]; + bool exec = ((m >> lane) & 1u) != 0u; + uint acc = tid; + uint base = (tgi & 31u) * 4u; + for (uint it = 0u; it < prm.x; it++) + { + uint a = base + (it & 7u) * 128u; + lds[exec ? min(((((a) + 1360u) >> 2) & 2047u), 1279u) : 1280u] = acc + 0u; + lds[exec ? min(((((a) + 1364u) >> 2) & 2047u), 1279u) : 1280u] = acc + 1u; + lds[exec ? min(((((a) + 1368u) >> 2) & 2047u), 1279u) : 1280u] = acc + 2u; + lds[exec ? min(((((a) + 1372u) >> 2) & 2047u), 1279u) : 1280u] = acc + 3u; + lds[exec ? min(((((a) + 1376u) >> 2) & 2047u), 1279u) : 1280u] = acc + 4u; + lds[exec ? min(((((a) + 1380u) >> 2) & 2047u), 1279u) : 1280u] = acc + 5u; + lds[exec ? min(((((a) + 1384u) >> 2) & 2047u), 1279u) : 1280u] = acc + 6u; + lds[exec ? min(((((a) + 1388u) >> 2) & 2047u), 1279u) : 1280u] = acc + 7u; + lds[exec ? min(((((a) + 1392u) >> 2) & 2047u), 1279u) : 1280u] = acc + 8u; + lds[exec ? min(((((a) + 1396u) >> 2) & 2047u), 1279u) : 1280u] = acc + 9u; + lds[exec ? min(((((a) + 1400u) >> 2) & 2047u), 1279u) : 1280u] = acc + 10u; + lds[exec ? min(((((a) + 1404u) >> 2) & 2047u), 1279u) : 1280u] = acc + 11u; + lds[exec ? min(((((a) + 1408u) >> 2) & 2047u), 1279u) : 1280u] = acc + 12u; + lds[exec ? min(((((a) + 1412u) >> 2) & 2047u), 1279u) : 1280u] = acc + 13u; + lds[exec ? min(((((a) + 1416u) >> 2) & 2047u), 1279u) : 1280u] = acc + 14u; + lds[exec ? min(((((a) + 1420u) >> 2) & 2047u), 1279u) : 1280u] = acc + 15u; + threadgroup_barrier(mem_flags::mem_threadgroup); + { uint r = lds[min(((((a) + 1360u) >> 2) & 2047u), 1279u)]; if (exec) { acc ^= r; } } + { uint r = lds[min(((((a) + 1364u) >> 2) & 2047u), 1279u)]; if (exec) { acc ^= r; } } + { uint r = lds[min(((((a) + 1368u) >> 2) & 2047u), 1279u)]; if (exec) { acc ^= r; } } + { uint r = lds[min(((((a) + 1372u) >> 2) & 2047u), 1279u)]; if (exec) { acc ^= r; } } + { uint r = lds[min(((((a) + 1376u) >> 2) & 2047u), 1279u)]; if (exec) { acc ^= r; } } + { uint r = lds[min(((((a) + 1380u) >> 2) & 2047u), 1279u)]; if (exec) { acc ^= r; } } + { uint r = lds[min(((((a) + 1384u) >> 2) & 2047u), 1279u)]; if (exec) { acc ^= r; } } + { uint r = lds[min(((((a) + 1388u) >> 2) & 2047u), 1279u)]; if (exec) { acc ^= r; } } + { uint r = lds[min(((((a) + 1392u) >> 2) & 2047u), 1279u)]; if (exec) { acc ^= r; } } + { uint r = lds[min(((((a) + 1396u) >> 2) & 2047u), 1279u)]; if (exec) { acc ^= r; } } + { uint r = lds[min(((((a) + 1400u) >> 2) & 2047u), 1279u)]; if (exec) { acc ^= r; } } + { uint r = lds[min(((((a) + 1404u) >> 2) & 2047u), 1279u)]; if (exec) { acc ^= r; } } + { uint r = lds[min(((((a) + 1408u) >> 2) & 2047u), 1279u)]; if (exec) { acc ^= r; } } + { uint r = lds[min(((((a) + 1412u) >> 2) & 2047u), 1279u)]; if (exec) { acc ^= r; } } + { uint r = lds[min(((((a) + 1416u) >> 2) & 2047u), 1279u)]; if (exec) { acc ^= r; } } + { uint r = lds[min(((((a) + 1420u) >> 2) & 2047u), 1279u)]; if (exec) { acc ^= r; } } + threadgroup_barrier(mem_flags::mem_threadgroup); + } + out[tid] = acc; +} +""", + ["lds_mask_only"] = """ +// bench group=lds_index variant=mask_only threads=262144 tg=32 iters=32 check=1 +#include +using namespace metal; +kernel void bench(device uint* data [[buffer(0)]], device uint* out [[buffer(1)]], constant uint4& prm [[buffer(2)]], + uint tid [[thread_position_in_grid]], uint lane [[thread_index_in_simdgroup]], uint tgi [[thread_index_in_threadgroup]]) +{ + threadgroup uint lds[1281]; + uint m = data[0]; + bool exec = ((m >> lane) & 1u) != 0u; + uint acc = tid; + uint base = (tgi & 31u) * 4u; + for (uint it = 0u; it < prm.x; it++) + { + uint a = base + (it & 7u) * 128u; + lds[((((a) + 1360u) >> 2) & 2047u)] = acc + 0u; + lds[((((a) + 1364u) >> 2) & 2047u)] = acc + 1u; + lds[((((a) + 1368u) >> 2) & 2047u)] = acc + 2u; + lds[((((a) + 1372u) >> 2) & 2047u)] = acc + 3u; + lds[((((a) + 1376u) >> 2) & 2047u)] = acc + 4u; + lds[((((a) + 1380u) >> 2) & 2047u)] = acc + 5u; + lds[((((a) + 1384u) >> 2) & 2047u)] = acc + 6u; + lds[((((a) + 1388u) >> 2) & 2047u)] = acc + 7u; + lds[((((a) + 1392u) >> 2) & 2047u)] = acc + 8u; + lds[((((a) + 1396u) >> 2) & 2047u)] = acc + 9u; + lds[((((a) + 1400u) >> 2) & 2047u)] = acc + 10u; + lds[((((a) + 1404u) >> 2) & 2047u)] = acc + 11u; + lds[((((a) + 1408u) >> 2) & 2047u)] = acc + 12u; + lds[((((a) + 1412u) >> 2) & 2047u)] = acc + 13u; + lds[((((a) + 1416u) >> 2) & 2047u)] = acc + 14u; + lds[((((a) + 1420u) >> 2) & 2047u)] = acc + 15u; + threadgroup_barrier(mem_flags::mem_threadgroup); + acc ^= lds[((((a) + 1360u) >> 2) & 2047u)]; + acc ^= lds[((((a) + 1364u) >> 2) & 2047u)]; + acc ^= lds[((((a) + 1368u) >> 2) & 2047u)]; + acc ^= lds[((((a) + 1372u) >> 2) & 2047u)]; + acc ^= lds[((((a) + 1376u) >> 2) & 2047u)]; + acc ^= lds[((((a) + 1380u) >> 2) & 2047u)]; + acc ^= lds[((((a) + 1384u) >> 2) & 2047u)]; + acc ^= lds[((((a) + 1388u) >> 2) & 2047u)]; + acc ^= lds[((((a) + 1392u) >> 2) & 2047u)]; + acc ^= lds[((((a) + 1396u) >> 2) & 2047u)]; + acc ^= lds[((((a) + 1400u) >> 2) & 2047u)]; + acc ^= lds[((((a) + 1404u) >> 2) & 2047u)]; + acc ^= lds[((((a) + 1408u) >> 2) & 2047u)]; + acc ^= lds[((((a) + 1412u) >> 2) & 2047u)]; + acc ^= lds[((((a) + 1416u) >> 2) & 2047u)]; + acc ^= lds[((((a) + 1420u) >> 2) & 2047u)]; + threadgroup_barrier(mem_flags::mem_threadgroup); + } + out[tid] = acc; +} +""", + ["lds_noclamp_guard"] = """ +// bench group=lds_index variant=noclamp_guard threads=262144 tg=32 iters=32 check=1 +#include +using namespace metal; +kernel void bench(device uint* data [[buffer(0)]], device uint* out [[buffer(1)]], constant uint4& prm [[buffer(2)]], + uint tid [[thread_position_in_grid]], uint lane [[thread_index_in_simdgroup]], uint tgi [[thread_index_in_threadgroup]]) +{ + threadgroup uint lds[1281]; + uint m = data[0]; + bool exec = ((m >> lane) & 1u) != 0u; + uint acc = tid; + uint base = (tgi & 31u) * 4u; + for (uint it = 0u; it < prm.x; it++) + { + uint a = base + (it & 7u) * 128u; + if (exec) { lds[((((a) + 1360u) >> 2) & 2047u)] = acc + 0u; } + if (exec) { lds[((((a) + 1364u) >> 2) & 2047u)] = acc + 1u; } + if (exec) { lds[((((a) + 1368u) >> 2) & 2047u)] = acc + 2u; } + if (exec) { lds[((((a) + 1372u) >> 2) & 2047u)] = acc + 3u; } + if (exec) { lds[((((a) + 1376u) >> 2) & 2047u)] = acc + 4u; } + if (exec) { lds[((((a) + 1380u) >> 2) & 2047u)] = acc + 5u; } + if (exec) { lds[((((a) + 1384u) >> 2) & 2047u)] = acc + 6u; } + if (exec) { lds[((((a) + 1388u) >> 2) & 2047u)] = acc + 7u; } + if (exec) { lds[((((a) + 1392u) >> 2) & 2047u)] = acc + 8u; } + if (exec) { lds[((((a) + 1396u) >> 2) & 2047u)] = acc + 9u; } + if (exec) { lds[((((a) + 1400u) >> 2) & 2047u)] = acc + 10u; } + if (exec) { lds[((((a) + 1404u) >> 2) & 2047u)] = acc + 11u; } + if (exec) { lds[((((a) + 1408u) >> 2) & 2047u)] = acc + 12u; } + if (exec) { lds[((((a) + 1412u) >> 2) & 2047u)] = acc + 13u; } + if (exec) { lds[((((a) + 1416u) >> 2) & 2047u)] = acc + 14u; } + if (exec) { lds[((((a) + 1420u) >> 2) & 2047u)] = acc + 15u; } + threadgroup_barrier(mem_flags::mem_threadgroup); + if (exec) { acc ^= lds[((((a) + 1360u) >> 2) & 2047u)]; } + if (exec) { acc ^= lds[((((a) + 1364u) >> 2) & 2047u)]; } + if (exec) { acc ^= lds[((((a) + 1368u) >> 2) & 2047u)]; } + if (exec) { acc ^= lds[((((a) + 1372u) >> 2) & 2047u)]; } + if (exec) { acc ^= lds[((((a) + 1376u) >> 2) & 2047u)]; } + if (exec) { acc ^= lds[((((a) + 1380u) >> 2) & 2047u)]; } + if (exec) { acc ^= lds[((((a) + 1384u) >> 2) & 2047u)]; } + if (exec) { acc ^= lds[((((a) + 1388u) >> 2) & 2047u)]; } + if (exec) { acc ^= lds[((((a) + 1392u) >> 2) & 2047u)]; } + if (exec) { acc ^= lds[((((a) + 1396u) >> 2) & 2047u)]; } + if (exec) { acc ^= lds[((((a) + 1400u) >> 2) & 2047u)]; } + if (exec) { acc ^= lds[((((a) + 1404u) >> 2) & 2047u)]; } + if (exec) { acc ^= lds[((((a) + 1408u) >> 2) & 2047u)]; } + if (exec) { acc ^= lds[((((a) + 1412u) >> 2) & 2047u)]; } + if (exec) { acc ^= lds[((((a) + 1416u) >> 2) & 2047u)]; } + if (exec) { acc ^= lds[((((a) + 1420u) >> 2) & 2047u)]; } + threadgroup_barrier(mem_flags::mem_threadgroup); + } + out[tid] = acc; +} +""", + ["lds_plain"] = """ +// bench group=lds_index variant=plain threads=262144 tg=32 iters=32 check=1 +#include +using namespace metal; +kernel void bench(device uint* data [[buffer(0)]], device uint* out [[buffer(1)]], constant uint4& prm [[buffer(2)]], + uint tid [[thread_position_in_grid]], uint lane [[thread_index_in_simdgroup]], uint tgi [[thread_index_in_threadgroup]]) +{ + threadgroup uint lds[1281]; + uint m = data[0]; + bool exec = ((m >> lane) & 1u) != 0u; + uint acc = tid; + uint base = (tgi & 31u) * 4u; + for (uint it = 0u; it < prm.x; it++) + { + uint a = base + (it & 7u) * 128u; + lds[(((a) + 1360u) >> 2)] = acc + 0u; + lds[(((a) + 1364u) >> 2)] = acc + 1u; + lds[(((a) + 1368u) >> 2)] = acc + 2u; + lds[(((a) + 1372u) >> 2)] = acc + 3u; + lds[(((a) + 1376u) >> 2)] = acc + 4u; + lds[(((a) + 1380u) >> 2)] = acc + 5u; + lds[(((a) + 1384u) >> 2)] = acc + 6u; + lds[(((a) + 1388u) >> 2)] = acc + 7u; + lds[(((a) + 1392u) >> 2)] = acc + 8u; + lds[(((a) + 1396u) >> 2)] = acc + 9u; + lds[(((a) + 1400u) >> 2)] = acc + 10u; + lds[(((a) + 1404u) >> 2)] = acc + 11u; + lds[(((a) + 1408u) >> 2)] = acc + 12u; + lds[(((a) + 1412u) >> 2)] = acc + 13u; + lds[(((a) + 1416u) >> 2)] = acc + 14u; + lds[(((a) + 1420u) >> 2)] = acc + 15u; + threadgroup_barrier(mem_flags::mem_threadgroup); + acc ^= lds[(((a) + 1360u) >> 2)]; + acc ^= lds[(((a) + 1364u) >> 2)]; + acc ^= lds[(((a) + 1368u) >> 2)]; + acc ^= lds[(((a) + 1372u) >> 2)]; + acc ^= lds[(((a) + 1376u) >> 2)]; + acc ^= lds[(((a) + 1380u) >> 2)]; + acc ^= lds[(((a) + 1384u) >> 2)]; + acc ^= lds[(((a) + 1388u) >> 2)]; + acc ^= lds[(((a) + 1392u) >> 2)]; + acc ^= lds[(((a) + 1396u) >> 2)]; + acc ^= lds[(((a) + 1400u) >> 2)]; + acc ^= lds[(((a) + 1404u) >> 2)]; + acc ^= lds[(((a) + 1408u) >> 2)]; + acc ^= lds[(((a) + 1412u) >> 2)]; + acc ^= lds[(((a) + 1416u) >> 2)]; + acc ^= lds[(((a) + 1420u) >> 2)]; + threadgroup_barrier(mem_flags::mem_threadgroup); + } + out[tid] = acc; +} +""", + ["mad_contract_on"] = """ +// bench group=contract variant=pragma_on_relaxed threads=1048576 tg=64 iters=64 +#include +using namespace metal; + +kernel void bench(device uint* data [[buffer(0)]], device uint* out [[buffer(1)]], constant uint4& prm [[buffer(2)]], + uint tid [[thread_position_in_grid]], uint lane [[thread_index_in_simdgroup]]) +{ + uint m = data[0]; + bool exec = ((m >> lane) & 1u) != 0u; + uint v0 = (data[(tid * 3u) & 0x3FFFFFu] >> 9) | 0x3F800000u, v1 = (data[(tid * 5u + 1u) & 0x3FFFFFu] >> 9) | 0x3F800000u; + uint v2 = (data[(tid * 7u + 2u) & 0x3FFFFFu] >> 9) | 0x3F800000u, v3 = (data[(tid * 11u + 3u) & 0x3FFFFFu] >> 9) | 0x3F800000u; + uint v4 = v0 ^ 0x1234u, v5 = v1 ^ 0x4321u, v6 = v2 ^ 0x777u, v7 = v3 ^ 0x999u; + uint s0 = 0x3F000000u | (data[1] & 0xFFFFu), s1 = 0x3F000000u | (data[2] & 0xFFFFu), s2 = 0x3E800000u | (data[3] & 0xFFFFu), s3 = 0x3F7F0000u; + for (uint it = 0u; it < prm.x; it++) + { + { float t = as_type(s0) * as_type(v0); float u = as_type(v1); uint r = as_type(t + u); v2 = r; } + { float t = as_type(s1) * as_type(v3); float u = as_type(v6); uint r = as_type(t + u); v1 = r; } + { float t = as_type(s2) * as_type(v6); float u = as_type(v3); uint r = as_type(t + u); v0 = r; } + { float t = as_type(s3) * as_type(v1); float u = as_type(v0); uint r = as_type(t + u); v7 = r; } + { float t = as_type(s0) * as_type(v4); float u = as_type(v5); uint r = as_type(t + u); v6 = r; } + { float t = as_type(s1) * as_type(v7); float u = as_type(v2); uint r = as_type(t + u); v5 = r; } + { float t = as_type(s2) * as_type(v2); float u = as_type(v7); uint r = as_type(t + u); v4 = r; } + { float t = as_type(s3) * as_type(v5); float u = as_type(v4); uint r = as_type(t + u); v3 = r; } + { float t = as_type(s0) * as_type(v0); float u = as_type(v1); uint r = as_type(t + u); v2 = r; } + { float t = as_type(s1) * as_type(v3); float u = as_type(v6); uint r = as_type(t + u); v1 = r; } + { float t = as_type(s2) * as_type(v6); float u = as_type(v3); uint r = as_type(t + u); v0 = r; } + { float t = as_type(s3) * as_type(v1); float u = as_type(v0); uint r = as_type(t + u); v7 = r; } + { float t = as_type(s0) * as_type(v4); float u = as_type(v5); uint r = as_type(t + u); v6 = r; } + { float t = as_type(s1) * as_type(v7); float u = as_type(v2); uint r = as_type(t + u); v5 = r; } + { float t = as_type(s2) * as_type(v2); float u = as_type(v7); uint r = as_type(t + u); v4 = r; } + { float t = as_type(s3) * as_type(v5); float u = as_type(v4); uint r = as_type(t + u); v3 = r; } + { float t = as_type(s0) * as_type(v0); float u = as_type(v1); uint r = as_type(t + u); v2 = r; } + { float t = as_type(s1) * as_type(v3); float u = as_type(v6); uint r = as_type(t + u); v1 = r; } + { float t = as_type(s2) * as_type(v6); float u = as_type(v3); uint r = as_type(t + u); v0 = r; } + { float t = as_type(s3) * as_type(v1); float u = as_type(v0); uint r = as_type(t + u); v7 = r; } + { float t = as_type(s0) * as_type(v4); float u = as_type(v5); uint r = as_type(t + u); v6 = r; } + { float t = as_type(s1) * as_type(v7); float u = as_type(v2); uint r = as_type(t + u); v5 = r; } + { float t = as_type(s2) * as_type(v2); float u = as_type(v7); uint r = as_type(t + u); v4 = r; } + { float t = as_type(s3) * as_type(v5); float u = as_type(v4); uint r = as_type(t + u); v3 = r; } + } + out[tid] = v0 ^ v1 ^ v2 ^ v3 ^ v4 ^ v5 ^ v6 ^ v7; +} +""", + ["mad_fma"] = """ +// bench group=contract variant=fma threads=1048576 tg=64 iters=64 +#include +using namespace metal; +#pragma clang fp contract(off) + +kernel void bench(device uint* data [[buffer(0)]], device uint* out [[buffer(1)]], constant uint4& prm [[buffer(2)]], + uint tid [[thread_position_in_grid]], uint lane [[thread_index_in_simdgroup]]) +{ + uint m = data[0]; + bool exec = ((m >> lane) & 1u) != 0u; + uint v0 = (data[(tid * 3u) & 0x3FFFFFu] >> 9) | 0x3F800000u, v1 = (data[(tid * 5u + 1u) & 0x3FFFFFu] >> 9) | 0x3F800000u; + uint v2 = (data[(tid * 7u + 2u) & 0x3FFFFFu] >> 9) | 0x3F800000u, v3 = (data[(tid * 11u + 3u) & 0x3FFFFFu] >> 9) | 0x3F800000u; + uint v4 = v0 ^ 0x1234u, v5 = v1 ^ 0x4321u, v6 = v2 ^ 0x777u, v7 = v3 ^ 0x999u; + uint s0 = 0x3F000000u | (data[1] & 0xFFFFu), s1 = 0x3F000000u | (data[2] & 0xFFFFu), s2 = 0x3E800000u | (data[3] & 0xFFFFu), s3 = 0x3F7F0000u; + for (uint it = 0u; it < prm.x; it++) + { + { uint r = as_type(fma(as_type(s0), as_type(v0), as_type(v1))); v2 = r; } + { uint r = as_type(fma(as_type(s1), as_type(v3), as_type(v6))); v1 = r; } + { uint r = as_type(fma(as_type(s2), as_type(v6), as_type(v3))); v0 = r; } + { uint r = as_type(fma(as_type(s3), as_type(v1), as_type(v0))); v7 = r; } + { uint r = as_type(fma(as_type(s0), as_type(v4), as_type(v5))); v6 = r; } + { uint r = as_type(fma(as_type(s1), as_type(v7), as_type(v2))); v5 = r; } + { uint r = as_type(fma(as_type(s2), as_type(v2), as_type(v7))); v4 = r; } + { uint r = as_type(fma(as_type(s3), as_type(v5), as_type(v4))); v3 = r; } + { uint r = as_type(fma(as_type(s0), as_type(v0), as_type(v1))); v2 = r; } + { uint r = as_type(fma(as_type(s1), as_type(v3), as_type(v6))); v1 = r; } + { uint r = as_type(fma(as_type(s2), as_type(v6), as_type(v3))); v0 = r; } + { uint r = as_type(fma(as_type(s3), as_type(v1), as_type(v0))); v7 = r; } + { uint r = as_type(fma(as_type(s0), as_type(v4), as_type(v5))); v6 = r; } + { uint r = as_type(fma(as_type(s1), as_type(v7), as_type(v2))); v5 = r; } + { uint r = as_type(fma(as_type(s2), as_type(v2), as_type(v7))); v4 = r; } + { uint r = as_type(fma(as_type(s3), as_type(v5), as_type(v4))); v3 = r; } + { uint r = as_type(fma(as_type(s0), as_type(v0), as_type(v1))); v2 = r; } + { uint r = as_type(fma(as_type(s1), as_type(v3), as_type(v6))); v1 = r; } + { uint r = as_type(fma(as_type(s2), as_type(v6), as_type(v3))); v0 = r; } + { uint r = as_type(fma(as_type(s3), as_type(v1), as_type(v0))); v7 = r; } + { uint r = as_type(fma(as_type(s0), as_type(v4), as_type(v5))); v6 = r; } + { uint r = as_type(fma(as_type(s1), as_type(v7), as_type(v2))); v5 = r; } + { uint r = as_type(fma(as_type(s2), as_type(v2), as_type(v7))); v4 = r; } + { uint r = as_type(fma(as_type(s3), as_type(v5), as_type(v4))); v3 = r; } + } + out[tid] = v0 ^ v1 ^ v2 ^ v3 ^ v4 ^ v5 ^ v6 ^ v7; +} +""", + ["mad_separate"] = """ +// bench group=contract variant=current_mul_add threads=1048576 tg=64 iters=64 +#include +using namespace metal; +#pragma clang fp contract(off) + +kernel void bench(device uint* data [[buffer(0)]], device uint* out [[buffer(1)]], constant uint4& prm [[buffer(2)]], + uint tid [[thread_position_in_grid]], uint lane [[thread_index_in_simdgroup]]) +{ + uint m = data[0]; + bool exec = ((m >> lane) & 1u) != 0u; + uint v0 = (data[(tid * 3u) & 0x3FFFFFu] >> 9) | 0x3F800000u, v1 = (data[(tid * 5u + 1u) & 0x3FFFFFu] >> 9) | 0x3F800000u; + uint v2 = (data[(tid * 7u + 2u) & 0x3FFFFFu] >> 9) | 0x3F800000u, v3 = (data[(tid * 11u + 3u) & 0x3FFFFFu] >> 9) | 0x3F800000u; + uint v4 = v0 ^ 0x1234u, v5 = v1 ^ 0x4321u, v6 = v2 ^ 0x777u, v7 = v3 ^ 0x999u; + uint s0 = 0x3F000000u | (data[1] & 0xFFFFu), s1 = 0x3F000000u | (data[2] & 0xFFFFu), s2 = 0x3E800000u | (data[3] & 0xFFFFu), s3 = 0x3F7F0000u; + for (uint it = 0u; it < prm.x; it++) + { + { float t = as_type(s0) * as_type(v0); float u = as_type(v1); uint r = as_type(t + u); v2 = r; } + { float t = as_type(s1) * as_type(v3); float u = as_type(v6); uint r = as_type(t + u); v1 = r; } + { float t = as_type(s2) * as_type(v6); float u = as_type(v3); uint r = as_type(t + u); v0 = r; } + { float t = as_type(s3) * as_type(v1); float u = as_type(v0); uint r = as_type(t + u); v7 = r; } + { float t = as_type(s0) * as_type(v4); float u = as_type(v5); uint r = as_type(t + u); v6 = r; } + { float t = as_type(s1) * as_type(v7); float u = as_type(v2); uint r = as_type(t + u); v5 = r; } + { float t = as_type(s2) * as_type(v2); float u = as_type(v7); uint r = as_type(t + u); v4 = r; } + { float t = as_type(s3) * as_type(v5); float u = as_type(v4); uint r = as_type(t + u); v3 = r; } + { float t = as_type(s0) * as_type(v0); float u = as_type(v1); uint r = as_type(t + u); v2 = r; } + { float t = as_type(s1) * as_type(v3); float u = as_type(v6); uint r = as_type(t + u); v1 = r; } + { float t = as_type(s2) * as_type(v6); float u = as_type(v3); uint r = as_type(t + u); v0 = r; } + { float t = as_type(s3) * as_type(v1); float u = as_type(v0); uint r = as_type(t + u); v7 = r; } + { float t = as_type(s0) * as_type(v4); float u = as_type(v5); uint r = as_type(t + u); v6 = r; } + { float t = as_type(s1) * as_type(v7); float u = as_type(v2); uint r = as_type(t + u); v5 = r; } + { float t = as_type(s2) * as_type(v2); float u = as_type(v7); uint r = as_type(t + u); v4 = r; } + { float t = as_type(s3) * as_type(v5); float u = as_type(v4); uint r = as_type(t + u); v3 = r; } + { float t = as_type(s0) * as_type(v0); float u = as_type(v1); uint r = as_type(t + u); v2 = r; } + { float t = as_type(s1) * as_type(v3); float u = as_type(v6); uint r = as_type(t + u); v1 = r; } + { float t = as_type(s2) * as_type(v6); float u = as_type(v3); uint r = as_type(t + u); v0 = r; } + { float t = as_type(s3) * as_type(v1); float u = as_type(v0); uint r = as_type(t + u); v7 = r; } + { float t = as_type(s0) * as_type(v4); float u = as_type(v5); uint r = as_type(t + u); v6 = r; } + { float t = as_type(s1) * as_type(v7); float u = as_type(v2); uint r = as_type(t + u); v5 = r; } + { float t = as_type(s2) * as_type(v2); float u = as_type(v7); uint r = as_type(t + u); v4 = r; } + { float t = as_type(s3) * as_type(v5); float u = as_type(v4); uint r = as_type(t + u); v3 = r; } + } + out[tid] = v0 ^ v1 ^ v2 ^ v3 ^ v4 ^ v5 ^ v6 ^ v7; +} +""", + ["minmax_current"] = """ +// bench group=minmax variant=current threads=1048576 tg=64 iters=64 +#include +using namespace metal; +#pragma clang fp contract(off) +kernel void bench(device uint* data [[buffer(0)]], device uint* out [[buffer(1)]], constant uint4& prm [[buffer(2)]], + uint tid [[thread_position_in_grid]], uint lane [[thread_index_in_simdgroup]]) +{ + uint v0 = (data[(tid * 3u) & 0x3FFFFFu] >> 9) | 0x3F800000u, v1 = (data[(tid * 5u + 1u) & 0x3FFFFFu] >> 9) | 0x3F000000u; + uint v2 = (data[(tid * 7u + 2u) & 0x3FFFFFu] >> 9) | 0x3E800000u, v3 = (data[(tid * 11u + 3u) & 0x3FFFFFu] >> 9) | 0x3F800000u; + for (uint it = 0u; it < prm.x; it++) + { + { uint ta = ((v0 & 0x7F800000u) == 0u ? (v0 & 0x80000000u) : v0); uint tb = ((v1 & 0x7F800000u) == 0u ? (v1 & 0x80000000u) : v1); v2 = isnan(as_type(ta)) ? tb : (as_type(ta) == as_type(tb) ? (ta | tb) : as_type(fmin(as_type(ta), as_type(tb)))); } + { uint ta = ((v1 & 0x7F800000u) == 0u ? (v1 & 0x80000000u) : v1); uint tb = ((v2 & 0x7F800000u) == 0u ? (v2 & 0x80000000u) : v2); v3 = isnan(as_type(ta)) ? tb : (as_type(ta) == as_type(tb) ? (ta & tb) : as_type(fmax(as_type(ta), as_type(tb)))); } + { uint ta = ((v2 & 0x7F800000u) == 0u ? (v2 & 0x80000000u) : v2); uint tb = ((v3 & 0x7F800000u) == 0u ? (v3 & 0x80000000u) : v3); v0 = isnan(as_type(ta)) ? tb : (as_type(ta) == as_type(tb) ? (ta | tb) : as_type(fmin(as_type(ta), as_type(tb)))); } + { uint ta = ((v3 & 0x7F800000u) == 0u ? (v3 & 0x80000000u) : v3); uint tb = ((v0 & 0x7F800000u) == 0u ? (v0 & 0x80000000u) : v0); v1 = isnan(as_type(ta)) ? tb : (as_type(ta) == as_type(tb) ? (ta & tb) : as_type(fmax(as_type(ta), as_type(tb)))); } + { uint ta = ((v0 & 0x7F800000u) == 0u ? (v0 & 0x80000000u) : v0); uint tb = ((v1 & 0x7F800000u) == 0u ? (v1 & 0x80000000u) : v1); v2 = isnan(as_type(ta)) ? tb : (as_type(ta) == as_type(tb) ? (ta | tb) : as_type(fmin(as_type(ta), as_type(tb)))); } + { uint ta = ((v1 & 0x7F800000u) == 0u ? (v1 & 0x80000000u) : v1); uint tb = ((v2 & 0x7F800000u) == 0u ? (v2 & 0x80000000u) : v2); v3 = isnan(as_type(ta)) ? tb : (as_type(ta) == as_type(tb) ? (ta & tb) : as_type(fmax(as_type(ta), as_type(tb)))); } + { uint ta = ((v2 & 0x7F800000u) == 0u ? (v2 & 0x80000000u) : v2); uint tb = ((v3 & 0x7F800000u) == 0u ? (v3 & 0x80000000u) : v3); v0 = isnan(as_type(ta)) ? tb : (as_type(ta) == as_type(tb) ? (ta | tb) : as_type(fmin(as_type(ta), as_type(tb)))); } + { uint ta = ((v3 & 0x7F800000u) == 0u ? (v3 & 0x80000000u) : v3); uint tb = ((v0 & 0x7F800000u) == 0u ? (v0 & 0x80000000u) : v0); v1 = isnan(as_type(ta)) ? tb : (as_type(ta) == as_type(tb) ? (ta & tb) : as_type(fmax(as_type(ta), as_type(tb)))); } + { uint ta = ((v0 & 0x7F800000u) == 0u ? (v0 & 0x80000000u) : v0); uint tb = ((v1 & 0x7F800000u) == 0u ? (v1 & 0x80000000u) : v1); v2 = isnan(as_type(ta)) ? tb : (as_type(ta) == as_type(tb) ? (ta | tb) : as_type(fmin(as_type(ta), as_type(tb)))); } + { uint ta = ((v1 & 0x7F800000u) == 0u ? (v1 & 0x80000000u) : v1); uint tb = ((v2 & 0x7F800000u) == 0u ? (v2 & 0x80000000u) : v2); v3 = isnan(as_type(ta)) ? tb : (as_type(ta) == as_type(tb) ? (ta & tb) : as_type(fmax(as_type(ta), as_type(tb)))); } + { uint ta = ((v2 & 0x7F800000u) == 0u ? (v2 & 0x80000000u) : v2); uint tb = ((v3 & 0x7F800000u) == 0u ? (v3 & 0x80000000u) : v3); v0 = isnan(as_type(ta)) ? tb : (as_type(ta) == as_type(tb) ? (ta | tb) : as_type(fmin(as_type(ta), as_type(tb)))); } + { uint ta = ((v3 & 0x7F800000u) == 0u ? (v3 & 0x80000000u) : v3); uint tb = ((v0 & 0x7F800000u) == 0u ? (v0 & 0x80000000u) : v0); v1 = isnan(as_type(ta)) ? tb : (as_type(ta) == as_type(tb) ? (ta & tb) : as_type(fmax(as_type(ta), as_type(tb)))); } + { uint ta = ((v0 & 0x7F800000u) == 0u ? (v0 & 0x80000000u) : v0); uint tb = ((v1 & 0x7F800000u) == 0u ? (v1 & 0x80000000u) : v1); v2 = isnan(as_type(ta)) ? tb : (as_type(ta) == as_type(tb) ? (ta | tb) : as_type(fmin(as_type(ta), as_type(tb)))); } + { uint ta = ((v1 & 0x7F800000u) == 0u ? (v1 & 0x80000000u) : v1); uint tb = ((v2 & 0x7F800000u) == 0u ? (v2 & 0x80000000u) : v2); v3 = isnan(as_type(ta)) ? tb : (as_type(ta) == as_type(tb) ? (ta & tb) : as_type(fmax(as_type(ta), as_type(tb)))); } + { uint ta = ((v2 & 0x7F800000u) == 0u ? (v2 & 0x80000000u) : v2); uint tb = ((v3 & 0x7F800000u) == 0u ? (v3 & 0x80000000u) : v3); v0 = isnan(as_type(ta)) ? tb : (as_type(ta) == as_type(tb) ? (ta | tb) : as_type(fmin(as_type(ta), as_type(tb)))); } + { uint ta = ((v3 & 0x7F800000u) == 0u ? (v3 & 0x80000000u) : v3); uint tb = ((v0 & 0x7F800000u) == 0u ? (v0 & 0x80000000u) : v0); v1 = isnan(as_type(ta)) ? tb : (as_type(ta) == as_type(tb) ? (ta & tb) : as_type(fmax(as_type(ta), as_type(tb)))); } + v0 += 0x10u; + } + out[tid] = v0 ^ v1 ^ v2 ^ v3; +} +""", + ["minmax_fminfmax"] = """ +// bench group=minmax variant=fminfmax threads=1048576 tg=64 iters=64 +#include +using namespace metal; +#pragma clang fp contract(off) +kernel void bench(device uint* data [[buffer(0)]], device uint* out [[buffer(1)]], constant uint4& prm [[buffer(2)]], + uint tid [[thread_position_in_grid]], uint lane [[thread_index_in_simdgroup]]) +{ + uint v0 = (data[(tid * 3u) & 0x3FFFFFu] >> 9) | 0x3F800000u, v1 = (data[(tid * 5u + 1u) & 0x3FFFFFu] >> 9) | 0x3F000000u; + uint v2 = (data[(tid * 7u + 2u) & 0x3FFFFFu] >> 9) | 0x3E800000u, v3 = (data[(tid * 11u + 3u) & 0x3FFFFFu] >> 9) | 0x3F800000u; + for (uint it = 0u; it < prm.x; it++) + { + { v2 = as_type(fmin(as_type(v0), as_type(v1))); } + { v3 = as_type(fmax(as_type(v1), as_type(v2))); } + { v0 = as_type(fmin(as_type(v2), as_type(v3))); } + { v1 = as_type(fmax(as_type(v3), as_type(v0))); } + { v2 = as_type(fmin(as_type(v0), as_type(v1))); } + { v3 = as_type(fmax(as_type(v1), as_type(v2))); } + { v0 = as_type(fmin(as_type(v2), as_type(v3))); } + { v1 = as_type(fmax(as_type(v3), as_type(v0))); } + { v2 = as_type(fmin(as_type(v0), as_type(v1))); } + { v3 = as_type(fmax(as_type(v1), as_type(v2))); } + { v0 = as_type(fmin(as_type(v2), as_type(v3))); } + { v1 = as_type(fmax(as_type(v3), as_type(v0))); } + { v2 = as_type(fmin(as_type(v0), as_type(v1))); } + { v3 = as_type(fmax(as_type(v1), as_type(v2))); } + { v0 = as_type(fmin(as_type(v2), as_type(v3))); } + { v1 = as_type(fmax(as_type(v3), as_type(v0))); } + v0 += 0x10u; + } + out[tid] = v0 ^ v1 ^ v2 ^ v3; +} +""", + ["minmax_noflush"] = """ +// bench group=minmax variant=noflush threads=1048576 tg=64 iters=64 +#include +using namespace metal; +#pragma clang fp contract(off) +kernel void bench(device uint* data [[buffer(0)]], device uint* out [[buffer(1)]], constant uint4& prm [[buffer(2)]], + uint tid [[thread_position_in_grid]], uint lane [[thread_index_in_simdgroup]]) +{ + uint v0 = (data[(tid * 3u) & 0x3FFFFFu] >> 9) | 0x3F800000u, v1 = (data[(tid * 5u + 1u) & 0x3FFFFFu] >> 9) | 0x3F000000u; + uint v2 = (data[(tid * 7u + 2u) & 0x3FFFFFu] >> 9) | 0x3E800000u, v3 = (data[(tid * 11u + 3u) & 0x3FFFFFu] >> 9) | 0x3F800000u; + for (uint it = 0u; it < prm.x; it++) + { + { uint ta = v0; uint tb = v1; v2 = isnan(as_type(ta)) ? tb : (as_type(ta) == as_type(tb) ? (ta | tb) : as_type(fmin(as_type(ta), as_type(tb)))); } + { uint ta = v1; uint tb = v2; v3 = isnan(as_type(ta)) ? tb : (as_type(ta) == as_type(tb) ? (ta & tb) : as_type(fmax(as_type(ta), as_type(tb)))); } + { uint ta = v2; uint tb = v3; v0 = isnan(as_type(ta)) ? tb : (as_type(ta) == as_type(tb) ? (ta | tb) : as_type(fmin(as_type(ta), as_type(tb)))); } + { uint ta = v3; uint tb = v0; v1 = isnan(as_type(ta)) ? tb : (as_type(ta) == as_type(tb) ? (ta & tb) : as_type(fmax(as_type(ta), as_type(tb)))); } + { uint ta = v0; uint tb = v1; v2 = isnan(as_type(ta)) ? tb : (as_type(ta) == as_type(tb) ? (ta | tb) : as_type(fmin(as_type(ta), as_type(tb)))); } + { uint ta = v1; uint tb = v2; v3 = isnan(as_type(ta)) ? tb : (as_type(ta) == as_type(tb) ? (ta & tb) : as_type(fmax(as_type(ta), as_type(tb)))); } + { uint ta = v2; uint tb = v3; v0 = isnan(as_type(ta)) ? tb : (as_type(ta) == as_type(tb) ? (ta | tb) : as_type(fmin(as_type(ta), as_type(tb)))); } + { uint ta = v3; uint tb = v0; v1 = isnan(as_type(ta)) ? tb : (as_type(ta) == as_type(tb) ? (ta & tb) : as_type(fmax(as_type(ta), as_type(tb)))); } + { uint ta = v0; uint tb = v1; v2 = isnan(as_type(ta)) ? tb : (as_type(ta) == as_type(tb) ? (ta | tb) : as_type(fmin(as_type(ta), as_type(tb)))); } + { uint ta = v1; uint tb = v2; v3 = isnan(as_type(ta)) ? tb : (as_type(ta) == as_type(tb) ? (ta & tb) : as_type(fmax(as_type(ta), as_type(tb)))); } + { uint ta = v2; uint tb = v3; v0 = isnan(as_type(ta)) ? tb : (as_type(ta) == as_type(tb) ? (ta | tb) : as_type(fmin(as_type(ta), as_type(tb)))); } + { uint ta = v3; uint tb = v0; v1 = isnan(as_type(ta)) ? tb : (as_type(ta) == as_type(tb) ? (ta & tb) : as_type(fmax(as_type(ta), as_type(tb)))); } + { uint ta = v0; uint tb = v1; v2 = isnan(as_type(ta)) ? tb : (as_type(ta) == as_type(tb) ? (ta | tb) : as_type(fmin(as_type(ta), as_type(tb)))); } + { uint ta = v1; uint tb = v2; v3 = isnan(as_type(ta)) ? tb : (as_type(ta) == as_type(tb) ? (ta & tb) : as_type(fmax(as_type(ta), as_type(tb)))); } + { uint ta = v2; uint tb = v3; v0 = isnan(as_type(ta)) ? tb : (as_type(ta) == as_type(tb) ? (ta | tb) : as_type(fmin(as_type(ta), as_type(tb)))); } + { uint ta = v3; uint tb = v0; v1 = isnan(as_type(ta)) ? tb : (as_type(ta) == as_type(tb) ? (ta & tb) : as_type(fmax(as_type(ta), as_type(tb)))); } + v0 += 0x10u; + } + out[tid] = v0 ^ v1 ^ v2 ^ v3; +} +""", + ["sdata_current"] = """ +// bench group=buf_sdata_prologue variant=current threads=1048576 tg=64 iters=1 check=1 +#include +using namespace metal; +#pragma clang fp contract(off) +kernel void bench(device uint* data [[buffer(0)]], device uint* out [[buffer(1)]], constant uint4& prm [[buffer(2)]], + constant uint* sd [[buffer(5)]], + uint tid [[thread_position_in_grid]]) +{ + uint words = prm.y; + uint s0 = (0u < words ? sd[0u] : 0u); + uint s1 = (1u < words ? sd[1u] : 0u); + uint s2 = (2u < words ? sd[2u] : 0u); + uint s3 = (3u < words ? sd[3u] : 0u); + uint s4 = (4u < words ? sd[4u] : 0u); + uint s5 = (5u < words ? sd[5u] : 0u); + uint s6 = (6u < words ? sd[6u] : 0u); + uint s7 = (7u < words ? sd[7u] : 0u); + uint s8 = (8u < words ? sd[8u] : 0u); + uint s9 = (9u < words ? sd[9u] : 0u); + uint s10 = (10u < words ? sd[10u] : 0u); + uint s11 = (11u < words ? sd[11u] : 0u); + uint s12 = (12u < words ? sd[12u] : 0u); + uint s13 = (13u < words ? sd[13u] : 0u); + uint s14 = (14u < words ? sd[14u] : 0u); + uint s15 = (15u < words ? sd[15u] : 0u); + uint s16 = (16u < words ? sd[16u] : 0u); + uint s17 = (17u < words ? sd[17u] : 0u); + uint s18 = (18u < words ? sd[18u] : 0u); + uint s19 = (19u < words ? sd[19u] : 0u); + uint s20 = (20u < words ? sd[20u] : 0u); + uint s21 = (21u < words ? sd[21u] : 0u); + uint s22 = (22u < words ? sd[22u] : 0u); + uint s23 = (23u < words ? sd[23u] : 0u); + uint s24 = (24u < words ? sd[24u] : 0u); + uint s25 = (25u < words ? sd[25u] : 0u); + uint s26 = (26u < words ? sd[26u] : 0u); + uint s27 = (27u < words ? sd[27u] : 0u); + uint s28 = (28u < words ? sd[28u] : 0u); + uint s29 = (29u < words ? sd[29u] : 0u); + uint s30 = (30u < words ? sd[30u] : 0u); + uint s31 = (31u < words ? sd[31u] : 0u); + uint v0 = (data[(tid * 3u) & 0x3FFFFFu] >> 9) | 0x3F800000u, v1 = (data[(tid * 5u + 1u) & 0x3FFFFFu] >> 9) | 0x3F000000u; + uint v2 = (data[(tid * 7u + 2u) & 0x3FFFFFu] >> 9) | 0x3E800000u, v3 = (data[(tid * 11u + 3u) & 0x3FFFFFu] >> 9) | 0x3F800000u; + { float t = as_type(s0) * as_type(v0); v1 = as_type(t + as_type(s3)); } + { float t = as_type(s1) * as_type(v1); v2 = as_type(t + as_type(s8)); } + { float t = as_type(s2) * as_type(v2); v3 = as_type(t + as_type(s13)); } + { float t = as_type(s3) * as_type(v3); v0 = as_type(t + as_type(s18)); } + { float t = as_type(s4) * as_type(v0); v1 = as_type(t + as_type(s23)); } + { float t = as_type(s5) * as_type(v1); v2 = as_type(t + as_type(s28)); } + { float t = as_type(s6) * as_type(v2); v3 = as_type(t + as_type(s1)); } + { float t = as_type(s7) * as_type(v3); v0 = as_type(t + as_type(s6)); } + { float t = as_type(s8) * as_type(v0); v1 = as_type(t + as_type(s11)); } + { float t = as_type(s9) * as_type(v1); v2 = as_type(t + as_type(s16)); } + { float t = as_type(s10) * as_type(v2); v3 = as_type(t + as_type(s21)); } + { float t = as_type(s11) * as_type(v3); v0 = as_type(t + as_type(s26)); } + { float t = as_type(s12) * as_type(v0); v1 = as_type(t + as_type(s31)); } + { float t = as_type(s13) * as_type(v1); v2 = as_type(t + as_type(s4)); } + { float t = as_type(s14) * as_type(v2); v3 = as_type(t + as_type(s9)); } + { float t = as_type(s15) * as_type(v3); v0 = as_type(t + as_type(s14)); } + { float t = as_type(s16) * as_type(v0); v1 = as_type(t + as_type(s19)); } + { float t = as_type(s17) * as_type(v1); v2 = as_type(t + as_type(s24)); } + { float t = as_type(s18) * as_type(v2); v3 = as_type(t + as_type(s29)); } + { float t = as_type(s19) * as_type(v3); v0 = as_type(t + as_type(s2)); } + { float t = as_type(s20) * as_type(v0); v1 = as_type(t + as_type(s7)); } + { float t = as_type(s21) * as_type(v1); v2 = as_type(t + as_type(s12)); } + { float t = as_type(s22) * as_type(v2); v3 = as_type(t + as_type(s17)); } + { float t = as_type(s23) * as_type(v3); v0 = as_type(t + as_type(s22)); } + { float t = as_type(s24) * as_type(v0); v1 = as_type(t + as_type(s27)); } + { float t = as_type(s25) * as_type(v1); v2 = as_type(t + as_type(s0)); } + { float t = as_type(s26) * as_type(v2); v3 = as_type(t + as_type(s5)); } + { float t = as_type(s27) * as_type(v3); v0 = as_type(t + as_type(s10)); } + { float t = as_type(s28) * as_type(v0); v1 = as_type(t + as_type(s15)); } + { float t = as_type(s29) * as_type(v1); v2 = as_type(t + as_type(s20)); } + { float t = as_type(s30) * as_type(v2); v3 = as_type(t + as_type(s25)); } + { float t = as_type(s31) * as_type(v3); v0 = as_type(t + as_type(s30)); } + { float t = as_type(s0) * as_type(v0); v1 = as_type(t + as_type(s3)); } + { float t = as_type(s1) * as_type(v1); v2 = as_type(t + as_type(s8)); } + { float t = as_type(s2) * as_type(v2); v3 = as_type(t + as_type(s13)); } + { float t = as_type(s3) * as_type(v3); v0 = as_type(t + as_type(s18)); } + { float t = as_type(s4) * as_type(v0); v1 = as_type(t + as_type(s23)); } + { float t = as_type(s5) * as_type(v1); v2 = as_type(t + as_type(s28)); } + { float t = as_type(s6) * as_type(v2); v3 = as_type(t + as_type(s1)); } + { float t = as_type(s7) * as_type(v3); v0 = as_type(t + as_type(s6)); } + out[tid] = v0 ^ v1 ^ v2 ^ v3; +} +""", + ["sdata_unchecked"] = """ +// bench group=buf_sdata_prologue variant=unchecked threads=1048576 tg=64 iters=1 check=1 +#include +using namespace metal; +#pragma clang fp contract(off) +kernel void bench(device uint* data [[buffer(0)]], device uint* out [[buffer(1)]], constant uint4& prm [[buffer(2)]], + constant uint* sd [[buffer(5)]], + uint tid [[thread_position_in_grid]]) +{ + uint words = prm.y; + uint s0 = sd[0u]; + uint s1 = sd[1u]; + uint s2 = sd[2u]; + uint s3 = sd[3u]; + uint s4 = sd[4u]; + uint s5 = sd[5u]; + uint s6 = sd[6u]; + uint s7 = sd[7u]; + uint s8 = sd[8u]; + uint s9 = sd[9u]; + uint s10 = sd[10u]; + uint s11 = sd[11u]; + uint s12 = sd[12u]; + uint s13 = sd[13u]; + uint s14 = sd[14u]; + uint s15 = sd[15u]; + uint s16 = sd[16u]; + uint s17 = sd[17u]; + uint s18 = sd[18u]; + uint s19 = sd[19u]; + uint s20 = sd[20u]; + uint s21 = sd[21u]; + uint s22 = sd[22u]; + uint s23 = sd[23u]; + uint s24 = sd[24u]; + uint s25 = sd[25u]; + uint s26 = sd[26u]; + uint s27 = sd[27u]; + uint s28 = sd[28u]; + uint s29 = sd[29u]; + uint s30 = sd[30u]; + uint s31 = sd[31u]; + uint v0 = (data[(tid * 3u) & 0x3FFFFFu] >> 9) | 0x3F800000u, v1 = (data[(tid * 5u + 1u) & 0x3FFFFFu] >> 9) | 0x3F000000u; + uint v2 = (data[(tid * 7u + 2u) & 0x3FFFFFu] >> 9) | 0x3E800000u, v3 = (data[(tid * 11u + 3u) & 0x3FFFFFu] >> 9) | 0x3F800000u; + { float t = as_type(s0) * as_type(v0); v1 = as_type(t + as_type(s3)); } + { float t = as_type(s1) * as_type(v1); v2 = as_type(t + as_type(s8)); } + { float t = as_type(s2) * as_type(v2); v3 = as_type(t + as_type(s13)); } + { float t = as_type(s3) * as_type(v3); v0 = as_type(t + as_type(s18)); } + { float t = as_type(s4) * as_type(v0); v1 = as_type(t + as_type(s23)); } + { float t = as_type(s5) * as_type(v1); v2 = as_type(t + as_type(s28)); } + { float t = as_type(s6) * as_type(v2); v3 = as_type(t + as_type(s1)); } + { float t = as_type(s7) * as_type(v3); v0 = as_type(t + as_type(s6)); } + { float t = as_type(s8) * as_type(v0); v1 = as_type(t + as_type(s11)); } + { float t = as_type(s9) * as_type(v1); v2 = as_type(t + as_type(s16)); } + { float t = as_type(s10) * as_type(v2); v3 = as_type(t + as_type(s21)); } + { float t = as_type(s11) * as_type(v3); v0 = as_type(t + as_type(s26)); } + { float t = as_type(s12) * as_type(v0); v1 = as_type(t + as_type(s31)); } + { float t = as_type(s13) * as_type(v1); v2 = as_type(t + as_type(s4)); } + { float t = as_type(s14) * as_type(v2); v3 = as_type(t + as_type(s9)); } + { float t = as_type(s15) * as_type(v3); v0 = as_type(t + as_type(s14)); } + { float t = as_type(s16) * as_type(v0); v1 = as_type(t + as_type(s19)); } + { float t = as_type(s17) * as_type(v1); v2 = as_type(t + as_type(s24)); } + { float t = as_type(s18) * as_type(v2); v3 = as_type(t + as_type(s29)); } + { float t = as_type(s19) * as_type(v3); v0 = as_type(t + as_type(s2)); } + { float t = as_type(s20) * as_type(v0); v1 = as_type(t + as_type(s7)); } + { float t = as_type(s21) * as_type(v1); v2 = as_type(t + as_type(s12)); } + { float t = as_type(s22) * as_type(v2); v3 = as_type(t + as_type(s17)); } + { float t = as_type(s23) * as_type(v3); v0 = as_type(t + as_type(s22)); } + { float t = as_type(s24) * as_type(v0); v1 = as_type(t + as_type(s27)); } + { float t = as_type(s25) * as_type(v1); v2 = as_type(t + as_type(s0)); } + { float t = as_type(s26) * as_type(v2); v3 = as_type(t + as_type(s5)); } + { float t = as_type(s27) * as_type(v3); v0 = as_type(t + as_type(s10)); } + { float t = as_type(s28) * as_type(v0); v1 = as_type(t + as_type(s15)); } + { float t = as_type(s29) * as_type(v1); v2 = as_type(t + as_type(s20)); } + { float t = as_type(s30) * as_type(v2); v3 = as_type(t + as_type(s25)); } + { float t = as_type(s31) * as_type(v3); v0 = as_type(t + as_type(s30)); } + { float t = as_type(s0) * as_type(v0); v1 = as_type(t + as_type(s3)); } + { float t = as_type(s1) * as_type(v1); v2 = as_type(t + as_type(s8)); } + { float t = as_type(s2) * as_type(v2); v3 = as_type(t + as_type(s13)); } + { float t = as_type(s3) * as_type(v3); v0 = as_type(t + as_type(s18)); } + { float t = as_type(s4) * as_type(v0); v1 = as_type(t + as_type(s23)); } + { float t = as_type(s5) * as_type(v1); v2 = as_type(t + as_type(s28)); } + { float t = as_type(s6) * as_type(v2); v3 = as_type(t + as_type(s1)); } + { float t = as_type(s7) * as_type(v3); v0 = as_type(t + as_type(s6)); } + out[tid] = v0 ^ v1 ^ v2 ^ v3; +} +""", + }; +} diff --git a/tests/SharpEmu.Gpu.Metal.Tests/MetalMslPatternBenchTests.cs b/tests/SharpEmu.Gpu.Metal.Tests/MetalMslPatternBenchTests.cs new file mode 100644 index 000000000..1b9096769 --- /dev/null +++ b/tests/SharpEmu.Gpu.Metal.Tests/MetalMslPatternBenchTests.cs @@ -0,0 +1,262 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using System.Globalization; +using System.Text.Json; +using System.Text.RegularExpressions; +using Xunit; +using Xunit.Abstractions; + +namespace SharpEmu.Gpu.Metal.Tests; + +/// +/// Micro-kernels for code shapes the Gen5 MSL translator emits, each compared with a cheaper +/// form of the same work. Standalone compute kernels (their source is in +/// MetalMslPatternBenchTests.Kernels.cs, header comment `bench group= variant= threads= tg= iters=`) +/// run with the compile options translated guest MSL uses; the GPU time is +/// GPUEndTime - GPUStartTime of one command buffer holding one dispatch. A pair marked check=1 +/// must produce the same output buffer in every variant of its group. Behind the exclusive +/// device gate, and only when asked: SHARPEMU_MSL_PATTERN_BENCH_OUTPUT=<json path>, +/// optionally SHARPEMU_MSL_PATTERN_BENCH_GROUPS=a,b to run only those groups. +/// +public sealed unsafe partial class MetalMslPatternBenchTests(ITestOutputHelper output) +{ + private const ulong GuestBase = 0x2_0000_0000; + private const int DataWords = 1 << 22; + private const int RangeEntries = 256; + private const int Warmup = 2; + private const int Reps = 9; + + private static readonly Lazy Device = new(MetalNative.MTLCreateSystemDefaultDevice); + + private static Dictionary ComposeKernels() => RawKernels.ToDictionary( + pair => pair.Key, + pair => pair.Value + .Replace("//@DEV_PRELUDE", DevPrelude, StringComparison.Ordinal) + .Replace("//@BUF_PRELUDE_CONSTANT", BufPrelude.Replace("device uint* b", "constant uint* b", StringComparison.Ordinal), StringComparison.Ordinal) + .Replace("//@BUF_PRELUDE", BufPrelude, StringComparison.Ordinal)); + + private sealed record Spec(string Name, string Source, string Group, string Variant, int Threads, int Tg, uint Iters, bool Check); + + [Fact] + public void MeasureMslPatternsWhenRequested() + { + var path = Environment.GetEnvironmentVariable("SHARPEMU_MSL_PATTERN_BENCH_OUTPUT"); + if (path is null || !RequireDevice()) return; + + var only = Environment.GetEnvironmentVariable("SHARPEMU_MSL_PATTERN_BENCH_GROUPS")? + .Split(',', StringSplitOptions.RemoveEmptyEntries); + var specs = ComposeKernels().Select(Parse).Where(s => only is null || only.Contains(s.Group)).ToList(); + + var device = Device.Value; + var queue = MetalNative.Send(device, MetalNative.Selector("newCommandQueue")); + var data = NewBuffer(device, (nuint)DataWords * 4); + var faults = NewBuffer(device, 1 << 16); + var ranges = NewBuffer(device, RangeEntries * 24); + var dataWords = (uint*)MetalNative.Send(data, MetalNative.Selector("contents")); + var state = 0x9E3779B9u; + for (var i = 0; i < DataWords; i++) + { + state ^= state << 13; state ^= state >> 17; state ^= state << 5; + dataWords[i] = state; + } + + dataWords[0] = 0xFFFFFFFFu; // every lane's exec bit set + var dataAddress = MetalNative.SendGpuAddress(data, MetalNative.Selector("gpuAddress")); + FillRanges((ulong*)MetalNative.Send(ranges, MetalNative.Selector("contents")), dataAddress); + + var results = new List(); + var outputs = new Dictionary(); + var failures = new List(); + var selSetBuffer = MetalNative.Selector("setBuffer:offset:atIndex:"); + foreach (var group in specs.GroupBy(s => s.Group)) + { + // Compile every variant of the group first, then take the repetitions round-robin + // across them, so a change in what else the GPU is doing lands on all variants alike. + var pool = MetalNative.objc_autoreleasePoolPush(); + var compiled = new List<(Spec Spec, nint Pso, nint Out, uint MaxThreads, uint StaticTg, List Samples, nint[] Handles)>(); + try + { + foreach (var spec in group) + { + var options = MetalNative.Send(MetalNative.Send(MetalNative.Class("MTLCompileOptions"), MetalNative.Selector("alloc")), MetalNative.Selector("init")); + MetalNative.ApplyMslCompileOptions(options); + nint error = 0; + var library = MetalNative.Send(device, MetalNative.Selector("newLibraryWithSource:options:error:"), MetalNative.NsString(spec.Source), options, ref error); + if (library == 0) { failures.Add($"{spec.Name}: {MetalNative.DescribeError(error)}"); continue; } + var function = MetalNative.Send(library, MetalNative.Selector("newFunctionWithName:"), MetalNative.NsString("bench")); + nint pipelineError = 0; + var pso = MetalNative.Send(device, MetalNative.Selector("newComputePipelineStateWithFunction:error:"), function, ref pipelineError); + if (pso == 0) { failures.Add($"{spec.Name}: pso {MetalNative.DescribeError(pipelineError)}"); continue; } + var outBuffer = NewBuffer(device, 1 << 22); + var stale = (uint*)MetalNative.Send(outBuffer, MetalNative.Selector("contents")); + for (var i = 0; i < spec.Threads; i++) stale[i] = 0xDEADBEEFu; + compiled.Add((spec, pso, outBuffer, + (uint)MetalNative.SendUInteger(pso, MetalNative.Selector("maxTotalThreadsPerThreadgroup")), + (uint)MetalNative.SendUInteger(pso, MetalNative.Selector("staticThreadgroupMemoryLength")), + new List(), [pso, function, library, options])); + } + + var parameters = stackalloc uint[4]; + for (var rep = 0; rep < Warmup + Reps; rep++) + { + foreach (var item in compiled) + { + var spec = item.Spec; + var buffer = MetalNative.Send(queue, MetalNative.Selector("commandBuffer")); + var encoder = MetalNative.Send(buffer, MetalNative.Selector("computeCommandEncoder")); + MetalNative.SendVoid(encoder, MetalNative.Selector("setComputePipelineState:"), item.Pso); + MetalNative.SendSetBuffer(encoder, selSetBuffer, data, 0, 0); + MetalNative.SendSetBuffer(encoder, selSetBuffer, item.Out, 0, 1); + MetalNative.SendSetBuffer(encoder, selSetBuffer, ranges, 0, 3); + MetalNative.SendSetBuffer(encoder, selSetBuffer, faults, 0, 4); + MetalNative.SendSetBuffer(encoder, selSetBuffer, data, 0, 5); + // x: iterations. y/z/w: the guest base (low, high) and range count for the + // device-address kernels; y is instead the byte length of the bound buffer + // for the buffer-load kernels. + parameters[0] = spec.Iters; + parameters[1] = spec.Group.StartsWith("buf_", StringComparison.Ordinal) ? (uint)(DataWords * 4) : (uint)(GuestBase & 0xFFFFFFFFu); + parameters[2] = (uint)(GuestBase >> 32); + parameters[3] = RangeEntries; + MetalNative.SendSetBytes(encoder, MetalNative.Selector("setBytes:length:atIndex:"), (nint)parameters, 16, 2); + MetalNative.SendDispatch(encoder, MetalNative.Selector("dispatchThreadgroups:threadsPerThreadgroup:"), + new MtlSize { Width = (nuint)(spec.Threads / spec.Tg), Height = 1, Depth = 1 }, + new MtlSize { Width = (nuint)spec.Tg, Height = 1, Depth = 1 }); + MetalNative.SendVoid(encoder, MetalNative.Selector("endEncoding")); + MetalNative.SendVoid(buffer, MetalNative.Selector("commit")); + MetalNative.SendVoid(buffer, MetalNative.Selector("waitUntilCompleted")); + // MTLCommandBufferStatusCompleted is 4; a kernel that faulted leaves its output stale. + var status = MetalNative.SendUInteger(buffer, MetalNative.Selector("status")); + if (status != 4) + { + failures.Add($"{spec.Name}: command buffer status {status}: {MetalNative.DescribeError(MetalNative.Send(buffer, MetalNative.Selector("error")))}"); + continue; + } + + var start = MetalNative.SendDouble(buffer, MetalNative.Selector("GPUStartTime")); + var end = MetalNative.SendDouble(buffer, MetalNative.Selector("GPUEndTime")); + if (rep >= Warmup) item.Samples.Add((end - start) * 1e6); + } + } + + foreach (var item in compiled) + { + var spec = item.Spec; + var samples = item.Samples; + if (samples.Count == 0) continue; + samples.Sort(); + var words = (uint*)MetalNative.Send(item.Out, MetalNative.Selector("contents")); + var snapshot = new uint[spec.Threads]; + for (var i = 0; i < snapshot.Length; i++) snapshot[i] = words[i]; + outputs[spec.Name] = snapshot; + results.Add(new + { + group = spec.Group, + variant = spec.Variant, + threads = spec.Threads, + threadgroup = spec.Tg, + iters = spec.Iters, + medianMicroseconds = Math.Round(samples[samples.Count / 2], 1), + minMicroseconds = Math.Round(samples[0], 1), + maxMicroseconds = Math.Round(samples[^1], 1), + maxThreadsPerThreadgroup = item.MaxThreads, + staticThreadgroupBytes = item.StaticTg, + checksum = Checksum(snapshot), + firstWords = snapshot.Take(4).Select(w => w.ToString("X8", CultureInfo.InvariantCulture)).ToArray(), + }); + output.WriteLine($"{spec.Group,-26} {spec.Variant,-24} median {samples[samples.Count / 2],9:F1} us min {samples[0],9:F1} maxTG {item.MaxThreads}"); + } + } + finally + { + foreach (var item in compiled) + { + foreach (var handle in item.Handles) MetalNative.SendVoid(handle, MetalNative.Selector("release")); + MetalNative.SendVoid(item.Out, MetalNative.Selector("release")); + } + + MetalNative.objc_autoreleasePoolPop(pool); + } + } + + + // Variants of a checked group agree with the group's first variant, element for element. + var mismatches = new List(); + foreach (var group in specs.Where(s => s.Check && outputs.ContainsKey(s.Name)).GroupBy(s => s.Group)) + { + var reference = group.First(); + foreach (var other in group.Skip(1)) + { + var a = outputs[reference.Name]; + var b = outputs[other.Name]; + var differing = 0; + for (var i = 0; i < a.Length; i++) if (a[i] != b[i]) differing++; + if (differing != 0) mismatches.Add($"{group.Key}: {other.Variant} differs from {reference.Variant} in {differing} of {a.Length} threads"); + } + } + + File.WriteAllText(path, JsonSerializer.Serialize(new + { + architecture = System.Runtime.InteropServices.RuntimeInformation.ProcessArchitecture.ToString(), + reps = Reps, + results, + mismatches, + failures, + }, new JsonSerializerOptions { WriteIndented = true })); + foreach (var line in mismatches) output.WriteLine("MISMATCH " + line); + foreach (var line in failures) output.WriteLine("FAILED " + line); + } + + // 256 ranges sorted by first page, a guest page being 16 KiB: lower one-page ranges that + // nothing reads, then two halves of the data buffer ("A" the first 8 MiB, "B" the next). + private static void FillRanges(ulong* table, ulong dataAddress) + { + var firstDataPage = GuestBase >> 14; + for (var i = 0; i < RangeEntries - 2; i++) + { + table[i * 3] = firstDataPage - 4096 + (ulong)i * 8; + table[i * 3 + 1] = 1; + table[i * 3 + 2] = dataAddress; + } + + const ulong halfPages = 0x800000 >> 14; + table[(RangeEntries - 2) * 3] = firstDataPage; + table[(RangeEntries - 2) * 3 + 1] = halfPages; + table[(RangeEntries - 2) * 3 + 2] = dataAddress; + table[(RangeEntries - 1) * 3] = firstDataPage + halfPages; + table[(RangeEntries - 1) * 3 + 1] = halfPages; + table[(RangeEntries - 1) * 3 + 2] = dataAddress + 0x800000; + } + + private static string Checksum(uint[] words) + { + ulong hash = 1469598103934665603UL; + foreach (var word in words) hash = (hash ^ word) * 1099511628211UL; + return hash.ToString("X16", CultureInfo.InvariantCulture); + } + + private static nint NewBuffer(nint device, nuint length) => + MetalNative.SendNewBuffer(device, MetalNative.Selector("newBufferWithLength:options:"), length, 0); + + private static Spec Parse(KeyValuePair kernel) + { + var header = Regex.Match(kernel.Value, @"^// bench (?.*)$", RegexOptions.Multiline).Groups["fields"].Value; + var fields = header.Split(' ', StringSplitOptions.RemoveEmptyEntries).Select(f => f.Split('=')).ToDictionary(p => p[0], p => p[1]); + return new Spec(kernel.Key, kernel.Value, fields["group"], fields["variant"], int.Parse(fields["threads"], CultureInfo.InvariantCulture), + int.Parse(fields["tg"], CultureInfo.InvariantCulture), uint.Parse(fields["iters"], CultureInfo.InvariantCulture), + fields.TryGetValue("check", out var check) && check == "1"); + } + + private bool RequireDevice() + { + if (Environment.GetEnvironmentVariable("SHARPEMU_TEST_REQUIRE_DEVICE") != "1") + { + output.WriteLine("MSL pattern micro-kernels run on the device only behind the exclusive device gate."); + return false; + } + + Assert.True(OperatingSystem.IsMacOS()); + Assert.NotEqual(0, Device.Value); + return true; + } +} From e2a8d4727fdb53148a4616eefa9bd8829f2c8d26 Mon Sep 17 00:00:00 2001 From: kuba Date: Wed, 28 Oct 2026 19:59:32 +0100 Subject: [PATCH 71/82] [Shader] Raw buffer dword loads read at a clamped index and select zero instead of branching around the load. sharpemu_load_dword and sharpemu_load_dwords (device and constant copies) always issue the load and select 0 for an index past the end, which answers what the per-dword bounds check answered, wrap included. Without the branch the compiler batches a shader's buffer loads: uniform four-dword loads took about a quarter of the time in the micro-kernel (387 -> 93 us), per-lane runs about two thirds (1939 -> 1277 us). SHARPEMU_MSL_BRANCHLESS_LOADS=0 keeps the branching helpers, byte for byte the old goldens. --- .../Gen5MslCodegenOptions.cs | 9 ++ .../Gen5MslTranslator.cs | 21 ++- .../Templates/dword_loads_branchless.msl | 32 +++++ .../Templates/dword_loads_checked.msl | 34 +++++ .../Templates/prelude.msl | 36 +---- .../Templates/scalar_constant_prelude.msl | 35 +---- .../Gen5MslBranchlessLoadsTests.cs | 125 ++++++++++++++++++ .../Goldens/exec-store-layout.msl | 32 +++-- .../Goldens/fmac-layout.msl | 32 +++-- .../Goldens/lds-layout.msl | 32 +++-- .../Goldens/loop-layout.msl | 32 +++-- .../Goldens/muls-layout.msl | 32 +++-- .../Goldens/pixel-layout.msl | 32 +++-- .../Goldens/typed-load-layout.msl | 32 +++-- .../Goldens/typed-store-layout.msl | 32 +++-- .../Goldens/vertex-layout.msl | 32 +++-- 16 files changed, 352 insertions(+), 228 deletions(-) create mode 100644 src/SharpEmu.ShaderCompiler.Metal/Templates/dword_loads_branchless.msl create mode 100644 src/SharpEmu.ShaderCompiler.Metal/Templates/dword_loads_checked.msl create mode 100644 tests/SharpEmu.ShaderCompiler.Metal.Tests/Gen5MslBranchlessLoadsTests.cs diff --git a/src/SharpEmu.ShaderCompiler.Metal/Gen5MslCodegenOptions.cs b/src/SharpEmu.ShaderCompiler.Metal/Gen5MslCodegenOptions.cs index cce94657d..548284c2c 100644 --- a/src/SharpEmu.ShaderCompiler.Metal/Gen5MslCodegenOptions.cs +++ b/src/SharpEmu.ShaderCompiler.Metal/Gen5MslCodegenOptions.cs @@ -187,6 +187,14 @@ public sealed record Gen5MslCodegenOptions( /// in mesh shaders without device-memory writes. SHARPEMU_MSL_SCALAR_CONSTANT_BUFFERS=0 disables this. public bool ScalarConstantBuffers { get; init; } = true; + /// Read a raw buffer dword (sharpemu_load_dword and the runs sharpemu_load_dwords makes) + /// at an index clamped into the binding and select zero for one past the end, instead of + /// branching around the load. Every dword is what the checked form reads; only the branch + /// per load is gone, which lets the compiler issue a shader's constant-buffer loads together + /// (about a quarter of the time of the branching form on uniform four-dword loads in a + /// device micro-benchmark). SHARPEMU_MSL_BRANCHLESS_LOADS=0 keeps the branch. + public bool BranchlessDwordLoads { get; init; } = true; + /// Experimental compact threadgroup bank for frequently addressed LDS /// regions, retaining device storage for other addresses. SHARPEMU_MSL_MESH_HOT_LDS=1. public bool MeshHotLds { get; init; } @@ -226,6 +234,7 @@ public sealed record Gen5MslCodegenOptions( MeshLdsForwarding = Environment.GetEnvironmentVariable("SHARPEMU_MSL_MESH_LDS_FORWARDING") == "1", ScalarConstantBuffers = Environment.GetEnvironmentVariable("SHARPEMU_MSL_SCALAR_CONSTANT_BUFFERS") != "0", MeshHotLds = Environment.GetEnvironmentVariable("SHARPEMU_MSL_MESH_HOT_LDS") == "1", + BranchlessDwordLoads = Environment.GetEnvironmentVariable("SHARPEMU_MSL_BRANCHLESS_LOADS") != "0", StaticScratchSlots = Environment.GetEnvironmentVariable("SHARPEMU_MSL_STATIC_SCRATCH") != "0", MeshExitJoin = Environment.GetEnvironmentVariable("SHARPEMU_MSL_MESH_EXIT_JOIN") == "1", MeshDeviceLdsThresholdBytes = uint.TryParse( diff --git a/src/SharpEmu.ShaderCompiler.Metal/Gen5MslTranslator.cs b/src/SharpEmu.ShaderCompiler.Metal/Gen5MslTranslator.cs index e0b4212a2..9bea28e65 100644 --- a/src/SharpEmu.ShaderCompiler.Metal/Gen5MslTranslator.cs +++ b/src/SharpEmu.ShaderCompiler.Metal/Gen5MslTranslator.cs @@ -32,7 +32,10 @@ public static partial class Gen5MslTranslator /// Every helper block a module with carries, in emission /// order, without the per-request resource struct (no prelude refers to it). /// - internal static string RenderModulePreludes(Gen5MslPreludes preludes) + internal static string RenderModulePreludes(Gen5MslPreludes preludes) => + RenderModulePreludes(preludes, Gen5MslCodegenOptions.FromEnvironment); + + internal static string RenderModulePreludes(Gen5MslPreludes preludes, Gen5MslCodegenOptions options) { var source = new StringBuilder(); source.AppendLine("#include "); @@ -40,7 +43,7 @@ internal static string RenderModulePreludes(Gen5MslPreludes preludes) source.AppendLine("using namespace metal;"); source.AppendLine(); CompilationContext.AppendLeadingPreludes(source, preludes); - CompilationContext.AppendTrailingPreludes(source, preludes); + CompilationContext.AppendTrailingPreludes(source, preludes, options.BranchlessDwordLoads); return source.ToString(); } @@ -776,7 +779,7 @@ private void EmitModule(StringBuilder source, IReadOnlyList blocks) // A request binds one argument buffer per stage; its lengths replace the uniforms. EmitResourceStruct(source); - AppendTrailingPreludes(source, preludes); + AppendTrailingPreludes(source, preludes, _options.BranchlessDwordLoads); if (UsesMipStatisticsConstant) { source.AppendLine( @@ -1344,15 +1347,21 @@ internal static void AppendLeadingPreludes(StringBuilder source, Gen5MslPreludes // The helpers after the resource struct. A pair lock reads through the coherent // helpers, so it always carries coherent_load_prelude. - internal static void AppendTrailingPreludes(StringBuilder source, Gen5MslPreludes preludes) + internal static void AppendTrailingPreludes(StringBuilder source, Gen5MslPreludes preludes, bool branchlessDwordLoads) { + var dwordLoads = branchlessDwordLoads ? "dword_loads_branchless" : "dword_loads_checked"; source.Append(MslTemplates.Render( "prelude", ("ballot_return", (preludes & Gen5MslPreludes.RealWaveLanes) != 0 ? "(uint)(uint64_t)simd_ballot(value)" - : "value ? 1u : 0u"))); + : "value ? 1u : 0u"), + ("dword_loads_device", MslTemplates.Render(dwordLoads, ("space", "device"))))); if ((preludes & Gen5MslPreludes.ScalarConstants) != 0) - source.Append(MslTemplates.Render("scalar_constant_prelude")); + { + source.Append(MslTemplates.Render( + "scalar_constant_prelude", + ("dword_loads_constant", MslTemplates.Render(dwordLoads, ("space", "constant"))))); + } if ((preludes & Gen5MslPreludes.BufferOffset) != 0) { diff --git a/src/SharpEmu.ShaderCompiler.Metal/Templates/dword_loads_branchless.msl b/src/SharpEmu.ShaderCompiler.Metal/Templates/dword_loads_branchless.msl new file mode 100644 index 000000000..d049cca40 --- /dev/null +++ b/src/SharpEmu.ShaderCompiler.Metal/Templates/dword_loads_branchless.msl @@ -0,0 +1,32 @@ +// A raw dword load: the address truncates to the dword it lands in, which is what the +// hardware does for S_BUFFER_LOAD_DWORD and BUFFER_LOAD_DWORD and what the SPIR-V +// backend emits. Out of range reads zero. The load is always issued, and a select discards +// it for an index past the end: an index past the end reads the binding's last dword (its +// first when it is empty, which the host still maps: the null buffer is 256 bytes, any other +// binding a page-granular window). A branch around the load would split a shader into one +// block per load, which the compiler cannot issue together. +static inline uint sharpemu_load_dword({{space}} uint* b, uint bytes, uint addr) +{ + uint index = addr >> 2; + uint words = bytes >> 2; + uint last = max(words, 1u) - 1u; + uint value = b[index > last ? last : index]; + return index < words ? value : 0u; +} + +// Up to four consecutive raw dwords of an S_BUFFER_LOAD/BUFFER_LOAD_DWORDXn, each what +// sharpemu_load_dword reads at addr + 4 * i, wrapping with the address as it does. +template +static inline uint4 sharpemu_load_dwords({{space}} uint* b, uint bytes, uint addr) +{ + uint4 result = uint4(0u); + uint words = bytes >> 2; + uint last = max(words, 1u) - 1u; + for (int i = 0; i < N; i++) + { + uint index = ((addr >> 2) + (uint)i) & 0x3FFFFFFFu; + uint value = b[index > last ? last : index]; + result[i] = index < words ? value : 0u; + } + return result; +} diff --git a/src/SharpEmu.ShaderCompiler.Metal/Templates/dword_loads_checked.msl b/src/SharpEmu.ShaderCompiler.Metal/Templates/dword_loads_checked.msl new file mode 100644 index 000000000..423f5c283 --- /dev/null +++ b/src/SharpEmu.ShaderCompiler.Metal/Templates/dword_loads_checked.msl @@ -0,0 +1,34 @@ +// A raw dword load: the address truncates to the dword it lands in, which is what the +// hardware does for S_BUFFER_LOAD_DWORD and BUFFER_LOAD_DWORD and what the SPIR-V +// backend emits. Out of range reads zero. +static inline uint sharpemu_load_dword({{space}} uint* b, uint bytes, uint addr) +{ + uint index = addr >> 2; + return index < (bytes >> 2) ? b[index] : 0u; +} + +// Up to four consecutive raw dwords of an S_BUFFER_LOAD/BUFFER_LOAD_DWORDXn, each what +// sharpemu_load_dword reads at addr + 4 * i. A run that lies inside the binding is one +// bounds check and adjacent loads; one that crosses the end reads dword by dword. +template +static inline uint4 sharpemu_load_dwords({{space}} uint* b, uint bytes, uint addr) +{ + uint4 result = uint4(0u); + uint index = addr >> 2; + uint words = bytes >> 2; + if (index < words && (uint)N <= words - index) + { + for (int i = 0; i < N; i++) + { + result[i] = b[index + (uint)i]; + } + } + else + { + for (int i = 0; i < N; i++) + { + result[i] = sharpemu_load_dword(b, bytes, addr + (uint)(4 * i)); + } + } + return result; +} diff --git a/src/SharpEmu.ShaderCompiler.Metal/Templates/prelude.msl b/src/SharpEmu.ShaderCompiler.Metal/Templates/prelude.msl index ab9396da3..579be8d5e 100644 --- a/src/SharpEmu.ShaderCompiler.Metal/Templates/prelude.msl +++ b/src/SharpEmu.ShaderCompiler.Metal/Templates/prelude.msl @@ -1,38 +1,4 @@ -// A raw dword load: the address truncates to the dword it lands in, which is what the -// hardware does for S_BUFFER_LOAD_DWORD and BUFFER_LOAD_DWORD and what the SPIR-V -// backend emits. Out of range reads zero. -static inline uint sharpemu_load_dword(device uint* b, uint bytes, uint addr) -{ - uint index = addr >> 2; - return index < (bytes >> 2) ? b[index] : 0u; -} - -// Up to four consecutive raw dwords of an S_BUFFER_LOAD/BUFFER_LOAD_DWORDXn, each what -// sharpemu_load_dword reads at addr + 4 * i. A run that lies inside the binding is one -// bounds check and adjacent loads; one that crosses the end reads dword by dword. -template -static inline uint4 sharpemu_load_dwords(device uint* b, uint bytes, uint addr) -{ - uint4 result = uint4(0u); - uint index = addr >> 2; - uint words = bytes >> 2; - if (index < words && (uint)N <= words - index) - { - for (int i = 0; i < N; i++) - { - result[i] = b[index + (uint)i]; - } - } - else - { - for (int i = 0; i < N; i++) - { - result[i] = sharpemu_load_dword(b, bytes, addr + (uint)(4 * i)); - } - } - return result; -} - +{{dword_loads_device}} // A dword a format access may straddle, assembled from the two it spans. Only the // format path needs this: a descriptor's stride is a byte count, so a component can // start at any byte. diff --git a/src/SharpEmu.ShaderCompiler.Metal/Templates/scalar_constant_prelude.msl b/src/SharpEmu.ShaderCompiler.Metal/Templates/scalar_constant_prelude.msl index b3d7fdd93..852c911a6 100644 --- a/src/SharpEmu.ShaderCompiler.Metal/Templates/scalar_constant_prelude.msl +++ b/src/SharpEmu.ShaderCompiler.Metal/Templates/scalar_constant_prelude.msl @@ -3,37 +3,4 @@ // Scalar-only read-only bindings use the uniform memory path, with identical // dword truncation, wrapping and partial-run bounds semantics. -// A raw dword load: the address truncates to the dword it lands in, which is what the -// hardware does for S_BUFFER_LOAD_DWORD and BUFFER_LOAD_DWORD and what the SPIR-V -// backend emits. Out of range reads zero. -static inline uint sharpemu_load_dword(constant uint* b, uint bytes, uint addr) -{ - uint index = addr >> 2; - return index < (bytes >> 2) ? b[index] : 0u; -} - -// Up to four consecutive raw dwords of an S_BUFFER_LOAD/BUFFER_LOAD_DWORDXn, each what -// sharpemu_load_dword reads at addr + 4 * i. A run that lies inside the binding is one -// bounds check and adjacent loads; one that crosses the end reads dword by dword. -template -static inline uint4 sharpemu_load_dwords(constant uint* b, uint bytes, uint addr) -{ - uint4 result = uint4(0u); - uint index = addr >> 2; - uint words = bytes >> 2; - if (index < words && (uint)N <= words - index) - { - for (int i = 0; i < N; i++) - { - result[i] = b[index + (uint)i]; - } - } - else - { - for (int i = 0; i < N; i++) - { - result[i] = sharpemu_load_dword(b, bytes, addr + (uint)(4 * i)); - } - } - return result; -} +{{dword_loads_constant}} diff --git a/tests/SharpEmu.ShaderCompiler.Metal.Tests/Gen5MslBranchlessLoadsTests.cs b/tests/SharpEmu.ShaderCompiler.Metal.Tests/Gen5MslBranchlessLoadsTests.cs new file mode 100644 index 000000000..34c34b9c4 --- /dev/null +++ b/tests/SharpEmu.ShaderCompiler.Metal.Tests/Gen5MslBranchlessLoadsTests.cs @@ -0,0 +1,125 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later +using Xunit; + +namespace SharpEmu.ShaderCompiler.Metal.Tests; + +// sharpemu_load_dword and sharpemu_load_dwords read through a clamped index and a select +// (Gen5MslCodegenOptions.BranchlessDwordLoads) rather than a branch around the load. Either +// form has to answer what a per-dword bounds check answers, for every address and size, +// including a run that crosses the end, an address that wraps and an empty binding. +public sealed class Gen5MslBranchlessLoadsTests +{ + private const int DataWords = 32; + private const int CaseBase = 32; + private const int OutputBase = 256; + private const int OutputStride = 8; + + private const string Kernel = """ + + kernel void branchless_loads_test(device uint* b [[buffer(0)]], uint t [[thread_position_in_threadgroup]]) + { + uint addr = b[32u + t * 2u]; + uint bytes = b[33u + t * 2u]; + uint o = 256u + t * 8u; + uint4 four = sharpemu_load_dwords<4>(b, bytes, addr); + uint4 two = sharpemu_load_dwords<2>(b, bytes, addr); + for (uint i = 0u; i < 4u; i++) + { + b[o + i] = four[i]; + } + b[o + 4u] = two[0]; + b[o + 5u] = two[1]; + b[o + 6u] = sharpemu_load_dword(b, bytes, addr); + } + """; + + // addr, bytes: inside, interior, the last dwords, past the end, an odd bias, an empty binding, + // a binding shorter than a dword, runs that cross the end, and addresses that wrap. + public static TheoryData Forms => new() { true, false }; + + private static readonly (uint Address, uint Bytes)[] Cases = + [ + (0u, 128u), (16u, 128u), (112u, 128u), (116u, 128u), (120u, 128u), (124u, 128u), (128u, 128u), + (2u, 128u), (1u, 128u), (7u, 128u), (0u, 0u), (0u, 3u), (4u, 8u), (8u, 16u), (12u, 16u), (16u, 16u), + (10u, 16u), (1u, 3u), (3u, 7u), (0xFFFF_FFFCu, 128u), (0xFFFF_FFF8u, 128u), (0xFFFF_FFFFu, 128u), + (0u, 132u), (0u, 5u), (4u, 4u), + ]; + + [Theory] + [MemberData(nameof(Forms))] + public void TheSwitchPicksTheBranchingOrTheBranchlessHelpers(bool branchless) + { + var source = Gen5MslTranslator.RenderModulePreludes( + Gen5MslPreludes.ScalarConstants, new Gen5MslCodegenOptions { BranchlessDwordLoads = branchless }); + Assert.Equal(branchless, source.Contains("max(words, 1u) - 1u", StringComparison.Ordinal)); + Assert.Equal(!branchless, source.Contains("(uint)N <= words - index", StringComparison.Ordinal)); + // The constant-space copy follows the same form. + Assert.Contains("static inline uint4 sharpemu_load_dwords(constant uint* b", source, StringComparison.Ordinal); + } + + [Fact] + public void EnvironmentSwitchDefaultsOn() + { + Assert.Equal( + Environment.GetEnvironmentVariable("SHARPEMU_MSL_BRANCHLESS_LOADS") != "0", + Gen5MslCodegenOptions.FromEnvironment.BranchlessDwordLoads); + } + + [Theory] + [MemberData(nameof(Forms))] + public void EveryCaseMatchesAPerDwordBoundsCheck(bool branchless) + { + if (!MetalNative.IsAvailable) + { + Assert.False( + OperatingSystem.IsMacOS() && Environment.GetEnvironmentVariable("SHARPEMU_TEST_REQUIRE_DEVICE") == "1", + "The required gate cannot run the branchless-loads test without a Metal device."); + return; + } + + var source = Gen5MslTranslator.RenderModulePreludes( + Gen5MslPreludes.None, new Gen5MslCodegenOptions { BranchlessDwordLoads = branchless }) + Kernel; + Assert.True(MetalNative.TryCompileLibrary(source, out var library, out var compileError), compileError); + + var words = new uint[OutputBase + Cases.Length * OutputStride]; + for (var index = 0; index < DataWords; index++) + { + words[index] = 0x01010101u * (uint)(index + 1) ^ 0xA5A5_0000u; + } + + for (var index = 0; index < Cases.Length; index++) + { + words[CaseBase + index * 2] = Cases[index].Address; + words[CaseBase + index * 2 + 1] = Cases[index].Bytes; + } + + var contents = new byte[words.Length * sizeof(uint)]; + Buffer.BlockCopy(words, 0, contents, 0, contents.Length); + Assert.True( + MetalNative.TryExecuteOnBuffer(library, "branchless_loads_test", contents, (uint)Cases.Length, out var result, out var error), + error); + var output = new uint[words.Length]; + Buffer.BlockCopy(result, 0, output, 0, result.Length); + + uint Expected(uint address, uint bytes, int dword) + { + var index = unchecked(address + (uint)(4 * dword)) >> 2; + return index < (bytes >> 2) ? words[index] : 0u; + } + + for (var index = 0; index < Cases.Length; index++) + { + var (address, bytes) = Cases[index]; + var at = OutputBase + index * OutputStride; + for (var dword = 0; dword < 4; dword++) + { + Assert.True(Expected(address, bytes, dword) == output[at + dword], $"four-dword run, case {index} ({address:X}, {bytes}), dword {dword}"); + } + + Assert.True(Expected(address, bytes, 0) == output[at + 4], $"two-dword run, case {index}, dword 0"); + Assert.True(Expected(address, bytes, 1) == output[at + 5], $"two-dword run, case {index}, dword 1"); + Assert.True(Expected(address, bytes, 0) == output[at + 6], $"single dword, case {index}"); + } + } +} diff --git a/tests/SharpEmu.ShaderCompiler.Metal.Tests/Goldens/exec-store-layout.msl b/tests/SharpEmu.ShaderCompiler.Metal.Tests/Goldens/exec-store-layout.msl index 5d71a4c1e..fa550d95c 100644 --- a/tests/SharpEmu.ShaderCompiler.Metal.Tests/Goldens/exec-store-layout.msl +++ b/tests/SharpEmu.ShaderCompiler.Metal.Tests/Goldens/exec-store-layout.msl @@ -272,35 +272,33 @@ struct Gen5StageResources // A raw dword load: the address truncates to the dword it lands in, which is what the // hardware does for S_BUFFER_LOAD_DWORD and BUFFER_LOAD_DWORD and what the SPIR-V -// backend emits. Out of range reads zero. +// backend emits. Out of range reads zero. The load is always issued, and a select discards +// it for an index past the end: an index past the end reads the binding's last dword (its +// first when it is empty, which the host still maps: the null buffer is 256 bytes, any other +// binding a page-granular window). A branch around the load would split a shader into one +// block per load, which the compiler cannot issue together. static inline uint sharpemu_load_dword(device uint* b, uint bytes, uint addr) { uint index = addr >> 2; - return index < (bytes >> 2) ? b[index] : 0u; + uint words = bytes >> 2; + uint last = max(words, 1u) - 1u; + uint value = b[index > last ? last : index]; + return index < words ? value : 0u; } // Up to four consecutive raw dwords of an S_BUFFER_LOAD/BUFFER_LOAD_DWORDXn, each what -// sharpemu_load_dword reads at addr + 4 * i. A run that lies inside the binding is one -// bounds check and adjacent loads; one that crosses the end reads dword by dword. +// sharpemu_load_dword reads at addr + 4 * i, wrapping with the address as it does. template static inline uint4 sharpemu_load_dwords(device uint* b, uint bytes, uint addr) { uint4 result = uint4(0u); - uint index = addr >> 2; uint words = bytes >> 2; - if (index < words && (uint)N <= words - index) - { - for (int i = 0; i < N; i++) - { - result[i] = b[index + (uint)i]; - } - } - else + uint last = max(words, 1u) - 1u; + for (int i = 0; i < N; i++) { - for (int i = 0; i < N; i++) - { - result[i] = sharpemu_load_dword(b, bytes, addr + (uint)(4 * i)); - } + uint index = ((addr >> 2) + (uint)i) & 0x3FFFFFFFu; + uint value = b[index > last ? last : index]; + result[i] = index < words ? value : 0u; } return result; } diff --git a/tests/SharpEmu.ShaderCompiler.Metal.Tests/Goldens/fmac-layout.msl b/tests/SharpEmu.ShaderCompiler.Metal.Tests/Goldens/fmac-layout.msl index cf5dde091..2714a42bb 100644 --- a/tests/SharpEmu.ShaderCompiler.Metal.Tests/Goldens/fmac-layout.msl +++ b/tests/SharpEmu.ShaderCompiler.Metal.Tests/Goldens/fmac-layout.msl @@ -270,35 +270,33 @@ struct Gen5StageResources // A raw dword load: the address truncates to the dword it lands in, which is what the // hardware does for S_BUFFER_LOAD_DWORD and BUFFER_LOAD_DWORD and what the SPIR-V -// backend emits. Out of range reads zero. +// backend emits. Out of range reads zero. The load is always issued, and a select discards +// it for an index past the end: an index past the end reads the binding's last dword (its +// first when it is empty, which the host still maps: the null buffer is 256 bytes, any other +// binding a page-granular window). A branch around the load would split a shader into one +// block per load, which the compiler cannot issue together. static inline uint sharpemu_load_dword(device uint* b, uint bytes, uint addr) { uint index = addr >> 2; - return index < (bytes >> 2) ? b[index] : 0u; + uint words = bytes >> 2; + uint last = max(words, 1u) - 1u; + uint value = b[index > last ? last : index]; + return index < words ? value : 0u; } // Up to four consecutive raw dwords of an S_BUFFER_LOAD/BUFFER_LOAD_DWORDXn, each what -// sharpemu_load_dword reads at addr + 4 * i. A run that lies inside the binding is one -// bounds check and adjacent loads; one that crosses the end reads dword by dword. +// sharpemu_load_dword reads at addr + 4 * i, wrapping with the address as it does. template static inline uint4 sharpemu_load_dwords(device uint* b, uint bytes, uint addr) { uint4 result = uint4(0u); - uint index = addr >> 2; uint words = bytes >> 2; - if (index < words && (uint)N <= words - index) - { - for (int i = 0; i < N; i++) - { - result[i] = b[index + (uint)i]; - } - } - else + uint last = max(words, 1u) - 1u; + for (int i = 0; i < N; i++) { - for (int i = 0; i < N; i++) - { - result[i] = sharpemu_load_dword(b, bytes, addr + (uint)(4 * i)); - } + uint index = ((addr >> 2) + (uint)i) & 0x3FFFFFFFu; + uint value = b[index > last ? last : index]; + result[i] = index < words ? value : 0u; } return result; } diff --git a/tests/SharpEmu.ShaderCompiler.Metal.Tests/Goldens/lds-layout.msl b/tests/SharpEmu.ShaderCompiler.Metal.Tests/Goldens/lds-layout.msl index b55c4b5b9..82a4811d8 100644 --- a/tests/SharpEmu.ShaderCompiler.Metal.Tests/Goldens/lds-layout.msl +++ b/tests/SharpEmu.ShaderCompiler.Metal.Tests/Goldens/lds-layout.msl @@ -272,35 +272,33 @@ struct Gen5StageResources // A raw dword load: the address truncates to the dword it lands in, which is what the // hardware does for S_BUFFER_LOAD_DWORD and BUFFER_LOAD_DWORD and what the SPIR-V -// backend emits. Out of range reads zero. +// backend emits. Out of range reads zero. The load is always issued, and a select discards +// it for an index past the end: an index past the end reads the binding's last dword (its +// first when it is empty, which the host still maps: the null buffer is 256 bytes, any other +// binding a page-granular window). A branch around the load would split a shader into one +// block per load, which the compiler cannot issue together. static inline uint sharpemu_load_dword(device uint* b, uint bytes, uint addr) { uint index = addr >> 2; - return index < (bytes >> 2) ? b[index] : 0u; + uint words = bytes >> 2; + uint last = max(words, 1u) - 1u; + uint value = b[index > last ? last : index]; + return index < words ? value : 0u; } // Up to four consecutive raw dwords of an S_BUFFER_LOAD/BUFFER_LOAD_DWORDXn, each what -// sharpemu_load_dword reads at addr + 4 * i. A run that lies inside the binding is one -// bounds check and adjacent loads; one that crosses the end reads dword by dword. +// sharpemu_load_dword reads at addr + 4 * i, wrapping with the address as it does. template static inline uint4 sharpemu_load_dwords(device uint* b, uint bytes, uint addr) { uint4 result = uint4(0u); - uint index = addr >> 2; uint words = bytes >> 2; - if (index < words && (uint)N <= words - index) - { - for (int i = 0; i < N; i++) - { - result[i] = b[index + (uint)i]; - } - } - else + uint last = max(words, 1u) - 1u; + for (int i = 0; i < N; i++) { - for (int i = 0; i < N; i++) - { - result[i] = sharpemu_load_dword(b, bytes, addr + (uint)(4 * i)); - } + uint index = ((addr >> 2) + (uint)i) & 0x3FFFFFFFu; + uint value = b[index > last ? last : index]; + result[i] = index < words ? value : 0u; } return result; } diff --git a/tests/SharpEmu.ShaderCompiler.Metal.Tests/Goldens/loop-layout.msl b/tests/SharpEmu.ShaderCompiler.Metal.Tests/Goldens/loop-layout.msl index 9f108ba94..ddf5b8bb1 100644 --- a/tests/SharpEmu.ShaderCompiler.Metal.Tests/Goldens/loop-layout.msl +++ b/tests/SharpEmu.ShaderCompiler.Metal.Tests/Goldens/loop-layout.msl @@ -272,35 +272,33 @@ struct Gen5StageResources // A raw dword load: the address truncates to the dword it lands in, which is what the // hardware does for S_BUFFER_LOAD_DWORD and BUFFER_LOAD_DWORD and what the SPIR-V -// backend emits. Out of range reads zero. +// backend emits. Out of range reads zero. The load is always issued, and a select discards +// it for an index past the end: an index past the end reads the binding's last dword (its +// first when it is empty, which the host still maps: the null buffer is 256 bytes, any other +// binding a page-granular window). A branch around the load would split a shader into one +// block per load, which the compiler cannot issue together. static inline uint sharpemu_load_dword(device uint* b, uint bytes, uint addr) { uint index = addr >> 2; - return index < (bytes >> 2) ? b[index] : 0u; + uint words = bytes >> 2; + uint last = max(words, 1u) - 1u; + uint value = b[index > last ? last : index]; + return index < words ? value : 0u; } // Up to four consecutive raw dwords of an S_BUFFER_LOAD/BUFFER_LOAD_DWORDXn, each what -// sharpemu_load_dword reads at addr + 4 * i. A run that lies inside the binding is one -// bounds check and adjacent loads; one that crosses the end reads dword by dword. +// sharpemu_load_dword reads at addr + 4 * i, wrapping with the address as it does. template static inline uint4 sharpemu_load_dwords(device uint* b, uint bytes, uint addr) { uint4 result = uint4(0u); - uint index = addr >> 2; uint words = bytes >> 2; - if (index < words && (uint)N <= words - index) - { - for (int i = 0; i < N; i++) - { - result[i] = b[index + (uint)i]; - } - } - else + uint last = max(words, 1u) - 1u; + for (int i = 0; i < N; i++) { - for (int i = 0; i < N; i++) - { - result[i] = sharpemu_load_dword(b, bytes, addr + (uint)(4 * i)); - } + uint index = ((addr >> 2) + (uint)i) & 0x3FFFFFFFu; + uint value = b[index > last ? last : index]; + result[i] = index < words ? value : 0u; } return result; } diff --git a/tests/SharpEmu.ShaderCompiler.Metal.Tests/Goldens/muls-layout.msl b/tests/SharpEmu.ShaderCompiler.Metal.Tests/Goldens/muls-layout.msl index 1a83885c1..61726e586 100644 --- a/tests/SharpEmu.ShaderCompiler.Metal.Tests/Goldens/muls-layout.msl +++ b/tests/SharpEmu.ShaderCompiler.Metal.Tests/Goldens/muls-layout.msl @@ -270,35 +270,33 @@ struct Gen5StageResources // A raw dword load: the address truncates to the dword it lands in, which is what the // hardware does for S_BUFFER_LOAD_DWORD and BUFFER_LOAD_DWORD and what the SPIR-V -// backend emits. Out of range reads zero. +// backend emits. Out of range reads zero. The load is always issued, and a select discards +// it for an index past the end: an index past the end reads the binding's last dword (its +// first when it is empty, which the host still maps: the null buffer is 256 bytes, any other +// binding a page-granular window). A branch around the load would split a shader into one +// block per load, which the compiler cannot issue together. static inline uint sharpemu_load_dword(device uint* b, uint bytes, uint addr) { uint index = addr >> 2; - return index < (bytes >> 2) ? b[index] : 0u; + uint words = bytes >> 2; + uint last = max(words, 1u) - 1u; + uint value = b[index > last ? last : index]; + return index < words ? value : 0u; } // Up to four consecutive raw dwords of an S_BUFFER_LOAD/BUFFER_LOAD_DWORDXn, each what -// sharpemu_load_dword reads at addr + 4 * i. A run that lies inside the binding is one -// bounds check and adjacent loads; one that crosses the end reads dword by dword. +// sharpemu_load_dword reads at addr + 4 * i, wrapping with the address as it does. template static inline uint4 sharpemu_load_dwords(device uint* b, uint bytes, uint addr) { uint4 result = uint4(0u); - uint index = addr >> 2; uint words = bytes >> 2; - if (index < words && (uint)N <= words - index) - { - for (int i = 0; i < N; i++) - { - result[i] = b[index + (uint)i]; - } - } - else + uint last = max(words, 1u) - 1u; + for (int i = 0; i < N; i++) { - for (int i = 0; i < N; i++) - { - result[i] = sharpemu_load_dword(b, bytes, addr + (uint)(4 * i)); - } + uint index = ((addr >> 2) + (uint)i) & 0x3FFFFFFFu; + uint value = b[index > last ? last : index]; + result[i] = index < words ? value : 0u; } return result; } diff --git a/tests/SharpEmu.ShaderCompiler.Metal.Tests/Goldens/pixel-layout.msl b/tests/SharpEmu.ShaderCompiler.Metal.Tests/Goldens/pixel-layout.msl index 0001548a8..cae755e75 100644 --- a/tests/SharpEmu.ShaderCompiler.Metal.Tests/Goldens/pixel-layout.msl +++ b/tests/SharpEmu.ShaderCompiler.Metal.Tests/Goldens/pixel-layout.msl @@ -270,35 +270,33 @@ struct Gen5StageResources // A raw dword load: the address truncates to the dword it lands in, which is what the // hardware does for S_BUFFER_LOAD_DWORD and BUFFER_LOAD_DWORD and what the SPIR-V -// backend emits. Out of range reads zero. +// backend emits. Out of range reads zero. The load is always issued, and a select discards +// it for an index past the end: an index past the end reads the binding's last dword (its +// first when it is empty, which the host still maps: the null buffer is 256 bytes, any other +// binding a page-granular window). A branch around the load would split a shader into one +// block per load, which the compiler cannot issue together. static inline uint sharpemu_load_dword(device uint* b, uint bytes, uint addr) { uint index = addr >> 2; - return index < (bytes >> 2) ? b[index] : 0u; + uint words = bytes >> 2; + uint last = max(words, 1u) - 1u; + uint value = b[index > last ? last : index]; + return index < words ? value : 0u; } // Up to four consecutive raw dwords of an S_BUFFER_LOAD/BUFFER_LOAD_DWORDXn, each what -// sharpemu_load_dword reads at addr + 4 * i. A run that lies inside the binding is one -// bounds check and adjacent loads; one that crosses the end reads dword by dword. +// sharpemu_load_dword reads at addr + 4 * i, wrapping with the address as it does. template static inline uint4 sharpemu_load_dwords(device uint* b, uint bytes, uint addr) { uint4 result = uint4(0u); - uint index = addr >> 2; uint words = bytes >> 2; - if (index < words && (uint)N <= words - index) - { - for (int i = 0; i < N; i++) - { - result[i] = b[index + (uint)i]; - } - } - else + uint last = max(words, 1u) - 1u; + for (int i = 0; i < N; i++) { - for (int i = 0; i < N; i++) - { - result[i] = sharpemu_load_dword(b, bytes, addr + (uint)(4 * i)); - } + uint index = ((addr >> 2) + (uint)i) & 0x3FFFFFFFu; + uint value = b[index > last ? last : index]; + result[i] = index < words ? value : 0u; } return result; } diff --git a/tests/SharpEmu.ShaderCompiler.Metal.Tests/Goldens/typed-load-layout.msl b/tests/SharpEmu.ShaderCompiler.Metal.Tests/Goldens/typed-load-layout.msl index 9d1385081..68030bdd1 100644 --- a/tests/SharpEmu.ShaderCompiler.Metal.Tests/Goldens/typed-load-layout.msl +++ b/tests/SharpEmu.ShaderCompiler.Metal.Tests/Goldens/typed-load-layout.msl @@ -444,35 +444,33 @@ struct Gen5StageResources // A raw dword load: the address truncates to the dword it lands in, which is what the // hardware does for S_BUFFER_LOAD_DWORD and BUFFER_LOAD_DWORD and what the SPIR-V -// backend emits. Out of range reads zero. +// backend emits. Out of range reads zero. The load is always issued, and a select discards +// it for an index past the end: an index past the end reads the binding's last dword (its +// first when it is empty, which the host still maps: the null buffer is 256 bytes, any other +// binding a page-granular window). A branch around the load would split a shader into one +// block per load, which the compiler cannot issue together. static inline uint sharpemu_load_dword(device uint* b, uint bytes, uint addr) { uint index = addr >> 2; - return index < (bytes >> 2) ? b[index] : 0u; + uint words = bytes >> 2; + uint last = max(words, 1u) - 1u; + uint value = b[index > last ? last : index]; + return index < words ? value : 0u; } // Up to four consecutive raw dwords of an S_BUFFER_LOAD/BUFFER_LOAD_DWORDXn, each what -// sharpemu_load_dword reads at addr + 4 * i. A run that lies inside the binding is one -// bounds check and adjacent loads; one that crosses the end reads dword by dword. +// sharpemu_load_dword reads at addr + 4 * i, wrapping with the address as it does. template static inline uint4 sharpemu_load_dwords(device uint* b, uint bytes, uint addr) { uint4 result = uint4(0u); - uint index = addr >> 2; uint words = bytes >> 2; - if (index < words && (uint)N <= words - index) - { - for (int i = 0; i < N; i++) - { - result[i] = b[index + (uint)i]; - } - } - else + uint last = max(words, 1u) - 1u; + for (int i = 0; i < N; i++) { - for (int i = 0; i < N; i++) - { - result[i] = sharpemu_load_dword(b, bytes, addr + (uint)(4 * i)); - } + uint index = ((addr >> 2) + (uint)i) & 0x3FFFFFFFu; + uint value = b[index > last ? last : index]; + result[i] = index < words ? value : 0u; } return result; } diff --git a/tests/SharpEmu.ShaderCompiler.Metal.Tests/Goldens/typed-store-layout.msl b/tests/SharpEmu.ShaderCompiler.Metal.Tests/Goldens/typed-store-layout.msl index 404d02c71..98337102f 100644 --- a/tests/SharpEmu.ShaderCompiler.Metal.Tests/Goldens/typed-store-layout.msl +++ b/tests/SharpEmu.ShaderCompiler.Metal.Tests/Goldens/typed-store-layout.msl @@ -444,35 +444,33 @@ struct Gen5StageResources // A raw dword load: the address truncates to the dword it lands in, which is what the // hardware does for S_BUFFER_LOAD_DWORD and BUFFER_LOAD_DWORD and what the SPIR-V -// backend emits. Out of range reads zero. +// backend emits. Out of range reads zero. The load is always issued, and a select discards +// it for an index past the end: an index past the end reads the binding's last dword (its +// first when it is empty, which the host still maps: the null buffer is 256 bytes, any other +// binding a page-granular window). A branch around the load would split a shader into one +// block per load, which the compiler cannot issue together. static inline uint sharpemu_load_dword(device uint* b, uint bytes, uint addr) { uint index = addr >> 2; - return index < (bytes >> 2) ? b[index] : 0u; + uint words = bytes >> 2; + uint last = max(words, 1u) - 1u; + uint value = b[index > last ? last : index]; + return index < words ? value : 0u; } // Up to four consecutive raw dwords of an S_BUFFER_LOAD/BUFFER_LOAD_DWORDXn, each what -// sharpemu_load_dword reads at addr + 4 * i. A run that lies inside the binding is one -// bounds check and adjacent loads; one that crosses the end reads dword by dword. +// sharpemu_load_dword reads at addr + 4 * i, wrapping with the address as it does. template static inline uint4 sharpemu_load_dwords(device uint* b, uint bytes, uint addr) { uint4 result = uint4(0u); - uint index = addr >> 2; uint words = bytes >> 2; - if (index < words && (uint)N <= words - index) - { - for (int i = 0; i < N; i++) - { - result[i] = b[index + (uint)i]; - } - } - else + uint last = max(words, 1u) - 1u; + for (int i = 0; i < N; i++) { - for (int i = 0; i < N; i++) - { - result[i] = sharpemu_load_dword(b, bytes, addr + (uint)(4 * i)); - } + uint index = ((addr >> 2) + (uint)i) & 0x3FFFFFFFu; + uint value = b[index > last ? last : index]; + result[i] = index < words ? value : 0u; } return result; } diff --git a/tests/SharpEmu.ShaderCompiler.Metal.Tests/Goldens/vertex-layout.msl b/tests/SharpEmu.ShaderCompiler.Metal.Tests/Goldens/vertex-layout.msl index 2d6178478..1d102f91c 100644 --- a/tests/SharpEmu.ShaderCompiler.Metal.Tests/Goldens/vertex-layout.msl +++ b/tests/SharpEmu.ShaderCompiler.Metal.Tests/Goldens/vertex-layout.msl @@ -270,35 +270,33 @@ struct Gen5StageResources // A raw dword load: the address truncates to the dword it lands in, which is what the // hardware does for S_BUFFER_LOAD_DWORD and BUFFER_LOAD_DWORD and what the SPIR-V -// backend emits. Out of range reads zero. +// backend emits. Out of range reads zero. The load is always issued, and a select discards +// it for an index past the end: an index past the end reads the binding's last dword (its +// first when it is empty, which the host still maps: the null buffer is 256 bytes, any other +// binding a page-granular window). A branch around the load would split a shader into one +// block per load, which the compiler cannot issue together. static inline uint sharpemu_load_dword(device uint* b, uint bytes, uint addr) { uint index = addr >> 2; - return index < (bytes >> 2) ? b[index] : 0u; + uint words = bytes >> 2; + uint last = max(words, 1u) - 1u; + uint value = b[index > last ? last : index]; + return index < words ? value : 0u; } // Up to four consecutive raw dwords of an S_BUFFER_LOAD/BUFFER_LOAD_DWORDXn, each what -// sharpemu_load_dword reads at addr + 4 * i. A run that lies inside the binding is one -// bounds check and adjacent loads; one that crosses the end reads dword by dword. +// sharpemu_load_dword reads at addr + 4 * i, wrapping with the address as it does. template static inline uint4 sharpemu_load_dwords(device uint* b, uint bytes, uint addr) { uint4 result = uint4(0u); - uint index = addr >> 2; uint words = bytes >> 2; - if (index < words && (uint)N <= words - index) - { - for (int i = 0; i < N; i++) - { - result[i] = b[index + (uint)i]; - } - } - else + uint last = max(words, 1u) - 1u; + for (int i = 0; i < N; i++) { - for (int i = 0; i < N; i++) - { - result[i] = sharpemu_load_dword(b, bytes, addr + (uint)(4 * i)); - } + uint index = ((addr >> 2) + (uint)i) & 0x3FFFFFFFu; + uint value = b[index > last ? last : index]; + result[i] = index < words ? value : 0u; } return result; } From a488f74768b817a6c7ee5dc47cf573ebd36de2e6 Mon Sep 17 00:00:00 2001 From: kuba Date: Wed, 28 Oct 2026 21:13:27 +0100 Subject: [PATCH 72/82] [Tools] The shader compile bench times every cached shader on the GPU with synthetic bindings. mode=gpu builds each program of a title's recorded lists, binds a synthetic argument buffer, buffers, textures and device-address page table the way Metal v2 does, and times a fixed dispatch or draw per shader on the GPU, with compile-option and translator variants (and MSL dumped from another translator setting, alts=) interleaved in one process; check=1 hashes the written buffers and targets so two variants can be compared for output. mode=gpureport ranks the results. Zero or patterned inputs change control flow, so the times are a cost proxy. --- .../GpuBench.cs | 1457 +++++++++++++++++ .../GpuBenchReport.cs | 276 ++++ .../Program.cs | 12 + 3 files changed, 1745 insertions(+) create mode 100644 tools/SharpEmu.Tools.ShaderCompileBench/GpuBench.cs create mode 100644 tools/SharpEmu.Tools.ShaderCompileBench/GpuBenchReport.cs diff --git a/tools/SharpEmu.Tools.ShaderCompileBench/GpuBench.cs b/tools/SharpEmu.Tools.ShaderCompileBench/GpuBench.cs new file mode 100644 index 000000000..7dd861901 --- /dev/null +++ b/tools/SharpEmu.Tools.ShaderCompileBench/GpuBench.cs @@ -0,0 +1,1457 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using System.Buffers.Binary; +using System.Collections.Concurrent; +using System.Diagnostics; +using System.Globalization; +using System.Runtime.InteropServices; +using System.Text; +using System.Text.RegularExpressions; +using SharpEmu.Gpu.Metal; +using SharpEmu.Gpu.Metal.V2; +using SharpEmu.Libs.Gpu.Pipelines; +using SharpEmu.ShaderCompiler.Metal; + +// mode=gpu: a synthetic GPU-time benchmark of every translated shader of a title's prewarm +// lists. Every shader is built into a pipeline and run on synthetic, zero (or patterned) +// inputs: a compute kernel is dispatched over a fixed grid, a vertex or mesh program draws a +// fixed vertex count behind a trivial consumer pixel function, and a pixel program draws a +// full-screen quad (several times over) from a synthetic vertex function. The time is the +// command buffer's GPUEndTime - GPUStartTime, one shader per command buffer. +// +// The inputs are NOT the title's: zero or patterned data changes control flow (loop trip +// counts, early-outs, which texels and buffers are touched), so a number is a cost proxy that +// ranks shaders and compares compile variants; it is not production time. +// +// Arguments (key=value, on top of dir=): +// kinds=cs,vs,ms,ps which programs (default all) only= +// variants=base,mathsafe,fastmath,fastmath_fnprecise,contract,noinvariance,optsize,early,basez,earlyd,basezd +// reps=5 warm=2 median of reps after warm-up fill=zero|pattern +// groups=256 verts=393216 quads=4 width=1920 height=1080 the fixed workload +// deep=30 deepreps=15 re-measure the slowest base shaders with more repetitions +// reps_from= run exactly the item indices of an earlier run (translator variants by environment) +// dumpmsl= [dumponly=1] write every item's MSL as .metal; alts=:,... variants made of such a dump +// check=1 hash what each variant stores (buffers and colour targets) to find a change of output +// alias= write the base variant under another name (a run under a translator environment variable) +// tag= out= workers=3 limit=0 +internal static unsafe class GpuBench +{ + private const nuint ResourceShared = 0; + private const nuint UsageRead = 1; + private const nuint UsageWrite = 2; + private const nuint StagesAll = 15; + + private static Func _arg = null!; + private static nint _device; + private static nint _queue; + private static Synth _synth = null!; + + private static string Arg(string key, string fallback) => _arg(key, fallback); + + private static int Int(string key, int fallback) => int.Parse(Arg(key, fallback.ToString(CultureInfo.InvariantCulture)), CultureInfo.InvariantCulture); + + private sealed class Item + { + public int Index; + public string Kind = ""; + public ulong Hash; + public string Source = ""; + public Gen5MslShader Shader = null!; + public string Body = ""; + public int Specialization; + public string Id => $"{Kind}=0x{Hash:X16}#{Index}"; + public bool IsPixel => Kind == "ps"; + } + + // One way of building a shader: compile options, a source patch, and the scenario it is run in. + private sealed record Variant(string Name, string Scenario, Func Applies, Func Patch, Action Options) + { + public string? AltDirectory { get; init; } + } + + private sealed class Built + { + public Variant Variant = null!; + public nint Library, Function, State; + public string? Error; + public double CompileMs; + public string Notes = ""; + public uint MaxThreads, ExecWidth, TgBytes; + public string Source = ""; + } + + private sealed class Row + { + public string Phase = "", Variant = "", Scenario = "", Status = "ok", Note = ""; + public Item Item = null!; + public double Median, Min, Max; + public int Reps; + public double Scale = 1; + } + + private static readonly nint SelRelease = MetalNative.Selector("release"); + private static readonly nint SelAlloc = MetalNative.Selector("alloc"); + private static readonly nint SelInit = MetalNative.Selector("init"); + private static readonly nint SelObjectAt = MetalNative.Selector("objectAtIndexedSubscript:"); + + public static int Run(string dir, Func arg) + { + _arg = arg; + MetalGpuBackend.Register(); + MetalRenderHost.ApplyTranslationSwitches(); + _device = MetalNative.MTLCreateSystemDefaultDevice(); + _queue = MetalNative.Send(_device, MetalNative.Selector("newCommandQueue")); + _synth = new Synth(_device, Arg("fill", "zero") == "pattern"); + var kinds = Arg("kinds", "cs,vs,ms,ps").Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries).ToHashSet(); + var only = Arg("only", "").Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries) + .Select(text => text.ToUpperInvariant()).ToArray(); + var items = LoadItems(dir, kinds, only); + if (Arg("dumpmsl", "") is { Length: > 0 } dump) + { + // The MSL of every item by index: a run under a translator environment variable writes its set here, and a later + // run reads it back as a variant (alts=), so both translations are measured in one process, interleaved. + Directory.CreateDirectory(dump); + foreach (var item in items) + { + File.WriteAllText(Path.Combine(dump, $"{item.Index}.metal"), item.Source); + } + + if (Arg("dumponly", "0") == "1") + { + return 0; + } + } + + var variants = Variants(); + foreach (var alt in Arg("alts", "").Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)) + { + var split = alt.IndexOf(':'); + var altDir = alt[(split + 1)..]; + variants[alt[..split]] = new Variant(alt[..split], "none", + item => File.Exists(Path.Combine(altDir, $"{item.Index}.metal")) && File.ReadAllText(Path.Combine(altDir, $"{item.Index}.metal")) != item.Source, + _ => throw new InvalidOperationException(), _ => { }) { AltDirectory = altDir }; + } + + var names = Arg("variants", "base").Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + var chosen = names.Select(name => variants.TryGetValue(name, out var found) ? found : throw new ArgumentException($"unknown variant {name}")).ToList(); + var tag = Arg("tag", "run"); + var outDir = Arg("out", "gpu-bench"); + Directory.CreateDirectory(outDir); + Console.Error.WriteLine($"gpu bench: items={items.Count} variants={string.Join(',', names)} fill={Arg("fill", "zero")} tag={tag}"); + WriteStatic(Path.Combine(outDir, $"static-{tag}.csv"), items); + + var reps = Int("reps", 5); + var rows = new List(); + var csv = Path.Combine(outDir, $"gpu-{tag}.csv"); + RunPhase("full", items, chosen, reps, rows, csv); + var deep = Int("deep", 0); + if (deep > 0) + { + var baseName = chosen[0].Name; + var top = rows.Where(row => row.Phase == "full" && row.Variant == baseName && row.Status == "ok") + .OrderByDescending(row => row.Median).Take(deep).Select(row => row.Item).ToList(); + RunPhase("deep", top, chosen, Int("deepreps", 15), rows, csv); + } + + return 0; + } + + // ---- items ------------------------------------------------------------------------------- + + private static List LoadItems(string dir, HashSet kinds, string[] only) + { + var compiler = SharpEmu.Libs.Gpu.GuestGpu.Current; + var items = new List(); + var seen = new HashSet(); + HashSet? wanted = null; + if (Arg("reps_from", "") is { Length: > 0 } from) + { + wanted = File.ReadLines(from).Skip(1).Select(line => line.Split(',')).Where(f => f.Length > 3 && f[0] == "full").Select(f => int.Parse(f[1], CultureInfo.InvariantCulture)).ToHashSet(); + } + + void Add(int index, ulong hash, IGuestCompiledShaderAdapter shader) + { + var metal = shader.Shader; + var kind = metal.Stage switch + { + Gen5MslStage.Compute => "cs", + Gen5MslStage.Vertex => "vs", + Gen5MslStage.Mesh => "ms", + _ => "ps", + }; + if (!kinds.Contains(kind) || (only.Length > 0 && !only.Any(prefix => $"{hash:X16}".StartsWith(prefix, StringComparison.Ordinal)))) + { + return; + } + + if (wanted is null) + { + var key = MetalLibraryPrewarm.Key(metal.Source, metal.UsesPerVertexInputs) ^ (ulong)metal.Stage << 60; + if (!seen.Add(key)) + { + return; + } + } + else if (!wanted.Contains(index)) + { + return; + } + + var entry = $"{metal.EntryPoint}("; + var at = metal.Source.IndexOf(" " + entry, StringComparison.Ordinal); + items.Add(new Item + { + Index = index, + Kind = kind, + Hash = hash, + Source = metal.Source, + Shader = metal, + Body = at >= 0 ? metal.Source[at..] : metal.Source, + }); + } + + using (var list = ShaderPrewarmList.Open(dir) ?? throw new InvalidOperationException("no compute list")) + { + var computes = list.LoadedComputes(); + for (var index = 0; index < computes.Count; index++) + { + var (record, code) = computes[index]; + if (ShaderPrewarmCompiler.TryCompilePrewarm(record, code, compiler, + MetalRenderHost.PrewarmSharedInt64Atomics, MetalRenderHost.PrewarmExecGuardElision, + MetalRenderHost.PrewarmSignedZeroInfNanPreserve, out var compiled, out _, out _) && + compiled is MetalCompiledGuestShader metal) + { + Add(index, record.Hash, new IGuestCompiledShaderAdapter(metal.Shader)); + } + } + } + + using (var list = GraphicsPrewarmList.Open(dir) ?? throw new InvalidOperationException("no graphics list")) + { + var graphics = list.Loaded(); + for (var index = 0; index < graphics.Count; index++) + { + var record = GraphicsPrewarmList.Decode(graphics[index]); + if (ShaderPrewarmCompiler.TryCompileGraphicsPrewarm(graphics[index], compiler, out var compiled, out _) && + compiled is MetalCompiledGuestShader metal) + { + Add(10000 + index, record.Hash, new IGuestCompiledShaderAdapter(metal.Shader)); + } + } + } + + var limit = Int("limit", 0); + return limit > 0 ? items.Take(limit).ToList() : items; + } + + private readonly record struct IGuestCompiledShaderAdapter(Gen5MslShader Shader); + + // ---- static metrics ----------------------------------------------------------------------- + + private static readonly Regex SampleRx = new(@"\.(sample|sample_compare|gather|gather_compare)\(", RegexOptions.Compiled); + private static readonly Regex ReadRx = new(@"\.read\(", RegexOptions.Compiled); + private static readonly Regex WriteRx = new(@"\.write\(", RegexOptions.Compiled); + private static readonly Regex LoadRx = new(@"sharpemu_load_|\bb\d+\[|sharpemu_resolve_device_", RegexOptions.Compiled); + private static readonly Regex StoreRx = new(@"sharpemu_store_|\bb\d+\[[^\]]*\]\s*=[^=]|atomic_[a-z_]*explicit\(|sharpemu_atomic", RegexOptions.Compiled); + private static readonly Regex LoopRx = new(@"\b(while|for)\s*\(", RegexOptions.Compiled); + private static readonly Regex BranchRx = new(@"\bif\s*\(", RegexOptions.Compiled); + private static readonly Regex BarrierRx = new(@"threadgroup_barrier|simdgroup_barrier", RegexOptions.Compiled); + private static readonly Regex AtomicRx = new(@"atomic_[a-z_]*explicit\(", RegexOptions.Compiled); + private static readonly Regex SimdRx = new(@"\bsimd_[a-z_]+\(", RegexOptions.Compiled); + + private static void WriteStatic(string path, List items) + { + var text = new StringBuilder("idx,id,kind,hash,msl_bytes,lines,tex_samples,tex_reads,tex_writes,dev_loads,stores_atomics,loops,branches,barriers,simd_ops,discard,depth_out,early_attr,tg_size,mesh_tg_bytes\n"); + foreach (var item in items) + { + var body = item.Body; + var shader = item.Shader; + text.Append(CultureInfo.InvariantCulture, + $"{item.Index},{item.Id},{item.Kind},0x{item.Hash:X16},{item.Source.Length},{item.Source.Count(c => c == '\n')}," + + $"{SampleRx.Count(body)},{ReadRx.Count(body)},{WriteRx.Count(body)},{LoadRx.Count(body)},{StoreRx.Count(body) + WriteRx.Count(body)}," + + $"{LoopRx.Count(body)},{BranchRx.Count(body)},{BarrierRx.Count(body)},{SimdRx.Count(body)},{(body.Contains("discard_fragment", StringComparison.Ordinal) ? 1 : 0)}," + + $"{(item.Source.Contains("[[depth(", StringComparison.Ordinal) ? 1 : 0)},{(item.Source.Contains("[[early_fragment_tests]]", StringComparison.Ordinal) ? 1 : 0)}," + + $"{shader.ThreadgroupSizeX}x{shader.ThreadgroupSizeY}x{shader.ThreadgroupSizeZ},{shader.MeshInfo?.ThreadgroupMemoryBytes ?? 0}\n"); + } + + File.WriteAllText(path, text.ToString()); + } + + // ---- variants ----------------------------------------------------------------------------- + + private static Dictionary Variants() + { + static void SetMode(nint options, nuint mode, nuint functions) + { + if (MetalNative.SendBool(options, MetalNative.Selector("respondsToSelector:"), MetalNative.Selector("setMathMode:"))) + { + MetalNative.SendVoidNUInt(options, MetalNative.Selector("setMathMode:"), mode); + if (MetalNative.SendBool(options, MetalNative.Selector("respondsToSelector:"), MetalNative.Selector("setMathFloatingPointFunctions:"))) + { + MetalNative.SendVoidNUInt(options, MetalNative.Selector("setMathFloatingPointFunctions:"), functions); + } + } + } + + static bool EarlyCandidate(Item item) => + item.IsPixel && !item.Source.Contains("[[early_fragment_tests]]", StringComparison.Ordinal) && + !item.Body.Contains("discard_fragment", StringComparison.Ordinal) && + !item.Source.Contains("[[depth(", StringComparison.Ordinal) && !item.Source.Contains("[[sample_mask", StringComparison.Ordinal); + + // The same, for the pixel functions that do discard: the translator leaves those on Metal's late order, since a discarded + // fragment would already have written depth. Only a cost bound - not a semantics-preserving change where depth is written. + static bool EarlyDiscardCandidate(Item item) => + item.IsPixel && !item.Source.Contains("[[early_fragment_tests]]", StringComparison.Ordinal) && + item.Body.Contains("discard_fragment", StringComparison.Ordinal) && + !item.Source.Contains("[[depth(", StringComparison.Ordinal) && !item.Source.Contains("[[sample_mask", StringComparison.Ordinal); + + var all = new[] + { + new Variant("earlyd", "zreject", EarlyDiscardCandidate, + s => s.Replace("fragment Gen5PsOut gen5_ps(", "[[early_fragment_tests]] fragment Gen5PsOut gen5_ps(", StringComparison.Ordinal), _ => { }), + new Variant("basezd", "zreject", EarlyDiscardCandidate, s => s, _ => { }), + // The emulator's configuration: MTLMathModeRelaxed, precise library functions, invariance preserved. + new Variant("base", "none", _ => true, s => s, _ => { }), + // SHARPEMU_MSL_MATHMODE=safe. + new Variant("mathsafe", "none", _ => true, s => s, o => SetMode(o, 0, 1)), + // setFastMathEnabled:true, the MTLMathModeFast of the old switch: unsafe for NaN/inf/signed zero. + new Variant("fastmath", "none", _ => true, s => s, o => + { + MetalNative.SendVoidBool(o, MetalNative.Selector("setFastMathEnabled:"), true); + SetMode(o, 2, 0); + }), + // Fast mode with the transcendentals kept precise (the part the NaN/inf semantics rely on). + new Variant("fastmath_fnprecise", "none", _ => true, s => s, o => SetMode(o, 2, 1)), + // The source's #pragma clang fp contract(off) removed: multiply-add fusion under the relaxed mode. + new Variant("contract", "none", item => item.Source.Contains("fp contract(off)", StringComparison.Ordinal), + s => s.Replace("#pragma clang fp contract(off)", "#pragma clang fp contract(fast)", StringComparison.Ordinal), _ => { }), + // preserveInvariance off: the positions are no longer invariant across pipelines. + new Variant("noinvariance", "none", item => item.Kind is "vs" or "ms", s => s, + o => MetalNative.SendVoidBool(o, MetalNative.Selector("setPreserveInvariance:"), false)), + new Variant("optsize", "none", _ => true, s => s, + o => MetalNative.SendVoidNUInt(o, MetalNative.Selector("setOptimizationLevel:"), 1)), + // [[early_fragment_tests]] on a pixel function that has neither discard nor a depth/mask export, run against a + // depth buffer that rejects every fragment (the best case), and its baseline in the same scenario. + new Variant("early", "zreject", EarlyCandidate, + s => s.Replace("fragment Gen5PsOut gen5_ps(", "[[early_fragment_tests]] fragment Gen5PsOut gen5_ps(", StringComparison.Ordinal), _ => { }), + new Variant("basez", "zreject", EarlyCandidate, s => s, _ => { }), + }; + return all.ToDictionary(variant => variant.Name); + } + + // ---- phases ------------------------------------------------------------------------------- + + private static void RunPhase(string phase, List items, List variants, int reps, List rows, string csvPath) + { + var warm = Int("warm", 2); + var workers = Int("workers", 3); + var ahead = 2; + var pending = new Dictionary>(); + var scheduler = new ConcurrentExclusiveSchedulerPair(TaskScheduler.Default, workers).ConcurrentScheduler; + Task Start(int index) => Task.Factory.StartNew(() => CompileItem(items[index], variants, workers), CancellationToken.None, TaskCreationOptions.LongRunning, scheduler); + for (var i = 0; i < Math.Min(ahead, items.Count); i++) + { + pending[i] = Start(i); + } + + var started = Stopwatch.StartNew(); + for (var index = 0; index < items.Count; index++) + { + var item = items[index]; + if (index + ahead < items.Count) + { + pending[index + ahead] = Start(index + ahead); + } + + var built = pending[index].Result; + pending.Remove(index); + var produced = new List(); + try + { + produced.AddRange(Measure(phase, item, built, warm, reps)); + } + catch (Exception exception) + { + produced.Add(new Row { Phase = phase, Item = item, Variant = "base", Status = "exception", Note = exception.Message }); + } + finally + { + foreach (var b in built) + { + foreach (var handle in (ReadOnlySpan)[b.State, b.Function, b.Library]) + { + if (handle != 0) + { + MetalNative.SendVoid(handle, SelRelease); + } + } + } + } + + rows.AddRange(produced); + var best = produced.Where(r => r.Status == "ok").Select(r => r.Median).DefaultIfEmpty(0).First(); + Console.Error.WriteLine(string.Create(CultureInfo.InvariantCulture, + $"[{phase} {index + 1}/{items.Count} {started.Elapsed.TotalSeconds:F0}s] {item.Id} {string.Join(' ', produced.Select(r => r.Status == "ok" ? $"{r.Variant}={r.Median:F0}us" : $"{r.Variant}:{r.Status}"))}")); + if ((index + 1) % 25 == 0 || index + 1 == items.Count) + { + WriteCsv(csvPath, rows); + } + } + } + + private static void WriteCsv(string path, List rows) + { + var text = new StringBuilder("phase,idx,id,kind,hash,variant,scenario,median_us,min_us,max_us,reps,scale,status,note\n"); + foreach (var row in rows) + { + text.Append(CultureInfo.InvariantCulture, + $"{row.Phase},{row.Item.Index},{row.Item.Id},{row.Item.Kind},0x{row.Item.Hash:X16},{(row.Variant == "base" && Arg("alias", "").Length > 0 ? Arg("alias", "") : row.Variant)},{row.Scenario},{row.Median:F1},{row.Min:F1},{row.Max:F1},{row.Reps},{row.Scale:F1},{row.Status},\"{row.Note.Replace('"', '\'').Replace('\n', ' ')}\"\n"); + } + + File.WriteAllText(path, text.ToString()); + } + + // ---- compile ------------------------------------------------------------------------------ + + private static Built[] CompileItem(Item item, List variants, int workers) + { + var result = new Built[variants.Count]; + Parallel.For(0, variants.Count, new ParallelOptions { MaxDegreeOfParallelism = Math.Max(1, workers) }, i => + { + var variant = variants[i]; + var built = new Built { Variant = variant }; + result[i] = built; + if (!variant.Applies(item)) + { + built.Notes = "n/a"; + return; + } + + var pool = MetalNative.objc_autoreleasePoolPush(); + try + { + var source = variant.AltDirectory is { } altDirectory ? File.ReadAllText(Path.Combine(altDirectory, $"{item.Index}.metal")) : variant.Patch(item.Source); + built.Source = source; + var t0 = Stopwatch.GetTimestamp(); + built.Library = CompileLibrary(source, item.Shader.UsesPerVertexInputs, variant.Options, out built.Error); + if (built.Library == 0) + { + return; + } + + nint error = 0; + if (item.Shader.UsesMipStatisticsConstant) + { + built.Function = MetalRenderHost.NewMipStatisticsFunction(built.Library, item.Shader.EntryPoint, false, out error); + } + else + { + built.Function = MetalNative.Send(built.Library, MetalNative.Selector("newFunctionWithName:"), MetalNative.NsString(item.Shader.EntryPoint)); + } + + if (built.Function == 0) + { + built.Error = "no entry point " + MetalNative.DescribeError(error); + return; + } + + built.State = item.Kind == "cs" ? NewComputeState(built) : NewRenderState(item, built); + built.CompileMs = (Stopwatch.GetTimestamp() - t0) * 1000.0 / Stopwatch.Frequency; + } + catch (Exception exception) + { + built.Error = exception.Message; + } + finally + { + MetalNative.objc_autoreleasePoolPop(pool); + } + }); + return result; + } + + private static nint CompileLibrary(string source, bool perVertex, Action adjust, out string? error) + { + error = null; + var options = MetalNative.Send(MetalNative.Send(MetalNative.Class("MTLCompileOptions"), SelAlloc), SelInit); + try + { + MetalNative.ApplyMslCompileOptions(options); + if (perVertex) + { + MetalNative.SetMslLanguageVersion(options, Gen5MslShader.PerVertexInputsLanguageVersion); + } + + adjust(options); + nint compileError = 0; + var library = MetalNative.Send(_device, MetalNative.Selector("newLibraryWithSource:options:error:"), MetalNative.NsString(source), options, ref compileError); + if (library == 0) + { + error = "library: " + MetalNative.DescribeError(compileError); + } + + return library; + } + finally + { + MetalNative.SendVoid(options, SelRelease); + } + } + + private static readonly ConcurrentDictionary Partners = new(); + + // A small helper library (the synthetic vertex or consumer pixel function) shared by every item that needs the same text. + private static nint PartnerFunction(string source, string entry) + { + var library = Partners.GetOrAdd(source, text => + { + var lib = CompileLibrary(text, false, _ => { }, out var error); + if (lib == 0) + { + throw new InvalidOperationException("partner " + error); + } + + return lib; + }); + return MetalNative.Send(library, MetalNative.Selector("newFunctionWithName:"), MetalNative.NsString(entry)); + } + + private static nint NewComputeState(Built built) + { + nint error = 0; + var state = MetalNative.Send(_device, MetalNative.Selector("newComputePipelineStateWithFunction:error:"), built.Function, ref error); + if (state == 0) + { + built.Error = "pipeline: " + MetalNative.DescribeError(error); + return 0; + } + + built.MaxThreads = (uint)MetalNative.SendUInteger(state, MetalNative.Selector("maxTotalThreadsPerThreadgroup")); + built.ExecWidth = (uint)MetalNative.SendUInteger(state, MetalNative.Selector("threadExecutionWidth")); + built.TgBytes = (uint)MetalNative.SendUInteger(state, MetalNative.Selector("staticThreadgroupMemoryLength")); + return state; + } + + // ---- synthetic partner stages ------------------------------------------------------------- + + private static List<(string Type, string Name, string Attrs)> ParseStruct(string source, string name) + { + var members = new List<(string, string, string)>(); + var start = source.IndexOf($"struct {name}\n", StringComparison.Ordinal); + if (start < 0) + { + return members; + } + + var end = source.IndexOf("};", start, StringComparison.Ordinal); + foreach (var line in source[start..end].Split('\n').Skip(2)) + { + var match = Regex.Match(line, @"^\s+([A-Za-z0-9_:<>]+)\s+([A-Za-z0-9_]+)\s*((?:\[\[[^\]]*\]\]\s*)*)(?:\[\d+\])?\s*;"); + if (match.Success) + { + members.Add((match.Groups[1].Value, match.Groups[2].Value, match.Groups[3].Value.Trim())); + } + } + + return members; + } + + private static string UserAttr(string attrs) => Regex.Match(attrs, @"user\(locn(\d+)\)") is { Success: true } m ? m.Groups[1].Value : ""; + + private static nint NewRenderState(Item item, Built built) + { + var shader = item.Shader; + var mesh = item.Kind == "ms"; + var descriptor = MetalNative.Send(MetalNative.Send(MetalNative.Class(mesh ? "MTLMeshRenderPipelineDescriptor" : "MTLRenderPipelineDescriptor"), SelAlloc), SelInit); + try + { + var colors = new List(); + var depth = false; + if (item.Kind == "ps") + { + foreach (var (type, name, attrs) in ParseStruct(item.Source, "Gen5PsOut")) + { + if (Regex.Match(attrs, @"color\((\d+)\)") is { Success: true } color) + { + var slot = int.Parse(color.Groups[1].Value, CultureInfo.InvariantCulture); + while (colors.Count <= slot) + { + colors.Add(0); + } + + colors[slot] = type.StartsWith("uint", StringComparison.Ordinal) ? 73u : type.StartsWith("int", StringComparison.Ordinal) ? 74u : 70u; + } + else if (attrs.Contains("depth(", StringComparison.Ordinal)) + { + depth = true; + } + } + + var vs = SyntheticVertex(item); + MetalNative.SendVoid(descriptor, MetalNative.Selector("setVertexFunction:"), PartnerFunction(vs, "bench_vs")); + MetalNative.SendVoid(descriptor, MetalNative.Selector("setFragmentFunction:"), built.Function); + if (ParseStruct(item.Source, "Gen5PsIn").Any(m => m.Attrs.Contains("render_target_array_index", StringComparison.Ordinal))) + { + MetalV2Native.SendVoidValue(descriptor, MetalNative.Selector("setInputPrimitiveTopology:"), 3); + } + } + else + { + colors.Add(70); + if (mesh) + { + var info = shader.MeshInfo!; + MetalNative.SendVoid(descriptor, MetalNative.Selector("setMeshFunction:"), built.Function); + MetalV2Native.SendVoidValue(descriptor, MetalNative.Selector("setMaxTotalThreadsPerMeshThreadgroup:"), Math.Max(info.ThreadsPerThreadgroup, 1)); + } + else + { + MetalNative.SendVoid(descriptor, MetalNative.Selector("setVertexFunction:"), built.Function); + var vertexDescriptor = VertexDescriptor(item); + if (vertexDescriptor != 0) + { + MetalNative.SendVoid(descriptor, MetalNative.Selector("setVertexDescriptor:"), vertexDescriptor); + } + } + + MetalNative.SendVoid(descriptor, MetalNative.Selector("setFragmentFunction:"), PartnerFunction(ConsumerPixel(item), "bench_ps")); + if (!mesh && (shader.WritesLayer || shader.WritesViewportIndex)) + { + MetalV2Native.SendVoidValue(descriptor, MetalNative.Selector("setInputPrimitiveTopology:"), 3); + } + } + + var attachments = MetalNative.Send(descriptor, MetalNative.Selector("colorAttachments")); + for (var slot = 0; slot < colors.Count; slot++) + { + if (colors[slot] != 0) + { + MetalV2Native.SendVoidValue(MetalNative.SendAtIndex(attachments, SelObjectAt, (nuint)slot), MetalNative.Selector("setPixelFormat:"), colors[slot]); + } + } + + var zreject = built.Variant.Scenario == "zreject"; + if (depth || zreject) + { + MetalV2Native.SendVoidValue(descriptor, MetalNative.Selector("setDepthAttachmentPixelFormat:"), 252); + } + + nint error = 0; + var state = MetalNative.SendNewMeshPipeline( + _device, + MetalNative.Selector(mesh ? "newRenderPipelineStateWithMeshDescriptor:options:reflection:error:" : "newRenderPipelineStateWithDescriptor:options:reflection:error:"), + descriptor, 0, 0, ref error); + if (state == 0) + { + built.Error = "pipeline: " + MetalNative.DescribeError(error); + return 0; + } + + if (mesh) + { + built.MaxThreads = (uint)shader.MeshInfo!.ThreadsPerThreadgroup; + built.TgBytes = shader.MeshInfo.ThreadgroupMemoryBytes; + } + + built.Notes = string.Join('/', colors.Select(c => c.ToString(CultureInfo.InvariantCulture))) + (depth ? "+z" : ""); + if (mesh) + { + var info = shader.MeshInfo!; + built.Notes = $"groups={Gen5MslMeshLaunch.SubgroupCount(info, Gen5MslMeshInputTopology.TriangleList, MeshVertices(info, (uint)Int("verts", 393216)))} tg={info.ThreadsPerThreadgroup} verts={info.MaxVertices} prims={info.MaxPrimitives} amplified={info.Amplified} gs={info.HasGeometryHalf} devicelds={info.DeviceLds} wave64={info.Wave64}"; + } + return state; + } + finally + { + MetalNative.SendVoid(descriptor, SelRelease); + } + } + + // A vertex function that draws a full-screen quad and feeds every interpolant the pixel program declares. + private static string SyntheticVertex(Item item) + { + var input = ParseStruct(item.Source, "Gen5PsIn"); + var text = new StringBuilder("#include \nusing namespace metal;\nstruct V {\n float4 position [[position]];\n"); + var users = input.Select(m => UserAttr(m.Attrs)).Where(n => n.Length > 0).Distinct().ToList(); + foreach (var user in users) + { + text.Append(CultureInfo.InvariantCulture, $" float4 a{user} [[user(locn{user})]];\n"); + } + + if (input.Any(m => m.Attrs.Contains("render_target_array_index", StringComparison.Ordinal))) + { + text.Append(" uint layer [[render_target_array_index]];\n"); + } + + text.Append("};\nvertex V bench_vs(uint vid [[vertex_id]]) {\n"); + text.Append(" const float2 q[6] = { float2(-1,-1), float2(1,-1), float2(-1,1), float2(1,-1), float2(1,1), float2(-1,1) };\n"); + text.Append(" float2 p = q[vid % 6u]; V o; o.position = float4(p, 0.5, 1.0); float4 v = float4(p * 0.5 + 0.5, 0.5, 1.0);\n"); + foreach (var user in users) + { + text.Append(CultureInfo.InvariantCulture, $" o.a{user} = v;\n"); + } + + if (input.Any(m => m.Attrs.Contains("render_target_array_index", StringComparison.Ordinal))) + { + text.Append(" o.layer = 0u;\n"); + } + + return text.Append(" return o;\n}\n").ToString(); + } + + // A pixel function that reads every interpolant a vertex or mesh program exports, so none of its output is dead. + private static string ConsumerPixel(Item item) + { + var outputs = ParseStruct(item.Source, item.Kind == "ms" ? "Gen5MeshVertex" : "Gen5VsOut"); + var users = outputs.Select(m => UserAttr(m.Attrs)).Where(n => n.Length > 0).Distinct().ToList(); + var text = new StringBuilder("#include \nusing namespace metal;\nstruct F {\n float4 position [[position]];\n"); + foreach (var user in users) + { + text.Append(CultureInfo.InvariantCulture, $" float4 a{user} [[user(locn{user})]];\n"); + } + + text.Append("};\nfragment float4 bench_ps(F in [[stage_in]]) {\n float4 s = in.position * 0.0;\n"); + foreach (var user in users) + { + text.Append(CultureInfo.InvariantCulture, $" s += in.a{user};\n"); + } + + return text.Append(" return s;\n}\n").ToString(); + } + + private static nint VertexDescriptor(Item item) + { + var attributes = ParseStruct(item.Source, "Gen5VsIn"); + if (attributes.Count == 0) + { + return 0; + } + + var descriptor = MetalNative.Send(MetalNative.Send(MetalNative.Class("MTLVertexDescriptor"), SelAlloc), SelInit); + var attributeArray = MetalNative.Send(descriptor, MetalNative.Selector("attributes")); + var layouts = MetalNative.Send(descriptor, MetalNative.Selector("layouts")); + foreach (var (type, _, attrs) in attributes) + { + var match = Regex.Match(attrs, @"attribute\((\d+)\)"); + if (!match.Success) + { + continue; + } + + var index = int.Parse(match.Groups[1].Value, CultureInfo.InvariantCulture); + if (index + VertexSlotBase >= 31) + { + throw new InvalidOperationException("too many vertex attributes"); + } + + var attribute = MetalNative.SendAtIndex(attributeArray, SelObjectAt, (nuint)index); + MetalV2Native.SendVoidValue(attribute, MetalNative.Selector("setFormat:"), VertexFormat(type)); + MetalV2Native.SendVoidValue(attribute, MetalNative.Selector("setOffset:"), 0); + MetalV2Native.SendVoidValue(attribute, MetalNative.Selector("setBufferIndex:"), (nuint)(index + VertexSlotBase)); + var layout = MetalNative.SendAtIndex(layouts, SelObjectAt, (nuint)(index + VertexSlotBase)); + MetalV2Native.SendVoidValue(layout, MetalNative.Selector("setStride:"), 16); + MetalV2Native.SendVoidValue(layout, MetalNative.Selector("setStepFunction:"), 1); + } + + return descriptor; + } + + private const int VertexSlotBase = 2; + + private static nuint VertexFormat(string type) + { + var lanes = type[^1] is >= '1' and <= '4' ? type[^1] - '0' : 1; + if (type.StartsWith("half", StringComparison.Ordinal)) + { + return (nuint)(25 + Math.Max(lanes, 2) - 2); + } + + var family = type.StartsWith("uint", StringComparison.Ordinal) ? 36 : type.StartsWith("int", StringComparison.Ordinal) ? 32 : 28; + return (nuint)(family + lanes - 1); + } + + // ---- synthetic bindings ------------------------------------------------------------------- + + private sealed class Synth + { + public const int PoolBuffers = 4; + public const uint PoolBytes = 32 << 20; + private readonly nint _device; + private readonly bool _pattern; + private readonly List _resources = []; + private readonly Dictionary _textures = new(); + private readonly object _gate = new(); + public readonly nint[] Pool = new nint[PoolBuffers]; + public nint Gds, Fault, Flattened, ShaderData, MipStats, StepReport, PairLocks, PageTable, Push, VertexStream, Indices, LdsPool, Meshlets, DrawInfo; + public nint Sampler; + public ulong SamplerId; + public uint PageTableEntries; + public nint[] Targets = []; + public nint Depth; + public nint SmallTarget; + + public Synth(nint device, bool pattern) + { + _device = device; + _pattern = pattern; + for (var i = 0; i < PoolBuffers; i++) + { + Pool[i] = Buffer(PoolBytes, patternable: true); + } + + Gds = Buffer(64 << 10, true); + Fault = Buffer(64 << 10, false); + Flattened = Buffer(16 << 10, true); + ShaderData = Buffer(2 << 10, true); + MipStats = Buffer(1 << 20, false); + StepReport = Buffer(4 << 10, false); + PairLocks = Buffer((uint)Gen5MslArgumentLayout.PairLockTableWords * 4, false); + Push = Buffer(4 << 10, true); + VertexStream = Buffer(16u * 1_048_576, true); + Indices = Buffer(16, false); + Meshlets = Buffer(16, false); + DrawInfo = Buffer(256, false); + LdsPool = Buffer((uint)Gen5MslMeshInfo.LdsPoolByteSize, false); + // 64 ranges of 16 MB, all backed by the first 16 MB of one pool buffer: every guest address under 1 GB + // resolves (through a real binary search over 64 entries), none reaches past the backing. + const int entries = 64; + const ulong pages = (16UL << 20) >> 14; + PageTable = Buffer(entries * 24, false); + var span = new Span((void*)MetalNative.Send(PageTable, MetalNative.Selector("contents")), entries * 24); + var baseAddress = MetalNative.SendGpuAddress(Pool[0], MetalNative.Selector("gpuAddress")); + for (var i = 0; i < entries; i++) + { + BinaryPrimitives.WriteUInt64LittleEndian(span[(i * 24)..], (ulong)i * pages); + BinaryPrimitives.WriteUInt64LittleEndian(span[(i * 24 + 8)..], pages); + BinaryPrimitives.WriteUInt64LittleEndian(span[(i * 24 + 16)..], baseAddress); + } + + PageTableEntries = entries; + var sampler = MetalNative.Send(MetalNative.Send(MetalNative.Class("MTLSamplerDescriptor"), SelAlloc), SelInit); + MetalNative.SendVoidBool(sampler, MetalNative.Selector("setSupportArgumentBuffers:"), true); + MetalNative.SendVoidNUInt(sampler, MetalNative.Selector("setMinFilter:"), 1); + MetalNative.SendVoidNUInt(sampler, MetalNative.Selector("setMagFilter:"), 1); + MetalNative.SendVoidNUInt(sampler, MetalNative.Selector("setMipFilter:"), 2); + MetalNative.SendVoidNUInt(sampler, MetalNative.Selector("setSAddressMode:"), 2); + MetalNative.SendVoidNUInt(sampler, MetalNative.Selector("setTAddressMode:"), 2); + MetalNative.SendVoidNUInt(sampler, MetalNative.Selector("setRAddressMode:"), 2); + Sampler = MetalNative.Send(_device, MetalNative.Selector("newSamplerStateWithDescriptor:"), sampler); + SamplerId = MetalNative.SendGpuResourceId(Sampler, MetalNative.Selector("gpuResourceID")); + SmallTarget = RenderTarget(70, 256, 256, 0); + } + + public nint Buffer(uint bytes, bool patternable) + { + var buffer = MetalNative.SendNewBuffer(_device, MetalNative.Selector("newBufferWithLength:options:"), bytes, ResourceShared); + var contents = (byte*)MetalNative.Send(buffer, MetalNative.Selector("contents")); + new Span(contents, (int)Math.Min(bytes, int.MaxValue)).Clear(); + if (_pattern && patternable) + { + Fill(contents, bytes); + } + + _resources.Add(buffer); + return buffer; + } + + // Words of 0..7: small enough that a loop bound or an offset read from memory stays bounded. + private static void Fill(byte* contents, uint bytes) + { + var words = (uint*)contents; + for (uint i = 0; i < bytes / 4; i++) + { + words[i] = (i * 2654435761u) >> 29; + } + } + + private nint[] Writable => [.. Pool, Gds, Fault, MipStats, StepReport, PairLocks]; + + // The buffers a shader can write back to their start state: zero, or the pattern. + public void ResetOutputs() + { + foreach (var buffer in Writable) + { + var length = (int)MetalNative.SendUInteger(buffer, MetalNative.Selector("length")); + var contents = (byte*)MetalNative.Send(buffer, MetalNative.Selector("contents")); + if (_pattern && (buffer == Gds || Pool.Contains(buffer))) + { + Fill(contents, (uint)length); + } + else + { + new Span(contents, length).Clear(); + } + } + } + + public string OutputHash() + { + var hash = new System.IO.Hashing.XxHash64(); + foreach (var buffer in Writable) + { + var length = (int)MetalNative.SendUInteger(buffer, MetalNative.Selector("length")); + hash.Append(new ReadOnlySpan((void*)MetalNative.Send(buffer, MetalNative.Selector("contents")), length)); + } + + return hash.GetCurrentHashAsUInt64().ToString("X16", CultureInfo.InvariantCulture); + } + + public void Refill() + { + if (!_pattern) + { + return; + } + + foreach (var buffer in Pool) + { + Fill((byte*)MetalNative.Send(buffer, MetalNative.Selector("contents")), PoolBytes); + } + } + + public nint RenderTarget(nuint format, nuint width, nuint height, nuint arrayLength) + { + var descriptor = MetalNative.SendTextureDescriptor(MetalNative.Class("MTLTextureDescriptor"), MetalNative.Selector("texture2DDescriptorWithPixelFormat:width:height:mipmapped:"), format, width, height, false); + MetalNative.SendVoidNUInt(descriptor, MetalNative.Selector("setUsage:"), 5); + MetalNative.SendVoidNUInt(descriptor, MetalNative.Selector("setStorageMode:"), ShareTargets ? 0u : 2u); + var texture = MetalNative.Send(_device, MetalNative.Selector("newTextureWithDescriptor:"), descriptor); + _resources.Add(texture); + return texture; + } + + public void EnsureTargets(int count, nuint width, nuint height) + { + if (Targets.Length >= count && Depth != 0) + { + return; + } + + if (Depth == 0) + { + Depth = RenderTarget(252, width, height, 0); + } + + var targets = new nint[Math.Max(count, 1)]; + Targets.CopyTo(targets, 0); + for (var i = Targets.Length; i < targets.Length; i++) + { + targets[i] = RenderTarget(70, width, height, 0); + } + + Targets = targets; + } + + // A texture for an argument-buffer field of the given name (class_numeric_dimension). + public nint Texture(string field) + { + lock (_gate) + { + if (_textures.TryGetValue(field, out var found)) + { + return found; + } + + var parts = field.Split('_'); + var compare = field.StartsWith("sampled_compare", StringComparison.Ordinal); + var storage = field.StartsWith("storage", StringComparison.Ordinal) || field.StartsWith("atomic", StringComparison.Ordinal); + var numeric = parts.Contains("uint") || parts.Contains("uint64") ? "uint" : parts.Contains("sint") ? "sint" : "float"; + var format = compare ? 252u : numeric == "uint" ? 123u : numeric == "sint" ? 124u : 70u; + var dimension = string.Join('_', parts.SkipWhile(p => p is not ("1d" or "2d" or "3d" or "cube"))); + var size = 512u; + var descriptor = MetalNative.SendTextureDescriptor(MetalNative.Class("MTLTextureDescriptor"), MetalNative.Selector("texture2DDescriptorWithPixelFormat:width:height:mipmapped:"), format, size, dimension.StartsWith("1d", StringComparison.Ordinal) ? 1 : size, !dimension.Contains("ms", StringComparison.Ordinal) && !compare); + switch (dimension) + { + case "1d": + MetalNative.SendVoidNUInt(descriptor, MetalNative.Selector("setTextureType:"), 0); + break; + case "1d_array": + MetalNative.SendVoidNUInt(descriptor, MetalNative.Selector("setTextureType:"), 1); + MetalNative.SendVoidNUInt(descriptor, MetalNative.Selector("setArrayLength:"), 4); + break; + case "2d_array" or "cube": + MetalNative.SendVoidNUInt(descriptor, MetalNative.Selector("setTextureType:"), 3); + MetalNative.SendVoidNUInt(descriptor, MetalNative.Selector("setArrayLength:"), 6); + break; + case "3d": + MetalNative.SendVoidNUInt(descriptor, MetalNative.Selector("setTextureType:"), 7); + MetalNative.SendVoidNUInt(descriptor, MetalNative.Selector("setDepth:"), 64); + MetalNative.SendVoidNUInt(descriptor, MetalNative.Selector("setWidth:"), 64); + MetalNative.SendVoidNUInt(descriptor, MetalNative.Selector("setHeight:"), 64); + MetalNative.SendVoidNUInt(descriptor, MetalNative.Selector("setMipmapLevelCount:"), 7); + break; + case "2d_ms": + MetalNative.SendVoidNUInt(descriptor, MetalNative.Selector("setTextureType:"), 4); + MetalNative.SendVoidNUInt(descriptor, MetalNative.Selector("setSampleCount:"), 4); + break; + case "2d_ms_array": + MetalNative.SendVoidNUInt(descriptor, MetalNative.Selector("setTextureType:"), 8); + MetalNative.SendVoidNUInt(descriptor, MetalNative.Selector("setSampleCount:"), 4); + MetalNative.SendVoidNUInt(descriptor, MetalNative.Selector("setArrayLength:"), 4); + break; + } + + MetalNative.SendVoidNUInt(descriptor, MetalNative.Selector("setUsage:"), storage ? 3u : 1u); + MetalNative.SendVoidNUInt(descriptor, MetalNative.Selector("setStorageMode:"), dimension.Contains("ms", StringComparison.Ordinal) ? 2u : 0u); + var texture = MetalNative.Send(_device, MetalNative.Selector("newTextureWithDescriptor:"), descriptor); + if (texture == 0) + { + throw new InvalidOperationException($"no texture for {field}"); + } + + _resources.Add(texture); + _textures[field] = texture; + return texture; + } + } + + public nint[] Resources() + { + lock (_gate) + { + return _resources.ToArray(); + } + } + } + + // Fills the stage argument buffer of a program from its layout, the way the host's WriteArgumentBuffer does. + private static nint ArgumentBuffer(Gen5MslShader shader) + { + var layout = shader.ArgumentLayout; + if (layout is null || layout.Fields.Count == 0) + { + return 0; + } + + var buffer = _synth.Buffer(Math.Max(layout.ByteSize, 16u), false); + var span = new Span((void*)MetalNative.Send(buffer, MetalNative.Selector("contents")), (int)Math.Max(layout.ByteSize, 16u)); + var gpu = (nint handle) => MetalNative.SendGpuAddress(handle, MetalNative.Selector("gpuAddress")); + foreach (var field in layout.Fields) + { + var at = span.Slice((int)field.ByteOffset, (int)field.ByteSize); + switch (field.FieldKind) + { + case MslArgumentFieldKind.BufferPointers: + for (var i = 0; i < field.Count; i++) + { + BinaryPrimitives.WriteUInt64LittleEndian(at[(i * 8)..], gpu(_synth.Pool[i % Synth.PoolBuffers])); + } + + break; + case MslArgumentFieldKind.BufferByteCounts: + for (var i = 0; i < field.Count; i++) + { + BinaryPrimitives.WriteUInt32LittleEndian(at[(i * 4)..], Synth.PoolBytes); + } + + break; + case MslArgumentFieldKind.Textures: + var texture = _synth.Texture(field.Name); + var id = MetalNative.SendGpuResourceId(texture, MetalNative.Selector("gpuResourceID")); + for (var i = 0; i < field.Count; i++) + { + BinaryPrimitives.WriteUInt64LittleEndian(at[(i * 8)..], id); + } + + break; + case MslArgumentFieldKind.Samplers: + for (var i = 0; i < field.Count; i++) + { + BinaryPrimitives.WriteUInt64LittleEndian(at[(i * 8)..], _synth.SamplerId); + } + + break; + case MslArgumentFieldKind.Pointer: + BinaryPrimitives.WriteUInt64LittleEndian(at, gpu(field.Kind switch + { + SharpEmu.ShaderCompiler.Resources.DescriptorBindingKind.GlobalDataShare => _synth.Gds, + SharpEmu.ShaderCompiler.Resources.DescriptorBindingKind.DeviceAddressPageTable => _synth.PageTable, + SharpEmu.ShaderCompiler.Resources.DescriptorBindingKind.FaultBuffer => _synth.Fault, + SharpEmu.ShaderCompiler.Resources.DescriptorBindingKind.FlattenedResourceTable => _synth.Flattened, + SharpEmu.ShaderCompiler.Resources.DescriptorBindingKind.ShaderData => _synth.ShaderData, + _ => _synth.MipStats, + })); + break; + case MslArgumentFieldKind.StepReport: + BinaryPrimitives.WriteUInt64LittleEndian(at, gpu(_synth.StepReport)); + break; + case MslArgumentFieldKind.PairLocks: + BinaryPrimitives.WriteUInt64LittleEndian(at, gpu(_synth.PairLocks)); + break; + case MslArgumentFieldKind.BindlessImageTables: + throw new InvalidOperationException("bindless image tables are not synthesized"); + default: + BinaryPrimitives.WriteUInt32LittleEndian(at, field.Kind switch + { + SharpEmu.ShaderCompiler.Resources.DescriptorBindingKind.GlobalDataShare => 64u << 10, + SharpEmu.ShaderCompiler.Resources.DescriptorBindingKind.DeviceAddressPageTable => _synth.PageTableEntries, + SharpEmu.ShaderCompiler.Resources.DescriptorBindingKind.FaultBuffer => 16u << 10, + SharpEmu.ShaderCompiler.Resources.DescriptorBindingKind.FlattenedResourceTable => 4096u, + SharpEmu.ShaderCompiler.Resources.DescriptorBindingKind.ShaderData => 512u, + _ => 0u, + }); + break; + } + } + + return buffer; + } + + // ---- measurement -------------------------------------------------------------------------- + + private static IEnumerable Measure(string phase, Item item, Built[] built, int warm, int reps) + { + var rows = new List(); + var live = built.Where(b => b.State != 0).ToList(); + foreach (var b in built.Where(b => b.State == 0)) + { + rows.Add(new Row { Phase = phase, Item = item, Variant = b.Variant.Name, Scenario = b.Variant.Scenario, Status = b.Notes == "n/a" ? "n/a" : "compile-failed", Note = b.Error ?? b.Notes }); + } + + if (live.Count == 0) + { + return rows; + } + + _synth.Refill(); + var arguments = ArgumentBuffer(item.Shader); + var samples = live.ToDictionary(b => b, _ => new List()); + var scale = 1.0; + var failure = new Dictionary(); + // A probe at 1/64 of the work first: a runaway shader is stopped before it holds the GPU. + var probe = Run(item, live[0], arguments, 1.0 / 64, out var probeStatus); + if (probeStatus != "ok") + { + foreach (var b in live) + { + rows.Add(new Row { Phase = phase, Item = item, Variant = b.Variant.Name, Scenario = b.Variant.Scenario, Status = "run-failed", Note = probeStatus }); + } + + return rows; + } + + var fraction = probe * 64 > 1_500_000 ? 1.0 / 16 : 1.0; + if (probe * 64 > 20_000_000) + { + fraction = 1.0 / 64; + } + + scale = 1.0 / fraction; + // check=1: the bytes every writable synthetic buffer holds after one run from a reset state, per variant, so a + // translator change that alters what a shader stores shows as a different out= hash (atomics racing between + // threads can differ run to run; the same program run twice is the reference). + var outputs = new Dictionary(); + if (Arg("check", "0") == "1") + { + foreach (var b in live) + { + _clearTargets = true; + var targets = item.IsPixel ? ColorCount(item) : 0; + _synth.ResetOutputs(); + var status = ""; + Run(item, b, arguments, fraction, out status); + var first = status == "ok" ? _synth.OutputHash() + (targets > 0 ? "+" + TargetHash(targets) : "") : status; + _synth.ResetOutputs(); + Run(item, b, arguments, fraction, out status); + var second = status == "ok" ? _synth.OutputHash() + (targets > 0 ? "+" + TargetHash(targets) : "") : status; + _clearTargets = false; + outputs[b] = first == second ? first : $"{first}~unstable"; + } + } + + for (var round = 0; round < warm + reps; round++) + { + for (var k = 0; k < live.Count; k++) + { + var b = live[(k + round) % live.Count]; + if (failure.ContainsKey(b)) + { + continue; + } + + var micros = Run(item, b, arguments, fraction, out var status); + if (status != "ok") + { + failure[b] = status; + continue; + } + + if (round >= warm) + { + samples[b].Add(micros); + } + } + } + + foreach (var b in live) + { + var list = samples[b]; + if (failure.TryGetValue(b, out var message) || list.Count == 0) + { + rows.Add(new Row { Phase = phase, Item = item, Variant = b.Variant.Name, Scenario = b.Variant.Scenario, Status = "run-failed", Note = message ?? "no samples" }); + continue; + } + + list.Sort(); + rows.Add(new Row + { + Phase = phase, + Item = item, + Variant = b.Variant.Name, + Scenario = b.Variant.Scenario, + Median = list[list.Count / 2] * scale, + Min = list[0] * scale, + Max = list[^1] * scale, + Reps = list.Count, + Scale = scale, + Note = $"{b.Notes} max_threads={b.MaxThreads} exec_width={b.ExecWidth} tg_bytes={b.TgBytes} compile_ms={b.CompileMs:F0} msl={MetalLibraryPrewarm.Key(b.Source, item.Shader.UsesPerVertexInputs):X16} out={outputs.GetValueOrDefault(b, "-")}", + }); + } + + return rows; + } + + // One command buffer, one dispatch or draw; the GPU time in microseconds of the fraction of the work asked for. + private static double Run(Item item, Built b, nint arguments, double fraction, out string status) + { + var pool = MetalNative.objc_autoreleasePoolPush(); + try + { + var commands = MetalNative.Send(_queue, MetalNative.Selector("commandBuffer")); + var resources = _synth.Resources(); + fixed (nint* handles = resources) + { + switch (item.Kind) + { + case "cs": + EncodeCompute(item, b, arguments, commands, handles, resources.Length, fraction); + break; + case "ps": + EncodePixel(item, b, arguments, commands, handles, resources.Length, fraction); + break; + default: + EncodeGeometry(item, b, arguments, commands, handles, resources.Length, fraction); + break; + } + } + + MetalNative.SendVoid(commands, MetalNative.Selector("commit")); + MetalNative.SendVoid(commands, MetalNative.Selector("waitUntilCompleted")); + var state = MetalNative.SendUInteger(commands, MetalNative.Selector("status")); + if (state != 4) + { + status = "command buffer " + MetalNative.DescribeError(MetalNative.Send(commands, MetalNative.Selector("error"))); + return 0; + } + + status = "ok"; + return (MetalNative.SendDouble(commands, MetalNative.Selector("GPUEndTime")) - MetalNative.SendDouble(commands, MetalNative.Selector("GPUStartTime"))) * 1e6; + } + catch (Exception exception) + { + status = exception.Message; + return 0; + } + finally + { + MetalNative.objc_autoreleasePoolPop(pool); + } + } + + private static void EncodeCompute(Item item, Built b, nint arguments, nint commands, nint* handles, int count, double fraction) + { + var encoder = MetalNative.Send(commands, MetalNative.Selector("computeCommandEncoder")); + MetalNative.SendVoid(encoder, MetalNative.Selector("setComputePipelineState:"), b.State); + var setBuffer = MetalNative.Selector("setBuffer:offset:atIndex:"); + MetalNative.SendSetBuffer(encoder, setBuffer, arguments, 0, 0); + MetalNative.SendSetBuffer(encoder, setBuffer, _synth.Push, 0, 1); + MetalNative.SendUseResources(encoder, MetalNative.Selector("useResources:count:usage:"), handles, (nuint)count, UsageRead | UsageWrite); + var shader = item.Shader; + var x = Math.Max(shader.ThreadgroupSizeX, 1u); + var y = Math.Max(shader.ThreadgroupSizeY, 1u); + var z = Math.Max(shader.ThreadgroupSizeZ, 1u); + var groups = (nuint)Math.Max(1, (int)(Int("groups", 256) * fraction)); + MetalNative.SendDispatch(encoder, MetalNative.Selector("dispatchThreadgroups:threadsPerThreadgroup:"), + new MtlSize { Width = groups, Height = 1, Depth = 1 }, new MtlSize { Width = x, Height = y, Depth = z }); + MetalNative.SendVoid(encoder, MetalNative.Selector("endEncoding")); + } + + private static nint BeginRender(nint commands, int colors, bool depth, nuint width, nuint height, bool zreject) + { + var pass = MetalNative.Send(MetalNative.Class("MTLRenderPassDescriptor"), MetalNative.Selector("renderPassDescriptor")); + var attachments = MetalNative.Send(pass, MetalNative.Selector("colorAttachments")); + for (var i = 0; i < colors; i++) + { + var attachment = MetalNative.SendAtIndex(attachments, SelObjectAt, (nuint)i); + MetalNative.SendVoid(attachment, MetalNative.Selector("setTexture:"), width == 256 ? _synth.SmallTarget : _synth.Targets[i]); + MetalNative.SendVoidNUInt(attachment, MetalNative.Selector("setLoadAction:"), _clearTargets ? 2u : 0u); + MetalNative.SendVoidNUInt(attachment, MetalNative.Selector("setStoreAction:"), 1); + } + + if (depth || zreject) + { + var attachment = MetalNative.Send(pass, MetalNative.Selector("depthAttachment")); + MetalNative.SendVoid(attachment, MetalNative.Selector("setTexture:"), _synth.Depth); + MetalNative.SendVoidNUInt(attachment, MetalNative.Selector("setLoadAction:"), 2); + MetalNative.SendVoidNUInt(attachment, MetalNative.Selector("setStoreAction:"), 0); + MetalNative.SendVoidDouble(attachment, MetalNative.Selector("setClearDepth:"), zreject ? 0.0 : 1.0); + } + + if (colors == 0 && !depth && !zreject) + { + MetalNative.SendVoidNUInt(pass, MetalNative.Selector("setRenderTargetWidth:"), width); + MetalNative.SendVoidNUInt(pass, MetalNative.Selector("setRenderTargetHeight:"), height); + MetalNative.SendVoidNUInt(pass, MetalNative.Selector("setDefaultRasterSampleCount:"), 1); + } + + return MetalNative.Send(commands, MetalNative.Selector("renderCommandEncoderWithDescriptor:"), pass); + } + + private static void EncodePixel(Item item, Built b, nint arguments, nint commands, nint* handles, int count, double fraction) + { + var width = (nuint)Int("width", 1920); + var height = (nuint)Int("height", 1080); + var colors = ColorCount(item); + var depth = item.Source.Contains("[[depth(", StringComparison.Ordinal); + var zreject = b.Variant.Scenario == "zreject"; + _synth.EnsureTargets(Math.Max(colors, 1), width, height); + var encoder = BeginRender(commands, colors, depth, width, height, zreject); + MetalNative.SendVoid(encoder, MetalNative.Selector("setRenderPipelineState:"), b.State); + if (zreject) + { + MetalNative.SendVoid(encoder, MetalNative.Selector("setDepthStencilState:"), DepthState()); + } + + var setFragment = MetalNative.Selector("setFragmentBuffer:offset:atIndex:"); + MetalNative.SendSetBuffer(encoder, setFragment, arguments, 0, 0); + MetalNative.SendSetBuffer(encoder, setFragment, _synth.Push, 0, 1); + MetalNative.SendUseResourcesStages(encoder, MetalNative.Selector("useResources:count:usage:stages:"), handles, (nuint)count, UsageRead | UsageWrite, StagesAll); + if (fraction < 1) + { + var side = Math.Sqrt(fraction); + MetalNative.SendVoidScissor(encoder, MetalNative.Selector("setScissorRect:"), new MtlScissorRect { X = 0, Y = 0, Width = Math.Max(1, (nuint)(width * side)), Height = Math.Max(1, (nuint)(height * side)) }); + } + + MetalNative.SendDrawPrimitivesInstanced(encoder, MetalNative.Selector("drawPrimitives:vertexStart:vertexCount:instanceCount:"), 3, 0, 6, (nuint)Int("quads", 4)); + MetalNative.SendVoid(encoder, MetalNative.Selector("endEncoding")); + } + + private static nint _depthState; + private static bool _clearTargets; + + private static bool ShareTargets => Arg("check", "0") == "1"; + + [DllImport("/usr/lib/libobjc.A.dylib", EntryPoint = "objc_msgSend")] + private static extern void GetBytes(nint texture, nint selector, nint destination, nuint bytesPerRow, MtlRegion region, nuint level); + + // The colour targets a pixel run wrote, read back (they are shared storage under check=1). + private static string TargetHash(int colors) + { + var hash = new System.IO.Hashing.XxHash64(); + var width = (nuint)Int("width", 1920); + var height = (nuint)Int("height", 1080); + var bytes = new byte[width * height * 4]; + for (var i = 0; i < colors && i < _synth.Targets.Length; i++) + { + fixed (byte* at = bytes) + { + GetBytes(_synth.Targets[i], MetalNative.Selector("getBytes:bytesPerRow:fromRegion:mipmapLevel:"), (nint)at, width * 4, + new MtlRegion { X = 0, Y = 0, Z = 0, Width = width, Height = height, Depth = 1 }, 0); + } + + hash.Append(bytes); + } + + return hash.GetCurrentHashAsUInt64().ToString("X16", CultureInfo.InvariantCulture); + } + + private static nint DepthState() + { + if (_depthState == 0) + { + var descriptor = MetalNative.Send(MetalNative.Send(MetalNative.Class("MTLDepthStencilDescriptor"), SelAlloc), SelInit); + MetalNative.SendVoidNUInt(descriptor, MetalNative.Selector("setDepthCompareFunction:"), 1); + MetalNative.SendVoidBool(descriptor, MetalNative.Selector("setDepthWriteEnabled:"), false); + _depthState = MetalNative.Send(_device, MetalNative.Selector("newDepthStencilStateWithDescriptor:"), descriptor); + } + + return _depthState; + } + + private static int ColorCount(Item item) + { + var count = 0; + foreach (var (_, _, attrs) in ParseStruct(item.Source, "Gen5PsOut")) + { + if (Regex.Match(attrs, @"color\((\d+)\)") is { Success: true } color) + { + count = Math.Max(count, int.Parse(color.Groups[1].Value, CultureInfo.InvariantCulture) + 1); + } + } + + return count; + } + + // A mesh draw whose grid is wider than 65535 threadgroups is dropped by Metal without an error (it ran in 14 us), so the + // vertex count is halved until the grid fits; the count then differs by program, and the groups are in the row's note. + private static uint MeshVertices(Gen5MslMeshInfo info, uint vertices) + { + while (vertices > 3 && Gen5MslMeshLaunch.SubgroupCount(info, Gen5MslMeshInputTopology.TriangleList, vertices) > 30000) + { + vertices = Math.Max(vertices / 2 / 3 * 3, 3u); + } + + return vertices; + } + + // A vertex or mesh program: a fixed vertex count of triangles behind the consumer pixel function. + private static void EncodeGeometry(Item item, Built b, nint arguments, nint commands, nint* handles, int count, double fraction) + { + var vertices = Math.Max((uint)(Int("verts", 393216) * fraction) / 3 * 3, 3u); + if (item.Kind == "ms") + { + vertices = MeshVertices(item.Shader.MeshInfo!, vertices); + } + + var encoder = BeginRender(commands, 1, false, 256, 256, false); + MetalNative.SendVoid(encoder, MetalNative.Selector("setRenderPipelineState:"), b.State); + MetalNative.SendUseResourcesStages(encoder, MetalNative.Selector("useResources:count:usage:stages:"), handles, (nuint)count, UsageRead | UsageWrite, StagesAll); + if (item.Kind == "vs") + { + var set = MetalNative.Selector("setVertexBuffer:offset:atIndex:"); + MetalNative.SendSetBuffer(encoder, set, arguments, 0, 0); + MetalNative.SendSetBuffer(encoder, set, _synth.Push, 0, 1); + for (var slot = VertexSlotBase; slot < 31; slot++) + { + MetalNative.SendSetBuffer(encoder, set, _synth.VertexStream, 0, (nuint)slot); + } + + MetalNative.SendDrawPrimitivesInstanced(encoder, MetalNative.Selector("drawPrimitives:vertexStart:vertexCount:instanceCount:"), 3, 0, vertices, 1); + } + else + { + var info = item.Shader.MeshInfo!; + var set = MetalNative.Selector("setMeshBuffer:offset:atIndex:"); + MetalNative.SendSetBuffer(encoder, set, arguments, 0, 0); + MetalNative.SendSetBuffer(encoder, set, _synth.Push, 0, 1); + var words = new Span((void*)MetalNative.Send(_synth.DrawInfo, MetalNative.Selector("contents")), (int)Gen5MslMeshInfo.DrawInfoDwordCount); + words.Clear(); + Gen5MslMeshLaunch.FillDrawInfo(info, Gen5MslMeshInputTopology.TriangleList, words, vertices, 1, false, 1, 0, 0, 0); + MetalNative.SendSetBuffer(encoder, set, _synth.DrawInfo, 0, info.DrawInfoBufferIndex); + MetalNative.SendSetBuffer(encoder, set, _synth.Indices, 0, info.IndexBufferIndex); + MetalNative.SendSetBuffer(encoder, set, _synth.Meshlets, 0, Gen5MslMeshInfo.MeshletBufferIndex); + if (info.DeviceLds) + { + MetalNative.SendSetBuffer(encoder, set, _synth.LdsPool, 0, Gen5MslMeshInfo.LdsPoolBufferIndex); + } + + var groups = Gen5MslMeshLaunch.SubgroupCount(info, Gen5MslMeshInputTopology.TriangleList, vertices); + MetalNative.SendDrawMesh(encoder, MetalNative.Selector("drawMeshThreadgroups:threadsPerObjectThreadgroup:threadsPerMeshThreadgroup:"), + new MtlSize { Width = Math.Max(groups, 1), Height = 1, Depth = 1 }, + new MtlSize { Width = 1, Height = 1, Depth = 1 }, + new MtlSize { Width = Math.Max(info.ThreadsPerThreadgroup, 1), Height = 1, Depth = 1 }); + } + + MetalNative.SendVoid(encoder, MetalNative.Selector("endEncoding")); + } +} diff --git a/tools/SharpEmu.Tools.ShaderCompileBench/GpuBenchReport.cs b/tools/SharpEmu.Tools.ShaderCompileBench/GpuBenchReport.cs new file mode 100644 index 000000000..f7466bc54 --- /dev/null +++ b/tools/SharpEmu.Tools.ShaderCompileBench/GpuBenchReport.cs @@ -0,0 +1,276 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using System.Globalization; +using System.Text; + +// mode=gpureport: ranks the shaders of one or more mode=gpu runs and tabulates each variant against +// its baseline. Arguments: csv= static= out= top=20 floor=20 +// hot= production pairs whose synthetic cost is reported on their own, +// share being their percentage of profiled GPU time (an empty prefix leaves that half out). +// floor is the smallest baseline time (microseconds) a shader needs to count in a ratio statistic: below it +// the fixed cost of a command buffer is most of the number. +internal static class GpuBenchReport +{ + private sealed record Sample(string Phase, int Index, string Id, string Kind, string Hash, string Variant, string Status, double Median); + + public static int Run(Func arg) + { + var inputs = arg("csv", "").Split(',', StringSplitOptions.RemoveEmptyEntries); + var outDir = arg("out", "."); + var top = int.Parse(arg("top", "20"), CultureInfo.InvariantCulture); + var floor = double.Parse(arg("floor", "20"), CultureInfo.InvariantCulture); + Directory.CreateDirectory(outDir); + var samples = new List(); + foreach (var path in inputs) + { + foreach (var line in File.ReadLines(path).Skip(1)) + { + var f = SplitCsv(line); + if (f.Count < 13) + { + continue; + } + + samples.Add(new Sample(f[0], int.Parse(f[1], CultureInfo.InvariantCulture), f[2], f[3], f[4], f[5], f[12], + double.Parse(f[7], CultureInfo.InvariantCulture))); + } + } + + var statics = new Dictionary(); + string staticHeader = ""; + if (arg("static", "") is { Length: > 0 } staticPath) + { + var lines = File.ReadAllLines(staticPath); + staticHeader = lines[0]; + foreach (var line in lines.Skip(1)) + { + var f = line.Split(','); + statics[int.Parse(f[0], CultureInfo.InvariantCulture)] = f; + } + } + + // The deep (more repetitions) number replaces the full one where there is one. + double? Value(int index, string variant, string phase) => + samples.Where(s => s.Index == index && s.Variant == variant && s.Phase == phase && s.Status == "ok").Select(s => (double?)s.Median).FirstOrDefault(); + var byIndex = samples.GroupBy(s => s.Index).ToDictionary(g => g.Key, g => g.First()); + double? Best(int index, string variant) => Value(index, variant, "deep") ?? Value(index, variant, "full"); + var ranked = byIndex.Keys.Select(i => (Index: i, Us: Best(i, "base"))).Where(x => x.Us.HasValue).OrderByDescending(x => x.Us!.Value).ToList(); + var variants = samples.Select(s => s.Variant).Distinct().ToList(); + + var md = new StringBuilder(); + md.AppendLine("> Synthetic GPU time: every shader runs on zero (or patterned) buffers, blank textures and a fixed workload. Zero or patterned inputs change control flow"); + md.AppendLine("> (trip counts, early-outs, touched memory), so these numbers are a cost proxy for ranking and for A/B of one shader against its own variant in the same"); + md.AppendLine("> process; they are not production times. Mesh programs run with a per-program vertex count (grid <= 30000 groups), so they do not compare across programs at equal vertices."); + md.AppendLine(); + md.AppendLine("## Synthetic GPU time by shader kind (base variant)"); + md.AppendLine(); + md.AppendLine("| kind | shaders | ran | failed | n/a | sum us | median us | p90 us | max us |"); + md.AppendLine("|---|---:|---:|---:|---:|---:|---:|---:|---:|"); + foreach (var kind in new[] { "cs", "vs", "ms", "ps" }) + { + var all = byIndex.Values.Where(s => s.Kind == kind).Select(s => s.Index).ToList(); + var ok = all.Select(i => Best(i, "base")).Where(v => v.HasValue).Select(v => v!.Value).Order().ToList(); + if (all.Count == 0) + { + continue; + } + + var failed = all.Count(i => !Best(i, "base").HasValue); + md.AppendLine(string.Create(CultureInfo.InvariantCulture, + $"| {kind} | {all.Count} | {ok.Count} | {failed} | 0 | {ok.Sum():F0} | {(ok.Count > 0 ? ok[ok.Count / 2] : 0):F0} | {(ok.Count > 0 ? ok[Math.Min(ok.Count - 1, (int)(ok.Count * 0.9))] : 0):F0} | {(ok.Count > 0 ? ok[^1] : 0):F0} |")); + } + + md.AppendLine(); + var failures = samples.Where(s => s.Phase == "full" && s.Variant == "base" && s.Status != "ok" && s.Status != "n/a").ToList(); + if (failures.Count > 0) + { + md.AppendLine($"Failed to run (base variant): {failures.Count}. By status: " + string.Join(", ", failures.GroupBy(s => s.Status).Select(g => $"{g.Key} x{g.Count()}"))); + md.AppendLine(); + } + + md.AppendLine($"## Top {top} by synthetic GPU time (base)"); + md.AppendLine(); + md.AppendLine("| # | shader | kind | us | msl kB | tex | loads | loops | branches | barriers | tg | notes |"); + md.AppendLine("|---:|---|---|---:|---:|---:|---:|---:|---:|---:|---|---|"); + var rank = 0; + foreach (var (index, us) in ranked.Take(top)) + { + var st = statics.GetValueOrDefault(index); + string S(int column) => st is null ? "" : st[column]; + md.AppendLine(string.Create(CultureInfo.InvariantCulture, + $"| {++rank} | {byIndex[index].Id} | {byIndex[index].Kind} | {us!.Value:F0} | {(st is null ? 0 : double.Parse(S(4), CultureInfo.InvariantCulture) / 1024):F0} | {S(6)} | {S(9)} | {S(11)} | {S(12)} | {S(13)} | {S(18)} | {Note(st)} |")); + } + + md.AppendLine(); + var csv = new StringBuilder("rank,idx,id,kind,hash,base_us"); + foreach (var variant in variants.Where(v => v != "base")) + { + csv.Append(',').Append(variant).Append("_us"); + } + + csv.AppendLine(staticHeader.Length > 0 ? "," + string.Join(',', staticHeader.Split(',').Skip(4)) : ""); + rank = 0; + foreach (var (index, us) in ranked) + { + csv.Append(CultureInfo.InvariantCulture, $"{++rank},{index},{byIndex[index].Id},{byIndex[index].Kind},{byIndex[index].Hash},{us!.Value:F1}"); + foreach (var variant in variants.Where(v => v != "base")) + { + csv.Append(',').Append(Best(index, variant) is { } v ? v.ToString("F1", CultureInfo.InvariantCulture) : ""); + } + + csv.AppendLine(statics.TryGetValue(index, out var st) ? "," + string.Join(',', st.Skip(4)) : ""); + } + + File.WriteAllText(Path.Combine(outDir, "ranking.csv"), csv.ToString()); + + // ---- variant tables ----------------------------------------------------------------- + md.AppendLine("## Variant A/B"); + md.AppendLine(); + md.AppendLine($"Ratios use shaders whose baseline is at least {floor:F0} us. `all` is every such shader (full phase, 5 reps); `top30` is the 30 slowest base shaders (deep phase, more reps). Delta is the change of the summed time, negative is faster; `geo` is the geometric-mean ratio per shader; `better/worse` count shaders more than 3% faster/slower. `early`/`earlyd` are compared with `basez`/`basezd` (same depth-rejecting scenario), every other variant with `base`."); + md.AppendLine(); + md.AppendLine("| variant | scope | n | sum base us | sum variant us | delta | geo ratio | better | worse |"); + md.AppendLine("|---|---|---:|---:|---:|---:|---:|---:|---:|"); + var top30 = ranked.Take(30).Select(x => x.Index).ToHashSet(); + var variantCsv = new StringBuilder("variant,scope,n,sum_base_us,sum_variant_us,delta_pct,geo_ratio,better,worse\n"); + foreach (var variant in variants.Where(v => v is not ("base" or "basez" or "basezd"))) + { + var reference = variant == "early" ? "basez" : variant == "earlyd" ? "basezd" : "base"; + foreach (var (scope, filter) in new (string, Func)[] + { + ("all", _ => true), ("top30", i => top30.Contains(i)), ("vs", i => byIndex[i].Kind == "vs"), ("ms", i => byIndex[i].Kind == "ms"), + ("ps", i => byIndex[i].Kind == "ps"), ("cs", i => byIndex[i].Kind == "cs"), + }) + { + var pairs = new List<(double Base, double Value)>(); + foreach (var index in byIndex.Keys.Where(filter)) + { + // The top-30 scope compares deep with deep where both exist, the others full with full. + var phase = scope == "top30" ? "deep" : "full"; + var baseline0 = scope == "top30" ? Best(index, reference) : Value(index, reference, phase); + var value0 = scope == "top30" ? Best(index, variant) : Value(index, variant, phase); + if (baseline0 is { } baseline && value0 is { } value && baseline >= floor) + { + pairs.Add((baseline, value)); + } + } + + if (pairs.Count == 0) + { + continue; + } + + var sumBase = pairs.Sum(p => p.Base); + var sumValue = pairs.Sum(p => p.Value); + var geo = Math.Exp(pairs.Average(p => Math.Log(p.Value / p.Base))); + var line = string.Create(CultureInfo.InvariantCulture, + $"| {variant} | {scope} | {pairs.Count} | {sumBase:F0} | {sumValue:F0} | {(sumValue / sumBase - 1) * 100:+0.0;-0.0}% | {geo:F3} | {pairs.Count(p => p.Value < p.Base * 0.97)} | {pairs.Count(p => p.Value > p.Base * 1.03)} |"); + md.AppendLine(line); + variantCsv.AppendLine(string.Create(CultureInfo.InvariantCulture, + $"{variant},{scope},{pairs.Count},{sumBase:F1},{sumValue:F1},{(sumValue / sumBase - 1) * 100:F2},{geo:F4},{pairs.Count(p => p.Value < p.Base * 0.97)},{pairs.Count(p => p.Value > p.Base * 1.03)}")); + } + } + + File.WriteAllText(Path.Combine(outDir, "variants.csv"), variantCsv.ToString()); + + // ---- hot pairs ---------------------------------------------------------------------- + if (arg("hot", "") is { Length: > 0 } hot) + { + md.AppendLine(); + md.AppendLine("## Production hot list (synthetic cost of each half, variant deltas of vs+ps summed)"); + md.AppendLine(); + var shown = variants.Where(v => v is not ("base" or "basez" or "basezd" or "early" or "earlyd")).ToList(); + md.AppendLine("| pair | prod share | vs us | ps us | " + string.Join(" | ", shown) + " |"); + md.AppendLine("|---|---:|---:|---:|" + string.Concat(shown.Select(_ => "---:|"))); + var weighted = shown.ToDictionary(v => v, _ => 0.0); + foreach (var entry in hot.Split(';', StringSplitOptions.RemoveEmptyEntries)) + { + var parts = entry.Split('|'); + var share = double.Parse(parts[3], CultureInfo.InvariantCulture); + var members = byIndex.Values.Where(s => + (parts[1].Length > 0 && s.Kind is "vs" or "ms" && s.Hash.StartsWith("0x" + parts[1], StringComparison.OrdinalIgnoreCase)) || + (parts[2].Length > 0 && s.Kind is "ps" or "cs" && s.Hash.StartsWith("0x" + parts[2], StringComparison.OrdinalIgnoreCase))).Select(s => s.Index).ToList(); + double Sum(string variant, params string[] kinds) => + members.Where(i => kinds.Contains(byIndex[i].Kind) && Best(i, variant).HasValue && Best(i, "base").HasValue).Sum(i => Best(i, variant)!.Value); + var vs = Sum("base", "vs", "ms", "cs"); + var ps = Sum("base", "ps"); + var cells = new List(); + foreach (var variant in shown) + { + var matched = members.Where(i => Best(i, variant).HasValue && Best(i, "base").HasValue).ToList(); + var baseline = matched.Sum(i => Best(i, "base")!.Value); + var value = matched.Sum(i => Best(i, variant)!.Value); + var delta = baseline > 0 ? (value / baseline - 1) * 100 : 0; + weighted[variant] += delta * share / 100; + cells.Add(string.Create(CultureInfo.InvariantCulture, $"{delta:+0.0;-0.0}%")); + } + + md.AppendLine(string.Create(CultureInfo.InvariantCulture, $"| {parts[0]} ({members.Count} items) | {share:F1}% | {vs:F0} | {ps:F0} | {string.Join(" | ", cells)} |")); + } + + md.AppendLine(string.Create(CultureInfo.InvariantCulture, + $"| share-weighted change of profiled GPU time (these entries only) | | | | {string.Join(" | ", shown.Select(v => $"{weighted[v]:+0.00;-0.00}%"))} |")); + } + + File.WriteAllText(Path.Combine(outDir, "tables.md"), md.ToString()); + Console.WriteLine(md.ToString()); + return 0; + } + + private static string Note(string[]? st) + { + if (st is null) + { + return ""; + } + + var notes = new List(); + if (st[15] == "1") + { + notes.Add("discard"); + } + + if (st[16] == "1") + { + notes.Add("depth-out"); + } + + if (st[17] == "1") + { + notes.Add("early-z"); + } + + if (st[10] != "0") + { + notes.Add("stores/atomics " + st[10]); + } + + return string.Join(", ", notes); + } + + private static List SplitCsv(string line) + { + var fields = new List(); + var text = new StringBuilder(); + var quoted = false; + foreach (var c in line) + { + if (c == '"') + { + quoted = !quoted; + } + else if (c == ',' && !quoted) + { + fields.Add(text.ToString()); + text.Clear(); + } + else + { + text.Append(c); + } + } + + fields.Add(text.ToString()); + return fields; + } +} diff --git a/tools/SharpEmu.Tools.ShaderCompileBench/Program.cs b/tools/SharpEmu.Tools.ShaderCompileBench/Program.cs index 0a8faaeb0..6b58a3b32 100644 --- a/tools/SharpEmu.Tools.ShaderCompileBench/Program.cs +++ b/tools/SharpEmu.Tools.ShaderCompileBench/Program.cs @@ -15,6 +15,8 @@ // Arguments are key=value: // dir=/metal-v2> (required; the list loader prunes files) // mode=translate|compute|libraries (default compute) +// mode=gpu synthetic GPU-time benchmark of every translated shader (see GpuBench.cs) +// mode=gpureport csv= out= ranking and variant A/B tables (GpuBenchReport.cs) // mode=analysis hash= reports graphics IR opportunities only // workers=4 tworkers=4 priority=normal|below limit=0 order=list|bigfirst // salt= prefixes every source with a comment, so the front end misses its cache @@ -72,6 +74,11 @@ private static int Main(string[] argv) return 0; } + if (Arg("mode", "") == "gpureport") + { + return GpuBenchReport.Run(Arg); + } + var dir = Arg("dir", ""); if (dir.Length == 0) { @@ -82,6 +89,11 @@ private static int Main(string[] argv) MetalNative.EnsureFrameworksLoaded(); var device = MetalNative.MTLCreateSystemDefaultDevice(); var mode = Arg("mode", "compute"); + if (mode == "gpu") + { + return GpuBench.Run(dir, Arg); + } + var listener = Arg("alloctypes", "") == "1" ? new AllocTypeListener() : null; var limit = int.Parse(Arg("limit", "0"), CultureInfo.InvariantCulture); var rows = new List(); From a6b43e64fa7ee8f6f63382a2c745a79336d455a1 Mon Sep 17 00:00:00 2001 From: kuba Date: Wed, 28 Oct 2026 21:04:10 +0100 Subject: [PATCH 73/82] [GPU] Held label copies land through the scatter dispatch across up to 512 buffers instead of 32. A command buffer's held occlusion snapshots read from one result buffer per render pass, so a submission of more than 32 passes declined the scatter and landed every copy as its own blit (about 5 us per draw in the production bench variation). Buffer lookups remember the last hit, since consecutive copies share their buffers. --- .../V2/MetalRenderHost.LabelScatter.cs | 41 +++++++++++++++---- .../Scenes/NativeOcclusionSceneTests.cs | 32 +++++++++++++++ 2 files changed, 65 insertions(+), 8 deletions(-) diff --git a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.LabelScatter.cs b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.LabelScatter.cs index 6fa51a9ba..6c0ee5afd 100644 --- a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.LabelScatter.cs +++ b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.LabelScatter.cs @@ -22,7 +22,10 @@ internal sealed partial class MetalRenderHost // Fewer copies than this cost less as blits than a compute encoder does. private const int LabelScatterMinCopies = 16; - private const int LabelScatterMaxBuffers = 32; + // A command buffer's held copies read the result buffer of every render pass that dumped occlusion counts + // into them (one per pass, up to NativeOcclusionBatchBudget of them), so a batch spans far more than a few + // buffers; declined, every copy of it is a blit of its own. + private const int LabelScatterMaxBuffers = 512; private const int LabelScatterThreads = 64; private const string LabelScatterSource = """ @@ -63,11 +66,12 @@ private unsafe bool TryEncodeLabelScatter(nint commandBuffer, ReadOnlySpan= 0 || - !NoteLabelScatterBuffer(copy.DestinationBuffer, sources, ref buffers)) + if (FindLabelScatterBuffer(copy.DestinationBuffer, 0, sources, ref sourceHint) >= 0 || + !NoteLabelScatterBuffer(copy.DestinationBuffer, sources, ref buffers, ref destinationHint)) { _labelScatterDeclined++; return false; @@ -98,10 +103,12 @@ private unsafe bool TryEncodeLabelScatter(nint commandBuffer, ReadOnlySpan= 0) + if ((uint)(hint - first) < (uint)count && _labelScatterBuffers[hint] == buffer) + { + return hint; + } + + var index = Array.IndexOf(_labelScatterBuffers, buffer, first, count); + if (index >= 0) + { + hint = index; + } + + return index; + } + + private bool NoteLabelScatterBuffer(nint buffer, int first, ref int count, ref int hint) + { + if (FindLabelScatterBuffer(buffer, first, count - first, ref hint) >= 0) { return true; } diff --git a/tests/SharpEmu.Gpu.Metal.Tests/Scenes/NativeOcclusionSceneTests.cs b/tests/SharpEmu.Gpu.Metal.Tests/Scenes/NativeOcclusionSceneTests.cs index ccf0a60fd..a91af1984 100644 --- a/tests/SharpEmu.Gpu.Metal.Tests/Scenes/NativeOcclusionSceneTests.cs +++ b/tests/SharpEmu.Gpu.Metal.Tests/Scenes/NativeOcclusionSceneTests.cs @@ -281,6 +281,38 @@ public void MoreThanOneBatchOfDumpsPreservesEverySnapshot() Assert.Equal(Ready | ((ulong)Size * Size * (i < 4096 ? 1UL : 2UL)), words[i * 32]); } + // Every pass dumps into a result buffer of its own, so one command buffer's held snapshots read from as many + // buffers as it has passes: past the scatter's old limit of 32 they still land through one dispatch. + [Fact] + public void DumpsFromMoreThanThirtyTwoPassesLandThroughOneScatter() + { + using var rig = CreateRig(); + if (rig is null) return; + rig.Host.NativeOcclusionEnabled = true; + const int passes = 48; + var targets = new[] { rig.ColorTarget(Size, Size), rig.ColorTarget(Size, Size) }; + var results = rig.Memory.Allocate(passes * 256, 64); + var before = rig.Host.LabelScatterTotals; + var pm4 = new ScenePm4(); + for (var i = 0; i < passes; i++) + { + var state = new SceneState(Size, Size).ColorTarget(0, targets[i % 2], Size, Size); + state.Context[Cx.DbCountControl] = 1u << 8; + rig.DrawMesh(pm4, state, SceneDraws.Quad(0.1f, 1, 0, 0)); + Dump(pm4, results + (ulong)i * 256); + } + rig.Submit(pm4); + var words = rig.Read(results, passes * 32); + for (var i = 0; i < passes; i++) + { + Assert.Equal(Ready | ((ulong)Size * Size * (ulong)(i + 1)), words[i * 32]); + for (var block = 1; block < 16; block++) Assert.Equal(Ready, words[i * 32 + block * 2]); + } + var after = rig.Host.LabelScatterTotals; + Assert.True(after.Batches > before.Batches, $"The dumps did not land through the scatter: {before} -> {after}."); + Assert.Equal(before.Declined, after.Declined); + } + [Fact] public void BatchBudgetWaitsForCompletedStorageWithoutLosingDrawsOrDumps() { From df118f2266eaa29fa61a6aef7cf30590d19d6d48 Mon Sep 17 00:00:00 2001 From: kuba Date: Wed, 28 Oct 2026 21:04:35 +0100 Subject: [PATCH 74/82] [GPU] An occlusion dump queues behind the deferred draws instead of draining them. A drain at every dump left the program resolver one draw of lead, so each draw resolved on the worker (about 1.2 us of waiting plus 2.3 us of caller resolution per draw with a dump after every draw). The dump now takes the same ordered slot a deferred barrier does, runs in issue order at the next drain, and keeps the draws batched. Dumps posted from the command thread still record where they stand. SHARPEMU_METAL_V2_DEFER_OCCLUSION_DUMPS=0 drains at every dump again. --- .../V2/MetalRenderHost.ConcurrentResolve.cs | 2 +- .../V2/MetalRenderHost.Occlusion.cs | 34 ++++++++++++++++--- .../V2/MetalRenderHost.Rendering.cs | 4 +-- 3 files changed, 33 insertions(+), 7 deletions(-) diff --git a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.ConcurrentResolve.cs b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.ConcurrentResolve.cs index 77d7f1a21..0b3a70c53 100644 --- a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.ConcurrentResolve.cs +++ b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.ConcurrentResolve.cs @@ -53,7 +53,7 @@ internal sealed partial class MetalRenderHost : IConcurrentResolveHost // a completion that reaches an entry point from another thread leaves the draws to it. private void DrainDraws([System.Runtime.CompilerServices.CallerMemberName] string caller = "") { - if (_translation is { HasPendingDraws: true } translation && _relay.IsGpuQueueThread) + if (_translation is { HasPendingDraws: true, IsDraining: false } translation && _relay.IsGpuQueueThread) { _drains++; if (MetalV2Diagnostics.CountDrainCallers) diff --git a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.Occlusion.cs b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.Occlusion.cs index d38b8ba51..855e1f79c 100644 --- a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.Occlusion.cs +++ b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.Occlusion.cs @@ -17,6 +17,16 @@ internal sealed partial class MetalRenderHost internal bool CoalesceOcclusionDumps { get; set; } = SharpEmu.ShaderCompiler.PerfSwitches.Enabled("SHARPEMU_METAL_V2_COALESCE_OCCLUSION_DUMPS"); + // SHARPEMU_METAL_V2_DEFER_OCCLUSION_DUMPS=0 drains the deferred draws at every dump again. + internal bool DeferOcclusionDumps { get; set; } = + SharpEmu.ShaderCompiler.PerfSwitches.Enabled("SHARPEMU_METAL_V2_DEFER_OCCLUSION_DUMPS"); + + private static ulong DumpRangeBytes(uint instanceMask, uint strideBytes) + { + var mask = instanceMask & 0xFF_FFFFu; + return mask == 0 ? 0 : checked((ulong)System.Numerics.BitOperations.Log2(mask) * strideBytes + sizeof(ulong)); + } + // A dump covers at most 24 blocks of at most 32 bytes; past this the blocks are not one result buffer. private const ulong MaxCoalescedDumpBytes = 1024; @@ -215,12 +225,28 @@ public bool DumpOcclusionCounter(int queueId, ulong address, uint instanceMask, if (OnCommandThread) { // No CPU result is needed here. Publish the pending guest range before - // returning so later producer reads cannot pass the queued GPU dump. - var mask = instanceMask & 0xFF_FFFFu; - var size = mask == 0 ? 0 : checked((ulong)System.Numerics.BitOperations.Log2(mask) * strideBytes + sizeof(ulong)); - _commandThread!.PostRange(() => DumpOcclusionCounter(queueId, address, instanceMask, strideBytes), address, size); + // returning so later producer reads cannot pass the queued GPU dump. The worker + // records it where it stands (deferring is for the worker's own stream): a producer + // read of the range is held only until this item has run. + _commandThread!.PostRange(() => DumpOcclusionCounter(queueId, address, instanceMask, strideBytes, defer: false), address, + DumpRangeBytes(instanceMask, strideBytes)); + return true; + } + + return DumpOcclusionCounter(queueId, address, instanceMask, strideBytes, defer: true); + } + + private bool DumpOcclusionCounter(int queueId, ulong address, uint instanceMask, uint strideBytes, bool defer) + { + // A dump only orders GPU work, so behind deferred draws it joins their queue rather than draining + // them: the program resolver keeps its lead over the draws and they record together at the next drain. + if (defer && DeferOcclusionDumps && TryDeferBehindDraws( + () => DumpOcclusionCounter(queueId, address, instanceMask, strideBytes, defer: false), address, + DumpRangeBytes(instanceMask, strideBytes), always: true)) + { return true; } + DrainDraws(); ValidateOcclusionQueue(queueId); InitializeNativeOcclusion(); diff --git a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.Rendering.cs b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.Rendering.cs index 3666aea10..96dd681f1 100644 --- a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.Rendering.cs +++ b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.Rendering.cs @@ -2455,9 +2455,9 @@ private void EmitOrderedGlobalBarrier() // A barrier or release only orders GPU commands, so behind deferred draws it joins their queue // instead of draining them: a drain at every barrier left the program resolver, which only // starts a draw's programs once it is submitted, no lead over the draws the drain runs. - private bool TryDeferBehindDraws(Action ordered, ulong writeAddress = 0, ulong writeSize = 0) + private bool TryDeferBehindDraws(Action ordered, ulong writeAddress = 0, ulong writeSize = 0, bool always = false) { - if (!MetalV2Diagnostics.DeferBarriers || _translation is not { HasPendingDraws: true, IsDraining: false } translation || + if (!(always || MetalV2Diagnostics.DeferBarriers) || _translation is not { HasPendingDraws: true, IsDraining: false } translation || !_relay.IsGpuQueueThread) { return false; From 5b0f09c0aac923f2a77d49794a47af7f923ec7dd Mon Sep 17 00:00:00 2001 From: kuba Date: Wed, 28 Oct 2026 21:41:05 +0100 Subject: [PATCH 75/82] [GPU] An occlusion batch reads its counts buffer's contents pointer once instead of once per dump. InitializeOcclusionSlot sent a contents message for every dump; a shared buffer's pointer holds for its life, so the batch keeps it from creation. --- .../SharpEmu.Gpu.Metal/V2/MetalRenderHost.Occlusion.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.Occlusion.cs b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.Occlusion.cs index 855e1f79c..d9c3b9522 100644 --- a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.Occlusion.cs +++ b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.Occlusion.cs @@ -45,6 +45,8 @@ private static ulong DumpRangeBytes(uint instanceMask, uint strideBytes) private sealed class OcclusionBatch(nint counts, nint results) { internal readonly nint Counts = counts, Results = results; + // A shared buffer's contents pointer holds for the buffer's life: one message send at creation, not one per dump. + internal readonly nint CountsContents = MetalNative.Send(counts, SelOcclusionContents); internal ulong Serial; internal int Slot, Queue; internal int VisibilityMode = -1, VisibilitySlot = -1; @@ -173,7 +175,7 @@ private unsafe void NativeOcclusionAttach(nint descriptor) private unsafe void InitializeOcclusionSlot(OcclusionBatch batch) { - ((ulong*)MetalNative.Send(batch.Counts, SelOcclusionContents))[batch.Slot] = 0; + ((ulong*)batch.CountsContents)[batch.Slot] = 0; _oqInitializedSlots++; } From cbf056473cf147f50886dc45545f0777c40d9766 Mon Sep 17 00:00:00 2001 From: kuba Date: Thu, 29 Oct 2026 06:44:37 +0100 Subject: [PATCH 76/82] [GPU] Draws, CP writes and occlusion dumps allocate less on the worker. A method that holds a capturing lambda anywhere allocates its closure on entry, on every path: the CP write hop, the in-order fill, the occlusion dump and the shader-write release now keep their lambdas in helpers, so the paths that do not post allocate nothing. A resource specialization creates its lists on first use, a failed materialization no longer builds one to throw away, and up to 64 sampler specializations live on the stack. In the worker frame bench this cut a draws-only submission from 666 to 651 KB and a dump from 351 to 305 bytes per packet, at the same timing; Silent Hill fps did not move measurably. The WAITS line reports the guest-memory window memo hits and misses. --- .../Resources/ResourceMaterializer.cs | 7 +++- .../Resources/ResourceSpecialization.cs | 21 +++++++--- .../V2/MetalCommandStream.cs | 38 ++++++++++++------- .../V2/MetalRenderHost.Occlusion.cs | 23 ++++++++--- .../SharpEmu.Gpu.Metal/V2/MetalRenderHost.cs | 2 + 5 files changed, 64 insertions(+), 27 deletions(-) diff --git a/src/SharpEmu.ShaderCompiler/Resources/ResourceMaterializer.cs b/src/SharpEmu.ShaderCompiler/Resources/ResourceMaterializer.cs index e8e05efdc..28c4c3ff2 100644 --- a/src/SharpEmu.ShaderCompiler/Resources/ResourceMaterializer.cs +++ b/src/SharpEmu.ShaderCompiler/Resources/ResourceMaterializer.cs @@ -1299,7 +1299,9 @@ private static int SamplerCompareFunction(ShaderResourceInfo info, MaterializedS private static bool SpecializeSamplers(ShaderResourceInfo info, MaterializedSnapshot snapshot, out List samplers) { samplers = []; - var specialized = new SamplerSpecialization[info.Samplers.Count]; + Span specialized = info.Samplers.Count <= 64 + ? stackalloc SamplerSpecialization[info.Samplers.Count] + : new SamplerSpecialization[info.Samplers.Count]; var any = false; for (var index = 0; index < info.Samplers.Count && index < snapshot.Samplers.Length; index++) { @@ -1363,7 +1365,8 @@ private static bool BuildSpecialization( using var specializationProfile = ResourceMaterializationProfile.Measure(ResourceMaterializationProfile.Phase.Specialization); failure = ResourceMaterializationFailure.Other; specializedSnapshot = snapshot; - specialization = new ResourceSpecialization(); + // Read only on success, which assigns it at the end. + specialization = null!; var info = plan.Info; var imageCount = info.Images.Count; var mappingWordCount = 0; diff --git a/src/SharpEmu.ShaderCompiler/Resources/ResourceSpecialization.cs b/src/SharpEmu.ShaderCompiler/Resources/ResourceSpecialization.cs index 36f84dedc..051140ab7 100644 --- a/src/SharpEmu.ShaderCompiler/Resources/ResourceSpecialization.cs +++ b/src/SharpEmu.ShaderCompiler/Resources/ResourceSpecialization.cs @@ -48,16 +48,25 @@ public sealed class ResourceSpecialization : IEquatable { // The plan's base buffer count; Buffers beyond it are materialized candidates. public int BaseBufferCount { get; init; } - public List Buffers { get; init; } = []; - public List Images { get; init; } = []; - public List BufferCandidateTables { get; init; } = []; - public List ImageDescriptorGroups { get; init; } = []; - public List SamplerDescriptorGroups { get; init; } = []; + // The lists start empty on first read rather than in the constructor: every draw builds one + // of these with all of them set, and eager initializers allocated six lists it replaced. + public List Buffers { get => _buffers ??= []; init => _buffers = value; } + public List Images { get => _images ??= []; init => _images = value; } + public List BufferCandidateTables { get => _bufferCandidateTables ??= []; init => _bufferCandidateTables = value; } + public List ImageDescriptorGroups { get => _imageDescriptorGroups ??= []; init => _imageDescriptorGroups = value; } + public List SamplerDescriptorGroups { get => _samplerDescriptorGroups ??= []; init => _samplerDescriptorGroups = value; } // One entry per plan sampler, or none when every entry would be the default, so the // common case keeps the specialization (and the cache keys built from it) it had // before samplers were specialized. - public List Samplers { get; init; } = []; + public List Samplers { get => _samplers ??= []; init => _samplers = value; } + + private List? _buffers; + private List? _images; + private List? _bufferCandidateTables; + private List? _imageDescriptorGroups; + private List? _samplerDescriptorGroups; + private List? _samplers; public bool Equals(ResourceSpecialization? other) => other is not null && BaseBufferCount == other.BaseBufferCount && Buffers.SequenceEqual(other.Buffers) && diff --git a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalCommandStream.cs b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalCommandStream.cs index 1155c4f24..c13d5e44e 100644 --- a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalCommandStream.cs +++ b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalCommandStream.cs @@ -971,6 +971,23 @@ private ulong EncodeOrderedCopies( internal bool QueueCommandProcessorWrites { get; set; } = Environment.GetEnvironmentVariable("SHARPEMU_METAL_V2_QUEUE_CP_WRITES") != "0"; + // The hops below are their own methods so the worker's own calls build no closure: a lambda + // over the parameters makes the compiler allocate its capture at entry of the method that holds it. + private void PostCommandProcessorWrite(ulong address, byte[] queued, string kind) + { + if (QueueCommandProcessorWrites) + { + // Reads overlapping this queued write wait until the worker installed + // its overlay. Unrelated packets can continue; prefetch's non-draw gate + // still orders future resource reads after the write. + _commandThread!.PostRange(() => RecordCommandProcessorWrite(address, queued, kind), address, (ulong)queued.Length); + } + else + { + RunTagged(() => RecordCommandProcessorWrite(address, queued, kind), nameof(RecordCommandProcessorWrite)); + } + } + public void RecordCommandProcessorWrite(ulong address, ReadOnlySpan source, string kind) { if (OnCommandThread) @@ -978,18 +995,7 @@ public void RecordCommandProcessorWrite(ulong address, ReadOnlySpan source if (source.IsEmpty) return; if (address == 0 || (ulong)source.Length > ulong.MaxValue - address) throw Fatal($"A command-processor write has an invalid range: address=0x{address:X16} bytes={source.Length}."); - var queued = source.ToArray(); - if (QueueCommandProcessorWrites) - { - // Reads overlapping this queued write wait until the worker installed - // its overlay. Unrelated packets can continue; prefetch's non-draw gate - // still orders future resource reads after the write. - _commandThread!.PostRange(() => RecordCommandProcessorWrite(address, queued, kind), address, (ulong)queued.Length); - } - else - { - RunTagged(() => RecordCommandProcessorWrite(address, queued, kind)); - } + PostCommandProcessorWrite(address, source.ToArray(), kind); return; } @@ -1284,12 +1290,15 @@ public bool TryWriteGuest(ulong address, ReadOnlySpan source, string kind // host fast path, which writes as parsed, is what the ordered path leaves out. private const ulong MaxStagedCommandProcessorFillBytes = 4096; + private void PostFillBufferInOrder(ulong address, ulong size, uint value) => + _commandThread!.PostRange(() => FillBufferInOrder(address, size, value), address, size); + /// A DMA fill of guest memory that lands in record order with the labels. public void FillBufferInOrder(ulong address, ulong size, uint value) { if (OnCommandThread) { - _commandThread!.PostRange(() => FillBufferInOrder(address, size, value), address, size); + PostFillBufferInOrder(address, size, value); return; } @@ -2575,7 +2584,7 @@ public void OnQueueReset(int queueId) public void ReleaseShaderWrites() { // Ordered behind the draws before it, and nothing the stream waits for: queued, not waited on. - if (PostToWorker(ReleaseShaderWrites)) + if (PostToWorker(_releaseShaderWrites ??= ReleaseShaderWrites)) { return; } @@ -2589,6 +2598,7 @@ public void ReleaseShaderWrites() _translation?.ReleaseShaderWrites(); } + private Action? _releaseShaderWrites; private Action? _releaseOrderedShaderWrites; /// diff --git a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.Occlusion.cs b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.Occlusion.cs index d9c3b9522..f219b7a73 100644 --- a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.Occlusion.cs +++ b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.Occlusion.cs @@ -230,25 +230,38 @@ public bool DumpOcclusionCounter(int queueId, ulong address, uint instanceMask, // returning so later producer reads cannot pass the queued GPU dump. The worker // records it where it stands (deferring is for the worker's own stream): a producer // read of the range is held only until this item has run. - _commandThread!.PostRange(() => DumpOcclusionCounter(queueId, address, instanceMask, strideBytes, defer: false), address, - DumpRangeBytes(instanceMask, strideBytes)); + PostOcclusionDump(queueId, address, instanceMask, strideBytes); return true; } return DumpOcclusionCounter(queueId, address, instanceMask, strideBytes, defer: true); } + private void PostOcclusionDump(int queueId, ulong address, uint instanceMask, uint strideBytes) => + _commandThread!.PostRange(() => DumpOcclusionCounterNow(queueId, address, instanceMask, strideBytes), address, + DumpRangeBytes(instanceMask, strideBytes)); + private bool DumpOcclusionCounter(int queueId, ulong address, uint instanceMask, uint strideBytes, bool defer) { // A dump only orders GPU work, so behind deferred draws it joins their queue rather than draining // them: the program resolver keeps its lead over the draws and they record together at the next drain. - if (defer && DeferOcclusionDumps && TryDeferBehindDraws( - () => DumpOcclusionCounter(queueId, address, instanceMask, strideBytes, defer: false), address, - DumpRangeBytes(instanceMask, strideBytes), always: true)) + if (defer && DeferOcclusionDumps && TryDeferOcclusionDump(queueId, address, instanceMask, strideBytes)) { return true; } + return DumpOcclusionCounterNow(queueId, address, instanceMask, strideBytes); + } + + // Its own method so the closure is only built when a dump is deferred: a lambda over the + // parameters makes the compiler allocate its capture at entry of the method that holds it. + private bool TryDeferOcclusionDump(int queueId, ulong address, uint instanceMask, uint strideBytes) => + TryDeferBehindDraws( + () => DumpOcclusionCounterNow(queueId, address, instanceMask, strideBytes), address, + DumpRangeBytes(instanceMask, strideBytes), always: true); + + private bool DumpOcclusionCounterNow(int queueId, ulong address, uint instanceMask, uint strideBytes) + { DrainDraws(); ValidateOcclusionQueue(queueId); InitializeNativeOcclusion(); diff --git a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.cs b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.cs index e8a2c92b7..6f8b62f49 100644 --- a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.cs +++ b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.cs @@ -2590,6 +2590,8 @@ public void ReportCounters() .Append(" data_replans=").Append(SharpEmu.ShaderCompiler.Resources.DeviceDataLoads.Replans) .Append(" resident=").Append(_guestMemory.ResidentBuffers.Length) .Append(" windows_touched=").Append(_guestMemory.TouchedWindows.Count) + .Append(" window_memo_hits=").Append(_guestMemory.WindowMemoCounts.Hits) + .Append(" window_memo_misses=").Append(_guestMemory.WindowMemoCounts.Misses) .Append(TakeWriteFenceCounters()); Console.Error.WriteLine($"[METAL-V2][WAITS]{reasons}"); var bindless = TakeBindlessCost(); From a6b92b4ceb485e532ea98183bec4d2c5d14ed745 Mon Sep 17 00:00:00 2001 From: kuba Date: Thu, 29 Oct 2026 06:44:52 +0100 Subject: [PATCH 77/82] [GPU] A depth target's stencil reaches the R8 image over it on the device instead of through guest memory. Every bind of the 8-bit image over a GPU-written D32S8 target's stencil plane wrote both planes back and reloaded the 8-bit image from guest memory: in Silent Hill's first room about 8 round trips a frame (wb_overlap and up_alias at about 160/s each). The stencil plane is now copied out of the depth texture into the 8-bit texture through a staging buffer (MTLBlitOptionStencilFromDepthStencil), and skipped while neither texture changed since the last copy. The depth target stays GPU-modified and owes guest memory its depth plane; once the 8-bit image takes the stencil over, the depth target's writeback leaves those bytes to it, and a depth target whose stencil taker is gone writes its depth plane back before it reloads. Silent Hill, two A/B pairs plus a traced pair: the round trips go to zero and GPU busy drops about 6% (845 to 772-791 ms/s, 10.6 to 10.2 us per draw), with identical frames. Guest fps went 24.0/24.5/24.0 to 22.5/22.9/23.5; the room is bound by the worker's CPU, and the runs differ mostly in guest stores stalled behind fog volume writebacks (7-60 ms/s across runs, either arm). SHARPEMU_METAL_V2_STENCIL_TO_COLOR_DEVICE=0 goes through guest memory again. The CHURN line counts the copies as stencil_to_color. --- .../SharpEmu.Gpu.Metal/V2/MetalCachedImage.cs | 28 +++++- .../SharpEmu.Gpu.Metal/V2/MetalImageCache.cs | 90 ++++++++++++++++--- .../V2/MetalImageTransfer.cs | 45 ++++++++++ .../SharpEmu.Gpu.Metal/V2/MetalRenderHost.cs | 2 +- .../SharpEmu.Gpu.Metal/V2/MetalV2Contracts.cs | 5 ++ ...talDccMaterializationTests.StencilPlane.cs | 66 ++++++++++++-- 6 files changed, 216 insertions(+), 20 deletions(-) diff --git a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalCachedImage.cs b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalCachedImage.cs index 7abcbac2a..cf905c970 100644 --- a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalCachedImage.cs +++ b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalCachedImage.cs @@ -249,7 +249,8 @@ public bool StillHoldsPublicationFrom(MetalCachedImage source) => /// /// Set when an image that writes this depth image's stencil bytes takes them over: the /// next time this image binds, its stencil plane is copied from that image on the - /// device, and the depth plane its texture holds stays as it is. + /// device, and the depth plane its texture holds stays as it is. The image may still be + /// GPU-modified for its depth plane; its writeback then leaves the stencil bytes alone. /// public bool StencilPlaneStale { get; set; } @@ -269,6 +270,31 @@ public bool HoldsStencilFrom(MetalCachedImage source) => Texture != 0 && source.Texture != 0 && _stencilSourceTexture == source.Texture && _stencilSourceVersion == source.ContentVersion; + // The other direction, on the 8-bit image over a depth image's stencil bytes: the depth + // texture its last device copy came from, that image's version then, and this image's + // own version once the copy landed. + private nint _stencilCopySourceTexture; + private long _stencilCopySourceVersion; + private long _stencilCopyOwnVersion; + + /// Records that this texture now holds a device copy of the depth image's stencil plane as it is. + public void NoteStencilCopyOf(MetalCachedImage depth) + { + _stencilCopySourceTexture = depth.Texture; + _stencilCopySourceVersion = depth.ContentVersion; + _stencilCopyOwnVersion = ContentVersion; + } + + /// + /// True while neither this texture nor the depth image has changed since the last + /// stencil copy out of it: a copy now would write the same texels. + /// + public bool StillHoldsStencilCopyOf(MetalCachedImage depth) => + !_cpuDirty && Texture != 0 && depth.Texture != 0 && + _stencilCopySourceTexture == depth.Texture && + _stencilCopySourceVersion == depth.ContentVersion && + _stencilCopyOwnVersion == ContentVersion; + public ImageDescription Description; public MetalTextureShape Shape { get; private set; } diff --git a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalImageCache.cs b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalImageCache.cs index 276f4700b..c5588103f 100644 --- a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalImageCache.cs +++ b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalImageCache.cs @@ -823,12 +823,18 @@ private void SynchronizeStencilPlanes( NoteForeignStencilShare(depth, image); var (start, end) = image.OverlapWithin(depth.Description.Stencil.Address, depth.Description.Stencil.Size); - if (depth.IsGpuModified) + var copiedOnDevice = false; + if (depth.IsGpuModified && !depth.StencilPlaneStale) { - // Both planes go out together: only the depth image's texture holds them. - WriteBackToGuest(depth, MetalWritebackCause.ImageOverlap); - MarkAliasRunDirty(image, start, end); - _stencilPlanePublishes++; + // Only the depth image's texture holds the stencil: copied straight into this + // texture when the device can, and otherwise both planes go out together. + copiedOnDevice = TryCopyStencilToColorOnDevice(depth, image); + if (!copiedOnDevice) + { + WriteBackToGuest(depth, MetalWritebackCause.ImageOverlap); + MarkAliasRunDirty(image, start, end); + _stencilPlanePublishes++; + } } if (takesGpuOwnership) @@ -839,7 +845,13 @@ private void SynchronizeStencilPlanes( // planes from guest memory instead round-trips the depth plane, and a reload // that does not give back exactly what was written left Black Myth: Wukong's // scene depth as garbage or zeros, every pass that tests or samples it with it. - depth.ClearGpuModified(); + // A depth image that handed its stencil over on the device, or had already + // given it up, still owes guest memory its depth plane and keeps it. + if (!copiedOnDevice && !depth.StencilPlaneStale) + { + depth.ClearGpuModified(); + } + if (StencilDeviceHandOver) { depth.StencilPlaneStale = true; @@ -893,7 +905,13 @@ private void SynchronizeStencilPlanes( if (image.StencilPlaneStale) { // The image that took the stencil over is gone (its bytes were published when it - // was released): only guest memory has them now. + // was released): only guest memory has them now. A depth plane still owed goes + // out first (the writeback leaves the stale stencil out), so the reload keeps it. + if (image.IsGpuModified) + { + WriteBackToGuest(image, MetalWritebackCause.ImageOverlap); + } + image.StencilPlaneStale = false; image.MarkCpuDirty(MetalImageDirtyCause.AliasWrite); } @@ -940,6 +958,57 @@ private bool TryCopyStencilOnDevice(MetalCachedImage depth, MetalCachedImage oth return true; } + // SHARPEMU_METAL_V2_STENCIL_TO_COLOR_DEVICE=0 publishes a depth image's GPU-written + // stencil to the 8-bit image over it through guest memory again (both planes written + // back, the 8-bit image reloaded). + private static readonly bool StencilToColorOnDevice = + Environment.GetEnvironmentVariable("SHARPEMU_METAL_V2_STENCIL_TO_COLOR_DEVICE") != "0"; + + private long _stencilColorCopies; + + // The stencil plane a GPU-modified depth image's texture holds, copied into the 8-bit + // image over its bytes. The depth image stays GPU-modified and owes guest memory both + // planes as before; the 8-bit image owes it nothing. Caller holds the lock. + private bool TryCopyStencilToColorOnDevice(MetalCachedImage depth, MetalCachedImage image) + { + if (!StencilToColorOnDevice || !StencilDeviceHandOver || depth.Texture == 0 || image.Texture == 0 || + depth.IsCpuDirty || image.IsGpuModified || + depth.IsWindowLinearView || image.IsWindowLinearView || + depth.FirstLevel != 0 || image.FirstLevel != 0 || + image.Description.TileMode != depth.Description.TileMode || + image.Description.Data.Address != depth.Description.Stencil.Address || + image.Description.Data.Size > depth.Description.Stencil.Size) + { + return false; + } + + if (image.StillHoldsStencilCopyOf(depth)) + { + return true; + } + + EnsureSparseBacked(image); + var context = _owner.BeginTransfer(MetalTransferCause.Writeback, depth.Texture, image.Texture); + if (!_transfer.TryEncodeColorFromStencil( + context, in depth.Description, depth.Texture, in image.Description, image.Texture)) + { + return false; + } + + if (context.WantsEncodedLabels) + { + context.ProfileEncoded(ProfileLabel("stencil-to-color", in image.Description)); + } + // The whole image is the plane's bytes, and its texture now holds them. + image.RefreshComplete(); + image.NoteStencilCopyOf(depth); + _stencilColorCopies++; + return true; + } + + /// Stencil planes copied into the 8-bit image over them on the device. + public long TakeStencilColorCopies() => Interlocked.Exchange(ref _stencilColorCopies, 0); + private long _stencilPlanePublishes; // The plane read on purpose is an 8-bit depth-tiled image at DB_STENCIL_*_BASE. Any @@ -2769,7 +2838,8 @@ private MetalWritebackOutcome WriteBackToGuest(MetalCachedImage image, MetalWrit ulong written; if (destination.IsValid && destination.Size >= description.Data.Size) { - var stencil = description.HasStencil + // A stencil plane another image took over is that image's to publish. + var stencil = description.HasStencil && !image.StencilPlaneStale ? _guestMemory.ResolveWritable(description.Stencil.Address, description.Stencil.Size) : default; var range = new SubresourceRange(0, description.Resources.Levels, 0, description.TransferLayers); @@ -2839,7 +2909,7 @@ private ulong WriteBackInPieces(IMetalSubmissionContext context, MetalCachedImag } var batch = (uint)Math.Clamp(TransferPieceBytes / layerBytes, 1, layers); - var stencilLayerBytes = description.HasStencil && layers != 0 ? description.Stencil.Size / layers : 0; + var stencilLayerBytes = description.HasStencil && !image.StencilPlaneStale && layers != 0 ? description.Stencil.Size / layers : 0; var written = 0UL; for (uint first = 0; first < layers; first += batch) { @@ -2891,7 +2961,7 @@ private ulong WriteBackInRuns(IMetalSubmissionContext context, MetalCachedImage // run's stencil bytes are its layers' stencil bytes. var layers = description.TransferLayers; var layerBytes = layers != 0 ? description.Data.Size / layers : 0; - var stencilLayerBytes = description.HasStencil && layers != 0 ? description.Stencil.Size / layers : 0; + var stencilLayerBytes = description.HasStencil && !image.StencilPlaneStale && layers != 0 ? description.Stencil.Size / layers : 0; var written = 0UL; StringBuilder? trace = null; foreach (var run in runs) diff --git a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalImageTransfer.cs b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalImageTransfer.cs index dc4e0e4b2..14fe7e44d 100644 --- a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalImageTransfer.cs +++ b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalImageTransfer.cs @@ -360,6 +360,51 @@ public bool TryEncodeStencilFromColor( return true; } + // The way back: a depth texture's stencil plane into the 8-bit image over its bytes, + // through the same staging buffer, out of the combined texture with + // MTLBlitOptionStencilFromDepthStencil. + public bool TryEncodeColorFromStencil( + IMetalSubmissionContext context, + in ImageDescription depthDescription, + nint depthTexture, + in ImageDescription colorDescription, + nint colorTexture) + { + if (_disposed || context.CommandBuffer == 0 || colorTexture == 0 || depthTexture == 0 || + colorDescription.IsBlock || colorDescription.BytesPerBlock != 1 || + colorDescription.Extent.Width != depthDescription.Extent.Width || + colorDescription.Extent.Height != depthDescription.Extent.Height || + colorDescription.Resources.Levels != 1 || colorDescription.TransferLayers != 1 || + depthDescription.TransferLayers != 1 || + colorDescription.Samples != 1 || depthDescription.Samples != 1 || + !MetalGuestFormats.TryDecodeHostFormat(depthDescription.PixelFormat, out var depthFormat) || + !MetalGuestFormats.HasStencilPlane(depthFormat)) + { + return false; + } + + var width = depthDescription.Extent.Width; + var height = depthDescription.Extent.Height; + var rowBytes = ((ulong)width + 255) & ~255UL; + var sliceBytes = rowBytes * height; + var staging = _tiler.Acquire(context, sliceBytes); + if (!staging.IsValid) + { + return false; + } + + var blit = _tiler.BeginBlit(context); + MetalNative.SendCopyTextureToBufferOptions(blit, SelCopyTextureToBuffer, + depthTexture, 0, 0, default, new MtlSize { Width = width, Height = height, Depth = 1 }, + staging.Buffer, (nuint)staging.Offset, (nuint)rowBytes, (nuint)sliceBytes, (nuint)MtlBlitOption.StencilFromDepthStencil); + MetalNative.SendCopyBufferToTextureOptions(blit, SelCopyBufferToTexture, + staging.Buffer, (nuint)staging.Offset, (nuint)rowBytes, (nuint)sliceBytes, + new MtlSize { Width = width, Height = height, Depth = 1 }, + colorTexture, 0, 0, default, (nuint)MtlBlitOption.None); + MetalTiler.EndEncoding(context, blit); + return true; + } + private static bool IsRawSingleChannel32(MtlPixelFormat format) => format is MtlPixelFormat.R32Float or MtlPixelFormat.R32Uint or MtlPixelFormat.R32Sint; diff --git a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.cs b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.cs index 6f8b62f49..add944b6c 100644 --- a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.cs +++ b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.cs @@ -2624,7 +2624,7 @@ public void ReportCounters() $"linear_views={_imageCache.LinearViewCount} linear_view_promotions={_imageCache.TakeLinearViewPromotions()} " + $"usage_promotions={_imageCache.TakeUsagePromotions()} " + $"alias_on_device={_imageCache.TakeDeviceAliasPublishes()} " + - $"stencil_planes={_imageCache.TakeStencilPlanePublishes()} stencil_device={_imageCache.TakeStencilDeviceHandOvers()} " + + $"stencil_planes={_imageCache.TakeStencilPlanePublishes()} stencil_device={_imageCache.TakeStencilDeviceHandOvers()} stencil_to_color={_imageCache.TakeStencilColorCopies()} " + $"alias_reused={_imageCache.TakeDeviceAliasReuses()} alias_syncs_skipped={_imageCache.TakeAliasSyncsSkipped()} untested_depth_writes={TakeUntestedDepthWrites()} " + $"part_up={subRange.Uploads} part_up_mb={subRange.UploadBytes / (1024 * 1024)} " + $"part_wb={subRange.Writebacks} part_wb_mb={subRange.WritebackBytes / (1024 * 1024)} " + diff --git a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalV2Contracts.cs b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalV2Contracts.cs index 464933417..37aee9fb0 100644 --- a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalV2Contracts.cs +++ b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalV2Contracts.cs @@ -181,6 +181,11 @@ bool TryEncodeStencilFromColor(IMetalSubmissionContext context, in ImageDescription colorDescription, nint colorTexture, in ImageDescription depthDescription, nint depthTexture) => false; + // The other way: a depth texture's stencil plane into the 8-bit image over its bytes. + bool TryEncodeColorFromStencil(IMetalSubmissionContext context, + in ImageDescription depthDescription, nint depthTexture, + in ImageDescription colorDescription, nint colorTexture) => false; + // Copies every slice and level of one texture into another of the same shape, on the // device. The two may differ in usage only. void EncodeWholeTextureCopy(IMetalSubmissionContext context, nint sourceTexture, nint targetTexture); diff --git a/tests/SharpEmu.Gpu.Metal.Tests/MetalDccMaterializationTests.StencilPlane.cs b/tests/SharpEmu.Gpu.Metal.Tests/MetalDccMaterializationTests.StencilPlane.cs index d3ed7de6b..bd8c344cf 100644 --- a/tests/SharpEmu.Gpu.Metal.Tests/MetalDccMaterializationTests.StencilPlane.cs +++ b/tests/SharpEmu.Gpu.Metal.Tests/MetalDccMaterializationTests.StencilPlane.cs @@ -49,8 +49,9 @@ private static ImageRequest StencilPlaneRequest(ImageRole role) } // A pass writes the stencil, then a shader samples the plane: the depth target owns - // the only copy, so it is written back and the R8 image reloads from guest memory. - // Sampled again with nothing written in between, nothing moves. + // the only copy, so its stencil plane is copied into the R8 texture on the device and + // the depth target stays GPU-modified, owing guest memory both planes. Sampled again + // with nothing written in between, nothing moves. [Fact] public void AStencilPlaneSampledAsR8SeesWhatTheDepthTargetWrote() { @@ -66,21 +67,69 @@ public void AStencilPlaneSampledAsR8SeesWhatTheDepthTargetWrote() var plane = rig.Cache.FindImage(ref planeRequest); Assert.NotEqual(depth, plane); rig.Cache.AcquireTextureView(plane, in planeRequest); - Assert.Equal(1, rig.Cache.TakeStencilPlanePublishes()); - Assert.False(rig.Cache.GetImage(depth).IsGpuModified); + Assert.Equal(0, rig.Cache.TakeStencilPlanePublishes()); + Assert.Equal(1, rig.Cache.TakeStencilColorCopies()); + Assert.True(rig.Cache.GetImage(depth).IsGpuModified); Assert.False(rig.Cache.GetImage(plane).IsCpuDirty); + Assert.False(rig.Cache.GetImage(plane).IsGpuModified); rig.Cache.AcquireTextureView(plane, in planeRequest); Assert.Equal(0, rig.Cache.TakeStencilPlanePublishes()); + Assert.Equal(0, rig.Cache.TakeStencilColorCopies()); // The next frame's stencil writes owe the R8 image another copy. rig.Cache.AcquireDepthTargetView(depth, in depthRequest); rig.Cache.AcquireTextureView(plane, in planeRequest); - Assert.Equal(1, rig.Cache.TakeStencilPlanePublishes()); + Assert.Equal(1, rig.Cache.TakeStencilColorCopies()); + Assert.Equal(0, rig.Cache.TakeStencilPlanePublishes()); rig.Harness.Submit(); } - // A shader that writes the plane through the R8 image takes it from the depth target. + // What the copies carry: the stencil the depth target loaded reaches the R8 texture, and + // what a shader then stores through the R8 image is what guest memory ends up with. The + // depth target still owes its depth plane then, and its writeback must leave the stencil + // bytes the R8 image took over alone. + [Fact] + public void AStencilPlaneCopiedToR8OnTheDeviceKeepsItsTexelsBothWays() + { + if (!RequireDevice()) return; + using var rig = new Rig(Device.Value); + var depthRequest = DepthAliasRequest(ImageRole.DepthTarget, stencil: true); + var stencilSize = (int)depthRequest.Description.Stencil.Size; + Assert.True(rig.Backing.TryWriteBacking(StencilPlaneAddress, Enumerable.Repeat((byte)0x11, stencilSize).ToArray())); + var depth = rig.Cache.FindImage(ref depthRequest); + rig.Cache.AcquireDepthTargetView(depth, in depthRequest); + rig.Harness.Submit(); + rig.Harness.Begin(); + // Guest memory forgets the stencil: only the depth texture has it now. + Assert.True(rig.Backing.TryWriteBacking(StencilPlaneAddress, new byte[stencilSize])); + + var planeRequest = StencilPlaneRequest(ImageRole.Texture); + var plane = rig.Cache.FindImage(ref planeRequest); + rig.Cache.AcquireTextureView(plane, in planeRequest); + Assert.Equal(1, rig.Cache.TakeStencilColorCopies()); + var sampled = ReadPlane(rig.Harness, planeRequest.Description, rig.Cache.GetImage(plane).Texture); + Assert.All(sampled, texel => Assert.Equal(0x11, texel)); + + var storageRequest = StencilPlaneRequest(ImageRole.StorageImage); + Assert.Equal(plane, rig.Cache.FindImage(ref storageRequest)); + rig.Cache.AcquireTextureView(plane, in storageRequest); + Assert.True(rig.Cache.GetImage(depth).IsGpuModified); + Assert.True(rig.Cache.GetImage(depth).StencilPlaneStale); + var texels = (int)(AliasWidth * AliasHeight); + FillPlane(rig.Harness, storageRequest.Description, rig.Cache.GetImage(plane).Texture, + Enumerable.Repeat((byte)0x5A, texels).ToArray()); + + rig.Cache.FlushAllGpuWrites(); + rig.Harness.Submit(); + var guest = new byte[stencilSize]; + Assert.True(rig.Backing.TryReadBacking(StencilPlaneAddress, guest)); + Assert.Equal(texels, guest.Count(b => b == 0x5A)); + Assert.DoesNotContain((byte)0x11, guest); + } + + // A shader that writes the plane through the R8 image takes it from the depth target, + // copied out of the depth texture on the device. // When the depth target binds again its stencil plane is copied from the R8 texture on // the device: the depth plane its texture holds is the newest and must not round-trip // through guest memory, which is what a reload of both planes did. @@ -96,8 +145,9 @@ public void AStencilPlaneWrittenAsR8IsHandedToTheDepthTargetOnTheDevice() var storageRequest = StencilPlaneRequest(ImageRole.StorageImage); var plane = rig.Cache.FindImage(ref storageRequest); rig.Cache.AcquireTextureView(plane, in storageRequest); - Assert.Equal(1, rig.Cache.TakeStencilPlanePublishes()); - Assert.False(rig.Cache.GetImage(depth).IsGpuModified); + Assert.Equal(0, rig.Cache.TakeStencilPlanePublishes()); + Assert.Equal(1, rig.Cache.TakeStencilColorCopies()); + Assert.True(rig.Cache.GetImage(depth).IsGpuModified); Assert.False(rig.Cache.GetImage(depth).IsCpuDirty); Assert.True(rig.Cache.GetImage(depth).StencilPlaneStale); Assert.True(rig.Cache.GetImage(plane).IsGpuModified); From c6679c60d6bf7e1cafa66c496a41b0de7403681f Mon Sep 17 00:00:00 2001 From: kuba Date: Thu, 29 Oct 2026 06:44:52 +0100 Subject: [PATCH 78/82] [GPU] Name the GPU-written images a guest store waits for, behind a trace switch. SHARPEMU_METAL_V2_STORE_TRACE= prints each store a writeback holds up with the image it waits for. In Silent Hill's first room they are 8-byte stores at the first address of the volumetric fog grid (a 74x42x96 RGBA16F volume the game reallocates every frame), blocking the storing thread about 14 ms each. --- .../V2/MetalImageCache.CpuStores.cs | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalImageCache.CpuStores.cs b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalImageCache.CpuStores.cs index 724305434..e07c9ab91 100644 --- a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalImageCache.CpuStores.cs +++ b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalImageCache.CpuStores.cs @@ -305,6 +305,12 @@ private bool HasGpuWrittenImageLocked(ulong address, ulong size, bool exact) // A free request filled with the GPU-written images over the range, at most a request's // worth; the next round takes the rest. Null when every request is in use. + // SHARPEMU_METAL_V2_STORE_TRACE= names the GPU-written images guest stores wait for. + private static readonly int StoreTraceLines = + int.TryParse(Environment.GetEnvironmentVariable("SHARPEMU_METAL_V2_STORE_TRACE"), out var lines) ? lines : 0; + + private static int _storeTraced; + private StoreRequest? RequestStorePublishLocked(ulong address, ulong size, bool exact) { StoreRequest? request = null; @@ -325,9 +331,18 @@ private bool HasGpuWrittenImageLocked(ulong address, ulong size, bool exact) var count = 0; foreach (var imageIdentifier in FindImagesInRange(address, size, pageOverlap: !exact)) { - if (count < MaxImagesPerStoreRequest && _slots.TryGet(imageIdentifier) is { IsGpuModified: true }) + if (count < MaxImagesPerStoreRequest && _slots.TryGet(imageIdentifier) is { IsGpuModified: true } image) { request.Images[count++] = imageIdentifier; + if (StoreTraceLines > 0 && Interlocked.Increment(ref _storeTraced) <= StoreTraceLines) + { + Console.Error.WriteLine( + $"[METAL-V2][STORE] store=0x{address:X}+0x{size:X} exact={exact} " + + $"image=0x{image.Description.Data.Address:X}+0x{image.Description.Data.Size:X} " + + $"stencil=0x{image.Description.Stencil.Address:X}+0x{image.Description.Stencil.Size:X} " + + $"{image.Description.PixelFormat} {image.Description.Extent.Width}x{image.Description.Extent.Height} " + + $"stale={image.StencilPlaneStale}"); + } } } From 2de518eb41d7b386eaab3d192b5d208834cc69ac Mon Sep 17 00:00:00 2001 From: kuba Date: Thu, 29 Oct 2026 09:54:21 +0100 Subject: [PATCH 79/82] [GPU] The Metal command stream runs on its own thread by default. The worker no longer interprets packets or writes registers; the command thread hands it the deferred draws, dispatches and host calls in stream order, and program resolution stays on the prefetch thread. In Silent Hill's first room, on top of the device stencil hand-over, the worker drops from about 9.3 to 8.6 s of CPU per 10 s and goes idle about 8% of the time, GPU busy drops to 660-690 ms/s, and guest fps rises from 22.9 to 24.9 and 25.0 (two A/B pairs; the second control ran at 19.9 with its guest RHI thread 70% busier and is left out). Frames are identical. The command thread waits about 29% of its time for synchronous host calls and 37% for queue space, and the worker about 13% for its program resolutions: every non-draw item gates the resolution of the draws after it. SHARPEMU_METAL_V2_COMMAND_THREAD=0 interprets the stream on the worker again. --- .../V2/MetalRenderHost.CommandThread.cs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.CommandThread.cs b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.CommandThread.cs index f3bff9488..f2ee1eede 100644 --- a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.CommandThread.cs +++ b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.CommandThread.cs @@ -7,7 +7,7 @@ namespace SharpEmu.Gpu.Metal.V2; /// -/// SHARPEMU_METAL_V2_COMMAND_THREAD=1: the command stream is interpreted on a thread of its own +/// The command stream is interpreted on a thread of its own /// (DeferredCommandThread, the Vulkan presenter's command thread), and the GPU worker runs what it /// hands over in stream order - the translation's deferred draws and dispatches, and the host calls /// the stream makes. The worker no longer reads packets or writes registers: Grand Theft Auto V spent @@ -19,11 +19,15 @@ namespace SharpEmu.Gpu.Metal.V2; /// before it (Run). Command memory is read on the command thread itself unless something the worker /// records may own the bytes. A deferred dispatch cannot suspend its packet, so the worker waits for /// a compute pipeline's compile in this mode (SupportsPipelineSuspend). +/// +/// On by default: in Silent Hill's first room the worker sheds the packets and register writes and +/// the title runs at about 25 fps instead of 23. SHARPEMU_METAL_V2_COMMAND_THREAD=0 interprets the +/// stream on the worker again. /// internal sealed partial class MetalRenderHost { internal static readonly bool CommandThreadEnabled = - Environment.GetEnvironmentVariable("SHARPEMU_METAL_V2_COMMAND_THREAD") == "1"; + Environment.GetEnvironmentVariable("SHARPEMU_METAL_V2_COMMAND_THREAD") != "0"; private static readonly bool CommandThreadQosEnabled = Environment.GetEnvironmentVariable("SHARPEMU_METAL_V2_COMMAND_THREAD_QOS") == "1"; From 7204d8f6dd7624304f972c492e2fb2d35660b970 Mon Sep 17 00:00:00 2001 From: kuba Date: Thu, 29 Oct 2026 11:33:22 +0100 Subject: [PATCH 80/82] [GPU] Barriers and queued writes no longer hold back the program resolutions of the draws after them. On the command thread every non-draw item held the prefetch gate, so the draws after a barrier, a fragment-write release, an occlusion dump, an end-of-pipe label or a command-processor write could not resolve their programs until the worker had run it. In Silent Hill's first room that is about one item per draw, and the worker waited some 110 ms a second for resolutions. Barriers and write releases now only order GPU commands (PostOrdering). Writes name the bytes they write: a resolution queued after one that reads those bytes while it is still pending is abandoned and redone on the worker, the same way it is for GPU-owned bytes (PendingRangeHolds, checked first in IsOwnedConcurrent). A write still waits for the resolutions before it, which read the bytes as they were. Silent Hill, first room, two A/B pairs: the worker's resolution wait drops from 195-275 ms to 55-77 ms per 2 s, about 120 resolutions in 170k are abandoned, and fps goes from 25.5/24.9 to 26.0/25.9. Screenshots match. The item-site counters (SHARPEMU_CTHREAD_ITEM_SITES=1) name what the items are. SHARPEMU_METAL_V2_UNGATED_BARRIERS=0 and SHARPEMU_METAL_V2_UNGATED_WRITES=0 restore the gates. --- .../DeferredCommandThread.Recycling.cs | 1 + .../Gpu/GpuCommands/DeferredCommandThread.cs | 107 +++++++++++++++++- .../Gpu/Rendering/ParallelProgramPrefetch.cs | 15 ++- .../V2/MetalCommandStream.cs | 20 ++-- .../V2/MetalRenderHost.CommandThread.cs | 37 +++++- .../V2/MetalRenderHost.ConcurrentResolve.cs | 8 ++ .../V2/MetalRenderHost.Occlusion.cs | 2 +- .../V2/MetalRenderHost.Rendering.cs | 2 +- .../GpuCommands/DeferredCommandThreadTests.cs | 82 ++++++++++++++ 9 files changed, 256 insertions(+), 18 deletions(-) diff --git a/src/SharpEmu.Libs/Gpu/GpuCommands/DeferredCommandThread.Recycling.cs b/src/SharpEmu.Libs/Gpu/GpuCommands/DeferredCommandThread.Recycling.cs index 5fe20e428..5b4353e5e 100644 --- a/src/SharpEmu.Libs/Gpu/GpuCommands/DeferredCommandThread.Recycling.cs +++ b/src/SharpEmu.Libs/Gpu/GpuCommands/DeferredCommandThread.Recycling.cs @@ -72,6 +72,7 @@ private void RecycleAsyncItem(Item item) item.Data = null; item.Failure = null; item.IsDraw = false; + item.Ungated = false; item.Address = 0; item.RangeSize = 0; item.JobsBefore = 0; diff --git a/src/SharpEmu.Libs/Gpu/GpuCommands/DeferredCommandThread.cs b/src/SharpEmu.Libs/Gpu/GpuCommands/DeferredCommandThread.cs index 48657f7e5..d5be66a07 100644 --- a/src/SharpEmu.Libs/Gpu/GpuCommands/DeferredCommandThread.cs +++ b/src/SharpEmu.Libs/Gpu/GpuCommands/DeferredCommandThread.cs @@ -36,6 +36,9 @@ private sealed class Item { public Kind Kind; public bool IsDraw; + // Orders GPU commands and changes nothing a program resolution reads: the draws after it + // resolve without waiting for it, as they do behind a draw. + public bool Ungated; public AgcExports.DeferredWork? Work; public Action? Action; public ulong Address; @@ -143,6 +146,41 @@ public void Start() public long Gate => _enqueuedNonDraw; + // SHARPEMU_CTHREAD_ITEM_SITES=1: the non-draw items by kind and source, in the PERF line. + private static readonly bool CountItemSites = + Environment.GetEnvironmentVariable("SHARPEMU_CTHREAD_ITEM_SITES") == "1"; + + private readonly Dictionary _itemSites = new(); + + private void NoteItemSite(Item item) + { + var name = item.Kind switch + { + Kind.Deferred => item.Work?.GetType().Name ?? "deferred", + Kind.Write => "write", + _ => (item.Sync ? "run:" : item.RangeSize != 0 ? "range:" : item.Ungated ? "ordering:" : "post:") + (item.Action?.Method.Name ?? "?"), + }; + lock (_itemSites) + { + System.Runtime.InteropServices.CollectionsMarshal.GetValueRefOrAddDefault(_itemSites, name, out _)++; + } + } + + public string TakeItemSites() + { + if (!CountItemSites) + { + return string.Empty; + } + + lock (_itemSites) + { + var top = string.Join(",", _itemSites.OrderByDescending(pair => pair.Value).Take(12).Select(pair => $"{pair.Key}={pair.Value}")); + _itemSites.Clear(); + return $" items_by=({top})"; + } + } + public bool HasWork => Volatile.Read(ref _outstanding) != 0 || _failure is not null; // Read by the render loop before it waits, so a queued item wakes it. @@ -170,6 +208,16 @@ public void Post(Action action) Enqueue(item); } + // A host call that only orders GPU commands (a barrier, a pass end): the resolutions of the + // draws after it are not held for it. + public void PostOrdering(Action action) + { + var item = RentAsyncItem(Kind.Call); + item.Action = action; + item.Ungated = true; + Enqueue(item); + } + public void Run(Action action) { var item = new Item { Kind = Kind.Call, Action = action, Sync = true }; @@ -202,12 +250,17 @@ public void PostWrite(ulong address, ReadOnlySpan data) } // A fill or copy the render thread runs in order; reads of its destination wait for it. - public void PostRange(Action action, ulong address, ulong size) + // An ungated one writes nothing but its range: the draws after it resolve their programs + // without waiting for it, and a resolution that reads the range while it is still queued is + // abandoned instead (PendingRangeHolds). + public void PostRange(Action action, ulong address, ulong size, bool ungated = false) { var item = RentAsyncItem(Kind.Call); item.Action = action; item.Address = address; item.RangeSize = size; + item.Ungated = ungated && size != 0; + item.JobsBefore = _prefetch?.Submitted ?? 0; if (size != 0) { lock (_writeLock) @@ -229,6 +282,32 @@ public void WaitForPendingRange(ulong address, ulong size) } } + /// + /// Whether an ungated range item queued before resolution job (its index + /// among the submitted jobs) still has to write bytes in the range. Any thread. + /// + public bool PendingRangeHolds(long job, ulong address, ulong size) + { + if (Volatile.Read(ref _pendingRangeCount) == 0) + { + return false; + } + + lock (_writeLock) + { + foreach (var pending in _pendingRanges) + { + if (pending.Ungated && pending.JobsBefore <= job && + address < pending.Address + pending.RangeSize && pending.Address < address + size) + { + return true; + } + } + } + + return false; + } + private bool OverlapsPendingRange(ulong address, ulong size) { lock (_writeLock) @@ -283,12 +362,17 @@ public bool ReadOverlay(ulong address, Span destination) private void Enqueue(Item item) { ThrowIfFailed(); - if (!item.IsDraw) + if (!item.IsDraw && !item.Ungated) { item.JobsBefore = _prefetch?.Submitted ?? 0; _enqueuedNonDraw++; } + if (CountItemSites && !item.IsDraw) + { + NoteItemSite(item); + } + Interlocked.Increment(ref _itemsEnqueued); if (Volatile.Read(ref _outstanding) >= MaxOutstanding) { @@ -443,7 +527,12 @@ private void Execute(Item item) return; } - _prefetch?.WaitFinished(item.JobsBefore); + // A write still waits for the resolutions before it: they read the bytes as they were. + if (!item.Ungated || item.RangeSize != 0) + { + _prefetch?.WaitFinished(item.JobsBefore); + } + try { switch (item.Kind) @@ -470,7 +559,11 @@ private void Execute(Item item) } finally { - _executedNonDraw++; + if (!item.Ungated) + { + _executedNonDraw++; + } + if (item.RangeSize != 0) { lock (_writeLock) @@ -480,7 +573,11 @@ private void Execute(Item item) } } - _prefetch?.AdvanceGate(_executedNonDraw); + if (!item.Ungated) + { + _prefetch?.AdvanceGate(_executedNonDraw); + } + if (item.Sync) { item.Done = true; diff --git a/src/SharpEmu.Libs/Gpu/Rendering/ParallelProgramPrefetch.cs b/src/SharpEmu.Libs/Gpu/Rendering/ParallelProgramPrefetch.cs index 108bcf8a8..67b6f96fd 100644 --- a/src/SharpEmu.Libs/Gpu/Rendering/ParallelProgramPrefetch.cs +++ b/src/SharpEmu.Libs/Gpu/Rendering/ParallelProgramPrefetch.cs @@ -23,6 +23,8 @@ public sealed class Job(RegisterBanks banks, long gate, bool indexed = false) internal GraphicsProgramInputs Inputs; internal GraphicsPrograms? Programs; internal object? Assumptions; + // The jobs submitted before this one. + internal long Index; // 0 queued, 1 resolved, 2 abandoned. internal volatile int State; } @@ -99,7 +101,7 @@ internal ParallelProgramPrefetch(RenderExecutor executor, IRenderHost host, ICon // thread, draws are submitted ahead of the work before them, which may change what they read. public Job Submit(RegisterBanks banks, long gate = 0, bool indexed = false) { - var job = new Job(banks, gate, indexed); + var job = new Job(banks, gate, indexed) { Index = Submitted }; lock (_lock) { _queue.Enqueue(job); @@ -421,9 +423,16 @@ private void Resolve(Job job) } } + /// The index of the job this thread resolves now, or -1. + [ThreadStatic] + private static long _resolvingJob; + + public static long ResolvingJob => _resolvingJob - 1; + private void ResolveCore(Job job) { _concurrent.BeginConcurrentResolve(); + _resolvingJob = job.Index + 1; try { job.Inputs = RenderExecutor.ProgramInputsOf(job.Banks, _host.FormatSupport, WorkerScratch.Enabled ? _fatal : _host.Fatal, job.Indexed); @@ -444,6 +453,10 @@ private void ResolveCore(Job job) job.State = 2; } + finally + { + _resolvingJob = 0; + } } public void Dispose() diff --git a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalCommandStream.cs b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalCommandStream.cs index c13d5e44e..56b76d60a 100644 --- a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalCommandStream.cs +++ b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalCommandStream.cs @@ -284,7 +284,7 @@ public void FillBuffer(ulong address, ulong size, uint value, bool isGds) { if (OnCommandThread) { - _commandThread!.PostRange(() => FillBuffer(address, size, value, isGds), address, isGds ? 0 : size); + _commandThread!.PostRange(() => FillBuffer(address, size, value, isGds), address, isGds ? 0 : size, UngatedWrites); return; } @@ -325,7 +325,7 @@ public void CopyBuffer(ulong destination, ulong source, ulong size, bool destina if (OnCommandThread) { _commandThread!.PostRange( - () => CopyBuffer(destination, source, size, destinationIsGds, sourceIsGds), destination, destinationIsGds ? 0 : size); + () => CopyBuffer(destination, source, size, destinationIsGds, sourceIsGds), destination, destinationIsGds ? 0 : size, UngatedWrites); return; } @@ -416,7 +416,9 @@ public void RecordEndOfPipe(in EndOfPipeWrite write) if (OnCommandThread) { var queued = write; - _commandThread!.Post(() => + // A label writes its destination and nothing else a program resolution reads. + var labelBytes = write.Destination == 0 ? 0UL : (ulong)EndOfPipeLabelBytes(write.Kind); + _commandThread!.PostRange(() => { if (PendingLabelsVisible && EndOfPipeLabelBytes(queued.Kind) != 0) { @@ -424,7 +426,7 @@ public void RecordEndOfPipe(in EndOfPipeWrite write) } RecordEndOfPipe(in queued); - }); + }, write.Destination, labelBytes == 0 ? 0 : Math.Max(sizeof(ulong), labelBytes), UngatedWrites); return; } @@ -980,7 +982,7 @@ private void PostCommandProcessorWrite(ulong address, byte[] queued, string kind // Reads overlapping this queued write wait until the worker installed // its overlay. Unrelated packets can continue; prefetch's non-draw gate // still orders future resource reads after the write. - _commandThread!.PostRange(() => RecordCommandProcessorWrite(address, queued, kind), address, (ulong)queued.Length); + _commandThread!.PostRange(() => RecordCommandProcessorWrite(address, queued, kind), address, (ulong)queued.Length, UngatedWrites); } else { @@ -1117,7 +1119,7 @@ public void RecordCommandProcessorQwords(ulong address, int count, ulong strideB (count == 1 || strideBytes <= (ulong.MaxValue - address - sizeof(ulong)) / (ulong)(count - 1))) { var size = (ulong)(count - 1) * strideBytes + sizeof(ulong); - _commandThread!.PostRange(() => RecordCommandProcessorQwords(address, count, strideBytes, value, kind), address, size); + _commandThread!.PostRange(() => RecordCommandProcessorQwords(address, count, strideBytes, value, kind), address, size, UngatedWrites); } else { @@ -1291,7 +1293,7 @@ public bool TryWriteGuest(ulong address, ReadOnlySpan source, string kind private const ulong MaxStagedCommandProcessorFillBytes = 4096; private void PostFillBufferInOrder(ulong address, ulong size, uint value) => - _commandThread!.PostRange(() => FillBufferInOrder(address, size, value), address, size); + _commandThread!.PostRange(() => FillBufferInOrder(address, size, value), address, size, UngatedWrites); /// A DMA fill of guest memory that lands in record order with the labels. public void FillBufferInOrder(ulong address, ulong size, uint value) @@ -1325,7 +1327,7 @@ public void CopyBufferInOrder(ulong destination, ulong source, ulong size) { if (OnCommandThread) { - _commandThread!.PostRange(() => CopyBufferInOrder(destination, source, size), destination, size); + _commandThread!.PostRange(() => CopyBufferInOrder(destination, source, size), destination, size, UngatedWrites); return; } @@ -2584,7 +2586,7 @@ public void OnQueueReset(int queueId) public void ReleaseShaderWrites() { // Ordered behind the draws before it, and nothing the stream waits for: queued, not waited on. - if (PostToWorker(_releaseShaderWrites ??= ReleaseShaderWrites)) + if (PostOrderingToWorker(_releaseShaderWrites ??= ReleaseShaderWrites)) { return; } diff --git a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.CommandThread.cs b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.CommandThread.cs index f2ee1eede..98015df54 100644 --- a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.CommandThread.cs +++ b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.CommandThread.cs @@ -95,7 +95,7 @@ private string FormatCommandThreadCounters() var culture = System.Globalization.CultureInfo.InvariantCulture; var (runs, runWaitMs, spaceWaitMs, items) = thread.TakeCounters(); - return $"runs={runs} run_wait_ms={runWaitMs.ToString("F1", culture)} space_wait_ms={spaceWaitMs.ToString("F1", culture)} items={items} sites=({TakeSyncSites()})"; + return $"runs={runs} run_wait_ms={runWaitMs.ToString("F1", culture)} space_wait_ms={spaceWaitMs.ToString("F1", culture)} items={items} sites=({TakeSyncSites()}){thread.TakeItemSites()}"; } private int _idleCommitQueued; @@ -199,6 +199,41 @@ private bool PostToWorker(Action action) return true; } + // Host work that only orders GPU commands, queued like PostToWorker without holding the program + // resolutions of the draws after it. Every non-draw item held them: in Silent Hill's first room + // two in five items are a barrier or a fragment-write release, and the worker waited about 13% of + // its time for resolutions the gates kept from starting. + // SHARPEMU_METAL_V2_UNGATED_BARRIERS=0 holds the resolutions behind them again. + private static readonly bool UngatedBarriers = + Environment.GetEnvironmentVariable("SHARPEMU_METAL_V2_UNGATED_BARRIERS") != "0"; + + // A write the command thread queues (a command-processor write, a fill or copy, an occlusion dump, + // an end-of-pipe label) names the only bytes it writes, so the draws after it resolve their + // programs without waiting for it either; a resolution that reads those bytes while the write is + // still queued is abandoned and done again on the worker (IsOwnedConcurrent). + // SHARPEMU_METAL_V2_UNGATED_WRITES=0 holds the resolutions behind them again. + private static readonly bool UngatedWrites = + Environment.GetEnvironmentVariable("SHARPEMU_METAL_V2_UNGATED_WRITES") != "0"; + + private bool PostOrderingToWorker(Action action) + { + if (!OnCommandThread) + { + return false; + } + + if (UngatedBarriers) + { + _commandThread!.PostOrdering(action); + } + else + { + _commandThread!.Post(action); + } + + return true; + } + // Host work whose result the stream needs: on the command thread it waits for the worker. private bool RunOnWorker(Action action, [System.Runtime.CompilerServices.CallerMemberName] string member = "", diff --git a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.ConcurrentResolve.cs b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.ConcurrentResolve.cs index 0b3a70c53..f1bb05737 100644 --- a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.ConcurrentResolve.cs +++ b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.ConcurrentResolve.cs @@ -278,7 +278,11 @@ internal bool IsConcurrentResolveStillCurrent(object assumptionsObject) return true; } + // A write the command thread queued ahead of the draw being resolved, still waiting for the worker, + // is checked first: the worker records a write's ownership before it lets go of the queued range, + // so the range found gone means the ownership below is already there. private bool IsOwnedConcurrent(ConcurrentQuery query, ulong address, ulong size) => + PendingCommandWriteHolds(address, size) || _imageCache.MayOwn(address, size) && _imageCache.HasGpuModifiedImageBytes(address, size) || (query == ConcurrentQuery.Word && IsGpuWritePendingConcurrent(address, size)) || (_cpOverlay.Count != 0 && @@ -286,6 +290,10 @@ private bool IsOwnedConcurrent(ConcurrentQuery query, ulong address, ulong size) (_labelOverlay.Count != 0 && _labelOverlay.Coverage(address, size, _signaledSerialReader ??= SignaledSerial) != MetalOverlayCoverage.None); + private bool PendingCommandWriteHolds(ulong address, ulong size) => + _commandThread is { } thread && ParallelProgramPrefetch.ResolvingJob is var job and >= 0 && + thread.PendingRangeHolds(job, address, size); + // The shared event's value, read from any thread: CompletedSerial also notes it in the worker's // completion state. private ulong SignaledSerial() => MetalV2Native.SendValue64(_sharedEvent, SelSignaledValue); diff --git a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.Occlusion.cs b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.Occlusion.cs index f219b7a73..b97975c78 100644 --- a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.Occlusion.cs +++ b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.Occlusion.cs @@ -239,7 +239,7 @@ public bool DumpOcclusionCounter(int queueId, ulong address, uint instanceMask, private void PostOcclusionDump(int queueId, ulong address, uint instanceMask, uint strideBytes) => _commandThread!.PostRange(() => DumpOcclusionCounterNow(queueId, address, instanceMask, strideBytes), address, - DumpRangeBytes(instanceMask, strideBytes)); + DumpRangeBytes(instanceMask, strideBytes), UngatedWrites); private bool DumpOcclusionCounter(int queueId, ulong address, uint instanceMask, uint strideBytes, bool defer) { diff --git a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.Rendering.cs b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.Rendering.cs index 96dd681f1..23612117d 100644 --- a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.Rendering.cs +++ b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.Rendering.cs @@ -2400,7 +2400,7 @@ private void RecordMemoryBarrier(nuint afterStages, /// public void EmitGlobalBarrier() { - if (PostToWorker(EmitGlobalBarrier)) + if (PostOrderingToWorker(EmitGlobalBarrier)) { return; } diff --git a/tests/SharpEmu.Libs.Tests/Gpu/GpuCommands/DeferredCommandThreadTests.cs b/tests/SharpEmu.Libs.Tests/Gpu/GpuCommands/DeferredCommandThreadTests.cs index 2679c5eab..ddd8d679c 100644 --- a/tests/SharpEmu.Libs.Tests/Gpu/GpuCommands/DeferredCommandThreadTests.cs +++ b/tests/SharpEmu.Libs.Tests/Gpu/GpuCommands/DeferredCommandThreadTests.cs @@ -52,6 +52,88 @@ public void ProducerIdentityAndOrderedConsumerWorkSurviveThreadSelection(bool us Assert.False(producer.IsAlive); } + // A barrier or pass end only orders GPU commands: it runs in stream order like any other + // item, but the draws after it do not wait for it to resolve their programs. + [Fact] + public void AnOrderingItemRunsInOrderWithoutHoldingTheGate() + { + var host = new RecordingCommandStreamHost(); + var queue = new CommandStreamQueue(host); + var producer = new DeferredCommandThread(queue, host.Memory, _ => { }, null, () => { }, false); + using var completed = new ManualResetEventSlim(); + var values = new List(); + long gateAfterPost = -1, gateAfterOrdering = -1; + Assert.True(queue.TryEnqueueControlBarrier(() => + { + producer.Post(() => values.Add(1)); + gateAfterPost = producer.Gate; + producer.PostOrdering(() => values.Add(2)); + gateAfterOrdering = producer.Gate; + producer.Run(() => values.Add(3)); + completed.Set(); + }, () => { })); + producer.Start(); + try + { + Assert.True(SpinWait.SpinUntil(() => + { + producer.RunBatch(long.MaxValue, 64, () => { }); + return completed.IsSet; + }, TimeSpan.FromSeconds(10))); + producer.ThrowIfFailed(); + Assert.Equal(new[] { 1, 2, 3 }, values); + Assert.Equal(gateAfterPost, gateAfterOrdering); + Assert.Equal(gateAfterPost + 1, producer.Gate); + } + finally + { + producer.Stop(() => producer.RunBatch(long.MaxValue, 64, () => { })); + } + } + + [Fact] + public void AnUngatedWriteHoldsOnlyItsRangeForTheResolutionsAfterIt() + { + var host = new RecordingCommandStreamHost(); + var queue = new CommandStreamQueue(host); + var producer = new DeferredCommandThread(queue, host.Memory, _ => { }, null, () => { }, false); + using var queued = new ManualResetEventSlim(); + long gateBefore = -1, gateAfter = -1; + bool heldInside = false, heldOutside = true, heldEarlierJob = true, heldGated = true; + Assert.True(queue.TryEnqueueControlBarrier(() => + { + gateBefore = producer.Gate; + producer.PostRange(() => { }, 0x1000, 0x100, ungated: true); + gateAfter = producer.Gate; + producer.PostRange(() => { }, 0x8000, 0x100); + heldInside = producer.PendingRangeHolds(0, 0x10F0, 0x20); + heldOutside = producer.PendingRangeHolds(0, 0x1100, 0x20); + heldEarlierJob = producer.PendingRangeHolds(-1, 0x1000, 0x10); + heldGated = producer.PendingRangeHolds(0, 0x8000, 0x10); + queued.Set(); + }, () => { })); + producer.Start(); + try + { + Assert.True(queued.Wait(TimeSpan.FromSeconds(10))); + Assert.Equal(gateBefore, gateAfter); + Assert.True(heldInside); + Assert.False(heldOutside); + Assert.False(heldEarlierJob); + Assert.False(heldGated); + Assert.True(SpinWait.SpinUntil(() => + { + producer.RunBatch(long.MaxValue, 64, () => { }); + return !producer.PendingRangeHolds(0, 0x1000, 0x100); + }, TimeSpan.FromSeconds(10))); + producer.ThrowIfFailed(); + } + finally + { + producer.Stop(() => producer.RunBatch(long.MaxValue, 64, () => { })); + } + } + [Theory] [InlineData(false)] [InlineData(true)] From d2f2e02e1f891298edf5fa7fa52be641324c5cf7 Mon Sep 17 00:00:00 2001 From: kuba Date: Thu, 29 Oct 2026 13:28:50 +0100 Subject: [PATCH 81/82] [GPU] A wait on a word a queued fill writes reads the fill's word instead of running the queue first. Silent Hill resets a fence word with a small DMA fill, records an end-of-pipe label to it and polls it, some 4000 times a second. On the command thread each poll found the fill still queued and ran the whole queue on the worker before reading, so the stream stopped and the worker then idled until the stream queued again. A small fill lands as a command-processor write of one repeated word, which is what the stream reads after it. A poll of bytes the last queued write over them fills completely, with no queued store over them and no image owning them, now reads that word (TryReadQueuedFill). End-of-pipe labels hold their range only for the program resolutions after them (PostLabel): the GPU writes a label once the work before it retires, so the stream reads the bytes as they are until then, as it did before labels had a range. Silent Hill, first room, two A/B pairs: the command thread's synchronous runs fall from about 11.9k to 2.9k per 2 s and its wait on them from about 620 to 200 ms; it now waits for queue room instead, the worker staying the limit. fps 26.45 / 22.47 -> 25.94 / 26.90, within noise. Screenshots match. SHARPEMU_METAL_V2_QUEUED_FILL_READS=0 runs the queue first again. With SHARPEMU_CTHREAD_ITEM_SITES=1 the PERF line also names the reads that still run the queue and the queued item that held them. --- .../DeferredCommandThread.Recycling.cs | 3 + .../Gpu/GpuCommands/DeferredCommandThread.cs | 138 +++++++++++++++++- .../V2/MetalCommandStream.cs | 15 +- .../V2/MetalRenderHost.CommandThread.cs | 23 ++- .../GpuCommands/DeferredCommandThreadTests.cs | 46 ++++++ 5 files changed, 218 insertions(+), 7 deletions(-) diff --git a/src/SharpEmu.Libs/Gpu/GpuCommands/DeferredCommandThread.Recycling.cs b/src/SharpEmu.Libs/Gpu/GpuCommands/DeferredCommandThread.Recycling.cs index 5b4353e5e..b176e8fb6 100644 --- a/src/SharpEmu.Libs/Gpu/GpuCommands/DeferredCommandThread.Recycling.cs +++ b/src/SharpEmu.Libs/Gpu/GpuCommands/DeferredCommandThread.Recycling.cs @@ -73,6 +73,9 @@ private void RecycleAsyncItem(Item item) item.Failure = null; item.IsDraw = false; item.Ungated = false; + item.IsFill = false; + item.IsLabel = false; + item.FillValue = 0; item.Address = 0; item.RangeSize = 0; item.JobsBefore = 0; diff --git a/src/SharpEmu.Libs/Gpu/GpuCommands/DeferredCommandThread.cs b/src/SharpEmu.Libs/Gpu/GpuCommands/DeferredCommandThread.cs index d5be66a07..8ebb7c0c2 100644 --- a/src/SharpEmu.Libs/Gpu/GpuCommands/DeferredCommandThread.cs +++ b/src/SharpEmu.Libs/Gpu/GpuCommands/DeferredCommandThread.cs @@ -47,6 +47,12 @@ private sealed class Item public ulong RangeSize; // Resolution jobs submitted before this non-draw item; they finish before it runs. public long JobsBefore; + // A range item that stores this word over its whole range (PostFill). + public bool IsFill; + public uint FillValue; + // An end-of-pipe label: the GPU writes it once the work before it retires, so a read on the + // stream's thread finds the bytes as they are until then (PostLabel). + public bool IsLabel; public bool Sync; public volatile bool Done; public Exception? Failure; @@ -175,9 +181,10 @@ public string TakeItemSites() lock (_itemSites) { - var top = string.Join(",", _itemSites.OrderByDescending(pair => pair.Value).Take(12).Select(pair => $"{pair.Key}={pair.Value}")); + var top = string.Join(",", _itemSites.Where(pair => !pair.Key.StartsWith("drain:")).OrderByDescending(pair => pair.Value).Take(12).Select(pair => $"{pair.Key}={pair.Value}")); + var drains = string.Join(",", _itemSites.Where(pair => pair.Key.StartsWith("drain:")).OrderByDescending(pair => pair.Value).Take(8).Select(pair => $"{pair.Key}={pair.Value}")); _itemSites.Clear(); - return $" items_by=({top})"; + return $" items_by=({top}) drains_by=({drains})"; } } @@ -273,11 +280,112 @@ public void PostRange(Action action, ulong address, ulong size, bool ungated = f Enqueue(item); } + // An end-of-pipe label write: it holds the range only for the program resolutions after it + // (PendingRangeHolds); the stream's own reads do not wait for it, as the label lands later. + public void PostLabel(Action action, ulong address, ulong size, bool ungated = false) + { + var item = RentAsyncItem(Kind.Call); + item.IsLabel = size != 0; + item.Action = action; + item.Address = address; + item.RangeSize = size; + item.Ungated = ungated && size != 0; + item.JobsBefore = _prefetch?.Submitted ?? 0; + if (size != 0) + { + lock (_writeLock) + { + _pendingRanges.Add(item); + _pendingRangeCount = _pendingRanges.Count; + } + } + + Enqueue(item); + } + + // A fill the worker stages as a command-processor write of one repeated word: a read of bytes it + // covers, with nothing queued after it over them, is answered from the word without waiting. + public void PostFill(Action action, ulong address, ulong size, uint value, bool ungated = false) + { + var item = RentAsyncItem(Kind.Call); + item.IsFill = size != 0; + item.FillValue = value; + item.Action = action; + item.Address = address; + item.RangeSize = size; + item.Ungated = ungated && size != 0; + item.JobsBefore = _prefetch?.Submitted ?? 0; + if (size != 0) + { + lock (_writeLock) + { + _pendingRanges.Add(item); + _pendingRangeCount = _pendingRanges.Count; + } + } + + Enqueue(item); + } + + /// + /// The bytes as the stream wrote them when the last queued range item over them is a fill that + /// covers them all and no queued store overlaps them; false otherwise. Command stream thread. + /// + public bool TryReadQueuedFill(ulong address, Span destination) + { + if (Volatile.Read(ref _pendingRangeCount) == 0 || destination.IsEmpty) + { + return false; + } + + var size = (ulong)destination.Length; + lock (_writeLock) + { + Item? last = null; + foreach (var pending in _pendingRanges) + { + if (!pending.IsLabel && address < pending.Address + pending.RangeSize && pending.Address < address + size) + { + last = pending; + } + } + + if (last is not { IsFill: true } fill || + address < fill.Address || address + size > fill.Address + fill.RangeSize) + { + return false; + } + + foreach (var write in _pendingWrites) + { + if (address < write.Address + (ulong)write.Data!.Length && write.Address < address + size) + { + return false; + } + } + + Span word = stackalloc byte[sizeof(uint)]; + System.Buffers.Binary.BinaryPrimitives.WriteUInt32LittleEndian(word, fill.FillValue); + var phase = (int)((address - fill.Address) % sizeof(uint)); + for (var index = 0; index < destination.Length; index++) + { + destination[index] = word[(phase + index) % sizeof(uint)]; + } + + return true; + } + } + // Runs every queued item when one of them is a fill or copy into this range. public void WaitForPendingRange(ulong address, ulong size) { if (_pendingRangeCount != 0 && OverlapsPendingRange(address, size)) { + if (CountItemSites) + { + NoteDrainSite(address, size); + } + Run(static () => { }); } } @@ -308,13 +416,37 @@ public bool PendingRangeHolds(long job, ulong address, ulong size) return false; } + // Which read drained the queue, and which queued item held the bytes (SHARPEMU_CTHREAD_ITEM_SITES=1). + private void NoteDrainSite(ulong address, ulong size) + { + var holder = "?"; + lock (_writeLock) + { + foreach (var pending in _pendingRanges) + { + if (!pending.IsLabel && address < pending.Address + pending.RangeSize && pending.Address < address + size) + { + holder = $"{pending.Action?.Method.Name ?? "?"}:fill={pending.IsFill}:+{pending.RangeSize}"; + } + } + } + + var frames = new System.Diagnostics.StackTrace(2, false).GetFrames(); + var callers = string.Join("<", frames.Take(4).Select(frame => frame.GetMethod()?.Name ?? "?")); + var name = $"drain:{callers}@{holder}/{size}"; + lock (_itemSites) + { + System.Runtime.InteropServices.CollectionsMarshal.GetValueRefOrAddDefault(_itemSites, name, out _)++; + } + } + private bool OverlapsPendingRange(ulong address, ulong size) { lock (_writeLock) { foreach (var pending in _pendingRanges) { - if (address < pending.Address + pending.RangeSize && pending.Address < address + size) + if (!pending.IsLabel && address < pending.Address + pending.RangeSize && pending.Address < address + size) { return true; } diff --git a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalCommandStream.cs b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalCommandStream.cs index 56b76d60a..30f40f461 100644 --- a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalCommandStream.cs +++ b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalCommandStream.cs @@ -418,7 +418,7 @@ public void RecordEndOfPipe(in EndOfPipeWrite write) var queued = write; // A label writes its destination and nothing else a program resolution reads. var labelBytes = write.Destination == 0 ? 0UL : (ulong)EndOfPipeLabelBytes(write.Kind); - _commandThread!.PostRange(() => + _commandThread!.PostLabel(() => { if (PendingLabelsVisible && EndOfPipeLabelBytes(queued.Kind) != 0) { @@ -1292,8 +1292,17 @@ public bool TryWriteGuest(ulong address, ReadOnlySpan source, string kind // host fast path, which writes as parsed, is what the ordered path leaves out. private const ulong MaxStagedCommandProcessorFillBytes = 4096; - private void PostFillBufferInOrder(ulong address, ulong size, uint value) => - _commandThread!.PostRange(() => FillBufferInOrder(address, size, value), address, size, UngatedWrites); + private void PostFillBufferInOrder(ulong address, ulong size, uint value) + { + if (size <= MaxStagedCommandProcessorFillBytes && size % sizeof(uint) == 0) + { + _commandThread!.PostFill(() => FillBufferInOrder(address, size, value), address, size, value, UngatedWrites); + } + else + { + _commandThread!.PostRange(() => FillBufferInOrder(address, size, value), address, size, UngatedWrites); + } + } /// A DMA fill of guest memory that lands in record order with the labels. public void FillBufferInOrder(ulong address, ulong size, uint value) diff --git a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.CommandThread.cs b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.CommandThread.cs index 98015df54..c169d6e09 100644 --- a/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.CommandThread.cs +++ b/src/graphics/SharpEmu.Gpu.Metal/V2/MetalRenderHost.CommandThread.cs @@ -95,7 +95,7 @@ private string FormatCommandThreadCounters() var culture = System.Globalization.CultureInfo.InvariantCulture; var (runs, runWaitMs, spaceWaitMs, items) = thread.TakeCounters(); - return $"runs={runs} run_wait_ms={runWaitMs.ToString("F1", culture)} space_wait_ms={spaceWaitMs.ToString("F1", culture)} items={items} sites=({TakeSyncSites()}){thread.TakeItemSites()}"; + return $"runs={runs} run_wait_ms={runWaitMs.ToString("F1", culture)} space_wait_ms={spaceWaitMs.ToString("F1", culture)} items={items} sites=({TakeSyncSites()}) fill_reads={Interlocked.Exchange(ref _queuedFillReads, 0)} fill_image_refusals={Interlocked.Exchange(ref _queuedFillImageRefusals, 0)}{thread.TakeItemSites()}"; } private int _idleCommitQueued; @@ -278,6 +278,16 @@ private string TakeSyncSites() return string.Join(",", top.Select(pair => $"{pair.Key}={pair.Value}")); } + // A polled word a queued DMA fill writes is the fill's word: the fill lands as a command-processor + // write, which is what the stream reads after it. Silent Hill fills a word and waits on it some + // 4000 times a second, and each wait ran the whole queue on the worker first, leaving the worker + // idle until the stream queued again. SHARPEMU_METAL_V2_QUEUED_FILL_READS=0 runs the queue first again. + private long _queuedFillReads; + private long _queuedFillImageRefusals; + + private static readonly bool QueuedFillReads = + Environment.GetEnvironmentVariable("SHARPEMU_METAL_V2_QUEUED_FILL_READS") != "0"; + // A polled word (WAIT_REG_MEM) on the command thread, read here: as the stream wrote it when a // command-processor write still owes it, else from memory. A label still in flight reads as it was, // the wait fails and parks, and lands with the commit the idle stream asks for (RequestIdleCommit) @@ -286,6 +296,17 @@ private string TakeSyncSites() private bool ReadWaitOperandOnCommandThread(ulong address, Span destination, bool commitPendingWrite) { var size = (ulong)destination.Length; + if (QueuedFillReads && !_imageCache.MayOwn(address, size) && _commandThread!.TryReadQueuedFill(address, destination)) + { + Interlocked.Increment(ref _queuedFillReads); + return true; + } + + if (QueuedFillReads && _imageCache.MayOwn(address, size)) + { + Interlocked.Increment(ref _queuedFillImageRefusals); + } + _commandThread!.WaitForPendingRange(address, size); if (_imageCache.MayOwn(address, size) && _imageCache.HasGpuModifiedImageBytes(address, size)) { diff --git a/tests/SharpEmu.Libs.Tests/Gpu/GpuCommands/DeferredCommandThreadTests.cs b/tests/SharpEmu.Libs.Tests/Gpu/GpuCommands/DeferredCommandThreadTests.cs index ddd8d679c..47b1ca0b4 100644 --- a/tests/SharpEmu.Libs.Tests/Gpu/GpuCommands/DeferredCommandThreadTests.cs +++ b/tests/SharpEmu.Libs.Tests/Gpu/GpuCommands/DeferredCommandThreadTests.cs @@ -134,6 +134,52 @@ public void AnUngatedWriteHoldsOnlyItsRangeForTheResolutionsAfterIt() } } + [Fact] + public void AQueuedFillAnswersReadsOfTheBytesItCoversUntilAWriteQueuedAfterItOverlaps() + { + var host = new RecordingCommandStreamHost(); + var queue = new CommandStreamQueue(host); + var producer = new DeferredCommandThread(queue, host.Memory, _ => { }, null, () => { }, false); + using var queued = new ManualResetEventSlim(); + var inside = new byte[4]; + var shifted = new byte[2]; + bool readInside = false, readShifted = false, readPartial = true, readCovered = true, readBehindLabel = false; + Assert.True(queue.TryEnqueueControlBarrier(() => + { + producer.PostFill(() => { }, 0x1000, 0x10, 0x44332211u, ungated: true); + readInside = producer.TryReadQueuedFill(0x1004, inside); + readShifted = producer.TryReadQueuedFill(0x1001, shifted); + readPartial = producer.TryReadQueuedFill(0x100E, new byte[4]); + producer.PostLabel(() => { }, 0x1000, 0x8, ungated: true); + readBehindLabel = producer.TryReadQueuedFill(0x1000, new byte[4]); + producer.PostRange(() => { }, 0x1008, 0x4, ungated: true); + readCovered = producer.TryReadQueuedFill(0x1008, new byte[4]); + queued.Set(); + }, () => { })); + producer.Start(); + try + { + Assert.True(queued.Wait(TimeSpan.FromSeconds(10))); + Assert.True(readInside); + Assert.Equal(new byte[] { 0x11, 0x22, 0x33, 0x44 }, inside); + Assert.True(readShifted); + Assert.Equal(new byte[] { 0x22, 0x33 }, shifted); + Assert.False(readPartial); + Assert.True(readBehindLabel); + Assert.False(readCovered); + Assert.True(SpinWait.SpinUntil(() => + { + producer.RunBatch(long.MaxValue, 64, () => { }); + return !producer.TryReadQueuedFill(0x1000, new byte[4]); + }, TimeSpan.FromSeconds(10))); + producer.ThrowIfFailed(); + } + finally + { + producer.Stop(() => producer.RunBatch(long.MaxValue, 64, () => { })); + } + } + [Theory] [InlineData(false)] [InlineData(true)] From 1dc741d8fe6b9dde9644e19b4830eae82100bb73 Mon Sep 17 00:00:00 2001 From: kuba Date: Thu, 29 Oct 2026 13:28:50 +0100 Subject: [PATCH 82/82] [Docs] List the command-thread, stencil, ungated and queued-fill defaults with their measurements. The defaults table named command threading as opt-in; it is now the default. The new rows give each switch, what it changes and its Silent Hill A/B result, including the ones within noise. --- docs/performance/performance-defaults.md | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/docs/performance/performance-defaults.md b/docs/performance/performance-defaults.md index 80c6d7c65..c539e251e 100644 --- a/docs/performance/performance-defaults.md +++ b/docs/performance/performance-defaults.md @@ -19,10 +19,14 @@ VSync, guest quality, or unrelated application setting is changed. | Unchanged-frame presentation reuse | `SHARPEMU_METAL_V2_REUSE_PRESENTED_FRAME` | Avoids repeated host presentations of an unchanged guest frame; movie updates, output changes, restoration and animated overlays invalidate the memo. Guest FPS is distinct from host Hz. | | Audio queue query leaf dispatch | `SHARPEMU_AUDIO_OUT2_QUEUE_LEAF` | Synthetic dispatch about 369 → 84.5 ns and 40 → 0 B/call; guest audio thread remains busy. Result/error and shutdown tests retain full-path behavior. | | Asynchronous command-item reuse | `SHARPEMU_COMMAND_ITEM_REUSE` | Native two-thread median 389.965 → 354.5175 ns; allocation 80.0008 → 0.026 B/item. Bounded 64-slot return ring; synchronous and failed items retain original ownership. No accepted runtime FPS gain. | -| Queued CP writes | `SHARPEMU_METAL_V2_QUEUE_CP_WRITES` | Removes synchronous handoffs for eligible writes when command threading is explicitly enabled. Pending ranges, prefetch gates and CPU-read fallbacks preserve ordering; producer-thread device tests cover byte/qword/sparse/predication cases. | +| Queued CP writes | `SHARPEMU_METAL_V2_QUEUE_CP_WRITES` | Removes synchronous handoffs for eligible writes on the command thread. Pending ranges, prefetch gates and CPU-read fallbacks preserve ordering; producer-thread device tests cover byte/qword/sparse/predication cases. | | Bounded vertex-layout history | `SHARPEMU_VERTEX_LAYOUT_HISTORY` | Exact byte-verified reuse across alternating layouts; adaptive backoff for low reuse, 512 slots and 2 MiB additional payload budget. Component timing/allocation evidence and miss costs are in [the measurements](vertex-layout-history.md); no isolated game-FPS claim. | | Caller resolution for short queues | `SHARPEMU_METAL_V2_CALLER_RESOLVE` | At most eight outstanding jobs; FIFO, gates, pause and ownership checks retained. Single-job ARM64 2,874 → 542 ns, x64 3,160 → 972 ns; ARM64 empty-program burst 32 costs about 27 ns/job. Same-payload room medians 16.49 enabled / 16.48 disabled, no isolated FPS gain. [Full measurements](caller-resolution.md). | | Cached pixel-input analysis | `SHARPEMU_PIXEL_INPUT_ANALYSIS` | Immutable decoded-program analysis with live draw controls. Warm 512-instruction sparse shader: ARM64 2,602 → 8.5 ns, x64 5,681 → 13.7 ns. Cold entries cost 32 extra bytes; smallest cold cases cost 0.9–2.3% time. [Full measurements and runtime limits](pixel-input-analysis.md). | +| Command stream on its own thread | `SHARPEMU_METAL_V2_COMMAND_THREAD` | The GPU worker no longer reads packets or writes registers. Silent Hill's first room, same binary: 22.9 → 24.9 / 25.0 FPS, screenshots unchanged. | +| Stencil plane copied on the device | `SHARPEMU_METAL_V2_STENCIL_TO_COLOR_DEVICE` | An R8 image over a D32S8 stencil plane is filled by blits instead of a guest-memory round trip. Writeback overlaps and alias uploads fell to zero; GPU busy 845 → 772–791 ms/s. FPS within noise; screenshots identical. Device test covers both directions. | +| Ungated barriers and queued writes | `SHARPEMU_METAL_V2_UNGATED_BARRIERS`, `SHARPEMU_METAL_V2_UNGATED_WRITES` | Barriers, write releases, CP writes, fills, copies, occlusion dumps and end-of-pipe labels no longer hold back the program resolutions behind them. A resolution that reads a still-queued write's bytes is abandoned and redone on the worker. Resolution wait 195–275 → 55–77 ms per 2 s; two A/B pairs 25.5 / 24.9 → 26.0 / 25.9 FPS; screenshots unchanged. | +| Polled words answered from queued fills | `SHARPEMU_METAL_V2_QUEUED_FILL_READS` | A wait on a word a queued small DMA fill writes reads the fill's word instead of running the whole queue on the worker first; end-of-pipe labels no longer make the stream wait at all. Command-thread synchronous runs 11.9k → 2.9k per 2 s. Two A/B pairs 26.45 / 22.47 → 25.94 / 26.90 FPS: within noise, as the worker stays the limit. Screenshots unchanged. | The existing negative label page filter, exact materialization comparison, dense register lookup, mip-layout construction and target memo fixes remain enabled. @@ -42,13 +46,12 @@ The three promoted shader properties also default on in explicitly constructed | `SHARPEMU_MSL_MESH_LDS_FORWARDING` | Same-binary shadow time 34.238 → 36.422 ms; title median 7.97 → 7.47 FPS. | | `SHARPEMU_MSL_MESH_HOT_LDS` | Same-binary total encoder time 108.483 → 109.451 ms; no benefit established. | | `SHARPEMU_MSL_MESH_EXIT_JOIN`, `SHARPEMU_MSL_UNOBSERVED_ALU` | Correctness coverage exists, but isolated performance evidence is inconclusive and their combined run had an unresolved mapping failure. | -| Command thread / command-thread QoS | Earlier command-thread overlap runs were slower than serial command-stream interpretation. Queued writes and recycling apply when command threading is requested; this change does not enable it globally. | +| Command-thread QoS | `SHARPEMU_METAL_V2_COMMAND_THREAD_QOS=1`; no gain established. | | Concurrent tiling and floating-point contraction | Representative writeback regressions or numerical differences remain; the original audit did not accept them. | Background program resolution is a separate existing default: it remains enabled -unless `SHARPEMU_METAL_V2_PARALLEL_RESOLVE=0`. It can run while command-stream -interpretation stays on the GPU worker. The earlier table incorrectly grouped -it with opt-in command threading; no runtime policy is changed by this correction. +unless `SHARPEMU_METAL_V2_PARALLEL_RESOLVE=0`. It also runs when command-stream +interpretation stays on the GPU worker (`SHARPEMU_METAL_V2_COMMAND_THREAD=0`). Diagnostics, detailed tracing, scripted-only input and visibility auditing stay off. Portable-buffer policy, device LDS placement threshold and asynchronous