feat: add isolated Wurst benchmark mode - #1218
Conversation
5148d42 to
d35b2a8
Compare
|
Final controller-generated benchmark evidence from the exact content trees later squashed into the PR commits (tree hashes were checked unchanged after squash).
Raw JSON artifact{
"schema": "wurst-benchmark-v1",
"disclaimer": "Benchmark results measure the JVM-hosted Wurst IL interpreter. They vary with the machine, JVM, compiler version, host load, and benchmark setup. Use them for controlled side-by-side comparisons under the same conditions, not as absolute Warcraft III, Jass, Lua, or in-game performance numbers.",
"environment": {
"os": "Linux",
"jvm": "25.0.2",
"compiler": "sha256:830c5a70409b76af8fa2a90889a4604712d5fba443e5036d1eea9af94aa9dd8b",
"grill": "1.4.0.0-nightly-master-4-g46b2d02",
"cpuCount": 16
},
"filter": "Polygon",
"forks": 5,
"warmup": 5,
"iterations": 20,
"benchmarks": [
{
"qualifiedName": "PolygonTests.benchmarkAcceleratedClassification",
"checksum": 7907,
"forks": [
{
"fork": 1,
"batchSize": 1,
"samplesNanos": [
368695448,
388876391,
359626453,
395386916,
370663259,
411735192,
360463508,
386470017,
364758746,
394589097,
364757655,
383784729,
369970288,
381094280,
362168867,
402393185,
364425923,
388975482,
367942356,
405378551
]
},
{
"fork": 2,
"batchSize": 1,
"samplesNanos": [
390771854,
458847486,
463081227,
510428099,
372039055,
414835844,
374836090,
409322444,
381503712,
424050419,
377278481,
416840215,
386946866,
426031436,
383267820,
410637414,
375440797,
412517585,
388279435,
387815615
]
},
{
"fork": 3,
"batchSize": 1,
"samplesNanos": [
384784109,
428797330,
402498415,
465103517,
386108011,
423878216,
385825022,
433709680,
396165032,
420415589,
409150086,
427668683,
392532353,
429282035,
406411227,
415995627,
394109590,
418071658,
409705249,
405370088
]
},
{
"fork": 4,
"batchSize": 1,
"samplesNanos": [
414072451,
396246406,
418434750,
390750933,
418060587,
398613829,
412787230,
394409881,
414325553,
402646238,
431413905,
401111636,
398001895,
414546813,
401323076,
422346471,
400226547,
430628003,
398766366,
428957415
]
},
{
"fork": 5,
"batchSize": 1,
"samplesNanos": [
369997541,
386462812,
369566293,
408023738,
372444783,
406191666,
369961088,
407032128,
379757878,
403177396,
361511843,
394286881,
365488847,
428558442,
390979809,
380326681,
385429100,
370491291,
387502066,
371115225
]
}
],
"statistics": {
"mean": 398884852.47,
"standardDeviation": 25354460.413373582,
"min": 359626453,
"max": 510428099,
"median": 396165032,
"p90": 428558442,
"p95": 431413905
}
},
{
"qualifiedName": "PolygonTests.benchmarkLinearClassification",
"checksum": 7907,
"forks": [
{
"fork": 1,
"batchSize": 1,
"samplesNanos": [
3132902675,
3067167963,
3166785904,
3182266664,
3114694566,
3203361512,
3162264104,
3138459019,
3133064619,
3164758554,
3109830442,
3139697189,
3173653490,
3078734573,
3117062534,
3239393029,
3117036597,
3107556801,
3127308615,
3103609486
]
},
{
"fork": 2,
"batchSize": 1,
"samplesNanos": [
3071094678,
3064336179,
3098448663,
3073401196,
3172979183,
3038884731,
3070162844,
3037044835,
3039965731,
3370020905,
3082659881,
3077294370,
3085079658,
3094553551,
3025968134,
3073774991,
3059496017,
3067219642,
3085895939,
3087553825
]
},
{
"fork": 3,
"batchSize": 1,
"samplesNanos": [
3153603425,
2983840982,
2964823825,
2959004012,
2974845813,
2936944315,
3008420737,
2981190371,
2968133714,
2959879918,
3108065743,
3026574451,
3049891475,
3010793584,
2921572924,
2928056935,
2949294249,
2949800043,
2952196727,
2955657770
]
},
{
"fork": 4,
"batchSize": 1,
"samplesNanos": [
3016008099,
3058396342,
3038951974,
2988359249,
3018712963,
3108180584,
2978482974,
3024881381,
3009479143,
3044290413,
3118117305,
3116563668,
3039004591,
3036238531,
3036610068,
3121108244,
3075213605,
3045029600,
3020826111,
3027068145
]
},
{
"fork": 5,
"batchSize": 1,
"samplesNanos": [
3137694776,
3095111533,
3152670934,
3136527288,
3117496025,
3163545958,
3244422324,
3081602943,
3095713912,
3133989786,
3091861231,
3101855693,
3067199969,
3104525899,
3073945232,
3066253393,
3199333699,
3085322659,
3085018665,
3089108087
]
}
],
"statistics": {
"mean": 3075427572.98,
"standardDeviation": 75177392.59450755,
"min": 2921572924,
"max": 3370020905,
"median": 3077294370,
"p90": 3163545958,
"p95": 3182266664
}
}
]
} |
|
@codex review |
There was a problem hiding this comment.
💡 Codex Review
Here are some automated review suggestions for this pull request.
Reviewed commit: d35b2a8fcc
ℹ️ About Codex in GitHub
Your team has set up Codex to review pull requests in this repo. Reviews are triggered when you
- Open a pull request for review
- Mark a draft as ready
- Comment "@codex review".
If Codex has suggestions, it will comment; otherwise it will react with 👍.
Codex can also answer questions or update the PR. Try commenting "@codex address that feedback".
| if (funcDef2.attrHasAnnotation("benchmark")) { | ||
| flags.add(IS_BENCHMARK); |
There was a problem hiding this comment.
Reject extension functions annotated as benchmarks
When a user writes an annotated extension function such as @benchmark function int.foo() returns int, the validator does not reject it because extension functions allow annotations and checkBenchmark is only called for FuncDef/NativeFunc; however this FuncDef-only flagging means the function is never given IS_BENCHMARK, so -benchmarkList silently omits it and exact execution reports it as missing. Please either reject ExtensionFuncDef explicitly as not parameterless/package benchmark material, or support invoking it with its receiver.
Useful? React with 👍 / 👎.
Adds a first-class isolated JVM benchmark worker mode to WurstScript.
@benchmarkfunctions returningint;@testand@benchmark, while keeping@compiletimeincompatible;wurst-benchmark-worker-v2contract with identity, checksum, batch size, and raw nanosecond samples only;-testFilterand-testTimeoutoptions in benchmark mode;Verification from
nix develop /home/daniel/Repositories/tever/tever_mainon the final tree:rtk gradlew test— full compiler suite passed;rtk gradlew shadowJar— passed;PolygonTestspassed 22/22 with 0 compiler errors and 0 warnings;The final benchmark measured a 396,165,032 ns accelerated median versus 3,077,294,370 ns linear median: 7.77× median speedup. These are relative JVM-hosted Wurst IL interpreter measurements, not Warcraft III runtime numbers.
Companion CLI/documentation PR: wurstscript/WurstSetup#75. The built-in annotation declaration is proposed separately in wurstscript/WurstStdlib2.