From f284302e5230d57680ae6a4d549fe93e187af0bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Havl=C3=AD=C4=8Dek?= Date: Wed, 5 Aug 2026 01:08:18 +0200 Subject: [PATCH] feat: add Grill JVM benchmark command --- README.md | 44 + .../kotlin/benchmark/BenchmarkCoordinator.kt | 246 ++++++ src/main/kotlin/benchmark/BenchmarkModels.kt | 95 +++ .../kotlin/benchmark/BenchmarkRenderer.kt | 100 +++ src/main/kotlin/file/CLICommand.kt | 6 + src/main/kotlin/file/SetupApp.kt | 269 ++++-- src/main/kotlin/file/SetupMain.kt | 100 +++ src/main/kotlin/global/InstallationManager.kt | 19 + src/test/kotlin/BenchmarkCommandTests.kt | 223 +++++ src/test/kotlin/BenchmarkCoordinatorTests.kt | 780 ++++++++++++++++++ 10 files changed, 1821 insertions(+), 61 deletions(-) create mode 100644 src/main/kotlin/benchmark/BenchmarkCoordinator.kt create mode 100644 src/main/kotlin/benchmark/BenchmarkModels.kt create mode 100644 src/main/kotlin/benchmark/BenchmarkRenderer.kt create mode 100644 src/test/kotlin/BenchmarkCommandTests.kt create mode 100644 src/test/kotlin/BenchmarkCoordinatorTests.kt diff --git a/README.md b/README.md index 343d051..4d187c8 100644 --- a/README.md +++ b/README.md @@ -70,6 +70,50 @@ The command exits with code `0` when dependencies are up to date and `1` when up > grill outdated ``` +### Benchmarking Wurst functions + +`grill benchmark` measures Wurst code in the JVM-hosted Wurst IL interpreter. A benchmark is a package-level, parameterless function annotated with `@benchmark` and returning `int`; its return value must be a stable, workload-derived checksum. + +```wurst +import Wurstunit + +@test @benchmark function benchmarkName() returns int + var checksum = 0 + for i = 0 to 999 + checksum += i + checksum.assertEquals(499500) + return checksum +``` + +The checksum should depend on the work being measured, not on a clock, random value, object identity, or mutable global state. When comparing two implementations, use the same inputs, operation count, and checksum calculation in both functions. A changing checksum indicates a correctness or benchmark-design problem and causes the comparison to fail. + +For very small operations, batch many fixed inputs inside one benchmark invocation and return one checksum for the complete batch. This makes the measured workload large enough to distinguish implementations; the runner also reports the calibrated invocation `batchSize` for each fork. A benchmark may also carry @test. In normal test mode its assertions run and the int return is ignored; in benchmark mode the same return is the checksum. Use benchmark-only functions when running the workload during every test suite would be too expensive. + +Run all benchmarks or select package/function names with an optional substring filter: + +```cmd +> grill benchmark +> grill benchmark Polygon +> grill benchmark Polygon --forks 5 --warmup 5 --iterations 20 +> grill benchmark Polygon --format json +> grill benchmark --help +``` + +Options are: + +- `[filter]` — optional substring used to select benchmark names. +- `--forks N` — positive number of isolated compiler JVMs per benchmark, serially (default `3`). +- `--warmup N` — non-negative number of unmeasured warmup samples per fork (default `5`). +- `--iterations N` — positive number of measured samples per fork (default `10`). +- `--format human|json` — concise comparison output or machine-readable `wurst-benchmark-v1` JSON (default `human`). +- `--help` — show benchmark-specific help without loading the project. + +Global options such as `-projectDir`, `--quiet`, and `--debug` remain available. + +Use JSON when a script needs raw samples, checksums, statistics, and environment metadata; diagnostics are kept off JSON stdout. The `environment.compiler` field is `sha256:` for the exact compiler JAR used by the workers. For credible relative results, keep the machine, OS, Java runtime, compiler and Grill versions, project inputs, fork/warmup/iteration settings, and background load consistent. Pin the process to dedicated CPU cores and avoid thermal or power-state changes where practical; `grill benchmark` does not itself control CPU affinity, frequency scaling, garbage collection, or other host-level noise. + +Benchmark results measure the JVM-hosted Wurst IL interpreter. They vary with the machine, JVM, compiler version, host load, and benchmark setup. Use them for controlled side-by-side comparisons under the same conditions, not as absolute Warcraft III, Jass, Lua, or in-game performance numbers. + ### Building the project diff --git a/src/main/kotlin/benchmark/BenchmarkCoordinator.kt b/src/main/kotlin/benchmark/BenchmarkCoordinator.kt new file mode 100644 index 0000000..1a7393b --- /dev/null +++ b/src/main/kotlin/benchmark/BenchmarkCoordinator.kt @@ -0,0 +1,246 @@ +package benchmark + +import com.fasterxml.jackson.databind.JsonNode +import com.fasterxml.jackson.databind.json.JsonMapper +import java.nio.file.Files +import java.nio.file.Path +import java.util.Comparator +import java.util.LinkedHashMap + +class BenchmarkCoordinator( + private val launcher: BenchmarkProcessLauncher, + commonArguments: List, + private val debug: Boolean = false, + private val compilerIdentity: String = "unknown", + private val grillIdentity: String = "unknown" +) { + private val commonArguments = commonArguments.toList() + private val mapper = JsonMapper.builder().build() + + fun run(request: BenchmarkRequest): BenchmarkReport { + val temporaryRoot = Files.createTempDirectory("grill-benchmark-") + try { + val discoveryOutput = temporaryRoot.resolve("discovery.json") + val filterArguments = request.filter + ?.takeIf { it.isNotBlank() } + ?.let { arrayOf("-benchmarkFilter", it) } + ?: emptyArray() + val discoveryArguments = workerArguments( + "-runbenchmarks", + "-benchmarkList", + *filterArguments, + "-benchmarkOutput", + discoveryOutput.toString() + ) + val selected = launchAndRead(discoveryArguments, discoveryOutput, "discovery", ::parseDiscovery).value + + val forksByBenchmark = LinkedHashMap>() + val checksums = mutableMapOf() + var executionIndex = 0 + repeat(request.forks) { forkRound -> + val roundNames = if (forkRound % 2 == 0) selected else selected.asReversed() + roundNames.forEach { qualifiedName -> + val output = temporaryRoot.resolve("execution-$executionIndex-$forkRound-${safeFileName(qualifiedName)}.json") + executionIndex++ + val arguments = workerArguments( + "-runbenchmarks", + "-benchmarkName", + qualifiedName, + "-benchmarkWarmup", + request.warmup.toString(), + "-benchmarkIterations", + request.iterations.toString(), + "-benchmarkOutput", + output.toString() + ) + val parsedExecution = launchAndRead(arguments, output, "execution for $qualifiedName") { + parseExecution(it, qualifiedName, request.iterations) + } + val result = parsedExecution.value + val previousChecksum = checksums.putIfAbsent(qualifiedName, result.checksum) + if (previousChecksum != null && previousChecksum != result.checksum) { + if (!parsedExecution.diagnosticsEmitted) { + parsedExecution.diagnostics.forEach(System.err::println) + } + check(false) { "benchmark checksum changed across forks for $qualifiedName" } + } + val forks = forksByBenchmark.getOrPut(qualifiedName) { mutableListOf() } + forks += BenchmarkFork(forkRound + 1, result.batchSize, result.samplesNanos) + } + } + + val aggregates = selected.map { qualifiedName -> + val forks = forksByBenchmark[qualifiedName].orEmpty().toList() + check(forks.size == request.forks) { + "benchmark $qualifiedName did not produce ${request.forks} fork results" + } + val allSamples = forks.flatMap { it.samplesNanos } + BenchmarkAggregate( + qualifiedName = qualifiedName, + checksum = checksums.getValue(qualifiedName), + forks = forks, + statistics = BenchmarkStatistics.fromSamples(allSamples) + ) + } + if (aggregates.size == 2) { + check(aggregates[0].checksum == aggregates[1].checksum) { + "benchmark checksums differ: ${aggregates[0].qualifiedName}=${aggregates[0].checksum}, " + + "${aggregates[1].qualifiedName}=${aggregates[1].checksum}" + } + } + return BenchmarkReport( + environment = BenchmarkEnvironment( + os = System.getProperty("os.name", "unknown"), + jvm = System.getProperty("java.version", "unknown"), + compiler = compilerIdentity, + grill = grillIdentity, + cpuCount = Runtime.getRuntime().availableProcessors() + ), + filter = request.filter, + forks = request.forks, + warmup = request.warmup, + iterations = request.iterations, + benchmarks = aggregates + ) + } finally { + deleteTree(temporaryRoot) + } + } + + fun run(filter: String?, forks: Int, warmup: Int, iterations: Int): BenchmarkReport { + return run(BenchmarkRequest(filter, forks, warmup, iterations)) + } + + private fun workerArguments(vararg extra: String): List { + val arguments = commonArguments.toMutableList() + if (!arguments.contains("-compactOutput")) { + arguments += "-compactOutput" + } + arguments += extra + return arguments + } + + private fun launchAndRead( + arguments: List, + output: Path, + phase: String, + parse: (JsonNode) -> T + ): ParsedWorker { + val result = launcher.run(arguments) + var diagnosticsEmitted = false + fun emitDiagnostics() { + if (!diagnosticsEmitted) { + result.output.forEach(System.err::println) + diagnosticsEmitted = true + } + } + if (debug) { + emitDiagnostics() + } + if (result.exitCode != 0) { + emitDiagnostics() + } + check(result.exitCode == 0) { "$phase worker exited with code ${result.exitCode}" } + if (!Files.isRegularFile(output)) { + emitDiagnostics() + check(false) { "$phase worker did not write ${output.fileName}" } + } + val document = try { + mapper.factory.createParser(Files.readString(output)).use { parser -> + val parsed = mapper.readTree(parser) + check(parser.nextToken() == null) { "$phase worker wrote trailing JSON content" } + parsed + } + } catch (exception: Exception) { + emitDiagnostics() + throw IllegalStateException("$phase worker wrote malformed JSON", exception) + } + if (document == null || !document.isObject) { + emitDiagnostics() + check(false) { "$phase worker JSON must be an object" } + } + return try { + ParsedWorker(parse(document), result.output.toList(), diagnosticsEmitted) + } catch (exception: Exception) { + emitDiagnostics() + throw exception + } + } + + private fun parseDiscovery(document: JsonNode): List { + requireExactFields(document, setOf("schema", "mode", "benchmarks"), "discovery") + check(document.requiredText("schema") == BENCHMARK_WORKER_SCHEMA) { "discovery worker schema is invalid" } + check(document.requiredText("mode") == "discovery") { "discovery worker mode is invalid" } + val benchmarks = document["benchmarks"] + check(benchmarks.isArray) { "discovery worker benchmarks must be an array" } + val names = benchmarks.map { + check(it.isTextual && it.textValue().isNotBlank()) { "discovery benchmark name must be non-empty text" } + it.textValue() + } + check(names.size == names.distinct().size) { "discovery returned duplicate benchmark names" } + check(names.isNotEmpty()) { "benchmark discovery selected no benchmarks" } + return names + } + + private fun parseExecution(document: JsonNode, expectedName: String, iterations: Int): WorkerExecution { + requireExactFields( + document, + setOf("schema", "mode", "qualifiedName", "checksum", "batchSize", "samplesNanos"), + "execution" + ) + check(document.requiredText("schema") == BENCHMARK_WORKER_SCHEMA) { "execution worker schema is invalid" } + check(document.requiredText("mode") == "execution") { "execution worker mode is invalid" } + check(document.requiredText("qualifiedName") == expectedName) { + "execution worker name does not match requested benchmark $expectedName" + } + val checksum = document["checksum"] + check(checksum.isIntegralNumber && checksum.canConvertToInt()) { "execution checksum must be an integer" } + val batchSize = document["batchSize"] + check(batchSize.isIntegralNumber && batchSize.canConvertToInt() && batchSize.intValue() > 0) { + "execution batchSize must be positive" + } + val samplesNode = document["samplesNanos"] + check(samplesNode.isArray && samplesNode.size() == iterations) { + "execution samplesNanos must contain exactly $iterations samples" + } + val samples = samplesNode.map { + check(it.isIntegralNumber && it.canConvertToLong() && it.longValue() >= 0) { + "execution samplesNanos must contain non-negative integers" + } + it.longValue() + } + return WorkerExecution(checksum.intValue(), batchSize.intValue(), samples) + } + + private fun requireExactFields(document: JsonNode, fields: Set, label: String) { + val actual = document.fieldNames().asSequence().toSet() + check(actual == fields) { "$label worker JSON fields are invalid: expected $fields, got $actual" } + } + + private fun JsonNode.requiredText(field: String): String { + val value = this[field] + check(value != null && value.isTextual) { "worker field $field must be text" } + return value.textValue() + } + + private fun safeFileName(name: String): String = name.replace(Regex("[^A-Za-z0-9_.-]"), "_") + + private fun deleteTree(path: Path) { + if (!Files.exists(path)) return + Files.walk(path).use { stream -> + stream.sorted(Comparator.reverseOrder()).forEach { Files.deleteIfExists(it) } + } + } + + private data class WorkerExecution( + val checksum: Int, + val batchSize: Int, + val samplesNanos: List + ) + + private data class ParsedWorker( + val value: T, + val diagnostics: List, + val diagnosticsEmitted: Boolean + ) +} diff --git a/src/main/kotlin/benchmark/BenchmarkModels.kt b/src/main/kotlin/benchmark/BenchmarkModels.kt new file mode 100644 index 0000000..cab7238 --- /dev/null +++ b/src/main/kotlin/benchmark/BenchmarkModels.kt @@ -0,0 +1,95 @@ +package benchmark + +const val BENCHMARK_WORKER_SCHEMA = "wurst-benchmark-worker-v2" +const val BENCHMARK_SCHEMA = "wurst-benchmark-v1" +const val BENCHMARK_DISCLAIMER = + "Benchmark results measure the JVM-hosted Wurst IL interpreter. They vary with the machine, JVM, compiler version, host load, and benchmark setup. Use them for controlled side-by-side comparisons under the same conditions, not as absolute Warcraft III, Jass, Lua, or in-game performance numbers." + +fun interface BenchmarkProcessLauncher { + fun run(arguments: List): BenchmarkProcessResult +} + +data class BenchmarkProcessResult(val exitCode: Int, val output: List) + +data class BenchmarkRequest( + val filter: String? = null, + val forks: Int = 3, + val warmup: Int = 5, + val iterations: Int = 10 +) { + init { + require(forks > 0) { "forks must be positive" } + require(warmup >= 0) { "warmup must be non-negative" } + require(iterations > 0) { "iterations must be positive" } + } +} + +data class BenchmarkEnvironment( + val os: String, + val jvm: String, + val compiler: String, + val grill: String, + val cpuCount: Int +) + +data class BenchmarkStatistics( + val mean: Double, + val standardDeviation: Double, + val min: Long, + val max: Long, + val median: Long, + val p90: Long, + val p95: Long +) { + companion object { + fun fromSamples(samples: List): BenchmarkStatistics { + require(samples.isNotEmpty()) { "at least one benchmark sample is required" } + require(samples.all { it >= 0 }) { "benchmark samples must be non-negative" } + val sorted = samples.sorted() + val mean = samples.average() + val variance = samples + .map { sample -> + val delta = sample - mean + delta * delta + } + .average() + fun nearestRank(percentile: Double): Long { + val rank = kotlin.math.ceil(percentile * sorted.size).toInt().coerceAtLeast(1) + return sorted[rank - 1] + } + return BenchmarkStatistics( + mean = mean, + standardDeviation = kotlin.math.sqrt(variance), + min = sorted.first(), + max = sorted.last(), + median = nearestRank(0.50), + p90 = nearestRank(0.90), + p95 = nearestRank(0.95) + ) + } + } +} + +data class BenchmarkFork( + val fork: Int, + val batchSize: Int, + val samplesNanos: List +) + +data class BenchmarkAggregate( + val qualifiedName: String, + val checksum: Int, + val forks: List, + val statistics: BenchmarkStatistics +) + +data class BenchmarkReport( + val schema: String = BENCHMARK_SCHEMA, + val disclaimer: String = BENCHMARK_DISCLAIMER, + val environment: BenchmarkEnvironment, + val filter: String?, + val forks: Int, + val warmup: Int, + val iterations: Int, + val benchmarks: List +) diff --git a/src/main/kotlin/benchmark/BenchmarkRenderer.kt b/src/main/kotlin/benchmark/BenchmarkRenderer.kt new file mode 100644 index 0000000..253d6cd --- /dev/null +++ b/src/main/kotlin/benchmark/BenchmarkRenderer.kt @@ -0,0 +1,100 @@ +package benchmark + +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.databind.node.ArrayNode +import com.fasterxml.jackson.databind.node.ObjectNode +import java.util.Locale + +object BenchmarkRenderer { + private val mapper = ObjectMapper() + + fun json(report: BenchmarkReport): String { + val root = mapper.createObjectNode() + root.put("schema", report.schema) + root.put("disclaimer", report.disclaimer) + root.set("environment", environment(report.environment)) + if (report.filter == null) root.putNull("filter") else root.put("filter", report.filter) + root.put("forks", report.forks) + root.put("warmup", report.warmup) + root.put("iterations", report.iterations) + val benchmarks = mapper.createArrayNode() + report.benchmarks.forEach { benchmarks.add(benchmark(it)) } + root.set("benchmarks", benchmarks) + return mapper.writeValueAsString(root) + } + + fun human(report: BenchmarkReport): String { + val lines = mutableListOf() + report.benchmarks.forEach { benchmark -> + val stats = benchmark.statistics + val sampleCount = benchmark.forks.sumOf { it.samplesNanos.size } + val batchSizes = benchmark.forks.joinToString(",") { "${it.fork}:${it.batchSize}" } + lines += "${benchmark.qualifiedName} median ${stats.median} ns/op p90 ${stats.p90} ns/op p95 ${stats.p95} ns/op " + + "min/max ${stats.min}/${stats.max} ns/op checksum ${benchmark.checksum} " + + "forks ${benchmark.forks.size} samples $sampleCount batchSizes $batchSizes" + } + if (report.benchmarks.size == 2) { + val first = report.benchmarks[0] + val second = report.benchmarks[1] + if (first.statistics.median == second.statistics.median) { + lines += "${first.qualifiedName} and ${second.qualifiedName} have equal median (${first.statistics.median} ns/op)" + } else { + val faster: BenchmarkAggregate + val slower: BenchmarkAggregate + if (first.statistics.median <= second.statistics.median) { + faster = first + slower = second + } else { + faster = second + slower = first + } + if (faster.statistics.median == 0L) { + lines += "${faster.qualifiedName} median is below timer resolution; no finite speedup ratio can be reported against ${slower.qualifiedName}" + } else { + val ratio = slower.statistics.median.toDouble() / faster.statistics.median + lines += "${faster.qualifiedName} is ${String.format(Locale.ROOT, "%.2fx", ratio)} faster than ${slower.qualifiedName}" + } + } + } + lines += "" + lines += report.disclaimer + return lines.joinToString("\n") + } + + private fun environment(environment: BenchmarkEnvironment): ObjectNode { + val node = mapper.createObjectNode() + node.put("os", environment.os) + node.put("jvm", environment.jvm) + node.put("compiler", environment.compiler) + node.put("grill", environment.grill) + node.put("cpuCount", environment.cpuCount) + return node + } + + private fun benchmark(benchmark: BenchmarkAggregate): ObjectNode { + val node = mapper.createObjectNode() + node.put("qualifiedName", benchmark.qualifiedName) + node.put("checksum", benchmark.checksum) + val forks = mapper.createArrayNode() + benchmark.forks.forEach { fork -> + val forkNode = mapper.createObjectNode() + forkNode.put("fork", fork.fork) + forkNode.put("batchSize", fork.batchSize) + val samples = mapper.createArrayNode() + fork.samplesNanos.forEach(samples::add) + forkNode.set("samplesNanos", samples) + forks.add(forkNode) + } + node.set("forks", forks) + val stats = mapper.createObjectNode() + stats.put("mean", benchmark.statistics.mean) + stats.put("standardDeviation", benchmark.statistics.standardDeviation) + stats.put("min", benchmark.statistics.min) + stats.put("max", benchmark.statistics.max) + stats.put("median", benchmark.statistics.median) + stats.put("p90", benchmark.statistics.p90) + stats.put("p95", benchmark.statistics.p95) + node.set("statistics", stats) + return node + } +} diff --git a/src/main/kotlin/file/CLICommand.kt b/src/main/kotlin/file/CLICommand.kt index 1080ff6..f62eb8e 100644 --- a/src/main/kotlin/file/CLICommand.kt +++ b/src/main/kotlin/file/CLICommand.kt @@ -13,9 +13,15 @@ enum class CLICommand { OUTDATED, BUILD, EXPORTOBJECTS, + BENCHMARK, SELF_UPDATE } +enum class BenchmarkFormat { + HUMAN, + JSON +} + enum class GlobalOptions(val optionName: String = "", val argCount: Int = 0) { REQ_CONFIRM("--request-confirmation") { override fun runOption(setupMain: SetupMain, args: List) { diff --git a/src/main/kotlin/file/SetupApp.kt b/src/main/kotlin/file/SetupApp.kt index bb92658..b3ca1fc 100644 --- a/src/main/kotlin/file/SetupApp.kt +++ b/src/main/kotlin/file/SetupApp.kt @@ -1,5 +1,11 @@ package file +import benchmark.BenchmarkCoordinator +import benchmark.BENCHMARK_DISCLAIMER +import benchmark.BenchmarkProcessLauncher +import benchmark.BenchmarkProcessResult +import benchmark.BenchmarkRenderer +import benchmark.BenchmarkRequest import config.CONFIG_FILE_NAME import config.ScriptMode import config.WurstProjectConfig @@ -30,6 +36,8 @@ object SetupApp { private val log = KotlinLogging.logger {} lateinit var setup: SetupMain + internal var benchmarkProcessLauncherOverride: BenchmarkProcessLauncher? = null + private data class WurstProcessResult(val exitCode: Int, val output: List) internal const val AGENTS_TEMPLATE_VERSION = "2026-06-22" @@ -39,23 +47,15 @@ object SetupApp { fun handleArgs(setup: SetupMain) { this.setup = setup + if (setup.command == CLICommand.BENCHMARK && setup.benchmarkHelp) { + println(benchmarkHelpText()) + ExitHandler.exit(0) + } DependencyManager.debug = setup.debug configureQuietLogging() updateGrillJar() if (setup.isGUILaunch) { - val helpText = """ - Grill is now CLI-first. Use the command line to interact with Grill. - - Example commands: - grill generate MyProject Generate a new Wurst project - grill generate MyProject --with-ci Include GitHub Actions workflow - grill generate MyProject --script-mode jass --wc3-patch pre1.29 - grill install Install/update project dependencies - grill install wurstscript Install the WurstScript compiler - grill build ExampleMap.w3x Build your project map - grill test Run project unit tests - grill help Show all available commands - """.trimIndent() + val helpText = guiHelpText() if (GraphicsEnvironment.isHeadless()) { log.info(helpText) } else { @@ -63,15 +63,92 @@ object SetupApp { } ExitHandler.exit(0) } else { - progress("🔥 Grill ${CompileTimeInfo.version}") + if (!(setup.command == CLICommand.BENCHMARK && setup.benchmarkFormat == BenchmarkFormat.JSON)) { + progress("🔥 Grill ${CompileTimeInfo.version}") + } handleCMD() } } + internal fun benchmarkHelpText(): String = """ + Grill benchmark — run selected @benchmark functions on the JVM-hosted Wurst IL interpreter. + + Usage: + grill benchmark [filter] [--forks N] [--warmup N] [--iterations N] [--format human|json] + + Options: + --forks N Positive number of JVM forks (default: 3) + --warmup N Non-negative warmup samples per fork (default: 5) + --iterations N Positive measured samples per fork (default: 10) + --format human|json Output format (default: human) + --help Show this benchmark-specific help + + JSON environment.compiler is sha256: for the exact compiler JAR. + + Global options such as -projectDir, --quiet, and --debug remain available. + + $BENCHMARK_DISCLAIMER + """.trimIndent() + + internal fun guiHelpText(): String = """ + Grill is now CLI-first. Use the command line to interact with Grill. + + Example commands: + grill generate MyProject Generate a new Wurst project + grill generate MyProject --with-ci Include GitHub Actions workflow + grill generate MyProject --script-mode jass --wc3-patch pre1.29 + grill install Install/update project dependencies + grill install wurstscript Install the WurstScript compiler + grill build ExampleMap.w3x Build your project map + grill test Run project unit tests + grill benchmark [filter] Compare @benchmark functions on the JVM + grill help Show all available commands + """.trimIndent() + + internal fun commandHelpText(): String = """ + |Common: + | grill generate MyProject + | grill install + | grill test + | grill benchmark [filter] + | grill build ExampleMap.w3x + | + |Project commands: + | install [dep|wurstscript|grill] Install/update dependencies, WurstScript compiler, or Grill itself + | remove [dep|wurstscript] Remove a dependency or uninstall WurstScript + | generate Generate a new Wurst project in a subfolder + | test [filter] Run unit tests, optionally filtered by package/function name + | benchmark [filter] Compare @benchmark functions on the JVM (`benchmark --help` for options) + | typecheck Typecheck the project without building a map + | outdated Check whether project dependencies are up to date + | build Build the project using the given input map + | exportobjects Export object editor data to Wurst source + | + |Global options: + | --quiet Suppress wurst output; only print errors and final result + | --debug Print full stack traces for troubleshooting + | + |Build options: + | --dev Build with compiletime isProductionBuild() = false + | + |Generate options: + | --script-mode lua|jass Script mode (default: lua) + | --wc3-patch WC3 patch target: reforged, pre1.29, or jass-history version + | --wc3-path Warcraft III install folder for VS Code/run + | --with-agents / --no-agents Include AGENTS.md (default: no) + | --with-ci / --no-ci Include GitHub Actions workflow (default: no) + | --with-dep Add a curated dependency (repeatable; ids: ${CuratedDependencies.ids.joinToString(", ")}) + """.trimMargin() + private fun configureQuietLogging() { val rootLogger = LoggerFactory.getLogger(org.slf4j.Logger.ROOT_LOGGER_NAME) if (rootLogger is ch.qos.logback.classic.Logger) { - rootLogger.level = if (setup.quiet) ch.qos.logback.classic.Level.ERROR else ch.qos.logback.classic.Level.INFO + rootLogger.level = when { + setup.command == CLICommand.BENCHMARK && setup.benchmarkFormat == BenchmarkFormat.JSON -> + ch.qos.logback.classic.Level.OFF + setup.quiet -> ch.qos.logback.classic.Level.ERROR + else -> ch.qos.logback.classic.Level.INFO + } } } @@ -115,6 +192,7 @@ object SetupApp { InstallationManager.verifyInstallation() } setup.command == CLICommand.TEST || + setup.command == CLICommand.BENCHMARK || setup.command == CLICommand.TYPECHECK || setup.command == CLICommand.BUILD || setup.command == CLICommand.EXPORTOBJECTS -> { @@ -133,40 +211,9 @@ object SetupApp { configData = WurstProjectConfig.loadProject(configFile)!! } - when { + when { setup.command == CLICommand.HELP -> { - log.info(""" - |Common: - | grill generate MyProject - | grill install - | grill test - | grill build ExampleMap.w3x - | - |Project commands: - | install [dep|wurstscript|grill] Install/update dependencies, WurstScript compiler, or Grill itself - | remove [dep|wurstscript] Remove a dependency or uninstall WurstScript - | generate Generate a new Wurst project in a subfolder - | test [filter] Run unit tests, optionally filtered by package/function name - | typecheck Typecheck the project without building a map - | outdated Check whether project dependencies are up to date - | build Build the project using the given input map - | exportobjects Export object editor data to Wurst source - | - |Global options: - | --quiet Suppress wurst output; only print errors and final result - | --debug Print full stack traces for troubleshooting - | - |Build options: - | --dev Build with compiletime isProductionBuild() = false - | - |Generate options: - | --script-mode lua|jass Script mode (default: lua) - | --wc3-patch WC3 patch target: reforged, pre1.29, or jass-history version - | --wc3-path Warcraft III install folder for VS Code/run - | --with-agents / --no-agents Include AGENTS.md (default: no) - | --with-ci / --no-ci Include GitHub Actions workflow (default: no) - | --with-dep Add a curated dependency (repeatable; ids: ${CuratedDependencies.ids.joinToString(", ")}) - """.trimMargin()) + log.info(commandHelpText()) } setup.command == CLICommand.INSTALL -> { if (setup.commandArg.isBlank()) { @@ -228,6 +275,16 @@ object SetupApp { printGenerateNextSteps(projectDir, projectConfig, setup.addAgents, setup.addGithubWorkflow, gameRoot) } } + setup.command == CLICommand.BENCHMARK -> { + if (configData == null) { + missingProject() + } else if (InstallationManager.status != InstallationManager.InstallationStatus.NOT_INSTALLED) { + benchmarkProject(configData) + } else { + System.err.println("❌ Wurst benchmark failed: compiler is not installed.") + ExitHandler.exit(1) + } + } setup.command == CLICommand.TEST -> { progress("⚗️ Running tests...") if (InstallationManager.status != InstallationManager.InstallationStatus.NOT_INSTALLED && configData != null) { @@ -307,9 +364,15 @@ object SetupApp { } private fun missingProject(): Nothing { - log.error("❌ This folder is not a Grill project.") - log.info("Expected: ${setup.projectRoot.resolve(CONFIG_FILE_NAME).toAbsolutePath()}") - log.info("Try: run `grill generate MyProject` to create a new project, or pass `-projectDir `.") + if (setup.command == CLICommand.BENCHMARK && setup.benchmarkFormat == BenchmarkFormat.JSON) { + System.err.println("❌ This folder is not a Grill project.") + System.err.println("Expected: ${setup.projectRoot.resolve(CONFIG_FILE_NAME).toAbsolutePath()}") + System.err.println("Try: run `grill generate MyProject` to create a new project, or pass `-projectDir `.") + } else { + log.error("❌ This folder is not a Grill project.") + log.info("Expected: ${setup.projectRoot.resolve(CONFIG_FILE_NAME).toAbsolutePath()}") + log.info("Try: run `grill generate MyProject` to create a new project, or pass `-projectDir `.") + } ExitHandler.exit(1) } @@ -1033,6 +1096,52 @@ object SetupApp { } } + private fun benchmarkProject(configData: WurstProjectConfigData) { + val args = commonArgs( + configData, + includeOutput = false, + includeCompileTimeFunctions = false + ) + if (!args.contains("-compactOutput")) { + args.add("-compactOutput") + } + val launcher = benchmarkProcessLauncherOverride ?: BenchmarkProcessLauncher { arguments -> + runBenchmarkWorkerProcess(arguments, compilerOutputDir()) + } + val coordinator = BenchmarkCoordinator( + launcher = launcher, + commonArguments = args, + debug = setup.debug, + compilerIdentity = InstallationManager.getCompilerIdentity(), + grillIdentity = CompileTimeInfo.version + ) + val rendered: String + try { + val report = coordinator.run( + BenchmarkRequest( + filter = setup.commandArg.ifBlank { null }, + forks = setup.benchmarkForks, + warmup = setup.benchmarkWarmup, + iterations = setup.benchmarkIterations + ) + ) + rendered = if (setup.benchmarkFormat == BenchmarkFormat.JSON) { + BenchmarkRenderer.json(report) + } else { + BenchmarkRenderer.human(report) + } + } catch (exception: Exception) { + if (setup.debug) { + exception.printStackTrace(System.err) + } else { + System.err.println("❌ Wurst benchmark failed: ${exception.message ?: exception.javaClass.simpleName}") + } + ExitHandler.exit(1) + } + println(rendered) + ExitHandler.exit(0) + } + private fun typecheckProject(configData: WurstProjectConfigData) { val args = commonArgs(configData) @@ -1069,9 +1178,39 @@ object SetupApp { return result } - private fun runWurstProcess(args: ArrayList, compactFallback: Boolean): WurstProcessResult { + private fun runWurstProcess( + args: ArrayList, + compactFallback: Boolean, + emitOutput: Boolean = true + ): WurstProcessResult { + return runProcess( + args = args, + outputDir = compilerOutputDir(), + emitOutput = emitOutput, + debug = setup.debug, + quiet = setup.quiet + ) + } + + internal fun runBenchmarkWorkerProcess(arguments: List, outputDir: Path): BenchmarkProcessResult { + val result = runProcess( + args = arguments, + outputDir = outputDir, + emitOutput = false, + debug = false, + quiet = true + ) + return BenchmarkProcessResult(result.exitCode, result.output) + } + + private fun runProcess( + args: List, + outputDir: Path, + emitOutput: Boolean, + debug: Boolean, + quiet: Boolean + ): WurstProcessResult { val pb = ProcessBuilder(args) - val outputDir = compilerOutputDir() Files.createDirectories(outputDir) pb.directory(outputDir.toFile()) pb.redirectErrorStream(true) @@ -1079,10 +1218,10 @@ object SetupApp { val output = ArrayList() p.inputStream.bufferedReader().forEachLine { line -> output.add(line) - if (!setup.debug && isNoisyCompilerVersionLine(line)) { + if (!debug && isNoisyCompilerVersionLine(line)) { return@forEachLine } - if (!setup.quiet) { + if (emitOutput && !quiet) { println(line) } } @@ -1090,7 +1229,11 @@ object SetupApp { return WurstProcessResult(exitCode, output) } - private fun commonArgs(configData: WurstProjectConfigData): ArrayList { + private fun commonArgs( + configData: WurstProjectConfigData, + includeOutput: Boolean = true, + includeCompileTimeFunctions: Boolean = true + ): ArrayList { val args = ArrayList(InstallationManager.compilerLaunchCommand().toList()) if (configData.scriptMode == ScriptMode.LUA) { @@ -1101,10 +1244,12 @@ object SetupApp { } val buildFolder = setup.projectRoot.resolve("_build") - val outputDir = compilerOutputDir() - Files.createDirectories(outputDir) - args.add("-out") - args.add(outputDir.resolve(outputFileName(configData)).toAbsolutePath().toString()) + if (includeOutput) { + val outputDir = compilerOutputDir() + Files.createDirectories(outputDir) + args.add("-out") + args.add(outputDir.resolve(outputFileName(configData)).toAbsolutePath().toString()) + } val jassdoc = buildFolder.resolve("dependencies").resolve("jassdoc") if (Files.exists(jassdoc)) { @@ -1120,7 +1265,9 @@ object SetupApp { } args.add(setup.projectRoot.resolve("wurst").toAbsolutePath().toString()) - args.add("-runcompiletimefunctions") + if (includeCompileTimeFunctions) { + args.add("-runcompiletimefunctions") + } if (setup.noPJass) { args.add("-noPJass") } diff --git a/src/main/kotlin/file/SetupMain.kt b/src/main/kotlin/file/SetupMain.kt index 0b3e45a..5f2054c 100644 --- a/src/main/kotlin/file/SetupMain.kt +++ b/src/main/kotlin/file/SetupMain.kt @@ -13,6 +13,16 @@ class SetupMain { var commandArg = "" + var benchmarkForks = 3 + + var benchmarkWarmup = 5 + + var benchmarkIterations = 10 + + var benchmarkFormat = BenchmarkFormat.HUMAN + + var benchmarkHelp = false + var measure = false var devBuild = false @@ -68,6 +78,10 @@ class SetupMain { try { command = CLICommand.valueOf(first.uppercase()) log.debug("found $command") + if (command == CLICommand.BENCHMARK) { + parseBenchmarkArgs(argsList.drop(1)) + return + } if (argsList.size > 1) { if (!argsList[1].startsWith("-")) { commandArg = argsList[1] @@ -83,6 +97,92 @@ class SetupMain { } } + private fun parseBenchmarkArgs(argsList: List) { + var i = 0 + var filterSeen = false + while (i < argsList.size) { + when (val arg = argsList[i]) { + "--help" -> { + benchmarkHelp = true + i++ + } + "--forks" -> { + benchmarkForks = parseBenchmarkPositiveInt(arg, benchmarkOptionValue(argsList, i, arg)) + i += 2 + } + "--warmup" -> { + benchmarkWarmup = parseBenchmarkNonNegativeInt(arg, benchmarkOptionValue(argsList, i, arg)) + i += 2 + } + "--iterations" -> { + benchmarkIterations = parseBenchmarkPositiveInt(arg, benchmarkOptionValue(argsList, i, arg)) + i += 2 + } + "--format" -> { + benchmarkFormat = parseBenchmarkFormat(arg, benchmarkOptionValue(argsList, i, arg)) + i += 2 + } + else -> { + val globalOption = GlobalOptions.values().firstOrNull { it.optionName == arg } + if (globalOption != null) { + val argEnd = i + 1 + globalOption.argCount + if (argEnd > argsList.size) { + benchmarkError("Option $arg requires ${globalOption.argCount} argument(s).") + } + globalOption.runOption(this, argsList.subList(i + 1, argEnd)) + i = argEnd + } else if (arg.startsWith("-")) { + benchmarkError("Unknown benchmark option <$arg>.") + } else if (filterSeen) { + benchmarkError("Unexpected extra benchmark argument <$arg>.") + } else { + commandArg = arg + filterSeen = true + i++ + } + } + } + } + } + + private fun benchmarkOptionValue(argsList: List, optionIndex: Int, option: String): String { + if (optionIndex + 1 >= argsList.size) { + benchmarkError("Option $option requires an argument.") + } + return argsList[optionIndex + 1] + } + + private fun parseBenchmarkPositiveInt(option: String, value: String): Int { + val parsed = value.toIntOrNull() + if (parsed == null || parsed <= 0) { + benchmarkError("Option $option requires a positive integer.") + } + return parsed + } + + private fun parseBenchmarkNonNegativeInt(option: String, value: String): Int { + val parsed = value.toIntOrNull() + if (parsed == null || parsed < 0) { + benchmarkError("Option $option requires a non-negative integer.") + } + return parsed + } + + private fun parseBenchmarkFormat(option: String, value: String): BenchmarkFormat { + return when (value.lowercase()) { + "human" -> BenchmarkFormat.HUMAN + "json" -> BenchmarkFormat.JSON + else -> { + benchmarkError("Option $option accepts only human or json.") + } + } + } + + private fun benchmarkError(message: String): Nothing { + log.error("❌ $message") + ExitHandler.exit(1) + } + private fun parseGlobalArgs(argsList: List, start: Int) { var i = start while (i < argsList.size) { diff --git a/src/main/kotlin/global/InstallationManager.kt b/src/main/kotlin/global/InstallationManager.kt index 0f0ba81..53613c7 100644 --- a/src/main/kotlin/global/InstallationManager.kt +++ b/src/main/kotlin/global/InstallationManager.kt @@ -9,6 +9,8 @@ import net.NetStatus import java.nio.file.Files import java.nio.file.Path import java.nio.file.Paths +import java.security.MessageDigest +import java.util.Locale import java.util.regex.Pattern @@ -171,6 +173,23 @@ object InstallationManager { return (detectCompilerJar() ?: compilerJar).toAbsolutePath().toString() } + fun getCompilerIdentity(): String { + val digest = MessageDigest.getInstance("SHA-256") + val compiler = detectCompilerJar() ?: compilerJar + Files.newInputStream(compiler).use { input -> + val buffer = ByteArray(DEFAULT_BUFFER_SIZE) + while (true) { + val read = input.read(buffer) + if (read < 0) break + digest.update(buffer, 0, read) + } + } + val hex = digest.digest().joinToString("") { + "%02x".format(Locale.ROOT, it.toInt() and 0xff) + } + return "sha256:$hex" + } + fun compilerLaunchCommand(vararg extraArgs: String): Array { val compiler = detectCompilerJar() ?: compilerJar val java = bundledJavaCommand() diff --git a/src/test/kotlin/BenchmarkCommandTests.kt b/src/test/kotlin/BenchmarkCommandTests.kt new file mode 100644 index 0000000..b214021 --- /dev/null +++ b/src/test/kotlin/BenchmarkCommandTests.kt @@ -0,0 +1,223 @@ +import file.BenchmarkFormat +import file.CLICommand +import file.ExitHandler +import file.SetupMain +import file.SetupApp +import org.testng.Assert +import org.testng.annotations.Test +import java.io.ByteArrayOutputStream +import java.io.PrintStream +import java.nio.file.Files +import java.nio.charset.StandardCharsets + +private const val CANONICAL_BENCHMARK_DISCLAIMER = + "Benchmark results measure the JVM-hosted Wurst IL interpreter. They vary with the machine, JVM, compiler version, host load, and benchmark setup. Use them for controlled side-by-side comparisons under the same conditions, not as absolute Warcraft III, Jass, Lua, or in-game performance numbers." + +private class BenchmarkExitException(val code: Int) : RuntimeException("exit $code") + +private fun benchmarkExitCode(block: () -> Unit): Int { + val previousHandler = ExitHandler.handler + return try { + ExitHandler.handler = { throw BenchmarkExitException(it) } + block() + -1 + } catch (exception: BenchmarkExitException) { + exception.code + } finally { + ExitHandler.handler = previousHandler + } +} + +private fun captureStdout(block: () -> Unit): String { + val previousOut = System.out + val output = ByteArrayOutputStream() + val capturedOut = PrintStream(output, true, StandardCharsets.UTF_8) + System.setOut(capturedOut) + try { + block() + } finally { + capturedOut.flush() + System.setOut(previousOut) + } + return output.toString(StandardCharsets.UTF_8) +} + +class BenchmarkCommandTests { + + @Test + fun generalHelpSurfacesBenchmarkCommand() { + Assert.assertTrue(SetupApp.guiHelpText().contains("grill benchmark [filter]")) + Assert.assertTrue(SetupApp.commandHelpText().contains("benchmark [filter]")) + } + + @Test + fun benchmarkAcceptsGlobalProjectAndDiagnosticOptions() { + val projectRoot = Files.createTempDirectory("grill-benchmark-global-options") + try { + val setup = SetupMain() + + val exitCode = benchmarkExitCode { + setup.parseArgs( + listOf( + "benchmark", + "Polygon", + "-projectDir", projectRoot.toString(), + "--quiet", + "--debug" + ) + ) + } + + Assert.assertEquals(exitCode, -1) + Assert.assertEquals(setup.projectRoot, projectRoot) + Assert.assertTrue(setup.quiet) + Assert.assertTrue(setup.debug) + Assert.assertEquals(setup.commandArg, "Polygon") + } finally { + Files.deleteIfExists(projectRoot) + } + } + + @Test + fun parsesBenchmarkOptionsAndFilter() { + val setup = SetupMain() + + setup.parseArgs( + listOf( + "benchmark", + "Polygon", + "--forks", "3", + "--warmup", "5", + "--iterations", "10", + "--format", "json" + ) + ) + + Assert.assertEquals(setup.command, CLICommand.BENCHMARK) + Assert.assertEquals(setup.commandArg, "Polygon") + Assert.assertEquals(setup.benchmarkForks, 3) + Assert.assertEquals(setup.benchmarkWarmup, 5) + Assert.assertEquals(setup.benchmarkIterations, 10) + Assert.assertEquals(setup.benchmarkFormat, BenchmarkFormat.JSON) + } + + @Test + fun usesBenchmarkDefaults() { + val setup = SetupMain() + + setup.parseArgs(listOf("benchmark")) + + Assert.assertEquals(setup.command, CLICommand.BENCHMARK) + Assert.assertTrue(setup.commandArg.isEmpty()) + Assert.assertEquals(setup.benchmarkForks, 3) + Assert.assertEquals(setup.benchmarkWarmup, 5) + Assert.assertEquals(setup.benchmarkIterations, 10) + Assert.assertEquals(setup.benchmarkFormat, BenchmarkFormat.HUMAN) + } + + @Test + fun acceptsZeroWarmupIterations() { + val setup = SetupMain() + + val exitCode = benchmarkExitCode { + setup.parseArgs(listOf("benchmark", "--warmup", "0")) + } + + Assert.assertEquals(exitCode, -1) + Assert.assertEquals(setup.benchmarkWarmup, 0) + } + + @Test + fun benchmarkHelpShowsOnlyBenchmarkHelpWithoutProjectSetup() { + val projectRoot = Files.createTempDirectory("grill-benchmark-help") + try { + Files.writeString(projectRoot.resolve("wurst.build"), "not valid project config") + val setup = SetupMain().apply { this.projectRoot = projectRoot } + + var exitCode = -1 + val output = captureStdout { + exitCode = benchmarkExitCode { + setup.doMain(arrayOf("benchmark", "--help")) + } + } + + Assert.assertEquals(exitCode, 0) + Assert.assertTrue(setup.benchmarkHelp) + Assert.assertTrue(output.contains("grill benchmark [filter]"), output) + Assert.assertTrue(output.contains("--forks N"), output) + Assert.assertTrue(output.contains("--warmup N"), output) + Assert.assertTrue(output.contains("--iterations N"), output) + Assert.assertTrue(output.contains("--format human|json"), output) + Assert.assertTrue(output.contains("environment.compiler"), output) + Assert.assertTrue(output.contains("sha256:"), output) + Assert.assertTrue(output.contains(CANONICAL_BENCHMARK_DISCLAIMER), output) + } finally { + Files.walk(projectRoot).sorted(Comparator.reverseOrder()).forEach(Files::deleteIfExists) + } + } + + @Test + fun rejectsEveryInvalidIntegerCategory() { + val invalidValuesByOption = mapOf( + "--forks" to listOf("0", "-1", "not-a-number"), + "--warmup" to listOf("-1", "not-a-number"), + "--iterations" to listOf("0", "-1", "not-a-number") + ) + + for ((option, invalidValues) in invalidValuesByOption) { + for (value in invalidValues) { + val code = benchmarkExitCode { + SetupMain().parseArgs(listOf("benchmark", option, value)) + } + Assert.assertEquals(code, 1, "$option $value should be rejected") + } + } + } + + @Test + fun rejectsMissingIntegerValues() { + for (option in listOf("--forks", "--warmup", "--iterations")) { + val code = benchmarkExitCode { + SetupMain().parseArgs(listOf("benchmark", option)) + } + Assert.assertEquals(code, 1, "$option without a value should be rejected") + } + } + + @Test + fun rejectsUnsupportedAndMissingFormats() { + for (args in listOf( + listOf("benchmark", "--format", "xml"), + listOf("benchmark", "--format") + )) { + val code = benchmarkExitCode { + SetupMain().parseArgs(args) + } + Assert.assertEquals(code, 1, "${args.joinToString(" ")} should be rejected") + } + } + + @Test + fun rejectsUnexpectedExtraPositionalArguments() { + val code = benchmarkExitCode { + SetupMain().parseArgs(listOf("benchmark", "first", "second")) + } + + Assert.assertEquals(code, 1) + } + + @Test + fun readmeDocumentsTheSameBenchmarkContractAsHelp() { + val readme = Files.readString(java.nio.file.Path.of("README.md"), StandardCharsets.UTF_8) + + Assert.assertTrue(readme.contains(CANONICAL_BENCHMARK_DISCLAIMER), "README must contain the canonical disclaimer") + Assert.assertTrue(readme.contains("@benchmark function benchmarkName() returns int"), readme) + Assert.assertTrue(readme.contains("--forks N"), readme) + Assert.assertTrue(readme.contains("--warmup N"), readme) + Assert.assertTrue(readme.contains("--iterations N"), readme) + Assert.assertTrue(readme.contains("--format human|json"), readme) + Assert.assertTrue(readme.contains("environment.compiler"), readme) + Assert.assertTrue(readme.contains("sha256:"), readme) + Assert.assertTrue(readme.contains("workload-derived checksum"), readme) + } +} diff --git a/src/test/kotlin/BenchmarkCoordinatorTests.kt b/src/test/kotlin/BenchmarkCoordinatorTests.kt new file mode 100644 index 0000000..8f8cc9d --- /dev/null +++ b/src/test/kotlin/BenchmarkCoordinatorTests.kt @@ -0,0 +1,780 @@ +import benchmark.BenchmarkCoordinator +import benchmark.BenchmarkProcessLauncher +import benchmark.BenchmarkProcessResult +import benchmark.BenchmarkRequest +import benchmark.BenchmarkRenderer +import com.fasterxml.jackson.databind.ObjectMapper +import file.ExitHandler +import file.SetupApp +import file.SetupMain +import org.testng.Assert +import org.testng.annotations.Test +import java.io.ByteArrayOutputStream +import java.io.PrintStream +import java.nio.charset.StandardCharsets +import java.nio.file.Files +import java.nio.file.Path +import java.util.Comparator + +private class CoordinatorExitException(val code: Int) : RuntimeException("exit $code") + +private class FixtureLauncher( + private val names: List = listOf("Bench.Fast", "Bench.Slow"), + private val samples: Map> = mapOf( + "Bench.Fast" to listOf(100L, 120L), + "Bench.Slow" to listOf(900L, 1_000L) + ), + private val checksums: Map = names.associateWith { 4950 }, + private val outputFor: (List, Int) -> String? = { arguments, _ -> + val name = arguments.optionValue("-benchmarkName") + if (name == null) { + val benchmarkJson = names.joinToString(",") { "\"$it\"" } + """{"schema":"wurst-benchmark-worker-v2","mode":"discovery","benchmarks":[$benchmarkJson]}""" + } else { + val values = samples.getValue(name) + workerExecutionJson(name, checksums.getValue(name), values, batchSize = 2) + } + }, + private val diagnosticsFor: (List, Int) -> List = { _, _ -> + listOf("worker log must stay captured") + } +) : BenchmarkProcessLauncher { + val calls = mutableListOf>() + val outputPaths = mutableListOf() + var active = 0 + var maxActive = 0 + var invocation = 0 + var exitCode = 0 + + override fun run(arguments: List): BenchmarkProcessResult { + Assert.assertEquals(active, 0, "worker launches must be serial") + active++ + maxActive = maxOf(maxActive, active) + calls += arguments + val output = Path.of(arguments.optionValue("-benchmarkOutput")!!) + outputPaths.add(output) + val callIndex = invocation++ + val json = outputFor(arguments, callIndex) + if (json != null) { + Files.writeString(output, json) + } + active-- + return BenchmarkProcessResult(exitCode, diagnosticsFor(arguments, callIndex)) + } +} + +private fun workerExecutionJson( + name: String, + checksum: Int = 1, + samples: List = listOf(1L), + batchSize: Int = 1 +): String = + """{"schema":"wurst-benchmark-worker-v2","mode":"execution","qualifiedName":"$name","checksum":$checksum,"batchSize":$batchSize,"samplesNanos":[${samples.joinToString(",")}]""" + "}" + +private fun List.optionValue(option: String): String? { + val index = indexOf(option) + return if (index >= 0 && index + 1 < size) this[index + 1] else null +} + +private fun expectCoordinatorFailure(block: () -> Unit) { + try { + block() + Assert.fail("expected benchmark coordination to fail") + } catch (_: IllegalStateException) { + } +} + +private fun deleteTree(path: Path) { + if (Files.exists(path)) { + Files.walk(path).use { stream -> + stream.sorted(Comparator.reverseOrder()).forEach(Files::deleteIfExists) + } + } +} + +private fun captureOutput(block: () -> Unit): Pair { + val oldOut = System.out + val oldErr = System.err + val out = ByteArrayOutputStream() + val err = ByteArrayOutputStream() + System.setOut(PrintStream(out, true, StandardCharsets.UTF_8)) + System.setErr(PrintStream(err, true, StandardCharsets.UTF_8)) + try { + block() + } finally { + System.setOut(oldOut) + System.setErr(oldErr) + } + return out.toString(StandardCharsets.UTF_8) to err.toString(StandardCharsets.UTF_8) +} + +class BenchmarkCoordinatorTests { + + @Test + fun realBenchmarkWorkerProcessCapturesOutputAndPreservesExitCodes() { + val processDirectory = Files.createTempDirectory("grill-real-worker-process") + val javaExecutable = Path.of( + System.getProperty("java.home"), + "bin", + if (System.getProperty("os.name").startsWith("Windows", ignoreCase = true)) "java.exe" else "java" + ).toString() + lateinit var success: BenchmarkProcessResult + lateinit var failure: BenchmarkProcessResult + + try { + val (stdout, stderr) = captureOutput { + success = SetupApp.runBenchmarkWorkerProcess( + listOf(javaExecutable, "-version"), + processDirectory + ) + failure = SetupApp.runBenchmarkWorkerProcess( + listOf(javaExecutable, "-definitely-not-a-real-java-option"), + processDirectory + ) + } + + Assert.assertEquals(success.exitCode, 0) + Assert.assertTrue(success.output.isNotEmpty()) + Assert.assertNotEquals(failure.exitCode, 0) + Assert.assertTrue(failure.output.isNotEmpty()) + Assert.assertEquals(stdout, "") + Assert.assertEquals(stderr, "") + } finally { + deleteTree(processDirectory) + } + } + + @Test + fun launchesDiscoveryThenSerialAlternatingWorkerRounds() { + val launcher = FixtureLauncher() + val report = BenchmarkCoordinator(launcher, listOf("java", "-jar", "compiler.jar")) + .run(BenchmarkRequest("Bench", forks = 2, warmup = 3, iterations = 2)) + + Assert.assertEquals(launcher.calls.size, 5) + Assert.assertNull(launcher.calls[0].optionValue("-benchmarkName")) + Assert.assertEquals( + launcher.calls.drop(1).mapNotNull { it.optionValue("-benchmarkName") }, + listOf("Bench.Fast", "Bench.Slow", "Bench.Slow", "Bench.Fast") + ) + val outputArguments = launcher.calls.map { it.optionValue("-benchmarkOutput")!! } + Assert.assertEquals(outputArguments.distinct().size, outputArguments.size, "every worker needs a unique output path") + Assert.assertEquals(launcher.maxActive, 1) + Assert.assertTrue(launcher.calls[0].containsAll(listOf("-compactOutput", "-runbenchmarks", "-benchmarkList", "-benchmarkFilter", "Bench"))) + Assert.assertTrue(launcher.calls.drop(1).all { args -> + args.containsAll(listOf("-compactOutput", "-runbenchmarks", "-benchmarkWarmup", "3", "-benchmarkIterations", "2")) + }) + Assert.assertEquals(report.benchmarks.map { it.qualifiedName }, listOf("Bench.Fast", "Bench.Slow")) + Assert.assertTrue(launcher.outputPaths.all { !Files.exists(it) }) + Assert.assertEquals(report.benchmarks.first().forks.size, 2) + } + + @Test + fun preservesRawSamplesAndBatchSizesAndRecomputesNearestRankAggregates() { + val launcher = FixtureLauncher( + names = listOf("Bench.Only"), + samples = mapOf("Bench.Only" to listOf(100L, 500L)), + checksums = mapOf("Bench.Only" to 42) + ) + val report = BenchmarkCoordinator(launcher, emptyList()) + .run(BenchmarkRequest(null, forks = 3, warmup = 0, iterations = 2)) + val benchmark = report.benchmarks.single() + + Assert.assertEquals(benchmark.checksum, 42) + Assert.assertEquals(benchmark.forks.flatMap { it.samplesNanos }, listOf(100L, 500L, 100L, 500L, 100L, 500L)) + Assert.assertTrue(benchmark.forks.all { it.batchSize == 2 }) + Assert.assertEquals(benchmark.statistics.median, 100L) + Assert.assertEquals(benchmark.statistics.p95, 500L) + Assert.assertEquals(benchmark.statistics.min, 100L) + Assert.assertEquals(benchmark.statistics.max, 500L) + } + + @Test + fun acceptsSamplesOnlyExecutionDocumentsAndDerivesStatistics() { + val launcher = FixtureLauncher( + names = listOf("Bench.Only"), + outputFor = { arguments, _ -> + val name = arguments.optionValue("-benchmarkName") + if (name == null) { + """{"schema":"wurst-benchmark-worker-v2","mode":"discovery","benchmarks":["Bench.Only"]}""" + } else { + """{"schema":"wurst-benchmark-worker-v2","mode":"execution","qualifiedName":"Bench.Only","checksum":42,"batchSize":1,"samplesNanos":[100,500]}""" + } + } + ) + + val report = BenchmarkCoordinator(launcher, emptyList()) + .run(BenchmarkRequest(null, forks = 1, warmup = 0, iterations = 2)) + val statistics = report.benchmarks.single().statistics + + Assert.assertEquals(statistics.median, 100L) + Assert.assertEquals(statistics.p95, 500L) + } + + @Test + fun acceptsZeroNanosecondWorkerSamples() { + val launcher = FixtureLauncher( + names = listOf("Bench.Only"), + outputFor = { arguments, _ -> + val name = arguments.optionValue("-benchmarkName") + if (name == null) { + """{"schema":"wurst-benchmark-worker-v2","mode":"discovery","benchmarks":["Bench.Only"]}""" + } else { + workerExecutionJson(name, samples = listOf(0L)) + } + } + ) + + val report = BenchmarkCoordinator(launcher, emptyList()) + .run(BenchmarkRequest(null, forks = 1, warmup = 0, iterations = 1)) + + Assert.assertEquals(report.benchmarks.single().forks.single().samplesNanos, listOf(0L)) + Assert.assertEquals(report.benchmarks.single().statistics.median, 0L) + } + + @Test + fun recomputesP90FromEveryRawSampleInsteadOfWorkerStatistics() { + val launcher = FixtureLauncher( + names = listOf("Bench.Only"), + samples = mapOf("Bench.Only" to (1L..10L).toList()) + ) + val report = BenchmarkCoordinator(launcher, emptyList()) + .run(BenchmarkRequest(null, forks = 1, warmup = 0, iterations = 10)) + + val statistics = ObjectMapper().readTree(BenchmarkRenderer.json(report))["benchmarks"][0]["statistics"] + Assert.assertEquals(statistics["median"].longValue(), 5L) + Assert.assertEquals(statistics["p90"].longValue(), 9L) + Assert.assertEquals(statistics["p95"].longValue(), 10L) + } + + @Test + fun rejectsEmptyDiscoveryNonzeroExitAndMalformedWorkerDocuments() { + val empty = FixtureLauncher(names = emptyList()) + val (emptyStdout, emptyStderr) = captureOutput { + expectCoordinatorFailure { + BenchmarkCoordinator(empty, emptyList()).run(BenchmarkRequest(null, 1, 0, 1)) + } + } + Assert.assertEquals(emptyStdout, "") + Assert.assertTrue(emptyStderr.contains("worker log must stay captured"), emptyStderr) + Assert.assertTrue(empty.outputPaths.all { !Files.exists(it) }) + + val failed = FixtureLauncher().apply { exitCode = 7 } + val (failedStdout, failedStderr) = captureOutput { + expectCoordinatorFailure { + BenchmarkCoordinator(failed, emptyList()).run(BenchmarkRequest(null, 1, 0, 1)) + } + } + Assert.assertEquals(failedStdout, "") + Assert.assertTrue(failedStderr.contains("worker log must stay captured"), failedStderr) + Assert.assertTrue(failed.outputPaths.all { !Files.exists(it) }) + + val missing = FixtureLauncher( + names = listOf("Bench.Only"), + samples = mapOf("Bench.Only" to listOf(1L)), + outputFor = { arguments, _ -> + if (arguments.optionValue("-benchmarkName") == null) { + """{"schema":"wurst-benchmark-worker-v2","mode":"discovery","benchmarks":["Bench.Only"]}""" + } else { + null + } + } + ) + val (missingStdout, missingStderr) = captureOutput { + expectCoordinatorFailure { + BenchmarkCoordinator(missing, emptyList()).run(BenchmarkRequest(null, 1, 0, 1)) + } + } + Assert.assertEquals(missingStdout, "") + Assert.assertTrue(missingStderr.contains("worker log must stay captured"), missingStderr) + Assert.assertTrue(missing.outputPaths.all { !Files.exists(it) }) + + val malformed = object : BenchmarkProcessLauncher { + val outputPaths = mutableListOf() + + override fun run(arguments: List): BenchmarkProcessResult { + val output = Path.of(arguments.optionValue("-benchmarkOutput")!!) + outputPaths.add(output) + Files.writeString(output, "not json") + return BenchmarkProcessResult(0, listOf("malformed worker diagnostic")) + } + } + val (malformedStdout, malformedStderr) = captureOutput { + expectCoordinatorFailure { + BenchmarkCoordinator(malformed, emptyList()).run(BenchmarkRequest(null, 1, 0, 1)) + } + } + Assert.assertEquals(malformedStdout, "") + Assert.assertTrue(malformedStderr.contains("malformed worker diagnostic"), malformedStderr) + Assert.assertTrue(malformed.outputPaths.all { !Files.exists(it) }) + } + + @Test + fun rejectsTrailingContentAfterWorkerJsonDocument() { + val launcher = FixtureLauncher( + outputFor = { arguments, _ -> + val name = arguments.optionValue("-benchmarkName") + if (name == null) { + """{"schema":"wurst-benchmark-worker-v2","mode":"discovery","benchmarks":["Bench.Only"]} {"extra":true}""" + } else { + workerExecutionJson(name) + } + } + ) + + val (_, stderr) = captureOutput { + expectCoordinatorFailure { + BenchmarkCoordinator(launcher, emptyList()).run(BenchmarkRequest(null, 1, 0, 1)) + } + } + + Assert.assertTrue(stderr.contains("worker log must stay captured"), stderr) + Assert.assertTrue(launcher.outputPaths.all { !Files.exists(it) }) + } + + @Test + fun rejectsWrongSchemaNameAndChecksumDrift() { + val wrongSchema = object : BenchmarkProcessLauncher { + val outputPaths = mutableListOf() + + override fun run(arguments: List): BenchmarkProcessResult { + val output = Path.of(arguments.optionValue("-benchmarkOutput")!!) + outputPaths.add(output) + val name = arguments.optionValue("-benchmarkName") + val json = if (name == null) { + """{"schema":"wrong","mode":"discovery","benchmarks":["Bench.Only"]}""" + } else { + """{"schema":"wurst-benchmark-worker-v2","mode":"execution","qualifiedName":"Wrong.Name","checksum":1,"batchSize":1,"samplesNanos":[1]}""" + } + Files.writeString(output, json) + return BenchmarkProcessResult(0, listOf("wrong schema diagnostic")) + } + } + val (wrongSchemaStdout, wrongSchemaStderr) = captureOutput { + expectCoordinatorFailure { + BenchmarkCoordinator(wrongSchema, emptyList()).run(BenchmarkRequest(null, 1, 0, 1)) + } + } + Assert.assertEquals(wrongSchemaStdout, "") + Assert.assertTrue(wrongSchemaStderr.contains("wrong schema diagnostic"), wrongSchemaStderr) + Assert.assertTrue(wrongSchema.outputPaths.all { !Files.exists(it) }) + + val drift = FixtureLauncher( + names = listOf("Bench.Only"), + checksums = mapOf("Bench.Only" to 1), + outputFor = { arguments, invocation -> + val name = arguments.optionValue("-benchmarkName") + if (name == null) { + """{"schema":"wurst-benchmark-worker-v2","mode":"discovery","benchmarks":["Bench.Only"]}""" + } else { + val checksum = if (invocation == 1) 2 else 1 + workerExecutionJson("Bench.Only", checksum = checksum) + } + } + ) + expectCoordinatorFailure { + BenchmarkCoordinator(drift, emptyList()).run(BenchmarkRequest(null, 2, 0, 1)) + } + Assert.assertTrue(drift.outputPaths.all { !Files.exists(it) }) + } + + @Test + fun rejectsOldAndMixedWorkerSchemas() { + val oldSchema = FixtureLauncher( + outputFor = { arguments, _ -> + if (arguments.optionValue("-benchmarkName") == null) { + """{"schema":"wurst-benchmark-worker-v1","mode":"discovery","benchmarks":["Bench.Only"]}""" + } else { + """{"schema":"wurst-benchmark-worker-v1","mode":"execution","qualifiedName":"Bench.Only","checksum":1,"batchSize":1,"samplesNanos":[1]}""" + } + } + ) + expectCoordinatorFailure { + BenchmarkCoordinator(oldSchema, emptyList()).run(BenchmarkRequest(null, 1, 0, 1)) + } + + val mixedSchema = FixtureLauncher( + outputFor = { arguments, _ -> + if (arguments.optionValue("-benchmarkName") == null) { + """{"schema":"wurst-benchmark-worker-v2","mode":"discovery","benchmarks":["Bench.Only"]}""" + } else { + """{"schema":"wurst-benchmark-worker-v1","mode":"execution","qualifiedName":"Bench.Only","checksum":1,"batchSize":1,"samplesNanos":[1]}""" + } + } + ) + expectCoordinatorFailure { + BenchmarkCoordinator(mixedSchema, emptyList()).run(BenchmarkRequest(null, 1, 0, 1)) + } + } + + @Test + fun rejectsTwoBenchmarkComparisonWhenChecksumsDiffer() { + val launcher = FixtureLauncher( + checksums = mapOf( + "Bench.Fast" to 10, + "Bench.Slow" to 11 + ) + ) + + expectCoordinatorFailure { + BenchmarkCoordinator(launcher, emptyList()).run(BenchmarkRequest(null, 1, 0, 2)) + } + + Assert.assertTrue(launcher.outputPaths.all { !Files.exists(it) }) + } + + @Test + fun emitsOnlyDriftingWorkerDiagnosticOnceAndSuppressesSuccessfulWorkerOutput() { + val launcher = FixtureLauncher( + names = listOf("Bench.Only"), + checksums = mapOf("Bench.Only" to 1), + diagnosticsFor = { arguments, invocation -> + if (arguments.optionValue("-benchmarkName") != null && invocation == 2) { + listOf("drifting worker diagnostic") + } else { + listOf("successful worker diagnostic") + } + }, + outputFor = { arguments, invocation -> + val name = arguments.optionValue("-benchmarkName") + if (name == null) { + """{"schema":"wurst-benchmark-worker-v2","mode":"discovery","benchmarks":["Bench.Only"]}""" + } else { + val checksum = if (invocation == 1) 2 else 1 + workerExecutionJson("Bench.Only", checksum = checksum) + } + } + ) + val (stdout, stderr) = captureOutput { + expectCoordinatorFailure { + BenchmarkCoordinator(launcher, emptyList()).run(BenchmarkRequest(null, 2, 0, 1)) + } + } + + Assert.assertEquals(stdout, "") + Assert.assertEquals(stderr.lines().count { it == "drifting worker diagnostic" }, 1, stderr) + Assert.assertFalse(stderr.contains("successful worker diagnostic"), stderr) + Assert.assertTrue(launcher.outputPaths.all { !Files.exists(it) }) + } + + @Test + fun rejectsNonPositiveBatchAndNegativeSamples() { + listOf( + """{"schema":"wurst-benchmark-worker-v2","mode":"execution","qualifiedName":"Bench.Only","checksum":1,"batchSize":0,"samplesNanos":[100]}""", + """{"schema":"wurst-benchmark-worker-v2","mode":"execution","qualifiedName":"Bench.Only","checksum":1,"batchSize":1,"samplesNanos":[-1]}""" + ).forEach { executionJson -> + val launcher = object : BenchmarkProcessLauncher { + val outputPaths = mutableListOf() + + override fun run(arguments: List): BenchmarkProcessResult { + val output = Path.of(arguments.optionValue("-benchmarkOutput")!!) + outputPaths.add(output) + val name = arguments.optionValue("-benchmarkName") + Files.writeString( + output, + if (name == null) { + """{"schema":"wurst-benchmark-worker-v2","mode":"discovery","benchmarks":["Bench.Only"]}""" + } else { + executionJson + } + ) + return BenchmarkProcessResult(0, listOf("invalid worker diagnostic")) + } + } + val (stdout, stderr) = captureOutput { + expectCoordinatorFailure { + BenchmarkCoordinator(launcher, emptyList()).run(BenchmarkRequest(null, 1, 0, 1)) + } + } + Assert.assertEquals(stdout, "") + Assert.assertTrue(stderr.contains("invalid worker diagnostic"), stderr) + Assert.assertTrue(launcher.outputPaths.all { !Files.exists(it) }) + } + } + + @Test + fun rejectsExecutionDocumentsContainingDerivedStatistics() { + val launcher = FixtureLauncher( + names = listOf("Bench.Only"), + outputFor = { arguments, _ -> + val name = arguments.optionValue("-benchmarkName") + if (name == null) { + """{"schema":"wurst-benchmark-worker-v2","mode":"discovery","benchmarks":["Bench.Only"]}""" + } else { + """{"schema":"wurst-benchmark-worker-v2","mode":"execution","qualifiedName":"Bench.Only","checksum":1,"batchSize":1,"samplesNanos":[100],"statistics":{"mean":100.0,"standardDeviation":0.0,"min":100,"max":100,"median":100,"p90":100,"p95":100}}""" + } + } + ) + + val (_, stderr) = captureOutput { + expectCoordinatorFailure { + BenchmarkCoordinator(launcher, emptyList()).run(BenchmarkRequest(null, 1, 0, 1)) + } + } + + Assert.assertTrue(stderr.contains("worker log must stay captured"), stderr) + Assert.assertTrue(launcher.outputPaths.all { !Files.exists(it) }) + } + + @Test + fun rendersCleanJsonAndHumanSpeedup() { + val launcher = FixtureLauncher() + val report = BenchmarkCoordinator(launcher, emptyList()) + .run(BenchmarkRequest(null, 1, 0, 2)) + val json = BenchmarkRenderer.json(report) + val expectedDisclaimer = + "Benchmark results measure the JVM-hosted Wurst IL interpreter. They vary with the machine, JVM, compiler version, host load, and benchmark setup. Use them for controlled side-by-side comparisons under the same conditions, not as absolute Warcraft III, Jass, Lua, or in-game performance numbers." + val parsed = ObjectMapper().readTree(json) + Assert.assertEquals(parsed["schema"].textValue(), "wurst-benchmark-v1") + Assert.assertEquals(parsed["disclaimer"].textValue(), expectedDisclaimer) + Assert.assertTrue(parsed["benchmarks"][0]["forks"][0]["samplesNanos"].isArray) + Assert.assertTrue(parsed["benchmarks"][0]["statistics"]["p90"].isIntegralNumber) + Assert.assertFalse(json.contains("worker log")) + + val human = BenchmarkRenderer.human(report) + Assert.assertTrue(human.contains("Bench.Fast")) + Assert.assertTrue(human.contains("median")) + Assert.assertTrue(human.contains("p90")) + Assert.assertTrue(human.contains("p95")) + Assert.assertTrue(human.contains("min/max")) + Assert.assertTrue(human.contains("checksum")) + Assert.assertTrue(human.contains("forks")) + Assert.assertTrue(human.contains("samples")) + Assert.assertTrue(human.contains("batchSizes 1:2"), human) + Assert.assertTrue(human.contains("Bench.Fast is")) + Assert.assertTrue(human.contains("faster than Bench.Slow")) + } + + @Test + fun rendersEqualMediansAsTie() { + val launcher = FixtureLauncher( + names = listOf("Bench.First", "Bench.Second"), + samples = mapOf( + "Bench.First" to listOf(100L), + "Bench.Second" to listOf(100L) + ) + ) + val report = BenchmarkCoordinator(launcher, emptyList()) + .run(BenchmarkRequest(null, 1, 0, 1)) + + val human = BenchmarkRenderer.human(report) + + Assert.assertTrue(human.contains("Bench.First and Bench.Second have equal median (100 ns/op)"), human) + Assert.assertFalse(human.contains("faster"), human) + } + + @Test + fun avoidsFiniteSpeedupClaimWhenMedianIsBelowTimerResolution() { + val launcher = FixtureLauncher( + names = listOf("Bench.Zero", "Bench.Measurable"), + samples = mapOf( + "Bench.Zero" to listOf(0L), + "Bench.Measurable" to listOf(100L) + ) + ) + val report = BenchmarkCoordinator(launcher, emptyList()) + .run(BenchmarkRequest(null, 1, 0, 1)) + + val human = BenchmarkRenderer.human(report) + + Assert.assertTrue(human.contains("Bench.Zero median is below timer resolution"), human) + Assert.assertFalse(human.contains("x faster"), human) + } + + @Test + fun integratedSetupAppUsesInstallationConfigAndKeepsJsonStdoutClean() { + val project = Files.createTempDirectory("grill-benchmark-project") + val install = Files.createTempDirectory("grill-benchmark-install") + val previousInstall = System.getProperty("wurst.install.dir") + val previousLauncher = SetupApp.benchmarkProcessLauncherOverride + val previousExitHandler = ExitHandler.handler + try { + Files.createDirectories(project.resolve("wurst")) + Files.createDirectories(project.resolve("_build")) + Files.writeString(project.resolve("wurst.build"), "name: Demo\ndependencies: []\nscriptMode: lua\nwc3Patch: v2.0\n") + Files.writeString(project.resolve("_build/core-jass.provenance"), "wc3Patch: v2.0\n") + Files.write(project.resolve("_build/common.j"), ByteArray(2048) { 'c'.code.toByte() }) + Files.write(project.resolve("_build/blizzard.j"), ByteArray(2048) { 'b'.code.toByte() }) + Files.createDirectories(install.resolve("wurst-compiler")) + Files.write(install.resolve("wurst-compiler/wurstscript.jar"), byteArrayOf(0)) + System.setProperty("wurst.install.dir", install.toString()) + val launcher = FixtureLauncher(names = listOf("Bench.Only"), samples = mapOf("Bench.Only" to listOf(1L, 2L))) + SetupApp.benchmarkProcessLauncherOverride = launcher + val setup = SetupMain().apply { projectRoot = project } + ExitHandler.handler = { throw CoordinatorExitException(it) } + val (stdout, stderr) = captureOutput { + try { + setup.doMain(arrayOf("benchmark", "--format", "json", "--iterations", "2")) + Assert.fail("benchmark should exit through ExitHandler") + } catch (exit: CoordinatorExitException) { + Assert.assertEquals(exit.code, 0) + } + } + val trimmed = stdout.trim() + val parser = ObjectMapper().factory.createParser(trimmed) + val parsed = parser.use { + val document = ObjectMapper().readTree(it) + Assert.assertNotNull(document) + Assert.assertNull(it.nextToken(), "JSON stdout must contain exactly one document") + document + } + Assert.assertEquals(parsed["schema"].textValue(), "wurst-benchmark-v1") + val canonicalDisclaimer = + "Benchmark results measure the JVM-hosted Wurst IL interpreter. They vary with the machine, JVM, compiler version, host load, and benchmark setup. Use them for controlled side-by-side comparisons under the same conditions, not as absolute Warcraft III, Jass, Lua, or in-game performance numbers." + Assert.assertEquals(parsed["disclaimer"].textValue(), canonicalDisclaimer) + Assert.assertEquals( + parsed.fieldNames().asSequence().toSet(), + setOf("schema", "disclaimer", "environment", "filter", "forks", "warmup", "iterations", "benchmarks") + ) + Assert.assertEquals( + parsed["environment"].fieldNames().asSequence().toSet(), + setOf("os", "jvm", "compiler", "grill", "cpuCount") + ) + Assert.assertEquals( + parsed["environment"]["compiler"].textValue(), + "sha256:6e340b9cffb37a989ca544e6bb780a2c78901d3fb33738768511a30617afa01d" + ) + val benchmark = parsed["benchmarks"][0] + Assert.assertEquals( + benchmark.fieldNames().asSequence().toSet(), + setOf("qualifiedName", "checksum", "forks", "statistics") + ) + Assert.assertEquals( + benchmark["statistics"].fieldNames().asSequence().toSet(), + setOf("mean", "standardDeviation", "min", "max", "median", "p90", "p95") + ) + Assert.assertEquals( + benchmark["forks"][0].fieldNames().asSequence().toSet(), + setOf("fork", "batchSize", "samplesNanos") + ) + Assert.assertEquals(stdout, trimmed + System.lineSeparator()) + Assert.assertFalse(stdout.contains("Grill"), stdout) + Assert.assertFalse(stdout.contains("worker log"), stdout) + Assert.assertFalse(stderr.contains("worker log"), stderr) + Assert.assertTrue(launcher.calls.first().contains("-compactOutput")) + } finally { + SetupApp.benchmarkProcessLauncherOverride = previousLauncher + ExitHandler.handler = previousExitHandler + if (previousInstall == null) System.clearProperty("wurst.install.dir") else System.setProperty("wurst.install.dir", previousInstall) + deleteTree(project) + deleteTree(install) + } + } + + @Test + fun benchmarkWorkersRejectOrdinaryModesButKeepCompilerProjectArguments() { + val project = Files.createTempDirectory("grill-benchmark-argument-validation") + val install = Files.createTempDirectory("grill-benchmark-argument-install") + val previousInstall = System.getProperty("wurst.install.dir") + val previousLauncher = SetupApp.benchmarkProcessLauncherOverride + val previousExitHandler = ExitHandler.handler + try { + Files.createDirectories(project.resolve("wurst")) + Files.createDirectories(project.resolve("_build/dependencies/dep")) + Files.createDirectories(project.resolve("_build")) + Files.writeString( + project.resolve("wurst.build"), + "name: Demo\ndependencies:\n - https://example.invalid/dep\nscriptMode: lua\nwc3Patch: v1.23a\n" + ) + Files.writeString(project.resolve("_build/core-jass.provenance"), "wc3Patch: v1.23a\n") + Files.write(project.resolve("_build/common.j"), ByteArray(2048) { 'c'.code.toByte() }) + Files.write(project.resolve("_build/blizzard.j"), ByteArray(2048) { 'b'.code.toByte() }) + Files.createDirectories(install.resolve("wurst-compiler")) + Files.write(install.resolve("wurst-compiler/wurstscript.jar"), byteArrayOf(0)) + System.setProperty("wurst.install.dir", install.toString()) + + val launcher = object : BenchmarkProcessLauncher { + val calls = mutableListOf>() + + override fun run(arguments: List): BenchmarkProcessResult { + val ordinaryModes = listOf("-out", "-runcompiletimefunctions") + check(ordinaryModes.none(arguments::contains)) { + "strict Wurst RunArgs rejected benchmark arguments: $arguments" + } + check(arguments.contains("-runbenchmarks")) + check(arguments.contains("-lua")) + check(arguments.contains(project.resolve("wurst").toAbsolutePath().toString())) + check(arguments.contains("-lib")) + check(arguments.contains(project.resolve("_build/dependencies/dep").toAbsolutePath().toString())) + check(arguments.contains(project.resolve("_build/common.j").toAbsolutePath().toString())) + check(arguments.contains(project.resolve("_build/blizzard.j").toAbsolutePath().toString())) + check(arguments.contains("-noPJass")) + check(arguments.contains("-legacyJassChecks")) + calls += arguments + + val output = Path.of(arguments.optionValue("-benchmarkOutput")!!) + val name = arguments.optionValue("-benchmarkName") + val json = if (name == null) { + """{"schema":"wurst-benchmark-worker-v2","mode":"discovery","benchmarks":["Bench.Only"]}""" + } else { + workerExecutionJson(name, samples = listOf(1L)) + } + Files.writeString(output, json) + return BenchmarkProcessResult(0, emptyList()) + } + } + SetupApp.benchmarkProcessLauncherOverride = launcher + val setup = SetupMain().apply { + projectRoot = project + parseArgs(listOf("benchmark")) + noPJass = true + benchmarkForks = 1 + benchmarkWarmup = 0 + benchmarkIterations = 1 + } + ExitHandler.handler = { throw CoordinatorExitException(it) } + + val exitCode = try { + SetupApp.handleArgs(setup) + -1 + } catch (exit: CoordinatorExitException) { + exit.code + } + + Assert.assertEquals(exitCode, 0) + Assert.assertEquals(launcher.calls.size, 2) + } finally { + SetupApp.benchmarkProcessLauncherOverride = previousLauncher + ExitHandler.handler = previousExitHandler + if (previousInstall == null) System.clearProperty("wurst.install.dir") else System.setProperty("wurst.install.dir", previousInstall) + deleteTree(project) + deleteTree(install) + } + } + + @Test + fun integratedSetupAppReturnsNonzeroOnCoordinatorFailure() { + val project = Files.createTempDirectory("grill-benchmark-failure-project") + val install = Files.createTempDirectory("grill-benchmark-failure-install") + val previousInstall = System.getProperty("wurst.install.dir") + val previousLauncher = SetupApp.benchmarkProcessLauncherOverride + val previousExitHandler = ExitHandler.handler + try { + Files.createDirectories(project.resolve("wurst")) + Files.writeString(project.resolve("wurst.build"), "name: Demo\ndependencies: []\nscriptMode: lua\nwc3Patch: v2.0\n") + Files.createDirectories(install.resolve("wurst-compiler")) + Files.write(install.resolve("wurst-compiler/wurstscript.jar"), byteArrayOf(0)) + System.setProperty("wurst.install.dir", install.toString()) + SetupApp.benchmarkProcessLauncherOverride = FixtureLauncher().apply { exitCode = 3 } + val setup = SetupMain().apply { projectRoot = project } + ExitHandler.handler = { throw CoordinatorExitException(it) } + val (stdout, stderr) = captureOutput { + val exitCode = try { + SetupApp.handleArgs(setup.apply { parseArgs(listOf("benchmark", "--format", "json")) }) + -1 + } catch (exit: CoordinatorExitException) { + exit.code + } + Assert.assertEquals(exitCode, 1) + } + Assert.assertEquals(stdout, "") + Assert.assertTrue(stderr.contains("worker log must stay captured"), stderr) + Assert.assertTrue(stderr.contains("Wurst benchmark failed"), stderr) + } finally { + SetupApp.benchmarkProcessLauncherOverride = previousLauncher + ExitHandler.handler = previousExitHandler + if (previousInstall == null) System.clearProperty("wurst.install.dir") else System.setProperty("wurst.install.dir", previousInstall) + deleteTree(project) + deleteTree(install) + } + } +}