diff --git a/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/CompilationProcess.java b/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/CompilationProcess.java index dc60e61fa..5227f2681 100644 --- a/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/CompilationProcess.java +++ b/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/CompilationProcess.java @@ -1,6 +1,10 @@ package de.peeeq.wurstio; import org.wurstscript.projectconfig.WurstProjectConfigData; +import de.peeeq.wurstio.benchmark.BenchmarkOptions; +import de.peeeq.wurstio.benchmark.BenchmarkResult; +import de.peeeq.wurstio.benchmark.BenchmarkWorkerOutput; +import de.peeeq.wurstio.benchmark.RunBenchmarks; import de.peeeq.wurstio.languageserver.requests.RunTests; import de.peeeq.wurstio.mpq.MpqEditor; import de.peeeq.wurstio.utils.FileUtils; @@ -19,6 +23,8 @@ import java.io.File; import java.io.IOException; import java.io.PrintStream; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.Optional; import java.util.function.Supplier; @@ -80,6 +86,11 @@ public CompilationProcess(WurstGui gui, RunArgs runArgs) { return null; } + if (runArgs.isRunBenchmarks()) { + timeTaker.measure("Run benchmark worker", () -> runBenchmarks(compiler)); + return null; + } + if (runArgs.isRunTests()) { timeTaker.measure("Run tests", () -> runTests(compiler.getImTranslator(), compiler, runArgs.getTestTimeout(), runArgs.getTestFilter())); @@ -116,6 +127,32 @@ public CompilationProcess(WurstGui gui, RunArgs runArgs) { return mapScript; } + private void runBenchmarks(WurstCompilerJassImpl compiler) { + try { + RunBenchmarks runner = new RunBenchmarks(); + Path output = Paths.get(runArgs.getBenchmarkOutput()); + if (runArgs.isBenchmarkList()) { + BenchmarkWorkerOutput.writeDiscovery( + output, + runner.discover(compiler.getImProg(), Optional.ofNullable(runArgs.getBenchmarkFilter()))); + } else { + BenchmarkResult result = runner.run( + compiler.getImTranslator(), + compiler.getImProg(), + runArgs.getBenchmarkName(), + new BenchmarkOptions(runArgs.getBenchmarkWarmup(), runArgs.getBenchmarkIterations(), 1_000_000L)); + BenchmarkWorkerOutput.writeResult(output, result); + } + } catch (Throwable e) { + String message = e.getMessage() == null ? e.getClass().getSimpleName() : e.getMessage(); + gui.sendError(new CompileError( + null, + "Benchmark worker failed: " + message, + CompileError.ErrorType.ERROR, + e)); + } + } + private boolean runPjass(File outputMapscript) { File commonJ = new File(outputMapscript.getParent(), "common.j"); File blizzJ = new File(outputMapscript.getParent(), "blizzard.j"); diff --git a/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/Main.java b/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/Main.java index 74181b452..5ee913bda 100644 --- a/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/Main.java +++ b/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/Main.java @@ -191,7 +191,10 @@ public static void main(String[] args) { compiledScript = compilationProcess.doCompilation(null, true); } - if (compiledScript != null) { + if (runArgs.isRunBenchmarks()) { + // Benchmark workers write their JSON result during compilation; + // a null script is the successful worker result, not a failure. + } else if (compiledScript != null) { File scriptFile = new File("compiled.j.txt"); Files.write(compiledScript.toString().getBytes(Charsets.UTF_8), scriptFile); } diff --git a/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/benchmark/BenchmarkClock.java b/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/benchmark/BenchmarkClock.java new file mode 100644 index 000000000..f635930d9 --- /dev/null +++ b/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/benchmark/BenchmarkClock.java @@ -0,0 +1,6 @@ +package de.peeeq.wurstio.benchmark; + +@FunctionalInterface +public interface BenchmarkClock { + long nanoTime(); +} diff --git a/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/benchmark/BenchmarkOptions.java b/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/benchmark/BenchmarkOptions.java new file mode 100644 index 000000000..5011d2903 --- /dev/null +++ b/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/benchmark/BenchmarkOptions.java @@ -0,0 +1,15 @@ +package de.peeeq.wurstio.benchmark; + +public record BenchmarkOptions(int warmupIterations, int measurementIterations, long minimumSampleNanos) { + public BenchmarkOptions { + if (warmupIterations < 0) { + throw new IllegalArgumentException("warmupIterations must be non-negative"); + } + if (measurementIterations <= 0) { + throw new IllegalArgumentException("measurementIterations must be positive"); + } + if (minimumSampleNanos < 0) { + throw new IllegalArgumentException("minimumSampleNanos must be non-negative"); + } + } +} diff --git a/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/benchmark/BenchmarkResult.java b/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/benchmark/BenchmarkResult.java new file mode 100644 index 000000000..b5f1b55ca --- /dev/null +++ b/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/benchmark/BenchmarkResult.java @@ -0,0 +1,27 @@ +package de.peeeq.wurstio.benchmark; + +import java.util.List; +import java.util.Objects; + +public record BenchmarkResult( + String qualifiedName, + int checksum, + int batchSize, + List samplesNanos +) { + public BenchmarkResult { + Objects.requireNonNull(qualifiedName, "qualifiedName"); + if (batchSize <= 0) { + throw new IllegalArgumentException("batchSize must be positive"); + } + samplesNanos = List.copyOf(Objects.requireNonNull(samplesNanos, "samplesNanos")); + if (samplesNanos.isEmpty()) { + throw new IllegalArgumentException("at least one benchmark sample is required"); + } + for (Long sample : samplesNanos) { + if (sample < 0) { + throw new IllegalArgumentException("benchmark samples must be non-negative"); + } + } + } +} diff --git a/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/benchmark/BenchmarkWorkerOutput.java b/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/benchmark/BenchmarkWorkerOutput.java new file mode 100644 index 000000000..f8130fa5d --- /dev/null +++ b/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/benchmark/BenchmarkWorkerOutput.java @@ -0,0 +1,109 @@ +package de.peeeq.wurstio.benchmark; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.AtomicMoveNotSupportedException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; +import java.util.List; +import java.util.Objects; + +/** Writes the machine-readable result of one isolated benchmark compiler worker. */ +public final class BenchmarkWorkerOutput { + public static final String SCHEMA = "wurst-benchmark-worker-v2"; + + @FunctionalInterface + public interface TemporaryFileWriter { + void write(Path temporary, String json) throws IOException; + } + + private static final Gson GSON = new GsonBuilder() + .disableHtmlEscaping() + .create(); + + private BenchmarkWorkerOutput() { + } + + public static void writeDiscovery(Path output, List benchmarkNames) throws IOException { + Objects.requireNonNull(benchmarkNames, "benchmarkNames"); + JsonObject json = envelope("discovery"); + JsonArray benchmarks = new JsonArray(); + for (String benchmarkName : benchmarkNames) { + benchmarks.add(Objects.requireNonNull(benchmarkName, "benchmarkName")); + } + json.add("benchmarks", benchmarks); + writeAtomically(output, GSON.toJson(json)); + } + + public static void writeResult(Path output, BenchmarkResult result) throws IOException { + Objects.requireNonNull(result, "result"); + JsonObject json = envelope("execution"); + json.addProperty("qualifiedName", result.qualifiedName()); + json.addProperty("checksum", result.checksum()); + json.addProperty("batchSize", result.batchSize()); + json.add("samplesNanos", GSON.toJsonTree(result.samplesNanos())); + writeAtomically(output, GSON.toJson(json)); + } + + /** + * Write a complete JSON document to a sibling temporary file, then rename it + * over the destination. Serialization happens before touching the destination. + */ + public static void writeAtomically(Path output, String json) throws IOException { + writeAtomically(output, json, BenchmarkWorkerOutput::writeTemporaryFile); + } + + public static void writeAtomically( + Path output, + String json, + TemporaryFileWriter temporaryFileWriter + ) throws IOException { + Objects.requireNonNull(output, "output"); + Objects.requireNonNull(json, "json"); + Objects.requireNonNull(temporaryFileWriter, "temporaryFileWriter"); + Path absoluteOutput = output.toAbsolutePath(); + Path parent = absoluteOutput.getParent(); + if (parent == null) { + throw new IOException("benchmark output has no parent directory: " + output); + } + Files.createDirectories(parent); + Path temporary = Files.createTempFile(parent, "wurst-benchmark-", ".tmp"); + try { + temporaryFileWriter.write(temporary, json); + try { + Files.move( + temporary, + absoluteOutput, + StandardCopyOption.ATOMIC_MOVE, + StandardCopyOption.REPLACE_EXISTING); + } catch (AtomicMoveNotSupportedException e) { + Files.move(temporary, absoluteOutput, StandardCopyOption.REPLACE_EXISTING); + } + } finally { + Files.deleteIfExists(temporary); + } + } + + private static void writeTemporaryFile(Path temporary, String json) throws IOException { + Files.writeString( + temporary, + json, + StandardCharsets.UTF_8, + StandardOpenOption.WRITE, + StandardOpenOption.TRUNCATE_EXISTING); + } + + private static JsonObject envelope(String mode) { + JsonObject json = new JsonObject(); + json.addProperty("schema", SCHEMA); + json.addProperty("mode", mode); + return json; + } +} diff --git a/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/benchmark/RunBenchmarks.java b/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/benchmark/RunBenchmarks.java new file mode 100644 index 000000000..fd6f49d3b --- /dev/null +++ b/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/benchmark/RunBenchmarks.java @@ -0,0 +1,273 @@ +package de.peeeq.wurstio.benchmark; + +import de.peeeq.wurstio.CompiletimeFunctionRunner; +import de.peeeq.wurstscript.ast.FuncDef; +import de.peeeq.wurstscript.gui.WurstGuiCliImpl; +import de.peeeq.wurstscript.intermediatelang.ILconst; +import de.peeeq.wurstscript.intermediatelang.ILconstInt; +import de.peeeq.wurstscript.intermediatelang.interpreter.NativesProvider; +import de.peeeq.wurstscript.jassIm.ImFunction; +import de.peeeq.wurstscript.jassIm.ImProg; +import de.peeeq.wurstscript.translation.imtranslation.FunctionFlagEnum; +import de.peeeq.wurstscript.translation.imtranslation.ImTranslator; +import org.wurstscript.projectconfig.WurstProjectConfigData; + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.Locale; +import java.util.Objects; +import java.util.Optional; + +import static de.peeeq.wurstio.CompiletimeFunctionRunner.FunctionFlagToRun.CompiletimeFunctions; + +public final class RunBenchmarks { + private final BenchmarkClock clock; + private final BenchmarkSessionFactory sessionFactory; + + @FunctionalInterface + public interface BenchmarkSessionFactory { + BenchmarkSession open(ImTranslator translator, ImProg program); + } + + public interface BenchmarkSession extends AutoCloseable { + void runCompiletime(); + + ILconst invoke(ImFunction function); + + @Override + void close(); + } + + public RunBenchmarks() { + this(System::nanoTime, RunBenchmarks::openDefaultSession); + } + + public RunBenchmarks(BenchmarkClock clock) { + this(clock, RunBenchmarks::openDefaultSession); + } + + public RunBenchmarks(BenchmarkClock clock, BenchmarkSessionFactory sessionFactory) { + this.clock = Objects.requireNonNull(clock, "clock"); + this.sessionFactory = Objects.requireNonNull(sessionFactory, "sessionFactory"); + } + + static BenchmarkSessionFactory defaultSessionFactory(NativesProvider additionalProvider) { + Objects.requireNonNull(additionalProvider, "additionalProvider"); + return (translator, program) -> openDefaultSession(translator, program, additionalProvider); + } + + public List discover(ImProg program, Optional filter) { + Objects.requireNonNull(program, "program"); + Objects.requireNonNull(filter, "filter"); + String normalizedFilter = filter.map(value -> value.toLowerCase(Locale.ROOT)).orElse(""); + List result = new ArrayList<>(); + for (ImFunction function : program.getFunctions()) { + if (!function.hasFlag(FunctionFlagEnum.IS_BENCHMARK)) { + continue; + } + Optional publicName = publicName(function); + if (publicName.isPresent() + && publicName.get().toLowerCase(Locale.ROOT).contains(normalizedFilter)) { + result.add(publicName.get()); + } + } + result.sort(Comparator.naturalOrder()); + return List.copyOf(result); + } + + public BenchmarkResult run( + ImTranslator translator, + ImProg program, + String qualifiedName, + BenchmarkOptions options + ) { + Objects.requireNonNull(translator, "translator"); + Objects.requireNonNull(program, "program"); + Objects.requireNonNull(qualifiedName, "qualifiedName"); + Objects.requireNonNull(options, "options"); + + ImFunction function = findBenchmark(program, qualifiedName); + + // Every run owns its interpreter and native providers. This keeps a + // benchmark isolated from any preceding benchmark and closes native + // resources on successful and exceptional exits alike. + try (BenchmarkSession session = sessionFactory.open(translator, program)) { + session.runCompiletime(); + Checksum checksum = new Checksum(); + + for (int i = 0; i < options.warmupIterations(); i++) { + runBatch(session, function, 1, checksum); + } + + int batchSize = calibrate(session, function, checksum, options.minimumSampleNanos()); + + List samples = new ArrayList<>(options.measurementIterations()); + for (int i = 0; i < options.measurementIterations(); i++) { + long start = clock.nanoTime(); + runBatch(session, function, batchSize, checksum); + long elapsed = clock.nanoTime() - start; + if (elapsed < 0) { + throw new IllegalStateException("benchmark clock moved backwards"); + } + samples.add(elapsed / batchSize); + } + + return new BenchmarkResult( + qualifiedName, + checksum.value(), + batchSize, + samples); + } + } + + private int calibrate( + BenchmarkSession session, + ImFunction function, + Checksum checksum, + long minimumSampleNanos + ) { + if (minimumSampleNanos == 0) { + return 1; + } + + int batchSize = 1; + while (true) { + long start = clock.nanoTime(); + runBatch(session, function, batchSize, checksum); + long elapsed = clock.nanoTime() - start; + if (elapsed < 0) { + throw new IllegalStateException("benchmark clock moved backwards during calibration"); + } + if (elapsed >= minimumSampleNanos) { + return batchSize; + } + if (batchSize > Integer.MAX_VALUE / 2) { + throw new IllegalStateException("benchmark batch size overflow during calibration"); + } + batchSize *= 2; + } + } + + private static void runBatch( + BenchmarkSession session, + ImFunction function, + int batchSize, + Checksum checksum + ) { + for (int i = 0; i < batchSize; i++) { + ILconst value = session.invoke(function); + if (!(value instanceof ILconstInt intValue)) { + throw new IllegalStateException( + "benchmark " + function.getName() + " returned " + + value.getClass().getSimpleName() + " instead of int"); + } + checksum.accept(intValue.getVal()); + } + } + + private static BenchmarkSession openDefaultSession(ImTranslator translator, ImProg program) { + return openDefaultSession(translator, program, null); + } + + private static BenchmarkSession openDefaultSession( + ImTranslator translator, + ImProg program, + NativesProvider additionalProvider + ) { + WurstGuiCliImpl gui = new WurstGuiCliImpl(true); + CompiletimeFunctionRunner compiletime = new CompiletimeFunctionRunner( + translator, + program, + Optional.empty(), + null, + gui, + CompiletimeFunctions, + WurstProjectConfigData.empty(), + false, + false); + if (additionalProvider != null) { + compiletime.getInterpreter().addNativeProvider(additionalProvider); + } + return new DefaultBenchmarkSession(compiletime, gui); + } + + private static final class DefaultBenchmarkSession implements BenchmarkSession { + private final CompiletimeFunctionRunner compiletime; + private final WurstGuiCliImpl gui; + + private DefaultBenchmarkSession(CompiletimeFunctionRunner compiletime, WurstGuiCliImpl gui) { + this.compiletime = compiletime; + this.gui = gui; + } + + @Override + public void runCompiletime() { + compiletime.run(); + if (gui.getErrorCount() > 0) { + throw new IllegalStateException("compiletime initialization failed: " + gui.getErrors()); + } + } + + @Override + public ILconst invoke(ImFunction function) { + return compiletime.getInterpreter().runFunc(function, null); + } + + @Override + public void close() { + compiletime.close(); + } + } + + private static ImFunction findBenchmark(ImProg program, String qualifiedName) { + List matches = new ArrayList<>(); + for (ImFunction function : program.getFunctions()) { + if (function.hasFlag(FunctionFlagEnum.IS_BENCHMARK) + && publicName(function).filter(qualifiedName::equals).isPresent()) { + matches.add(function); + } + } + if (matches.isEmpty()) { + throw new IllegalArgumentException("no benchmark named " + qualifiedName); + } + if (matches.size() > 1) { + throw new IllegalArgumentException("multiple benchmarks named " + qualifiedName); + } + return matches.get(0); + } + + private static Optional publicName(ImFunction function) { + if (!(function.getTrace() instanceof FuncDef funcDef)) { + return Optional.empty(); + } + if (funcDef.attrNearestPackage() == null + || funcDef.attrNearestPackage().tryGetNameDef() == null) { + return Optional.empty(); + } + return Optional.of( + funcDef.attrNearestPackage().tryGetNameDef().getName() + "." + funcDef.getName()); + } + + private static final class Checksum { + private boolean initialized; + private int value; + + private void accept(int next) { + if (!initialized) { + value = next; + initialized = true; + } else if (value != next) { + throw new IllegalStateException( + "benchmark returned changing checksums: expected " + value + ", got " + next); + } + } + + private int value() { + if (!initialized) { + throw new IllegalStateException("benchmark produced no samples"); + } + return value; + } + } +} diff --git a/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstscript/RunArgs.java b/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstscript/RunArgs.java index d6cae57e9..decdad712 100644 --- a/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstscript/RunArgs.java +++ b/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstscript/RunArgs.java @@ -26,6 +26,11 @@ public class RunArgs { private @Nullable String inputmap = null; private @Nullable int testTimeout = 20; private @Nullable String testFilter = null; + private @Nullable String benchmarkFilter = null; + private @Nullable String benchmarkName = null; + private @Nullable String benchmarkOutput = null; + private int benchmarkWarmup = 0; + private int benchmarkIterations = 1; private final List options = Lists.newArrayList(); private final List libDirs = Lists.newArrayList(); private final RunOption optionHelp; @@ -33,6 +38,13 @@ public class RunArgs { private final RunOption optionInline; private final RunOption optionLocalOptimizations; private final RunOption optionRuntests; + private final RunOption optionRunBenchmarks; + private final RunOption optionBenchmarkList; + private final RunOption optionBenchmarkFilter; + private final RunOption optionBenchmarkName; + private final RunOption optionBenchmarkWarmup; + private final RunOption optionBenchmarkIterations; + private final RunOption optionBenchmarkOutput; private final RunOption optionGui; private final RunOption optionAbout; private final RunOption optionShowErrors; @@ -58,6 +70,7 @@ public class RunArgs { private final RunOption optionHotStartmap; private final RunOption optionHotReload; private final RunOption optionTestTimeout; + private final RunOption optionTestFilter; private final RunOption optionDevBuild; private int functionSplitLimit = 10000; @@ -107,8 +120,15 @@ public RunArgs(String... args) { this.args = args; // interpreter optionRuntests = addOption("runtests", "Run all test functions found in the scripts."); + optionRunBenchmarks = addOption("runbenchmarks", "Run the hidden benchmark worker mode."); + optionBenchmarkList = addOption("benchmarkList", "List benchmark functions instead of executing one."); optionTestTimeout = addOptionWithArg("testTimeout", "Timeout in seconds after which tests will be cancelled and considered failed, if they did not yet succeed.", arg -> testTimeout = Integer.parseInt(arg)); - addOptionWithArg("testFilter", "Only run tests whose qualified name (Package.function) contains this string (case-insensitive).", arg -> testFilter = arg); + optionTestFilter = addOptionWithArg("testFilter", "Only run tests whose qualified name (Package.function) contains this string (case-insensitive).", arg -> testFilter = arg); + optionBenchmarkFilter = addOptionWithArg("benchmarkFilter", "Filter benchmark names during benchmark discovery.", arg -> benchmarkFilter = arg); + optionBenchmarkName = addOptionWithArg("benchmarkName", "Execute one exact qualified benchmark name.", arg -> benchmarkName = arg); + optionBenchmarkWarmup = addOptionWithArg("benchmarkWarmup", "Number of unmeasured benchmark warmup iterations.", arg -> benchmarkWarmup = parseBenchmarkInteger("benchmarkWarmup", arg)); + optionBenchmarkIterations = addOptionWithArg("benchmarkIterations", "Number of measured benchmark iterations.", arg -> benchmarkIterations = parseBenchmarkInteger("benchmarkIterations", arg)); + optionBenchmarkOutput = addOptionWithArg("benchmarkOutput", "Output path for benchmark worker JSON.", arg -> benchmarkOutput = arg); optionRunCompileTimeFunctions = addOption("runcompiletimefunctions", "Run all compiletime functions found in the scripts."); optionInjectCompiletimeObjects = addOption("injectobjects", "Injects the objects generated by compiletime functions into the map."); // optimization @@ -205,6 +225,67 @@ public RunArgs(String... args) { if (optionHelp.isSet) { printHelpAndExit(); } + + validateBenchmarkOptions(); + } + + private static int parseBenchmarkInteger(String option, String value) { + try { + return Integer.parseInt(value, 10); + } catch (NumberFormatException e) { + throw new RuntimeException("Invalid integer for -" + option + ": " + value, e); + } + } + + private void validateBenchmarkOptions() { + boolean hasBenchmarkOption = optionRunBenchmarks.isSet + || optionBenchmarkList.isSet + || optionBenchmarkFilter.isSet + || optionBenchmarkName.isSet + || optionBenchmarkWarmup.isSet + || optionBenchmarkIterations.isSet + || optionBenchmarkOutput.isSet; + if (!hasBenchmarkOption) { + return; + } + if (!optionRunBenchmarks.isSet) { + throw new RuntimeException("Benchmark options require -runbenchmarks."); + } + StringBuilder conflictingModes = new StringBuilder(); + if (optionRuntests.isSet) conflictingModes.append(" -runtests"); + if (optionRunCompileTimeFunctions.isSet) conflictingModes.append(" -runcompiletimefunctions"); + if (optionBuild.isSet) conflictingModes.append(" -build"); + if (optionDevBuild.isSet) conflictingModes.append(" -dev"); + if (outFile != null) conflictingModes.append(" -out"); + if (optionInjectCompiletimeObjects.isSet) conflictingModes.append(" -injectobjects"); + if (optionTestFilter.isSet) conflictingModes.append(" -testFilter"); + if (optionTestTimeout.isSet) conflictingModes.append(" -testTimeout"); + if (conflictingModes.length() > 0) { + throw new RuntimeException( + "-runbenchmarks cannot be combined with ordinary execution/output modes:" + conflictingModes); + } + if (benchmarkOutput == null || benchmarkOutput.isBlank()) { + throw new RuntimeException("-runbenchmarks requires -benchmarkOutput."); + } + if (benchmarkWarmup < 0) { + throw new RuntimeException("-benchmarkWarmup must be non-negative."); + } + if (benchmarkIterations <= 0) { + throw new RuntimeException("-benchmarkIterations must be positive."); + } + if (optionBenchmarkList.isSet && benchmarkName != null) { + throw new RuntimeException("-benchmarkList cannot be combined with -benchmarkName."); + } + if (optionBenchmarkList.isSet) { + if (optionBenchmarkWarmup.isSet || optionBenchmarkIterations.isSet) { + throw new RuntimeException("-benchmarkWarmup and -benchmarkIterations require -benchmarkName."); + } + } else if (benchmarkName == null) { + throw new RuntimeException("-runbenchmarks requires either -benchmarkList or -benchmarkName."); + } + if (benchmarkName != null && benchmarkFilter != null) { + throw new RuntimeException("-benchmarkFilter cannot be combined with -benchmarkName."); + } } private boolean isDoubleArg(String arg, RunOption option) { @@ -391,6 +472,34 @@ public boolean isRunTests() { return optionRuntests.isSet; } + public boolean isRunBenchmarks() { + return optionRunBenchmarks.isSet; + } + + public boolean isBenchmarkList() { + return optionBenchmarkList.isSet; + } + + public @Nullable String getBenchmarkFilter() { + return benchmarkFilter; + } + + public @Nullable String getBenchmarkName() { + return benchmarkName; + } + + public int getBenchmarkWarmup() { + return benchmarkWarmup; + } + + public int getBenchmarkIterations() { + return benchmarkIterations; + } + + public @Nullable String getBenchmarkOutput() { + return benchmarkOutput; + } + public boolean isPrettyPrint() { return optionPrettyPrint.isSet; } diff --git a/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstscript/intermediatelang/interpreter/ILInterpreter.java b/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstscript/intermediatelang/interpreter/ILInterpreter.java index 1f78165cc..0008d0602 100644 --- a/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstscript/intermediatelang/interpreter/ILInterpreter.java +++ b/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstscript/intermediatelang/interpreter/ILInterpreter.java @@ -499,6 +499,28 @@ public void runVoidFunc(ImFunction f, @Nullable Element trace) { runFunc(globalState, f, trace, args); } + /** + * Runs a source-level function and returns its value. + * + *

Source functions which are parameterless may still have one hidden + * stacktrace parameter after translation. Keep the same calling convention + * as {@link #runVoidFunc(ImFunction, Element)} for that case.

+ */ + public ILconst runFunc(ImFunction f, @Nullable Element trace) { + globalState.resetStackframes(); + if (f.getParameters().size() > 1) { + throw new IllegalArgumentException( + "expected a parameterless function with at most one hidden parameter, got " + + f.getParameters().size() + " parameters for " + f.getName()); + } + ILconst[] args = f.getParameters().isEmpty() + ? new ILconst[0] + : new ILconst[]{new ILconstString("initial call")}; + LocalState state = runFunc(globalState, f, trace, args); + return Objects.requireNonNull(state.getReturnVal(), + "function " + f.getName() + " returned no value"); + } + public Element getLastStatement() { return globalState.getLastStatement(); } diff --git a/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstscript/translation/imtranslation/FunctionFlagEnum.java b/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstscript/translation/imtranslation/FunctionFlagEnum.java index d619b3252..9d6f9cfdb 100644 --- a/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstscript/translation/imtranslation/FunctionFlagEnum.java +++ b/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstscript/translation/imtranslation/FunctionFlagEnum.java @@ -4,8 +4,9 @@ public enum FunctionFlagEnum implements FunctionFlag { IS_BJ, IS_NATIVE, IS_TEST, + IS_BENCHMARK, IS_COMPILETIME_NATIVE, IS_EXTERN, IS_VARARG -} \ No newline at end of file +} diff --git a/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstscript/translation/imtranslation/ImTranslator.java b/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstscript/translation/imtranslation/ImTranslator.java index 8551a3f8c..de49fb6b4 100644 --- a/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstscript/translation/imtranslation/ImTranslator.java +++ b/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstscript/translation/imtranslation/ImTranslator.java @@ -946,6 +946,9 @@ public ImFunction getFuncFor(TranslatedToImFunction funcDef) { if (funcDef2.attrHasAnnotation("test")) { flags.add(IS_TEST); } + if (funcDef2.attrHasAnnotation("benchmark")) { + flags.add(IS_BENCHMARK); + } } // Check if last parameter is vararg diff --git a/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstscript/validation/WurstValidator.java b/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstscript/validation/WurstValidator.java index 1b56d1ce5..0fc0503f1 100644 --- a/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstscript/validation/WurstValidator.java +++ b/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstscript/validation/WurstValidator.java @@ -447,6 +447,8 @@ private void check(Element e) { visit((ExtensionFuncDef) e); if (e instanceof FuncDef) visit((FuncDef) e); + if (e instanceof NativeFunc) + checkBenchmark((NativeFunc) e); if (e instanceof FuncRef) checkFuncRef((FuncRef) e); if (e instanceof FunctionLike) @@ -1727,6 +1729,7 @@ private void visit(FuncDef func) { func.getErrorHandler().setProgress(null, ProgressHelper.getValidatorPercent(visitedFunctions, functionCount)); checkFunctionName(func); + checkBenchmark(func); if (func.attrIsAbstract()) { if (!func.attrHasEmptyBody()) { func.addError("Abstract function " + func.getName() + " must not have a body."); @@ -1737,6 +1740,25 @@ private void visit(FuncDef func) { } } + private void checkBenchmark(FunctionDefinition func) { + if (!func.attrHasAnnotation("benchmark")) return; + if (func.attrNearestStructureDef() != null + || !func.getParameters().isEmpty() + || !func.attrReturnTyp().equalsType(WurstTypeInt.instance(), func)) { + func.addError("@benchmark functions must be package-level, parameterless functions returning int."); + } + if (func instanceof NativeFunc + || func.attrHasAnnotation("extern") + || func.attrHasAnnotation("compiletimenative") + || func.attrIsAbstract() + || (func instanceof FuncDef && ((FuncDef) func).attrHasEmptyBody())) { + func.addError("@benchmark functions must be executable by the compiletime IL interpreter; native, extern, and compiletimenative declarations are not supported."); + } + if (func.attrIsCompiletime()) { + func.addError("@benchmark cannot be combined with @compiletime."); + } + } + private void checkUninitializedVars(FunctionLike f) { if (!isHeavy()) { // Phase-1: collect only. Avoid triggering extra attributes in light validation. diff --git a/de.peeeq.wurstscript/src/test/java/de/peeeq/wurstio/benchmark/BenchmarkSessionTestSupport.java b/de.peeeq.wurstscript/src/test/java/de/peeeq/wurstio/benchmark/BenchmarkSessionTestSupport.java new file mode 100644 index 000000000..a4f32f0bc --- /dev/null +++ b/de.peeeq.wurstscript/src/test/java/de/peeeq/wurstio/benchmark/BenchmarkSessionTestSupport.java @@ -0,0 +1,15 @@ +package de.peeeq.wurstio.benchmark; + +import de.peeeq.wurstscript.intermediatelang.interpreter.NativesProvider; + +/** Test-only bridge for the package-private default-session seam. */ +public final class BenchmarkSessionTestSupport { + private BenchmarkSessionTestSupport() { + } + + public static RunBenchmarks.BenchmarkSessionFactory defaultSessionFactory( + NativesProvider additionalProvider + ) { + return RunBenchmarks.defaultSessionFactory(additionalProvider); + } +} diff --git a/de.peeeq.wurstscript/src/test/java/tests/wurstscript/tests/BenchmarkAnnotationTests.java b/de.peeeq.wurstscript/src/test/java/tests/wurstscript/tests/BenchmarkAnnotationTests.java new file mode 100644 index 000000000..d00796243 --- /dev/null +++ b/de.peeeq.wurstscript/src/test/java/tests/wurstscript/tests/BenchmarkAnnotationTests.java @@ -0,0 +1,159 @@ +package tests.wurstscript.tests; + +import de.peeeq.wurstio.WurstCompilerJassImpl; +import de.peeeq.wurstscript.RunArgs; +import de.peeeq.wurstscript.gui.WurstGuiCliImpl; +import de.peeeq.wurstscript.jassIm.ImFunction; +import de.peeeq.wurstscript.jassIm.ImProg; +import de.peeeq.wurstscript.translation.imtranslation.FunctionFlagEnum; +import de.peeeq.wurstscript.utils.Utils; +import de.peeeq.wurstscript.ast.WurstModel; +import org.testng.annotations.Test; + +import java.util.Collections; + +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertTrue; + +public class BenchmarkAnnotationTests extends WurstScriptTest { + + private static final String BENCHMARK_ERROR = + "@benchmark functions must be package-level, parameterless functions returning int."; + private static final String COMBINATION_ERROR = + "@benchmark cannot be combined with @compiletime."; + private static final String EXECUTION_ERROR = + "@benchmark functions must be executable by the compiletime IL interpreter; native, extern, and compiletimenative declarations are not supported."; + + private static final String VALID = """ + package Bench + @benchmark function classify() returns int + return 42 + """; + + @Test + public void validBenchmarkGetsDedicatedImFlag() { + Compilation compilation = compile(VALID); + assertTrue(compilation.gui.getErrorList().isEmpty(), compilation.gui.getErrors()); + + ImFunction function = compilation.imProg.getFunctions().stream() + .filter(f -> f.getName().contains("classify")) + .findFirst() + .orElseThrow(); + assertTrue(function.hasFlag(FunctionFlagEnum.IS_BENCHMARK)); + assertFalse(function.hasFlag(FunctionFlagEnum.IS_TEST)); + } + + @Test + public void benchmarkRejectsParameters() { + assertBenchmarkError(""" + package Bench + @benchmark function withParameter(int value) returns int + return value + """, BENCHMARK_ERROR); + } + + @Test + public void benchmarkRejectsNothingReturn() { + assertBenchmarkError(""" + package Bench + @benchmark function noReturn() + """, BENCHMARK_ERROR); + } + + @Test + public void benchmarkRejectsNonIntReturn() { + assertBenchmarkError(""" + package Bench + @benchmark function returnsString() returns string + return "wrong" + """, BENCHMARK_ERROR); + } + + @Test + public void benchmarkRejectsClassMethods() { + assertBenchmarkError(""" + package Bench + class C + @benchmark + function classFunction() returns int + return 42 + """, BENCHMARK_ERROR); + } + + @Test + public void benchmarkCanCombineWithTest() { + Compilation compilation = compile(""" + package Bench + @benchmark @test function testBenchmark() returns int + return 42 + """); + assertTrue(compilation.gui.getErrorList().isEmpty(), compilation.gui.getErrors()); + + ImFunction function = compilation.imProg.getFunctions().stream() + .filter(f -> f.getName().contains("testBenchmark")) + .findFirst() + .orElseThrow(); + assertTrue(function.hasFlag(FunctionFlagEnum.IS_TEST)); + assertTrue(function.hasFlag(FunctionFlagEnum.IS_BENCHMARK)); + } + + @Test + public void benchmarkCannotCombineWithCompiletime() { + assertBenchmarkError(""" + package Bench + @benchmark @compiletime function compiletimeBenchmark() returns int + return 42 + """, COMBINATION_ERROR); + } + + @Test + public void benchmarkRejectsNativeDeclaration() { + assertBenchmarkError(""" + package Bench + @benchmark native nativeBenchmark() returns int + """, EXECUTION_ERROR); + } + + @Test + public void benchmarkRejectsExternDeclaration() { + assertBenchmarkError(""" + package Bench + @benchmark @extern function externBenchmark() returns int + return 42 + """, EXECUTION_ERROR); + } + + @Test + public void benchmarkRejectsCompiletimeNativeDeclaration() { + assertBenchmarkError(""" + package Bench + @benchmark @compiletimenative function nativeBenchmark() returns int + return 42 + """, EXECUTION_ERROR); + } + + private void assertBenchmarkError(String source, String expectedMessage) { + Compilation compilation = compile(source); + assertTrue(compilation.gui.getErrors().contains(expectedMessage), compilation.gui.getErrors()); + } + + private Compilation compile(String source) { + WurstGuiCliImpl gui = new WurstGuiCliImpl(); + WurstCompilerJassImpl compiler = new WurstCompilerJassImpl(null, gui, null, new RunArgs()); + WurstModel model = parseFiles(null, + Collections.singletonList(new CU("benchmark", Utils.join(source.lines().toList(), "\n") + "\n")), + false, + compiler); + compiler.checkProg(model); + ImProg imProg = null; + if (gui.getErrorList().isEmpty()) { + imProg = compiler.translateProgToIm(model); + assertNotNull(imProg); + } + return new Compilation(gui, imProg); + } + + private record Compilation(WurstGuiCliImpl gui, ImProg imProg) { + } +} diff --git a/de.peeeq.wurstscript/src/test/java/tests/wurstscript/tests/BenchmarkCliTests.java b/de.peeeq.wurstscript/src/test/java/tests/wurstscript/tests/BenchmarkCliTests.java new file mode 100644 index 000000000..60145da59 --- /dev/null +++ b/de.peeeq.wurstscript/src/test/java/tests/wurstscript/tests/BenchmarkCliTests.java @@ -0,0 +1,326 @@ +package tests.wurstscript.tests; + +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import de.peeeq.wurstio.Main; +import de.peeeq.wurstio.benchmark.BenchmarkResult; +import de.peeeq.wurstio.benchmark.BenchmarkWorkerOutput; +import de.peeeq.wurstscript.RunArgs; +import org.testng.annotations.Test; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Stream; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertThrows; +import static org.testng.Assert.assertTrue; + +public class BenchmarkCliTests { + + @Test + public void parsesDiscoveryWorkerArguments() throws Exception { + Path output = Files.createTempFile("benchmark-worker", ".json"); + RunArgs args = new RunArgs( + "-runbenchmarks", + "-benchmarkList", + "-benchmarkFilter", "Polygon", + "-benchmarkOutput", output.toString()); + + assertTrue(args.isRunBenchmarks()); + assertTrue(args.isBenchmarkList()); + assertEquals(args.getBenchmarkFilter(), "Polygon"); + assertEquals(args.getBenchmarkOutput(), output.toString()); + } + + @Test + public void parsesExecutionWorkerArguments() throws Exception { + Path output = Files.createTempFile("benchmark-worker", ".json"); + RunArgs args = new RunArgs( + "-runbenchmarks", + "-benchmarkName", "Bench.work", + "-benchmarkWarmup", "4", + "-benchmarkIterations", "8", + "-benchmarkOutput", output.toString()); + + assertTrue(args.isRunBenchmarks()); + assertFalse(args.isBenchmarkList()); + assertEquals(args.getBenchmarkName(), "Bench.work"); + assertEquals(args.getBenchmarkWarmup(), 4); + assertEquals(args.getBenchmarkIterations(), 8); + assertEquals(args.getBenchmarkOutput(), output.toString()); + } + + @Test + public void rejectsInvalidBenchmarkArgumentCombinations() { + assertThrows(RuntimeException.class, () -> new RunArgs( + "-runbenchmarks", "-benchmarkList", "-benchmarkName", "Bench.work", + "-benchmarkOutput", "out.json")); + assertThrows(RuntimeException.class, () -> new RunArgs( + "-runbenchmarks", "-benchmarkList")); + assertThrows(RuntimeException.class, () -> new RunArgs( + "-runbenchmarks", "-benchmarkName", "Bench.work", + "-benchmarkWarmup", "-1", "-benchmarkIterations", "1", + "-benchmarkOutput", "out.json")); + assertThrows(RuntimeException.class, () -> new RunArgs( + "-runbenchmarks", "-benchmarkName", "Bench.work", + "-benchmarkWarmup", "0", "-benchmarkIterations", "0", + "-benchmarkOutput", "out.json")); + assertThrows(RuntimeException.class, () -> new RunArgs( + "-runbenchmarks", "-benchmarkName", "Bench.work", + "-benchmarkWarmup", "nope", "-benchmarkIterations", "1", + "-benchmarkOutput", "out.json")); + assertThrows(RuntimeException.class, () -> new RunArgs( + "-runbenchmarks", "-benchmarkList", + "-benchmarkWarmup", "0", "-benchmarkIterations", "1", + "-benchmarkOutput", "out.json")); + assertThrows(RuntimeException.class, () -> new RunArgs( + "-runbenchmarks", "-benchmarkName", "Bench.work", "-benchmarkOutput", "out.json", + "-runtests")); + assertThrows(RuntimeException.class, () -> new RunArgs( + "-runbenchmarks", "-benchmarkName", "Bench.work", "-benchmarkOutput", "out.json", + "-runcompiletimefunctions")); + assertThrows(RuntimeException.class, () -> new RunArgs( + "-runbenchmarks", "-benchmarkName", "Bench.work", "-benchmarkOutput", "out.json", + "-build")); + assertThrows(RuntimeException.class, () -> new RunArgs( + "-runbenchmarks", "-benchmarkName", "Bench.work", "-benchmarkOutput", "out.json", + "-dev")); + assertThrows(RuntimeException.class, () -> new RunArgs( + "-runbenchmarks", "-benchmarkName", "Bench.work", "-benchmarkOutput", "out.json", + "-out", "compiled.j.txt")); + assertThrows(RuntimeException.class, () -> new RunArgs( + "-runbenchmarks", "-benchmarkName", "Bench.work", "-benchmarkOutput", "out.json", + "-testFilter", "Bench")); + assertThrows(RuntimeException.class, () -> new RunArgs( + "-runbenchmarks", "-benchmarkName", "Bench.work", "-benchmarkOutput", "out.json", + "-testTimeout", "30")); + } + + @Test + public void preservesCompilerProjectOptionsInBenchmarkMode() { + RunArgs args = new RunArgs( + "-runbenchmarks", "-benchmarkName", "Bench.work", "-benchmarkOutput", "out.json", + "-lua", "-lib", "dependency", "-noPJass", "-legacyJassChecks"); + + assertTrue(args.isRunBenchmarks()); + assertTrue(args.isLua()); + assertTrue(args.isDisablePjass()); + assertTrue(args.isLegacyJassTypeChecks()); + assertEquals(args.getAdditionalLibDirs().size(), 1); + } + + @Test + public void writesDiscoveryJsonWithWorkerSchema() throws Exception { + Path output = Files.createTempFile("benchmark-worker", ".json"); + BenchmarkWorkerOutput.writeDiscovery(output, List.of("Bench.first", "Bench.second")); + + JsonObject json = JsonParser.parseString(Files.readString(output)).getAsJsonObject(); + assertEquals(json.get("schema").getAsString(), "wurst-benchmark-worker-v2"); + assertEquals(json.get("mode").getAsString(), "discovery"); + assertEquals(json.getAsJsonArray("benchmarks").size(), 2); + assertEquals(json.getAsJsonArray("benchmarks").get(0).getAsString(), "Bench.first"); + } + + @Test + public void writesExecutionJsonWithRawSamplesWithoutDerivedStatistics() throws Exception { + Path output = Files.createTempFile("benchmark-worker", ".json"); + List samples = List.of(100L, 120L, 110L); + BenchmarkResult result = new BenchmarkResult("Bench.work", 4950, 2, samples); + + BenchmarkWorkerOutput.writeResult(output, result); + + JsonObject json = JsonParser.parseString(Files.readString(output)).getAsJsonObject(); + assertEquals(json.get("schema").getAsString(), "wurst-benchmark-worker-v2"); + assertEquals(json.get("mode").getAsString(), "execution"); + assertEquals(json.get("qualifiedName").getAsString(), "Bench.work"); + assertEquals(json.get("checksum").getAsInt(), 4950); + assertEquals(json.get("batchSize").getAsInt(), 2); + assertEquals(json.getAsJsonArray("samplesNanos").get(1).getAsLong(), 120L); + assertFalse(json.has("statistics")); + } + + @Test + public void compilerWorkerDiscoversWithoutGeneratingScript() throws Exception { + Path directory = Files.createTempDirectory("benchmark-cli"); + Path source = writeBenchmarkSource(directory); + Path output = directory.resolve("discovery.json"); + + ProcessResult process = runCompiler(directory, + source, "-lua", "-runbenchmarks", "-benchmarkList", "-benchmarkFilter", "Bench", + "-benchmarkOutput", output.toString()); + + assertEquals(process.exitCode(), 0, process.output()); + assertTrue(Files.exists(output)); + assertFalse(Files.exists(directory.resolve("compiled.j.txt"))); + assertFalse(Files.exists(directory.resolve("temp/output.j"))); + JsonObject json = JsonParser.parseString(Files.readString(output)).getAsJsonObject(); + assertEquals(json.getAsJsonArray("benchmarks").get(0).getAsString(), "Bench.work"); + } + + @Test + public void compilerWorkerRejectsNonExecutableBenchmarkBeforeDiscovery() throws Exception { + Path directory = Files.createTempDirectory("benchmark-cli-invalid"); + Path source = directory.resolve("Bench.wurst"); + Files.writeString(source, """ + package Bench + @benchmark native nativeBenchmark() returns int + """); + Path output = directory.resolve("discovery.json"); + + ProcessResult process = runCompiler(directory, + source, "-runbenchmarks", "-benchmarkList", "-benchmarkFilter", "Bench", + "-benchmarkOutput", output.toString()); + + assertTrue(process.exitCode() != 0, process.output()); + assertTrue(process.output().contains( + "@benchmark functions must be executable by the compiletime IL interpreter; native, extern, and compiletimenative declarations are not supported."), process.output()); + assertFalse(Files.exists(output)); + } + + @Test + public void compilerWorkerExecutesOneBenchmarkAndPreservesOutputOnFailure() throws Exception { + Path directory = Files.createTempDirectory("benchmark-cli"); + Path source = writeBenchmarkSource(directory); + Path output = directory.resolve("execution.json"); + + ProcessResult success = runCompiler(directory, + source, "-runbenchmarks", "-benchmarkName", "Bench.work", + "-benchmarkWarmup", "0", "-benchmarkIterations", "1", + "-benchmarkOutput", output.toString()); + + assertEquals(success.exitCode(), 0, success.output()); + JsonObject json = JsonParser.parseString(Files.readString(output)).getAsJsonObject(); + assertEquals(json.get("checksum").getAsInt(), 42); + assertTrue(json.getAsJsonArray("samplesNanos").size() == 1); + assertFalse(Files.exists(directory.resolve("compiled.j.txt"))); + assertFalse(Files.exists(directory.resolve("temp/output.j"))); + + Files.writeString(output, "existing valid output"); + ProcessResult failure = runCompiler(directory, + source, "-runbenchmarks", "-benchmarkName", "Bench.missing", + "-benchmarkWarmup", "0", "-benchmarkIterations", "1", + "-benchmarkOutput", output.toString()); + + assertTrue(failure.exitCode() != 0, failure.output()); + assertEquals(Files.readString(output), "existing valid output"); + assertFalse(Files.exists(directory.resolve("compiled.j.txt"))); + assertFalse(Files.exists(directory.resolve("temp/output.j"))); + } + + @Test + public void compilerWorkerExecutesExactBenchmarkForLuaWithoutGeneratingScript() throws Exception { + Path directory = Files.createTempDirectory("benchmark-cli-lua"); + Path source = writeBenchmarkSource(directory); + Path output = directory.resolve("execution.json"); + + ProcessResult process = runCompiler(directory, + source, "-lua", "-runbenchmarks", "-benchmarkName", "Bench.work", + "-benchmarkWarmup", "0", "-benchmarkIterations", "1", + "-benchmarkOutput", output.toString()); + + assertEquals(process.exitCode(), 0, process.output()); + JsonObject json = JsonParser.parseString(Files.readString(output)).getAsJsonObject(); + assertEquals(json.get("checksum").getAsInt(), 42); + assertFalse(Files.exists(directory.resolve("compiled.j.txt"))); + assertFalse(Files.exists(directory.resolve("temp/output.j"))); + } + + @Test + public void exactNameWorkerExecutesOnlyRequestedBenchmark() throws Exception { + Path directory = Files.createTempDirectory("benchmark-cli-multiple"); + Path source = writeMultipleBenchmarkSource(directory); + Path output = directory.resolve("execution.json"); + + ProcessResult process = runCompiler(directory, + source, "-runbenchmarks", "-benchmarkName", "Bench.second", + "-benchmarkWarmup", "0", "-benchmarkIterations", "1", + "-benchmarkOutput", output.toString()); + + assertEquals(process.exitCode(), 0, process.output()); + JsonObject json = JsonParser.parseString(Files.readString(output)).getAsJsonObject(); + assertEquals(json.get("qualifiedName").getAsString(), "Bench.second"); + assertEquals(json.get("checksum").getAsInt(), 22, + "first benchmark would have changed the second benchmark checksum"); + } + + @Test + public void atomicWriterPreservesDestinationAndCleansTempAfterInjectedFailure() throws Exception { + Path directory = Files.createTempDirectory("benchmark-atomic"); + Path output = directory.resolve("worker.json"); + Files.writeString(output, "old output"); + + assertThrows(IOException.class, () -> BenchmarkWorkerOutput.writeAtomically( + output, + "new output", + (temporary, json) -> { + Files.writeString(temporary, json); + throw new IOException("forced temporary writer failure"); + })); + + assertEquals(Files.readString(output), "old output"); + try (Stream files = Files.list(directory)) { + assertTrue(files.noneMatch(path -> path.getFileName().toString().endsWith(".tmp"))); + } + } + + @Test + public void atomicWriterSupportsShortDestinationBasenames() throws Exception { + Path directory = Files.createTempDirectory("benchmark-atomic-short"); + Path output = directory.resolve("x"); + + BenchmarkWorkerOutput.writeDiscovery(output, List.of("Bench.work")); + + assertTrue(Files.exists(output)); + assertTrue(Files.readString(output).contains("Bench.work")); + } + + private static Path writeBenchmarkSource(Path directory) throws Exception { + Path source = directory.resolve("Bench.wurst"); + Files.writeString(source, """ + package Bench + @benchmark function work() returns int + return 42 + """); + return source; + } + + private static Path writeMultipleBenchmarkSource(Path directory) throws Exception { + Path source = directory.resolve("MultipleBench.wurst"); + Files.writeString(source, """ + package Bench + int firstCalls = 0 + int secondCalls = 0 + @benchmark function first() returns int + firstCalls += 1 + return 11 + @benchmark function second() returns int + secondCalls += 1 + return firstCalls * 1000 + 22 + """); + return source; + } + + private static ProcessResult runCompiler(Path directory, Path source, String... args) throws Exception { + List command = new ArrayList<>(); + command.add(Path.of(System.getProperty("java.home"), "bin", "java").toString()); + command.add("-cp"); + command.add(System.getProperty("java.class.path")); + command.add(Main.class.getName()); + command.add(source.toString()); + command.addAll(List.of(args)); + Process process = new ProcessBuilder(command) + .directory(directory.toFile()) + .redirectErrorStream(true) + .start(); + String output = new String(process.getInputStream().readAllBytes()); + return new ProcessResult(process.waitFor(), output); + } + + private record ProcessResult(int exitCode, String output) { + } +} diff --git a/de.peeeq.wurstscript/src/test/java/tests/wurstscript/tests/RunBenchmarksTests.java b/de.peeeq.wurstscript/src/test/java/tests/wurstscript/tests/RunBenchmarksTests.java new file mode 100644 index 000000000..521808ffc --- /dev/null +++ b/de.peeeq.wurstscript/src/test/java/tests/wurstscript/tests/RunBenchmarksTests.java @@ -0,0 +1,431 @@ +package tests.wurstscript.tests; + +import de.peeeq.wurstio.WurstCompilerJassImpl; +import de.peeeq.wurstio.benchmark.BenchmarkResult; +import de.peeeq.wurstio.benchmark.RunBenchmarks; +import de.peeeq.wurstio.benchmark.BenchmarkOptions; +import de.peeeq.wurstio.benchmark.BenchmarkSessionTestSupport; +import de.peeeq.wurstscript.RunArgs; +import de.peeeq.wurstscript.ast.WurstModel; +import de.peeeq.wurstscript.gui.WurstGuiCliImpl; +import de.peeeq.wurstscript.intermediatelang.ILconst; +import de.peeeq.wurstscript.intermediatelang.ILconstInt; +import de.peeeq.wurstscript.intermediatelang.interpreter.NativesProvider; +import de.peeeq.wurstscript.jassIm.ImProg; +import de.peeeq.wurstscript.translation.imtranslation.ImTranslator; +import de.peeeq.wurstscript.utils.Utils; +import org.testng.annotations.Test; + +import java.util.ArrayDeque; +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import java.io.PrintStream; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertThrows; +import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; + +public class RunBenchmarksTests extends WurstScriptTest { + + @Test + public void discoversPublicBenchmarkNamesAndExcludesTests() { + Compilation compilation = compile(""" + package Bench + native testSuccess() + @benchmark function work() returns int + return 4950 + @test function notABenchmark() + testSuccess() + """); + + RunBenchmarks runner = new RunBenchmarks(() -> 0L); + + assertEquals(runner.discover(compilation.program(), Optional.of("bench.work")), List.of("Bench.work")); + } + + @Test + public void dualAnnotatedFunctionRunsAsBenchmark() { + Compilation compilation = compile(""" + package Bench + @test @benchmark function checkedWork() returns int + return 4950 + """); + RunBenchmarks runner = new RunBenchmarks(new FakeClock(0L, 100L)); + + assertEquals(runner.discover(compilation.program(), Optional.empty()), List.of("Bench.checkedWork")); + BenchmarkResult result = runner.run( + compilation.translator(), compilation.program(), "Bench.checkedWork", + new BenchmarkOptions(0, 1, 0)); + assertEquals(result.checksum(), 4950); + } + + @Test + public void measuresIntegerNanosecondsAndCalculatesPercentiles() { + Compilation compilation = compile(""" + package Bench + @benchmark function work() returns int + return 4950 + """); + FakeClock clock = new FakeClock(0L, 100L, 100L, 220L, 220L, 330L); + RunBenchmarks runner = new RunBenchmarks(clock); + + BenchmarkResult result = runner.run( + compilation.translator(), compilation.program(), "Bench.work", + new BenchmarkOptions(0, 3, 0)); + + assertEquals(result.checksum(), 4950); + assertEquals(result.batchSize(), 1); + assertEquals(result.samplesNanos(), List.of(100L, 120L, 110L)); + } + + @Test + public void calibrationSamplesAreNotReported() { + Compilation compilation = compile(""" + package Bench + @benchmark function work() returns int + return 4950 + """); + FakeClock clock = new FakeClock(0L, 10L, 10L, 110L); + RunBenchmarks runner = new RunBenchmarks(clock); + + BenchmarkResult result = runner.run( + compilation.translator(), compilation.program(), "Bench.work", + new BenchmarkOptions(0, 1, 10)); + + assertEquals(result.batchSize(), 1); + assertEquals(result.samplesNanos(), List.of(100L)); + } + + @Test + public void adaptiveCalibrationNormalizesBatchesAndExcludesCalibrationSamples() { + Compilation compilation = compile(""" + package Bench + @benchmark function work() returns int + return 4950 + """); + TrackingSession session = new TrackingSession(false, false); + FakeClock clock = new FakeClock(0L, 4L, 4L, 14L, 14L, 25L); + + BenchmarkResult result = new RunBenchmarks( + clock, + (translator, program) -> session).run( + compilation.translator(), compilation.program(), "Bench.work", + new BenchmarkOptions(0, 1, 10)); + + assertEquals(result.batchSize(), 2); + assertEquals(result.samplesNanos(), List.of(5L)); + assertEquals(session.invokeCalls, 5); + } + + @Test + public void warmupsPrecedeCalibrationAndMeasurement() { + Compilation compilation = compile(""" + package Bench + @benchmark function work() returns int + return 4950 + """); + TrackingSession session = new TrackingSession(false, false); + EventClock clock = new EventClock(List.of(0L, 4L, 4L, 14L, 14L, 25L), session.events); + + BenchmarkResult result = new RunBenchmarks( + clock, + (translator, program) -> session).run( + compilation.translator(), compilation.program(), "Bench.work", + new BenchmarkOptions(2, 1, 10)); + + assertEquals(result.checksum(), 4950); + assertEquals(result.batchSize(), 2); + assertEquals(result.samplesNanos(), List.of(5L)); + assertEquals(session.events, List.of( + "initialize", "invoke", "invoke", "clock", "invoke", "clock", "clock", "invoke", "invoke", + "clock", "clock", "invoke", "invoke", "clock", "close")); + } + + @Test + public void changingChecksumsFailDuringWarmup() { + Compilation compilation = compile(""" + package Bench + @benchmark function changing() returns int + return 42 + """); + TrackingSession session = new TrackingSession(false, false, List.of(42, 43)); + + IllegalStateException failure; + try { + new RunBenchmarks( + new EventClock(List.of()), + (translator, program) -> session).run( + compilation.translator(), compilation.program(), "Bench.changing", + new BenchmarkOptions(2, 1, 0)); + fail("changing checksum should fail during warmup"); + return; + } catch (IllegalStateException e) { + failure = e; + } + + assertTrue(failure.getMessage().contains("changing checksums"), failure.getMessage()); + assertEquals(session.events, List.of("initialize", "invoke", "invoke", "close")); + } + + @Test + public void rawBenchmarkResultsRejectInvalidSamples() { + assertThrows(IllegalArgumentException.class, () -> + new BenchmarkResult("Bench.work", 1, 1, List.of())); + assertThrows(IllegalArgumentException.class, () -> + new BenchmarkResult("Bench.work", 1, 1, List.of(-1L))); + + BenchmarkResult result = new BenchmarkResult("Bench.work", 1, 1, List.of(1L)); + assertEquals(result.samplesNanos(), List.of(1L)); + } + + @Test + public void changingChecksumsFailDuringMeasurement() { + Compilation compilation = compile(""" + package Bench + int counter = 0 + @benchmark function changing() returns int + counter += 1 + return counter + """); + + IllegalStateException failure; + try { + new RunBenchmarks(new FakeClock(0L, 1L, 1L, 2L)).run( + compilation.translator(), compilation.program(), "Bench.changing", + new BenchmarkOptions(0, 2, 0)); + fail("changing checksum should fail"); + return; + } catch (IllegalStateException e) { + failure = e; + } + assertTrue(failure.getMessage().contains("changing checksums"), failure.getMessage()); + } + + @Test + public void interpreterExceptionsEscapeTheBenchmarkRun() { + Compilation compilation = compile(""" + package Bench + int denominator = 0 + @benchmark function failing() returns int + return 1 div denominator + """); + + assertThrows(RuntimeException.class, () -> + new RunBenchmarks(new FakeClock(0L, 1L)).run( + compilation.translator(), compilation.program(), "Bench.failing", + new BenchmarkOptions(0, 1, 0))); + } + + @Test + public void benchmarkSessionClosesOnSuccessAndFailure() { + Compilation compilation = compile(""" + package Bench + @benchmark function work() returns int + return 4950 + """); + + TrackingSession success = new TrackingSession(false, false); + BenchmarkResult result = new RunBenchmarks( + new FakeClock(0L, 100L), + (translator, program) -> success).run( + compilation.translator(), compilation.program(), "Bench.work", + new BenchmarkOptions(0, 1, 0)); + assertEquals(result.checksum(), 4950); + assertEquals(success.initializeCalls, 1); + assertEquals(success.invokeCalls, 1); + assertEquals(success.events, List.of("initialize", "invoke", "close")); + + TrackingSession initializationFailure = new TrackingSession(true, false); + assertThrows(IllegalStateException.class, () -> new RunBenchmarks( + new FakeClock(), + (translator, program) -> initializationFailure).run( + compilation.translator(), compilation.program(), "Bench.work", + new BenchmarkOptions(0, 1, 0))); + assertEquals(initializationFailure.initializeCalls, 1); + assertEquals(initializationFailure.invokeCalls, 0); + assertEquals(initializationFailure.events, List.of("initialize", "close")); + + TrackingSession invocationFailure = new TrackingSession(false, true); + assertThrows(IllegalStateException.class, () -> new RunBenchmarks( + new FakeClock(0L, 100L), + (translator, program) -> invocationFailure).run( + compilation.translator(), compilation.program(), "Bench.work", + new BenchmarkOptions(0, 1, 0))); + assertEquals(invocationFailure.initializeCalls, 1); + assertEquals(invocationFailure.invokeCalls, 1); + assertEquals(invocationFailure.events, List.of("initialize", "invoke", "close")); + } + + @Test + public void realInterpreterProviderClosesAfterSuccessAndBenchmarkFailure() { + Compilation successCompilation = compile(""" + package Bench + @compiletimenative function trackedNative() returns int + return 7 + @benchmark function work() returns int + return trackedNative() + """); + TrackingProvider successProvider = new TrackingProvider(false); + BenchmarkResult success = new RunBenchmarks( + new EventClock(List.of(0L, 100L)), + BenchmarkSessionTestSupport.defaultSessionFactory(successProvider)).run( + successCompilation.translator(), successCompilation.program(), "Bench.work", + new BenchmarkOptions(0, 1, 0)); + + assertEquals(success.checksum(), 7); + assertTrue(successProvider.invoked); + assertTrue(successProvider.closed); + + Compilation failureCompilation = compile(""" + package Bench + @compiletimenative function trackedNative() returns int + return 7 + @benchmark function work() returns int + return trackedNative() + """); + TrackingProvider failureProvider = new TrackingProvider(true); + + assertThrows(RuntimeException.class, () -> new RunBenchmarks( + new EventClock(List.of(0L)), + BenchmarkSessionTestSupport.defaultSessionFactory(failureProvider)).run( + failureCompilation.translator(), failureCompilation.program(), "Bench.work", + new BenchmarkOptions(0, 1, 0))); + assertTrue(failureProvider.invoked); + assertTrue(failureProvider.closed); + } + + private Compilation compile(String source) { + WurstGuiCliImpl gui = new WurstGuiCliImpl(); + WurstCompilerJassImpl compiler = new WurstCompilerJassImpl(null, gui, null, new RunArgs()); + WurstModel model = parseFiles(null, + Collections.singletonList(new CU("benchmark", Utils.join(source.lines().toList(), "\n") + "\n")), + false, + compiler); + compiler.checkProg(model); + assertTrue(gui.getErrorList().isEmpty(), gui.getErrorsAndWarnings().toString()); + ImProg program = compiler.translateProgToIm(model); + assertTrue(program != null, gui.getErrorsAndWarnings().toString()); + return new Compilation(compiler.getImTranslator(), program); + } + + private record Compilation(ImTranslator translator, ImProg program) { + } + + private static final class FakeClock implements de.peeeq.wurstio.benchmark.BenchmarkClock { + private final ArrayDeque values = new ArrayDeque<>(); + + private FakeClock(long... values) { + for (long value : values) { + this.values.add(value); + } + } + + @Override + public long nanoTime() { + if (values.isEmpty()) { + throw new AssertionError("fake clock exhausted"); + } + return values.removeFirst(); + } + } + + private static final class EventClock implements de.peeeq.wurstio.benchmark.BenchmarkClock { + private final ArrayDeque values = new ArrayDeque<>(); + private final List events; + + private EventClock(List values) { + this(values, new java.util.ArrayList<>()); + } + + private EventClock(List values, List events) { + this.values.addAll(values); + this.events = events; + } + + @Override + public long nanoTime() { + events.add("clock"); + if (values.isEmpty()) { + throw new AssertionError("fake clock exhausted"); + } + return values.removeFirst(); + } + } + + private static final class TrackingSession implements RunBenchmarks.BenchmarkSession { + private final boolean failInitialization; + private final boolean failInvocation; + private final List events = new java.util.ArrayList<>(); + private final ArrayDeque returnValues; + private int initializeCalls; + private int invokeCalls; + + private TrackingSession(boolean failInitialization, boolean failInvocation) { + this(failInitialization, failInvocation, List.of(4950)); + } + + private TrackingSession(boolean failInitialization, boolean failInvocation, List returnValues) { + this.failInitialization = failInitialization; + this.failInvocation = failInvocation; + this.returnValues = new ArrayDeque<>(returnValues); + } + + @Override + public void runCompiletime() { + events.add("initialize"); + initializeCalls++; + if (failInitialization) { + throw new IllegalStateException("initialization failed"); + } + } + + @Override + public ILconst invoke(de.peeeq.wurstscript.jassIm.ImFunction function) { + events.add("invoke"); + invokeCalls++; + if (failInvocation) { + throw new IllegalStateException("invocation failed"); + } + return ILconstInt.create(returnValues.isEmpty() ? 4950 : returnValues.removeFirst()); + } + + @Override + public void close() { + events.add("close"); + } + } + + private static final class TrackingProvider implements NativesProvider { + private final boolean fail; + private boolean invoked; + private boolean closed; + + private TrackingProvider(boolean fail) { + this.fail = fail; + } + + @Override + public ILconst invoke(String funcname, ILconst[] args) + throws de.peeeq.wurstscript.intermediatelang.interpreter.NoSuchNativeException { + if (!funcname.equals("trackedNative")) { + throw new de.peeeq.wurstscript.intermediatelang.interpreter.NoSuchNativeException(funcname); + } + invoked = true; + if (fail) { + throw new IllegalStateException("tracked provider failure"); + } + return ILconstInt.create(7); + } + + @Override + public void setOutStream(PrintStream outStream) { + } + + @Override + public void close() { + closed = true; + } + } +} diff --git a/de.peeeq.wurstscript/src/test/java/tests/wurstscript/tests/RunTestsOutputRedirectTests.java b/de.peeeq.wurstscript/src/test/java/tests/wurstscript/tests/RunTestsOutputRedirectTests.java index 4593eb5a4..c13a99898 100644 --- a/de.peeeq.wurstscript/src/test/java/tests/wurstscript/tests/RunTestsOutputRedirectTests.java +++ b/de.peeeq.wurstscript/src/test/java/tests/wurstscript/tests/RunTestsOutputRedirectTests.java @@ -55,12 +55,47 @@ public void testResultsAreStillReported() { assertTrue(output.contains("assertion message"), output); } + @Test + public void normalTestModeNeverExecutesBenchmarks() { + String output = runTests(false, new String[] { + "package test", + "native println(string msg)", + "@benchmark function benchmarkOnly() returns int", + "\tprintln(\"benchmark output\")", + "\treturn 42", + "@test function passingTest()", + "\tprintln(\"test output\")", + }); + + assertFalse(output.contains("benchmark output"), output); + assertTrue(output.contains("test output"), output); + assertTrue(output.contains("Tests succeeded: 1/1"), output); + } + + @Test + public void dualAnnotatedFunctionRunsAsNormalTest() { + String output = runTests(false, new String[] { + "package test", + "native println(string msg)", + "@test @benchmark function checkedWork() returns int", + "\tprintln(\"dual test output\")", + "\treturn 42", + }); + + assertTrue(output.contains("dual test output"), output); + assertTrue(output.contains("Tests succeeded: 1/1"), output); + } + private String runTests(boolean compactOutput) { + return runTests(compactOutput, PROGRAM); + } + + private String runTests(boolean compactOutput, String[] program) { RunArgs runArgs = new RunArgs(); WurstGui gui = new WurstGuiCliImpl(); WurstCompilerJassImpl compiler = new WurstCompilerJassImpl(null, gui, null, runArgs); WurstModel model = parseFiles(null, - Collections.singletonList(new CU("test", Utils.join(PROGRAM, "\n") + "\n")), false, compiler); + Collections.singletonList(new CU("test", Utils.join(program, "\n") + "\n")), false, compiler); compiler.checkProg(model); if (!gui.getErrorList().isEmpty()) { throw gui.getErrorList().get(0);