From dba4a25f395292c9f6e2bf0a8ff5aa3aa9f90cc1 Mon Sep 17 00:00:00 2001 From: Ifeanyi Shadrach Odom Date: Wed, 29 Jul 2026 15:28:24 +0100 Subject: [PATCH 1/5] Add --title option docs and usage example Document the new --title command-line option for customizing joined summary titles and export file names in benchmarks. Updated console-args.md and IntroJoin.md with descriptions and usage examples. --- docs/articles/guides/console-args.md | 1 + docs/articles/samples/IntroJoin.md | 8 ++++++++ 2 files changed, 9 insertions(+) diff --git a/docs/articles/guides/console-args.md b/docs/articles/guides/console-args.md index e6bf87f7fa..2af9e3042d 100644 --- a/docs/articles/guides/console-args.md +++ b/docs/articles/guides/console-args.md @@ -274,6 +274,7 @@ dotnet run -c Release -- --filter * --runtimes net6.0 net8.0 --statisticalTest 5 * `--anyCategories` Any Categories to run * `--attribute` Run all methods with given attribute (applied to class or method) * `--join` (Default: false) Prints single table with results for all benchmarks +* `--title` Custom title for the joined summary and the base name of its exported result files * `--keepFiles` (Default: false) Determines if all auto-generated files should be kept or removed after running the benchmarks. * `--noOverwrite` (Default: false) Determines if the exported result files should not be overwritten (be default they are overwritten). * `--counters` Hardware Counters diff --git a/docs/articles/samples/IntroJoin.md b/docs/articles/samples/IntroJoin.md index c4ba284c7a..5a8f5da242 100644 --- a/docs/articles/samples/IntroJoin.md +++ b/docs/articles/samples/IntroJoin.md @@ -16,6 +16,14 @@ If you are using `BenchmarkSwitcher` and want to run all the benchmarks with a c --join --allCategories=IntroJoinA ``` +By default the joined summary and its exported result files are named `BenchmarkRun-joined-{timestamp}`. +You can override this with a custom title via the `--title` option (or `ManualConfig.WithTitle(...)` in code), +which is handy when you run many sets of joined benchmarks and want to tell the exports apart: + +``` +--join --allCategories=IntroJoinA --title=MyBenchmarks +``` + ### Output ```markdown From e51d40f0c7958e48e2d443d360a061e42c2aa899 Mon Sep 17 00:00:00 2001 From: Ifeanyi Shadrach Odom Date: Wed, 29 Jul 2026 15:29:21 +0100 Subject: [PATCH 2/5] Add support for custom summary title in configs Introduce a Title property to IConfig and its implementations, enabling users to set a custom summary title and base name for exported result files when using joined summaries. Add fluent config methods and a --title CLI option. Update summary joining logic and config builders to support the new property. --- src/BenchmarkDotNet/Configs/ConfigExtensions.cs | 6 ++++++ src/BenchmarkDotNet/Configs/DebugConfig.cs | 2 ++ src/BenchmarkDotNet/Configs/DefaultConfig.cs | 2 ++ src/BenchmarkDotNet/Configs/IConfig.cs | 7 +++++++ src/BenchmarkDotNet/Configs/ImmutableConfig.cs | 3 +++ src/BenchmarkDotNet/Configs/ImmutableConfigBuilder.cs | 1 + src/BenchmarkDotNet/Configs/ManualConfig.cs | 8 ++++++++ .../ConsoleArguments/CommandLineOptions.cs | 3 +++ src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs | 3 +++ src/BenchmarkDotNet/Reports/Summary.cs | 4 ++-- src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs | 5 +++-- 11 files changed, 40 insertions(+), 4 deletions(-) diff --git a/src/BenchmarkDotNet/Configs/ConfigExtensions.cs b/src/BenchmarkDotNet/Configs/ConfigExtensions.cs index 9856e727b9..a44dba0fa8 100644 --- a/src/BenchmarkDotNet/Configs/ConfigExtensions.cs +++ b/src/BenchmarkDotNet/Configs/ConfigExtensions.cs @@ -42,6 +42,12 @@ public static class ConfigExtensions [PublicAPI] public static ManualConfig WithSummaryStyle(this IConfig config, SummaryStyle summaryStyle) => config.With(c => c.WithSummaryStyle(summaryStyle)); [PublicAPI] public static ManualConfig WithArtifactsPath(this IConfig config, string artifactsPath) => config.With(m => m.WithArtifactsPath(artifactsPath)); + + /// + /// sets the title of the summary and the base name of the exported result files produced when + /// is set + /// + [PublicAPI] public static ManualConfig WithTitle(this IConfig config, string title) => config.With(m => m.WithTitle(title)); [PublicAPI] public static ManualConfig WithUnionRule(this IConfig config, ConfigUnionRule unionRule) => config.With(m => m.WithUnionRule(unionRule)); [PublicAPI] public static ManualConfig WithCultureInfo(this IConfig config, CultureInfo cultureInfo) => config.With(m => m.CultureInfo = cultureInfo); diff --git a/src/BenchmarkDotNet/Configs/DebugConfig.cs b/src/BenchmarkDotNet/Configs/DebugConfig.cs index 3f9b3cd2a7..4c83303585 100644 --- a/src/BenchmarkDotNet/Configs/DebugConfig.cs +++ b/src/BenchmarkDotNet/Configs/DebugConfig.cs @@ -67,6 +67,8 @@ public abstract class DebugConfig : IConfig public string? ArtifactsPath => null; // DefaultConfig.ArtifactsPath will be used if the user does not specify it in explicit way + public string? Title => null; // the default "BenchmarkRun-joined-{timestamp}" title will be used if the user does not specify it in explicit way + public CultureInfo? CultureInfo => null; public IEnumerable GetLogicalGroupRules() => []; diff --git a/src/BenchmarkDotNet/Configs/DefaultConfig.cs b/src/BenchmarkDotNet/Configs/DefaultConfig.cs index cd9e1c2fe4..df47aeb52f 100644 --- a/src/BenchmarkDotNet/Configs/DefaultConfig.cs +++ b/src/BenchmarkDotNet/Configs/DefaultConfig.cs @@ -117,6 +117,8 @@ public string ArtifactsPath } } + public string? Title => null; + public IReadOnlyList ConfigAnalysisConclusion => emptyConclusion; public IEnumerable GetJobs() => []; diff --git a/src/BenchmarkDotNet/Configs/IConfig.cs b/src/BenchmarkDotNet/Configs/IConfig.cs index b554db3f7c..063a4e1d0f 100644 --- a/src/BenchmarkDotNet/Configs/IConfig.cs +++ b/src/BenchmarkDotNet/Configs/IConfig.cs @@ -41,6 +41,13 @@ public interface IConfig /// string? ArtifactsPath { get; } + /// + /// the title of the summary and the base name of the exported result files produced when + /// is set; when not specified, a default value + /// ("BenchmarkRun-joined-{timestamp}") is used + /// + string? Title { get; } + CultureInfo? CultureInfo { get; } /// diff --git a/src/BenchmarkDotNet/Configs/ImmutableConfig.cs b/src/BenchmarkDotNet/Configs/ImmutableConfig.cs index 5d644e7a93..5b6f56ef77 100644 --- a/src/BenchmarkDotNet/Configs/ImmutableConfig.cs +++ b/src/BenchmarkDotNet/Configs/ImmutableConfig.cs @@ -47,6 +47,7 @@ internal ImmutableConfig( ImmutableHashSet uniqueEventProcessors, ConfigUnionRule unionRule, string artifactsPath, + string? title, CultureInfo cultureInfo, IOrderer orderer, ICategoryDiscoverer categoryDiscoverer, @@ -70,6 +71,7 @@ internal ImmutableConfig( eventProcessors = uniqueEventProcessors; UnionRule = unionRule; ArtifactsPath = artifactsPath; + Title = title; CultureInfo = cultureInfo; Orderer = orderer; CategoryDiscoverer = categoryDiscoverer; @@ -82,6 +84,7 @@ internal ImmutableConfig( public ConfigUnionRule UnionRule { get; } public string ArtifactsPath { get; } + public string? Title { get; } public CultureInfo CultureInfo { get; } public ConfigOptions Options { get; } public IOrderer Orderer { get; } diff --git a/src/BenchmarkDotNet/Configs/ImmutableConfigBuilder.cs b/src/BenchmarkDotNet/Configs/ImmutableConfigBuilder.cs index c78efd6f02..18c1b65fd6 100644 --- a/src/BenchmarkDotNet/Configs/ImmutableConfigBuilder.cs +++ b/src/BenchmarkDotNet/Configs/ImmutableConfigBuilder.cs @@ -71,6 +71,7 @@ public static ImmutableConfig Create(IConfig source) uniqueEventProcessors, source.UnionRule, source.ArtifactsPath ?? DefaultConfig.Instance.ArtifactsPath!, + source.Title, source.CultureInfo ?? DefaultCultureInfo.Instance, source.Orderer ?? DefaultOrderer.Instance, source.CategoryDiscoverer ?? DefaultCategoryDiscoverer.Instance, diff --git a/src/BenchmarkDotNet/Configs/ManualConfig.cs b/src/BenchmarkDotNet/Configs/ManualConfig.cs index 1bf4518d7e..562b99167b 100644 --- a/src/BenchmarkDotNet/Configs/ManualConfig.cs +++ b/src/BenchmarkDotNet/Configs/ManualConfig.cs @@ -49,6 +49,7 @@ public class ManualConfig : IConfig [PublicAPI] public ConfigOptions Options { get; set; } [PublicAPI] public ConfigUnionRule UnionRule { get; set; } = ConfigUnionRule.Union; [PublicAPI] public string? ArtifactsPath { get; set; } + [PublicAPI] public string? Title { get; set; } [PublicAPI] public CultureInfo? CultureInfo { get; set; } [PublicAPI] public IOrderer? Orderer { get; set; } [PublicAPI] public ICategoryDiscoverer? CategoryDiscoverer { get; set; } @@ -82,6 +83,12 @@ public ManualConfig WithArtifactsPath(string artifactsPath) return this; } + public ManualConfig WithTitle(string title) + { + Title = title; + return this; + } + public ManualConfig WithSummaryStyle(SummaryStyle summaryStyle) { SummaryStyle = summaryStyle; @@ -221,6 +228,7 @@ public void Add(IConfig config) Orderer = config.Orderer ?? Orderer; CategoryDiscoverer = config.CategoryDiscoverer ?? CategoryDiscoverer; ArtifactsPath = config.ArtifactsPath ?? ArtifactsPath; + Title = config.Title ?? Title; CultureInfo = config.CultureInfo ?? CultureInfo; SummaryStyle = config.SummaryStyle ?? SummaryStyle; logicalGroupRules.AddRangeDistinct(config.GetLogicalGroupRules()); diff --git a/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs b/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs index d0b27d368b..54e0ce6433 100644 --- a/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs +++ b/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs @@ -81,6 +81,9 @@ public bool UseDisassemblyDiagnoser [Option("join", Required = false, Default = false, HelpText = "Prints single table with results for all benchmarks")] public bool Join { get; set; } + [Option("title", Required = false, HelpText = "Custom title for the joined summary and the base name of its exported result files")] + public string? Title { get; set; } + [Option("keepFiles", Required = false, Default = false, HelpText = "Determines if all auto-generated files should be kept or removed after running the benchmarks.")] public bool KeepBenchmarkFiles { get; set; } diff --git a/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs b/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs index 671383aee7..1e8ad98789 100644 --- a/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs +++ b/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs @@ -382,6 +382,9 @@ private static IConfig CreateConfig(CommandLineOptions options, IConfig? globalC if (options.ArtifactsDirectory != null) config.ArtifactsPath = options.ArtifactsDirectory.FullName; + if (options.Title.IsNotBlank()) + config.Title = options.Title; + var filters = GetFilters(options).ToArray(); if (filters.Length > 1) config.AddFilter(new UnionFilter(filters)); diff --git a/src/BenchmarkDotNet/Reports/Summary.cs b/src/BenchmarkDotNet/Reports/Summary.cs index 4bc77c1a2c..f4bcee959f 100644 --- a/src/BenchmarkDotNet/Reports/Summary.cs +++ b/src/BenchmarkDotNet/Reports/Summary.cs @@ -89,9 +89,9 @@ internal static Summary ValidationFailed(string title, string resultsDirectoryPa => new Summary(title, [], HostEnvironmentInfo.GetCurrent(), resultsDirectoryPath, logFilePath, TimeSpan.Zero, DefaultCultureInfo.Instance, validationErrors ?? [], []); - internal static Summary Join(List summaries, ClockSpan clockSpan) + internal static Summary Join(List summaries, ClockSpan clockSpan, string? title = null) => new Summary( - $"BenchmarkRun-joined-{DateTime.Now:yyyy-MM-dd-HH-mm-ss}", + title ?? $"BenchmarkRun-joined-{DateTime.Now:yyyy-MM-dd-HH-mm-ss}", summaries.SelectMany(summary => summary.Reports).ToImmutableArray(), HostEnvironmentInfo.GetCurrent(), summaries.First().ResultsDirectoryPath, diff --git a/src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs b/src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs index 70b02d75bb..948ada59db 100644 --- a/src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs +++ b/src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs @@ -172,9 +172,10 @@ private static async ValueTask RunCore(BenchmarkRunInfo[] benchmarkRu if (supportedBenchmarks.Any(b => b.Config.Options.IsSet(ConfigOptions.JoinSummary))) { - var joinedSummary = Summary.Join(results, runsChronometer.GetElapsed()); + var joinConfig = supportedBenchmarks.First(b => b.Config.Options.IsSet(ConfigOptions.JoinSummary)).Config; + var joinedSummary = Summary.Join(results, runsChronometer.GetElapsed(), joinConfig.Title); - await PrintSummary(compositeLogger, supportedBenchmarks.First(b => b.Config.Options.IsSet(ConfigOptions.JoinSummary)).Config, joinedSummary, cancellationToken).ConfigureAwait(); + await PrintSummary(compositeLogger, joinConfig, joinedSummary, cancellationToken).ConfigureAwait(); results.Clear(); results.Add(joinedSummary); From b685142b622bda7665000ce5377d9e6de4339e11 Mon Sep 17 00:00:00 2001 From: Ifeanyi Shadrach Odom Date: Wed, 29 Jul 2026 15:30:44 +0100 Subject: [PATCH 3/5] Add tests for custom Title handling in configs and summaries Added unit tests to verify correct parsing, preservation, and fallback behavior of the Title property in configuration and summary classes. Ensured command-line parsing, immutable config creation, and summary joining all handle custom and default titles as expected. --- .../ConfigParserTests.cs | 18 +++++++++++++++ .../Configs/ImmutableConfigTests.cs | 16 ++++++++++++++ .../Reports/SummaryTableTests.cs | 22 +++++++++++++++++++ 3 files changed, 56 insertions(+) diff --git a/tests/BenchmarkDotNet.Tests/ConfigParserTests.cs b/tests/BenchmarkDotNet.Tests/ConfigParserTests.cs index 09832a6c8d..1f88d4eeb4 100644 --- a/tests/BenchmarkDotNet.Tests/ConfigParserTests.cs +++ b/tests/BenchmarkDotNet.Tests/ConfigParserTests.cs @@ -399,6 +399,24 @@ public void WhenConfigOptionsFlagsAreNotSpecifiedTheyAreNotSet() Assert.Equal(ConfigOptions.Default, config.Options); } + [Fact] + public void TitleParsedCorrectly() + { + var config = ConfigParser.Parse(["--title", "MyCustomTitle"], new OutputLogger(Output)).config; + + Assert.NotNull(config); + Assert.Equal("MyCustomTitle", config.Title); + } + + [Fact] + public void WhenTitleIsNotSpecifiedItIsNull() + { + var config = ConfigParser.Parse([], new OutputLogger(Output)).config; + + Assert.NotNull(config); + Assert.Null(config.Title); + } + [Fact] public void PackagesPathParsedCorrectly() { diff --git a/tests/BenchmarkDotNet.Tests/Configs/ImmutableConfigTests.cs b/tests/BenchmarkDotNet.Tests/Configs/ImmutableConfigTests.cs index 09f04a18ba..ab78df1888 100644 --- a/tests/BenchmarkDotNet.Tests/Configs/ImmutableConfigTests.cs +++ b/tests/BenchmarkDotNet.Tests/Configs/ImmutableConfigTests.cs @@ -320,6 +320,22 @@ public void WhenArtifactsPathIsNullDefaultValueShouldBeUsed() Assert.Equal(final.ArtifactsPath, DefaultConfig.Instance.ArtifactsPath); } + [Fact] + public void WhenTitleIsNullItStaysNullSoTheDefaultIsUsedLater() + { + var mutable = ManualConfig.CreateEmpty(); + var final = ImmutableConfigBuilder.Create(mutable); + Assert.Null(final.Title); + } + + [Fact] + public void CustomTitleIsPreserved() + { + var mutable = ManualConfig.CreateEmpty().WithTitle("MyCustomTitle"); + var final = ImmutableConfigBuilder.Create(mutable); + Assert.Equal("MyCustomTitle", final.Title); + } + [Fact] public void WhenOrdererIsNullDefaultValueShouldBeUsed() { diff --git a/tests/BenchmarkDotNet.Tests/Reports/SummaryTableTests.cs b/tests/BenchmarkDotNet.Tests/Reports/SummaryTableTests.cs index f916798b07..0d91ec26f3 100644 --- a/tests/BenchmarkDotNet.Tests/Reports/SummaryTableTests.cs +++ b/tests/BenchmarkDotNet.Tests/Reports/SummaryTableTests.cs @@ -98,6 +98,28 @@ public void JoinedSummaryHandlesMissingColumnValues() Assert.Equal(["NA", "tagged"], joinedSummary.Table.Columns.Single(c => c.Header == "Tag").Content); } + [Fact] // Issue #2621 + public void JoinedSummaryUsesCustomTitleWhenProvided() + { + var taggedSummary = MockFactory.CreateSummary(typeof(TaggedBenchmark)); + var otherSummary = MockFactory.CreateSummary(typeof(OtherBenchmark)); + + var joinedSummary = Summary.Join([taggedSummary, otherSummary], default, "MyCustomTitle"); + + Assert.Equal("MyCustomTitle", joinedSummary.Title); + } + + [Fact] // Issue #2621 + public void JoinedSummaryFallsBackToDefaultTitleWhenNoneProvided() + { + var taggedSummary = MockFactory.CreateSummary(typeof(TaggedBenchmark)); + var otherSummary = MockFactory.CreateSummary(typeof(OtherBenchmark)); + + var joinedSummary = Summary.Join([taggedSummary, otherSummary], default); + + Assert.StartsWith("BenchmarkRun-joined-", joinedSummary.Title); + } + [Fact] // Issue #1070 public void CustomOrdererIsSupported() { From 3c6a7f7c386b869c708e694dae0fe03405ef7844 Mon Sep 17 00:00:00 2001 From: Ifeanyi Shadrach Odom Date: Thu, 30 Jul 2026 13:20:45 +0100 Subject: [PATCH 4/5] Improve naming of exported benchmark result files Refactor file naming to use summary titles derived from benchmarked types or custom config/CLI titles. Introduce TitleHelper to ensure unique, valid, and truncated artifact names, preventing overwrites. Update docs, tests, and related code to use the new naming logic. --- .../Configs/ConfigExtensions.cs | 3 +- src/BenchmarkDotNet/Configs/DebugConfig.cs | 2 +- src/BenchmarkDotNet/Configs/IConfig.cs | 7 +- .../ConsoleArguments/CommandLineOptions.cs | 2 +- src/BenchmarkDotNet/Exporters/ExporterBase.cs | 25 +--- src/BenchmarkDotNet/Helpers/TitleHelper.cs | 83 ++++++++++++ src/BenchmarkDotNet/Reports/Summary.cs | 2 +- .../Running/BenchmarkRunnerClean.cs | 61 ++++----- .../ExporterIOTests.cs | 33 +---- .../Helpers/TitleHelperTests.cs | 128 ++++++++++++++++++ .../Reports/SummaryTableTests.cs | 3 +- 11 files changed, 257 insertions(+), 92 deletions(-) create mode 100644 src/BenchmarkDotNet/Helpers/TitleHelper.cs create mode 100644 tests/BenchmarkDotNet.Tests/Helpers/TitleHelperTests.cs diff --git a/src/BenchmarkDotNet/Configs/ConfigExtensions.cs b/src/BenchmarkDotNet/Configs/ConfigExtensions.cs index a44dba0fa8..a2f11e4f45 100644 --- a/src/BenchmarkDotNet/Configs/ConfigExtensions.cs +++ b/src/BenchmarkDotNet/Configs/ConfigExtensions.cs @@ -44,8 +44,7 @@ public static class ConfigExtensions [PublicAPI] public static ManualConfig WithArtifactsPath(this IConfig config, string artifactsPath) => config.With(m => m.WithArtifactsPath(artifactsPath)); /// - /// sets the title of the summary and the base name of the exported result files produced when - /// is set + /// sets the title of the produced summaries and the base name of the result files exported for them /// [PublicAPI] public static ManualConfig WithTitle(this IConfig config, string title) => config.With(m => m.WithTitle(title)); [PublicAPI] public static ManualConfig WithUnionRule(this IConfig config, ConfigUnionRule unionRule) => config.With(m => m.WithUnionRule(unionRule)); diff --git a/src/BenchmarkDotNet/Configs/DebugConfig.cs b/src/BenchmarkDotNet/Configs/DebugConfig.cs index 4c83303585..8b75139b70 100644 --- a/src/BenchmarkDotNet/Configs/DebugConfig.cs +++ b/src/BenchmarkDotNet/Configs/DebugConfig.cs @@ -67,7 +67,7 @@ public abstract class DebugConfig : IConfig public string? ArtifactsPath => null; // DefaultConfig.ArtifactsPath will be used if the user does not specify it in explicit way - public string? Title => null; // the default "BenchmarkRun-joined-{timestamp}" title will be used if the user does not specify it in explicit way + public string? Title => null; // the title derived from the benchmarked types will be used if the user does not specify it in explicit way public CultureInfo? CultureInfo => null; public IEnumerable GetLogicalGroupRules() => []; diff --git a/src/BenchmarkDotNet/Configs/IConfig.cs b/src/BenchmarkDotNet/Configs/IConfig.cs index 063a4e1d0f..343f9faa46 100644 --- a/src/BenchmarkDotNet/Configs/IConfig.cs +++ b/src/BenchmarkDotNet/Configs/IConfig.cs @@ -42,9 +42,10 @@ public interface IConfig string? ArtifactsPath { get; } /// - /// the title of the summary and the base name of the exported result files produced when - /// is set; when not specified, a default value - /// ("BenchmarkRun-joined-{timestamp}") is used + /// the title of the produced summaries and the base name of the result files exported for them; + /// when the run reports more than one summary, the name of the benchmarked type is appended to keep + /// their files apart. When not specified, the name of the benchmarked type is used + /// (or "BenchmarkRun-joined" when is set) /// string? Title { get; } diff --git a/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs b/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs index 54e0ce6433..125e60e97d 100644 --- a/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs +++ b/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs @@ -81,7 +81,7 @@ public bool UseDisassemblyDiagnoser [Option("join", Required = false, Default = false, HelpText = "Prints single table with results for all benchmarks")] public bool Join { get; set; } - [Option("title", Required = false, HelpText = "Custom title for the joined summary and the base name of its exported result files")] + [Option("title", Required = false, HelpText = "Custom title for the produced summaries and the base name of the result files exported for them")] public string? Title { get; set; } [Option("keepFiles", Required = false, Default = false, HelpText = "Determines if all auto-generated files should be kept or removed after running the benchmarks.")] diff --git a/src/BenchmarkDotNet/Exporters/ExporterBase.cs b/src/BenchmarkDotNet/Exporters/ExporterBase.cs index fab0e08b4a..87eadc31e9 100644 --- a/src/BenchmarkDotNet/Exporters/ExporterBase.cs +++ b/src/BenchmarkDotNet/Exporters/ExporterBase.cs @@ -17,7 +17,6 @@ public abstract class ExporterBase : IExporter public async ValueTask ExportAsync(Summary summary, ILogger logger, CancellationToken cancellationToken) { - string fileName = GetFileName(summary); string filePath = GetArtifactFullName(summary); if (File.Exists(filePath)) { @@ -28,7 +27,7 @@ public async ValueTask ExportAsync(Summary summary, ILogger logger, Cancellation catch (IOException) { string uniqueString = DateTime.Now.ToString("yyyyMMdd-HHmmss"); - string alternativeFilePath = $"{Path.Combine(summary.ResultsDirectoryPath, fileName)}-{FileCaption}{FileNameSuffix}-{uniqueString}.{FileExtension}"; + string alternativeFilePath = $"{Path.Combine(summary.ResultsDirectoryPath, summary.Title)}-{FileCaption}{FileNameSuffix}-{uniqueString}.{FileExtension}"; logger.WriteLineError($"Could not overwrite file {filePath}. Exporting to {alternativeFilePath}"); filePath = alternativeFilePath; } @@ -41,23 +40,11 @@ public async ValueTask ExportAsync(Summary summary, ILogger logger, Cancellation logger.WriteLineInfo($" {filePath.GetBaseName(Directory.GetCurrentDirectory())}"); } + /// + /// the exported files are named after the title of the summary, which is guaranteed to be unique + /// within a single run and valid for a path (see ) + /// public string GetArtifactFullName(Summary summary) - { - string fileName = GetFileName(summary); - return $"{Path.Combine(summary.ResultsDirectoryPath, fileName)}-{FileCaption}{FileNameSuffix}.{FileExtension}"; - } - - private static string GetFileName(Summary summary) - { - // we can't use simple name here, because user might be running benchmarks for a library, which defines few types with the same name - // and reports the results per type, so every summary is going to contain just single benchmark - // and we can't tell here if there is a name conflict or not - var targets = summary.BenchmarksCases.Select(b => b.Descriptor.Type).Distinct().ToArray(); - - if (targets.Length == 1) - return FolderNameHelper.ToFolderName(targets.Single()); - - return summary.Title; - } + => $"{Path.Combine(summary.ResultsDirectoryPath, summary.Title)}-{FileCaption}{FileNameSuffix}.{FileExtension}"; } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Helpers/TitleHelper.cs b/src/BenchmarkDotNet/Helpers/TitleHelper.cs new file mode 100644 index 0000000000..6f0aa1e646 --- /dev/null +++ b/src/BenchmarkDotNet/Helpers/TitleHelper.cs @@ -0,0 +1,83 @@ +using BenchmarkDotNet.Extensions; +using BenchmarkDotNet.Running; + +namespace BenchmarkDotNet.Helpers +{ + /// + /// provides the titles that identify a run and every summary it produces + /// + /// + /// the title of a summary is also the base name of every file exported for it, so the titles of the + /// summaries produced by a single run have to be unique, otherwise the results would overwrite each other #529 + /// + internal static class TitleHelper + { + internal const string JoinedSummaryTitle = "BenchmarkRun-joined"; + internal const string MultipleTypesTitle = "BenchmarkRun"; + + private const int MinTruncatedLength = 3; + + /// + /// the title of the entire run, used as the base name of its log file + /// + internal static string GetRunTitle(BenchmarkRunInfo[] benchmarkRunInfos, int desiredMaxLength = int.MaxValue) + { + string? customTitle = benchmarkRunInfos + .Select(benchmark => benchmark.Config.Title) + .FirstOrDefault(title => title.IsNotBlank()); + + if (customTitle.IsNotBlank()) + return Truncate(Escape(customTitle), desiredMaxLength); + + var uniqueTargetTypes = benchmarkRunInfos + .SelectMany(info => info.BenchmarksCases.Select(benchmark => benchmark.Descriptor.Type)) + .Distinct() + .ToArray(); + + return Truncate( + uniqueTargetTypes.Length == 1 ? FolderNameHelper.ToFolderName(uniqueTargetTypes[0]) : MultipleTypesTitle, + desiredMaxLength); + } + + /// + /// the title of a summary that reports the benchmarks of a single type, + /// where tells whether the run produces just this one + /// + internal static string GetSummaryTitle(BenchmarkRunInfo benchmarkRunInfo, bool isTheOnlySummary, int desiredMaxLength = int.MaxValue) + { + // few types might have the same name: A.Name and B.Name would both report "Name", + // so we use the namespace-qualified name to tell their results apart #529 + string typeName = FolderNameHelper.ToFolderName(benchmarkRunInfo.Type); + string? customTitle = benchmarkRunInfo.Config.Title; + + if (customTitle.IsBlank()) + return Truncate(typeName, desiredMaxLength); + + // the title is defined by the config, so every summary of the run would be given the same one + return Truncate(isTheOnlySummary ? Escape(customTitle) : $"{Escape(customTitle)}-{typeName}", desiredMaxLength); + } + + /// + /// the title of the single summary that joins the results of all the types + /// + internal static string GetJoinedSummaryTitle(string? customTitle, int desiredMaxLength = int.MaxValue) + => Truncate(customTitle.IsBlank() ? JoinedSummaryTitle : Escape(customTitle), desiredMaxLength); + + // the title becomes a file name, so it can not contain characters that are invalid for a path + private static string Escape(string title) => FolderNameHelper.ToFolderName((object)title); + + /// + /// shortens the title so that the paths of the artifacts named after it don't exceed the limits of the OS + /// + private static string Truncate(string title, int desiredMaxLength) + { + if (title.Length <= desiredMaxLength || desiredMaxLength < MinTruncatedLength) + return title; + + int prefixLength = desiredMaxLength / 2; + int suffixLength = desiredMaxLength - prefixLength - 1; + + return title.Substring(0, prefixLength) + "-" + title.Substring(title.Length - suffixLength, suffixLength); + } + } +} diff --git a/src/BenchmarkDotNet/Reports/Summary.cs b/src/BenchmarkDotNet/Reports/Summary.cs index f4bcee959f..801ae3c9ee 100644 --- a/src/BenchmarkDotNet/Reports/Summary.cs +++ b/src/BenchmarkDotNet/Reports/Summary.cs @@ -91,7 +91,7 @@ internal static Summary ValidationFailed(string title, string resultsDirectoryPa internal static Summary Join(List summaries, ClockSpan clockSpan, string? title = null) => new Summary( - title ?? $"BenchmarkRun-joined-{DateTime.Now:yyyy-MM-dd-HH-mm-ss}", + title ?? TitleHelper.GetJoinedSummaryTitle(customTitle: null), summaries.SelectMany(summary => summary.Reports).ToImmutableArray(), HostEnvironmentInfo.GetCurrent(), summaries.First().ResultsDirectoryPath, diff --git a/src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs b/src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs index 948ada59db..b6d235e34c 100644 --- a/src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs +++ b/src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs @@ -31,6 +31,10 @@ internal static class BenchmarkRunnerClean { internal const string DateTimeFormat = "yyyyMMdd-HHmmss"; + // the artifacts of a run are named "{title}{suffix}", where the longest suffix we produce is + // "-{DateTimeFormat}.log" for the log file and "-report-full-compressed.json" for the exported results + private const int MaxArtifactNameSuffixLength = 32; + internal static readonly IResolver DefaultResolver = new CompositeResolver(EnvironmentResolver.Instance, InfrastructureResolver.Instance); internal static async ValueTask Run(BenchmarkRunInfo[] benchmarkRunInfos, CancellationToken cancellationToken) @@ -57,13 +61,13 @@ private static async ValueTask RunCore(BenchmarkRunInfo[] benchmarkRu var artifactsToCleanup = new List(); var rootArtifactsFolderPath = GetRootArtifactsFolderPath(benchmarkRunInfos); - var maxTitleLength = OsDetector.IsWindows() - ? 254 - rootArtifactsFolderPath.Length - : int.MaxValue; - var title = GetTitle(benchmarkRunInfos, maxTitleLength); var resultsFolderPath = GetResultsFolderPath(rootArtifactsFolderPath, benchmarkRunInfos); - var logFilePath = Path.Combine(rootArtifactsFolderPath, title + ".log"); - var idToResume = GetIdToResume(rootArtifactsFolderPath, title, benchmarkRunInfos); + var maxTitleLength = GetMaxTitleLength(rootArtifactsFolderPath, resultsFolderPath); + var title = TitleHelper.GetRunTitle(benchmarkRunInfos, maxTitleLength); + // in contrary to the exported results, every run gets its own log file, so that --resume can find the previous one + var logFileName = $"{title}-{DateTime.Now.ToString(DateTimeFormat)}"; + var logFilePath = Path.Combine(rootArtifactsFolderPath, logFileName + ".log"); + var idToResume = GetIdToResume(rootArtifactsFolderPath, title, logFileName, benchmarkRunInfos); using var streamLogger = new StreamLogger(GetLogFileStreamWriter(benchmarkRunInfos, logFilePath)); var compositeLogger = CreateCompositeLogger(benchmarkRunInfos, streamLogger); @@ -153,8 +157,9 @@ private static async ValueTask RunCore(BenchmarkRunInfo[] benchmarkRu } eventProcessor.OnStartRunBenchmarksInType(benchmarkRunInfo.Type, benchmarkRunInfo.BenchmarksCases); + var summaryTitle = TitleHelper.GetSummaryTitle(benchmarkRunInfo, supportedBenchmarks.Length == 1, maxTitleLength); (var summary, benchmarksToRunCount) = await Run(benchmarkRunInfo, benchmarkToBuildResult, resolver, compositeLogger, eventProcessor, artifactsToCleanup, - resultsFolderPath, logFilePath, totalBenchmarkCount, runsChronometer, benchmarksToRunCount, + summaryTitle, resultsFolderPath, logFilePath, totalBenchmarkCount, runsChronometer, benchmarksToRunCount, taskbarProgress, cancellationToken).ConfigureAwait(); eventProcessor.OnEndRunBenchmarksInType(benchmarkRunInfo.Type, summary); @@ -173,7 +178,7 @@ private static async ValueTask RunCore(BenchmarkRunInfo[] benchmarkRu if (supportedBenchmarks.Any(b => b.Config.Options.IsSet(ConfigOptions.JoinSummary))) { var joinConfig = supportedBenchmarks.First(b => b.Config.Options.IsSet(ConfigOptions.JoinSummary)).Config; - var joinedSummary = Summary.Join(results, runsChronometer.GetElapsed(), joinConfig.Title); + var joinedSummary = Summary.Join(results, runsChronometer.GetElapsed(), TitleHelper.GetJoinedSummaryTitle(joinConfig.Title, maxTitleLength)); await PrintSummary(compositeLogger, joinConfig, joinedSummary, cancellationToken).ConfigureAwait(); @@ -212,6 +217,7 @@ private static async ValueTask RunCore(BenchmarkRunInfo[] benchmarkRu ILogger logger, EventProcessor eventProcessor, List artifactsToCleanup, + string title, string resultsFolderPath, string logFilePath, int totalBenchmarkCount, @@ -227,7 +233,6 @@ private static async ValueTask RunCore(BenchmarkRunInfo[] benchmarkRu var config = benchmarkRunInfo.Config; var cultureInfo = config.CultureInfo ?? DefaultCultureInfo.Instance; var reports = new List(); - string title = GetTitle([benchmarkRunInfo]); using var consoleTitler = new ConsoleTitler($"{benchmarksToRunCount}/{totalBenchmarkCount} Remaining"); logger.WriteLineInfo($"// Found {benchmarks.Length} benchmarks:"); @@ -758,32 +763,13 @@ private static string GetRootArtifactsFolderPath(BenchmarkRunInfo[] benchmarkRun return customPath != default ? customPath.CreateIfNotExists() : defaultPath; } - private static string GetTitle(BenchmarkRunInfo[] benchmarkRunInfos, int desiredMaxLength = int.MaxValue) - { - // few types might have the same name: A.Name and B.Name will both report "Name" - // in that case, we can not use the type name as file name because they would be getting overwritten #529 - var uniqueTargetTypes = benchmarkRunInfos.SelectMany(info => info.BenchmarksCases.Select(benchmark => benchmark.Descriptor.Type)).Distinct().ToArray(); - - var fileNamePrefix = (uniqueTargetTypes.Length == 1) - ? FolderNameHelper.ToFolderName(uniqueTargetTypes[0]) - : "BenchmarkRun"; - string dateTimeSuffix = DateTime.Now.ToString(DateTimeFormat); - - int maxFileNamePrefixLength = desiredMaxLength - dateTimeSuffix.Length - 1; - if (maxFileNamePrefixLength <= 2) - return dateTimeSuffix; - - if (fileNamePrefix.Length > maxFileNamePrefixLength) - { - int length1 = maxFileNamePrefixLength / 2; - int length2 = maxFileNamePrefixLength - length1 - 1; - fileNamePrefix = fileNamePrefix.Substring(0, length1) + - "-" + - fileNamePrefix.Substring(fileNamePrefix.Length - length2, length2); - } - - return $"{fileNamePrefix}-{dateTimeSuffix}"; - } + /// + /// the budget left for the title once the folder and the suffix that the artifacts are named after are accounted for + /// + private static int GetMaxTitleLength(string rootArtifactsFolderPath, string resultsFolderPath) + => OsDetector.IsWindows() + ? 254 - Math.Max(rootArtifactsFolderPath.Length, resultsFolderPath.Length) - MaxArtifactNameSuffixLength + : int.MaxValue; private static string GetResultsFolderPath(string rootArtifactsFolderPath, BenchmarkRunInfo[] benchmarkRunInfos) { @@ -866,13 +852,14 @@ private static void PrintValidationErrors(ILogger logger, IEnumerable benchmark.Config.Options.IsSet(ConfigOptions.Resume))) { var directoryInfo = new DirectoryInfo(rootArtifactsFolderPath); + // the log files of the previous runs of the same benchmarks are named "{title}-{timestamp}.log" var logFilesExceptCurrent = directoryInfo - .GetFiles($"{currentLogFileName.Split('-')[0]}*") + .GetFiles($"{title}-*.log") .Where(file => Path.GetFileNameWithoutExtension(file.Name) != currentLogFileName) .ToArray(); diff --git a/tests/BenchmarkDotNet.IntegrationTests/ExporterIOTests.cs b/tests/BenchmarkDotNet.IntegrationTests/ExporterIOTests.cs index e8f8795895..d37acef4ff 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/ExporterIOTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/ExporterIOTests.cs @@ -69,36 +69,15 @@ public async Task ExporterWorksWhenFileIsLocked() } } - [Fact] - public async Task ExporterUsesFullyQualifiedTypeNameAsFileName() - { - string resultsDirectoryPath = Path.GetTempPath(); - var exporter = new MockExporter(); - var mockSummary = GetMockSummary(resultsDirectoryPath, config: null, typeof(Generic)); - var expectedFilePath = $"{Path.Combine(mockSummary.ResultsDirectoryPath, "BenchmarkDotNet.IntegrationTests.Generic_Int32_")}-report.txt"; - string? actualFilePath = null; - - try - { - await exporter.ExportAsync(mockSummary, NullLogger.Instance, CancellationToken.None); - actualFilePath = exporter.GetArtifactFullName(mockSummary); - - Assert.Equal(expectedFilePath, actualFilePath); - } - finally - { - if (File.Exists(actualFilePath)) - File.Delete(actualFilePath); - } - } - - [Fact] - public async Task ExporterUsesSummaryTitleAsFileNameWhenBenchmarksJoinedToSingleSummary() + // the file names are derived from the title alone, see TitleHelperTests for how the runner builds it + [Theory] + [InlineData(typeof(Generic))] // a summary that reports a single type + [InlineData(typeof(ClassA), typeof(ClassB))] // a summary that joins the results of many types + public async Task ExporterUsesSummaryTitleAsFileName(params Type[] typesWithBenchmarks) { string resultsDirectoryPath = Path.GetTempPath(); var exporter = new MockExporter(); - var joinConfig = ManualConfig.CreateEmpty().WithOptions(ConfigOptions.JoinSummary); - var mockSummary = GetMockSummary(resultsDirectoryPath, joinConfig, typeof(ClassA), typeof(ClassB)); + var mockSummary = GetMockSummary(resultsDirectoryPath, config: null, typesWithBenchmarks); var expectedFilePath = $"{Path.Combine(mockSummary.ResultsDirectoryPath, mockSummary.Title)}-report.txt"; string? actualFilePath = null; diff --git a/tests/BenchmarkDotNet.Tests/Helpers/TitleHelperTests.cs b/tests/BenchmarkDotNet.Tests/Helpers/TitleHelperTests.cs new file mode 100644 index 0000000000..d139ce251a --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Helpers/TitleHelperTests.cs @@ -0,0 +1,128 @@ +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Helpers; +using BenchmarkDotNet.Running; + +namespace BenchmarkDotNet.Tests.Helpers +{ + public class TitleHelperTests + { + private const string CustomTitle = "MyCustomTitle"; + private const string FirstTypeName = "BenchmarkDotNet.Tests.Helpers.FirstBenchmarks"; + private const string SecondTypeName = "BenchmarkDotNet.Tests.Helpers.SecondBenchmarks"; + + [Fact] + public void SummaryTitleIsTheFullyQualifiedTypeNameWhenNoTitleIsSet() + { + var runInfo = CreateRunInfo(typeof(FirstBenchmarks), title: null); + + Assert.Equal(FirstTypeName, TitleHelper.GetSummaryTitle(runInfo, isTheOnlySummary: true)); + } + + [Fact] + public void SummaryTitleEscapesTheCharactersThatAreInvalidForAPath() + { + var runInfo = CreateRunInfo(typeof(Generic), title: null); + + Assert.Equal("BenchmarkDotNet.Tests.Helpers.Generic_Int32_", TitleHelper.GetSummaryTitle(runInfo, isTheOnlySummary: true)); + } + + [Fact] + public void SummaryTitleIsTheCustomTitleWhenTheRunReportsASingleSummary() + { + var runInfo = CreateRunInfo(typeof(FirstBenchmarks), CustomTitle); + + Assert.Equal(CustomTitle, TitleHelper.GetSummaryTitle(runInfo, isTheOnlySummary: true)); + } + + [Fact] // the custom title is shared by all the summaries, so their exported files must not be named after it alone #529 + public void SummaryTitleCombinesTheCustomTitleWithTheTypeNameWhenTheRunReportsManySummaries() + { + var first = CreateRunInfo(typeof(FirstBenchmarks), CustomTitle); + var second = CreateRunInfo(typeof(SecondBenchmarks), CustomTitle); + + Assert.Equal($"{CustomTitle}-{FirstTypeName}", TitleHelper.GetSummaryTitle(first, isTheOnlySummary: false)); + Assert.Equal($"{CustomTitle}-{SecondTypeName}", TitleHelper.GetSummaryTitle(second, isTheOnlySummary: false)); + } + + [Theory] + [InlineData("with/slash", "with_slash")] + [InlineData("with:colon", "with_colon")] + public void CustomTitleIsEscaped(string title, string expected) + { + var runInfo = CreateRunInfo(typeof(FirstBenchmarks), title); + + Assert.Equal(expected, TitleHelper.GetSummaryTitle(runInfo, isTheOnlySummary: true)); + } + + [Fact] + public void TooLongTitleIsTruncated() + { + var runInfo = CreateRunInfo(typeof(FirstBenchmarks), title: null); + + var title = TitleHelper.GetSummaryTitle(runInfo, isTheOnlySummary: true, desiredMaxLength: 10); + + Assert.Equal(10, title.Length); + Assert.StartsWith("Bench", title); + Assert.EndsWith("arks", title); + } + + [Fact] + public void RunTitleIsTheTypeNameWhenAllTheBenchmarksBelongToASingleType() + { + var runInfos = new[] { CreateRunInfo(typeof(FirstBenchmarks), title: null) }; + + Assert.Equal(FirstTypeName, TitleHelper.GetRunTitle(runInfos)); + } + + [Fact] + public void RunTitleIsSharedByAllTheTypesWhenThereIsMoreThanOne() + { + var runInfos = new[] { CreateRunInfo(typeof(FirstBenchmarks), title: null), CreateRunInfo(typeof(SecondBenchmarks), title: null) }; + + Assert.Equal(TitleHelper.MultipleTypesTitle, TitleHelper.GetRunTitle(runInfos)); + } + + [Fact] + public void RunTitleIsTheCustomTitleWhenItIsSet() + { + var runInfos = new[] { CreateRunInfo(typeof(FirstBenchmarks), CustomTitle), CreateRunInfo(typeof(SecondBenchmarks), CustomTitle) }; + + Assert.Equal(CustomTitle, TitleHelper.GetRunTitle(runInfos)); + } + + [Fact] + public void JoinedSummaryTitleFallsBackToTheDefaultWhenNoTitleIsSet() + { + Assert.Equal(TitleHelper.JoinedSummaryTitle, TitleHelper.GetJoinedSummaryTitle(customTitle: null)); + Assert.Equal(TitleHelper.JoinedSummaryTitle, TitleHelper.GetJoinedSummaryTitle(customTitle: " ")); + } + + [Fact] + public void JoinedSummaryTitleIsTheCustomTitleWhenItIsSet() + { + Assert.Equal(CustomTitle, TitleHelper.GetJoinedSummaryTitle(CustomTitle)); + } + + private static BenchmarkRunInfo CreateRunInfo(Type type, string? title) + => BenchmarkConverter.TypeToBenchmarks(type, title == null ? ManualConfig.CreateEmpty() : ManualConfig.CreateEmpty().WithTitle(title)); + } + + public class FirstBenchmarks + { + [Benchmark] + public void Method() { } + } + + public class SecondBenchmarks + { + [Benchmark] + public void Method() { } + } + + public class Generic + { + [Benchmark] + public void Method() { } + } +} diff --git a/tests/BenchmarkDotNet.Tests/Reports/SummaryTableTests.cs b/tests/BenchmarkDotNet.Tests/Reports/SummaryTableTests.cs index 0d91ec26f3..2b175d4271 100644 --- a/tests/BenchmarkDotNet.Tests/Reports/SummaryTableTests.cs +++ b/tests/BenchmarkDotNet.Tests/Reports/SummaryTableTests.cs @@ -3,6 +3,7 @@ using BenchmarkDotNet.Configs; using BenchmarkDotNet.Diagnosers; using BenchmarkDotNet.Exporters; +using BenchmarkDotNet.Helpers; using BenchmarkDotNet.Loggers; using BenchmarkDotNet.Order; using BenchmarkDotNet.Reports; @@ -117,7 +118,7 @@ public void JoinedSummaryFallsBackToDefaultTitleWhenNoneProvided() var joinedSummary = Summary.Join([taggedSummary, otherSummary], default); - Assert.StartsWith("BenchmarkRun-joined-", joinedSummary.Title); + Assert.Equal(TitleHelper.JoinedSummaryTitle, joinedSummary.Title); } [Fact] // Issue #1070 From 88070f43ec49c3d733a6de42ef44617ca50d919c Mon Sep 17 00:00:00 2001 From: Ifeanyi Shadrach Odom Date: Thu, 30 Jul 2026 13:24:28 +0100 Subject: [PATCH 5/5] Add support for custom titles in exported result files and update related documentation --- docs/articles/configs/exporters.md | 10 ++ docs/articles/guides/console-args.md | 2 +- docs/articles/samples/IntroJoin.md | 2 +- .../ArtifactNamingTests.cs | 128 ++++++++++++++++++ 4 files changed, 140 insertions(+), 2 deletions(-) create mode 100644 tests/BenchmarkDotNet.IntegrationTests/ArtifactNamingTests.cs diff --git a/docs/articles/configs/exporters.md b/docs/articles/configs/exporters.md index 5e65b8e55e..d83a3e566f 100644 --- a/docs/articles/configs/exporters.md +++ b/docs/articles/configs/exporters.md @@ -10,6 +10,16 @@ By default, files with results will be located in `.\BenchmarkDotNet.Artifacts\results` directory, but this can be changed via the `ArtifactsPath` property in the `IConfig`. Default exporters are: csv, html and markdown. +The exported files are named after the title of the summary they belong to, which by default is the full name of +the benchmarked type (or `BenchmarkRun-joined` when the results of many types are joined into a single summary). +You can override it via the `Title` property in the `IConfig` (`--title` on the command line); when the run reports +more than one summary, the name of the benchmarked type is appended to the title, so that the results of one type +don't overwrite the results of another: + +```cs +ManualConfig.Create(DefaultConfig.Instance).WithTitle("MyBenchmarks"); // MyBenchmarks-report-github.md +``` + --- [!include[IntroExport](../samples/IntroExport.md)] diff --git a/docs/articles/guides/console-args.md b/docs/articles/guides/console-args.md index 2af9e3042d..46f19a6de2 100644 --- a/docs/articles/guides/console-args.md +++ b/docs/articles/guides/console-args.md @@ -274,7 +274,7 @@ dotnet run -c Release -- --filter * --runtimes net6.0 net8.0 --statisticalTest 5 * `--anyCategories` Any Categories to run * `--attribute` Run all methods with given attribute (applied to class or method) * `--join` (Default: false) Prints single table with results for all benchmarks -* `--title` Custom title for the joined summary and the base name of its exported result files +* `--title` Custom title for the produced summaries and the base name of the result files exported for them * `--keepFiles` (Default: false) Determines if all auto-generated files should be kept or removed after running the benchmarks. * `--noOverwrite` (Default: false) Determines if the exported result files should not be overwritten (be default they are overwritten). * `--counters` Hardware Counters diff --git a/docs/articles/samples/IntroJoin.md b/docs/articles/samples/IntroJoin.md index 5a8f5da242..6409d069e9 100644 --- a/docs/articles/samples/IntroJoin.md +++ b/docs/articles/samples/IntroJoin.md @@ -16,7 +16,7 @@ If you are using `BenchmarkSwitcher` and want to run all the benchmarks with a c --join --allCategories=IntroJoinA ``` -By default the joined summary and its exported result files are named `BenchmarkRun-joined-{timestamp}`. +By default the joined summary and its exported result files are named `BenchmarkRun-joined`. You can override this with a custom title via the `--title` option (or `ManualConfig.WithTitle(...)` in code), which is handy when you run many sets of joined benchmarks and want to tell the exports apart: diff --git a/tests/BenchmarkDotNet.IntegrationTests/ArtifactNamingTests.cs b/tests/BenchmarkDotNet.IntegrationTests/ArtifactNamingTests.cs new file mode 100644 index 0000000000..21f5b53535 --- /dev/null +++ b/tests/BenchmarkDotNet.IntegrationTests/ArtifactNamingTests.cs @@ -0,0 +1,128 @@ +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Columns; +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Exporters; +using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Loggers; +using BenchmarkDotNet.Reports; +using BenchmarkDotNet.Running; +using BenchmarkDotNet.Tests.Loggers; +using BenchmarkDotNet.Toolchains.InProcess.Emit; + +namespace BenchmarkDotNet.IntegrationTests +{ + public class ArtifactNamingTests : BenchmarkTestExecutor + { + private const string CustomTitle = "MyCustomTitle"; + private const string FirstTypeName = "BenchmarkDotNet.IntegrationTests.FirstBenchmarks"; + private const string SecondTypeName = "BenchmarkDotNet.IntegrationTests.SecondBenchmarks"; + + public ArtifactNamingTests(ITestOutputHelper output) : base(output) { } + + [Fact] + public void ArtifactsAreNamedAfterTheBenchmarkedTypesWhenNoTitleIsSet() + { + using var artifacts = new TempFolder(); + + RunBenchmarks(artifacts, title: null, join: false, typeof(FirstBenchmarks), typeof(SecondBenchmarks)); + + Assert.Equal([$"{FirstTypeName}-report-github.md", $"{SecondTypeName}-report-github.md"], GetExportedFiles(artifacts)); + // the log file of every run is unique, so that --resume can tell the previous run from the current one + Assert.Equal("BenchmarkRun", GetLogFileName(artifacts).Split('-')[0]); + } + + [Fact] + public void ArtifactsAreNamedAfterTheCustomTitle() + { + using var artifacts = new TempFolder(); + + RunBenchmarks(artifacts, CustomTitle, join: false, typeof(FirstBenchmarks)); + + Assert.Equal([$"{CustomTitle}-report-github.md"], GetExportedFiles(artifacts)); + Assert.StartsWith($"{CustomTitle}-", GetLogFileName(artifacts)); + } + + [Fact] // the title is defined by the config, so all the summaries of the run would otherwise share it + public void TypeNameIsAppendedToTheCustomTitleWhenTheRunReportsManySummaries() + { + using var artifacts = new TempFolder(); + + RunBenchmarks(artifacts, CustomTitle, join: false, typeof(FirstBenchmarks), typeof(SecondBenchmarks)); + + Assert.Equal( + [$"{CustomTitle}-{FirstTypeName}-report-github.md", $"{CustomTitle}-{SecondTypeName}-report-github.md"], + GetExportedFiles(artifacts)); + } + + [Fact] + public void JoinedResultsAreExportedToASingleFileNamedAfterTheCustomTitle() + { + using var artifacts = new TempFolder(); + + RunBenchmarks(artifacts, CustomTitle, join: true, typeof(FirstBenchmarks), typeof(SecondBenchmarks)); + + Assert.Equal([$"{CustomTitle}-report-github.md"], GetExportedFiles(artifacts)); + } + + [Fact] + public void JoinedResultsFallBackToADefaultFileNameWhenNoTitleIsSet() + { + using var artifacts = new TempFolder(); + + RunBenchmarks(artifacts, title: null, join: true, typeof(FirstBenchmarks), typeof(SecondBenchmarks)); + + Assert.Equal(["BenchmarkRun-joined-report-github.md"], GetExportedFiles(artifacts)); + } + + private void RunBenchmarks(TempFolder artifacts, string? title, bool join, params Type[] types) + { + var config = ManualConfig.CreateEmpty() + .AddJob(Job.Dry.WithToolchain(InProcessEmitToolchain.Default)) + .AddExporter(MarkdownExporter.GitHub) + .AddColumnProvider(DefaultColumnProviders.Instance) + .AddLogger(new OutputLogger(Output)) + .WithArtifactsPath(artifacts.Path); + + if (join) + config = config.WithOptions(ConfigOptions.JoinSummary); + if (title != null) + config = config.WithTitle(title); + + var summaries = BenchmarkRunner.Run(types, config); + + Assert.All(summaries, summary => Assert.False(summary.HasCriticalValidationErrors)); + } + + private static string[] GetExportedFiles(TempFolder artifacts) + => Directory.GetFiles(Path.Combine(artifacts.Path, "results")).Select(Path.GetFileName).OrderBy(name => name).ToArray()!; + + private static string GetLogFileName(TempFolder artifacts) + => Path.GetFileNameWithoutExtension(Directory.GetFiles(artifacts.Path, "*.log").Single()); + + private sealed class TempFolder : IDisposable + { + internal string Path { get; } = Directory.CreateDirectory(System.IO.Path.Combine(System.IO.Path.GetTempPath(), System.IO.Path.GetRandomFileName())).FullName; + + public void Dispose() + { + try + { + Directory.Delete(Path, recursive: true); + } + catch (IOException) { } // the artifacts are not worth failing the test over + } + } + } + + public class FirstBenchmarks + { + [Benchmark] + public void Method() { } + } + + public class SecondBenchmarks + { + [Benchmark] + public void Method() { } + } +}