Reorganize command line interface options/parameters #3182

docs/articles/guides/console-args.md

@@ -187,6 +187,56 @@ Now, the default settings are: `WarmupCount=1` but you might still overwrite it
 dotnet run -c Release -- --warmupCount 2
 ```
 
+## Benchmark profiles
+
+Most of the console arguments describe a *single* job, so they can not express "run these benchmarks
+for this particular combination of settings, and that other combination too". Instead of trying to
+spell such combinations out on the command line, you can define them in code, give each of them an
+`Id`, and then select them from the command line with `--profile`:
+
+* `--profile` - runs only the jobs whose `Id` matches. Glob patterns are supported and the matching
+is case insensitive. Not to be confused with `--profiler`.
+
+```cs
+static void Main(string[] args)
+    => BenchmarkSwitcher
+        .FromAssembly(typeof(Program).Assembly)
+        .Run(args, GetGlobalConfig());
+
+static IConfig GetGlobalConfig()
+    => DefaultConfig.Instance
+        .AddJob(Job.Default.WithToolchain(InProcessEmitToolchain.Instance).WithId("debug"))
+        .AddJob(Job.Default.WithRuntime(CoreRuntime.Core80).WithId("net8"))
+        .AddJob(Job.Default.WithRuntime(CoreRuntime.Core90).WithId("net9"));
+```
+
+Example: run the benchmarks in process, which is handy when you want to debug them.
+
+```log
+dotnet run -c Release -- --filter '*JsonSerialization*' --profile debug
+```
+
+Example: compare .NET 8.0 with .NET 9.0, without running the `debug` job. Just like `--filter`,
+`--profile` accepts more than one value, separated by spaces.
+
+```log
+dotnet run -c Release -- --filter '*' --profile net8 net9
+```
+
+The very same thing works for the jobs that are defined via attributes, because `[SimpleJob]` accepts
+an `id` argument:
+
+```cs
+[SimpleJob(RuntimeMoniker.Net80, id: "net8")]
+[SimpleJob(RuntimeMoniker.Net90, id: "net9")]
+public class JsonSerialization { /* ... */ }
+```
+
+`--profile` narrows down what the other filters have selected, so it can be freely combined with
+`--filter`, `--allCategories`, `--anyCategories` and `--attribute`. Jobs that were given no explicit
+`Id` are still selectable by their generated one (`DefaultJob`, or `Job-XXXXXX`); if nothing matches,
+the list of available profiles is printed for you.
+
 ## Response files support
 
 Benchmark.NET supports parsing parameters via response files. for example you can create file `run.rsp` with following content
@@ -252,6 +302,8 @@ dotnet run -c Release -- --filter * --runtimes net6.0 net8.0 --statisticalTest 5
     `--envVars ENV_VAR_KEY_1:value_1 ENV_VAR_KEY_2:value_2`
 * Hide Mean and Ratio columns (use double quotes for multi-word columns: "Alloc Ratio"):
     `-h Mean Ratio`
+* Run only the jobs whose Id is 'net8' or 'net9' (Ids are assigned in code, e.g. Job.Default.WithId("net8")):
+    `--filter * --profile net8 net9`
 
 ## More
 
@@ -264,6 +316,7 @@ dotnet run -c Release -- --filter * --runtimes net6.0 net8.0 --statisticalTest 5
 * `-d, --disasm`              (Default: false) Gets disassembly of benchmarked code
 * `-p, --profiler`            Profiles benchmarked code using selected profiler. Available options: EP/ETW/CV/NativeMemory
 * `-f, --filter`              Glob patterns
+* `--profile`                 Runs only the jobs with matching Id(s). Ids are assigned in code via Job.WithId(...) or the 'id' argument of [SimpleJob]. Glob patterns are supported. Not to be confused with --profiler.
 * `-h, --hide`                Hides columns by name
 * `-i, --inProcess`           (Default: false) Run benchmarks in Process
 * `-a, --artifacts`           Valid path to accessible directory

src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs

@@ -51,6 +51,9 @@ public bool UseDisassemblyDiagnoser
         [Option('f', "filter", Required = false, HelpText = "Glob patterns")]
         public IEnumerable<string> Filters { get; set; } = [];
 
+        [Option("profile", Required = false, HelpText = "Runs only the jobs with matching Id(s). Ids are assigned in code via Job.WithId(...) or the 'id' argument of [SimpleJob]. Glob patterns are supported. Not to be confused with --profiler.")]
+        public IEnumerable<string> Profiles { get; set; } = [];
+
         [Option('h', "hide", Required = false, HelpText = "Hides columns by name")]
         public IEnumerable<string> HiddenColumns { get; set; } = [];
 
@@ -268,6 +271,7 @@ public static IEnumerable<Example> Examples
                 yield return new Example("Run benchmarks using environment variables 'ENV_VAR_KEY_1' with value 'value_1' and 'ENV_VAR_KEY_2' with value 'value_2'", longName,
                     new CommandLineOptions { EnvironmentVariables = ["ENV_VAR_KEY_1:value_1", "ENV_VAR_KEY_2:value_2"] });
                 yield return new Example("Hide Mean and Ratio columns (use double quotes for multi-word columns: \"Alloc Ratio\")", shortName, new CommandLineOptions { HiddenColumns = ["Mean", "Ratio"], });
+                yield return new Example("Run only the jobs whose Id is 'net8' or 'net9' (Ids are assigned in code, e.g. Job.Default.WithId(\"net8\"))", longName, new CommandLineOptions { Filters = [Escape("*")], Profiles = ["net8", "net9"] });
             }
         }
 

src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs

@@ -388,6 +388,11 @@ private static IConfig CreateConfig(CommandLineOptions options, IConfig? globalC
             else
                 config.AddFilter(filters);
 
+            // the profile filter selects jobs, not benchmarks, so it's added separately to make sure
+            // that it's always combined with the filters above rather than being one of them
+            if (options.Profiles.Any())
+                config.AddFilter(new JobIdFilter([.. options.Profiles]));
+
             config.HideColumns(options.HiddenColumns.ToArray());
 
             config.WithOption(ConfigOptions.JoinSummary, options.Join);

src/BenchmarkDotNet/Filters/JobIdFilter.cs

@@ -0,0 +1,47 @@
+using BenchmarkDotNet.Running;
+using System.Text.RegularExpressions;
+
+namespace BenchmarkDotNet.Filters
+{
+    /// <summary>
+    /// filters benchmarks by the id of the job they belong to (glob patterns are supported)
+    /// </summary>
+    public class JobIdFilter : IFilter
+    {
+        private readonly Regex[] patterns;
+
+        // The available job ids are not known upfront because the jobs defined via attributes are discovered
+        // while the benchmark cases are being created. They are recorded here so that when no job matches we
+        // can tell the user which ids they could have used instead.
+        private readonly HashSet<string> observedJobIds = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
+        private readonly HashSet<string> matchedJobIds = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
+
+        public JobIdFilter(string[] jobIds) => patterns = GlobFilter.ToRegex(jobIds);
+
+        /// <summary>
+        /// the ids of all the jobs this filter has been asked about.
+        /// ImmutableConfigBuilder keeps the filters in an unordered set, so which benchmark cases reach
+        /// this filter depends on how it happens to be ordered against the other filters. That makes this
+        /// a lower bound of what the user could have typed, which is all we need it for.
+        /// </summary>
+        internal IReadOnlyCollection<string> ObservedJobIds => observedJobIds;
+
+        /// <summary>
+        /// the ids of the jobs that this filter has accepted
+        /// </summary>
+        internal IReadOnlyCollection<string> MatchedJobIds => matchedJobIds;
+
+        public bool Predicate(BenchmarkCase benchmarkCase)
+        {
+            string jobId = benchmarkCase.Job.ResolvedId;
+
+            observedJobIds.Add(jobId);
+
+            if (!patterns.Any(pattern => pattern.IsMatch(jobId)))
+                return false;
+
+            matchedJobIds.Add(jobId);
+            return true;
+        }
+    }
+}

src/BenchmarkDotNet/Running/BenchmarkSwitcher.cs

@@ -4,6 +4,7 @@
 using BenchmarkDotNet.Engines;
 using BenchmarkDotNet.Environments;
 using BenchmarkDotNet.Extensions;
+using BenchmarkDotNet.Filters;
 using BenchmarkDotNet.Jobs;
 using BenchmarkDotNet.Loggers;
 using BenchmarkDotNet.Parameters;
@@ -18,6 +19,8 @@ namespace BenchmarkDotNet.Running
 {
     public class BenchmarkSwitcher
     {
+        private const int MaxDisplayedProfiles = 40;
+
         private readonly IUserInteraction userInteraction = new UserInteraction();
         private readonly List<Type> types = [];
         private readonly List<Assembly> assemblies = [];
@@ -165,13 +168,41 @@ internal async ValueTask<IEnumerable<Summary>> RunWithDirtyAssemblyResolveHelper
 
             if (filteredBenchmarks.IsEmpty())
             {
-                userInteraction.PrintWrongFilterInfo(benchmarksToFilter, logger, [.. options.Filters]);
+                if (!TryPrintWrongProfileInfo(effectiveConfig, logger, options))
+                    userInteraction.PrintWrongFilterInfo(benchmarksToFilter, logger, [.. options.Filters]);
                 return [];
             }
 
             return await BenchmarkRunnerClean.Run(filteredBenchmarks, cancellationToken).ConfigureAwait(false);
         }
 
+        /// <summary>
+        /// explains that it was --profile which returned 0 benchmarks, so that the user is not shown
+        /// benchmark name suggestions for what is actually a mistyped job id.
+        /// </summary>
+        /// <returns>true if the profile was the reason why nothing was left to run</returns>
+        private static bool TryPrintWrongProfileInfo(IConfig effectiveConfig, ILogger logger, CommandLineOptions options)
+        {
+            var profiles = options.Profiles.ToArray();
+            if (profiles.IsEmpty())
+                return false;
+
+            var profileFilter = effectiveConfig.GetFilters().OfType<JobIdFilter>().FirstOrDefault();
+            if (profileFilter == null || profileFilter.ObservedJobIds.IsEmpty() || !profileFilter.MatchedJobIds.IsEmpty())
+                return false; // either no job was ever checked, or some did match, so --profile is not to blame
+
+            logger.WriteLineError($"{(profiles.Length == 1 ? "The profile" : "Profiles")} '{string.Join("', '", profiles)}' that you have provided returned 0 benchmarks.");
+            logger.WriteLineInfo("Please remember that --profile is applied to the Id of the job, which is assigned in code via Job.WithId(...) or the 'id' argument of [SimpleJob].");
+            logger.WriteLineInfo("Available profiles:");
+
+            foreach (string jobId in profileFilter.ObservedJobIds.OrderBy(id => id, StringComparer.OrdinalIgnoreCase).Take(MaxDisplayedProfiles))
+                logger.WriteLineInfo($"\t{jobId}");
+
+            logger.WriteLineInfo("To learn more about filtering use `--help`.");
+
+            return true;
+        }
+
         private static void PrintList(ILogger nonNullLogger, IConfig effectiveConfig, IReadOnlyList<Type> allAvailableTypesWithRunnableBenchmarks, CommandLineOptions options)
         {
             var printer = new BenchmarkCasesPrinter(options.ListBenchmarkCaseMode);

tests/BenchmarkDotNet.IntegrationTests/BenchmarkSwitcherTest.cs

@@ -1,5 +1,6 @@
 using BenchmarkDotNet.Attributes;
 using BenchmarkDotNet.Configs;
+using BenchmarkDotNet.Engines;
 using BenchmarkDotNet.Environments;
 using BenchmarkDotNet.Exporters;
 using BenchmarkDotNet.Helpers;
@@ -301,6 +302,90 @@ public void JobNotDefinedButStillBenchmarkIsExecuted()
             Assert.True(results.All(r => r.BenchmarksCases.All(bc => bc.Job == Job.Default)));
         }
 
+        [Fact]
+        public void WhenUserProvidesProfileOnlyTheJobsWithMatchingIdAreExecuted()
+        {
+            var types = new[] { typeof(ClassB) };
+            var switcher = new BenchmarkSwitcher(types);
+            var configWithNamedJobs = ManualConfig.CreateEmpty()
+                .AddJob(Job.Dry.WithId("first"))
+                .AddJob(Job.Dry.WithId("second"));
+
+            var results = switcher.Run(["--filter", "*Method3", "--profile", "second"], configWithNamedJobs);
+
+            var jobs = results.SelectMany(r => r.BenchmarksCases.Select(bc => bc.Job)).ToArray();
+            Assert.Equal("second", Assert.Single(jobs).ResolvedId);
+        }
+
+        [Fact]
+        public void WhenUserProvidesProfileGlobPatternAllTheMatchingJobsAreExecuted()
+        {
+            var types = new[] { typeof(ClassB) };
+            var switcher = new BenchmarkSwitcher(types);
+            var configWithNamedJobs = ManualConfig.CreateEmpty()
+                .AddJob(Job.Dry.WithId("net8"))
+                .AddJob(Job.Dry.WithId("net9"))
+                .AddJob(Job.Dry.WithId("debug"));
+
+            var results = switcher.Run(["--filter", "*Method3", "--profile", "net*"], configWithNamedJobs);
+
+            var jobIds = results.SelectMany(r => r.BenchmarksCases.Select(bc => bc.Job.ResolvedId)).OrderBy(id => id).ToArray();
+            Assert.Equal(["net8", "net9"], jobIds);
+        }
+
+        [Fact]
+        public void WhenUserProvidesProfileTheJobsDefinedViaAttributesAreSelectableToo()
+        {
+            var types = new[] { typeof(WithTwoNamedJobAttributes) };
+            var switcher = new BenchmarkSwitcher(types);
+            var config = ManualConfig.CreateEmpty();
+
+            var results = switcher.Run(["--filter", "*WithTwoNamedJobAttributes*", "--profile", "secondAttributeJob"], config);
+
+            var jobs = results.SelectMany(r => r.BenchmarksCases.Select(bc => bc.Job)).ToArray();
+            Assert.Equal("secondAttributeJob", Assert.Single(jobs).ResolvedId);
+        }
+
+        [Fact]
+        public void WhenProfileReturnsNothingTheAvailableProfilesAreDisplayedAndNoBenchmarksAreExecuted()
+        {
+            var logger = new OutputLogger(Output);
+            var configWithNamedJobs = ManualConfig.CreateEmpty().AddLogger(logger)
+                .AddJob(Job.Dry.WithId("first"))
+                .AddJob(Job.Dry.WithId("second"));
+
+            var summaries = BenchmarkSwitcher
+                .FromTypes([typeof(ClassB)])
+                .Run(["--filter", "*Method3", "--profile", "WRONG"], configWithNamedJobs);
+
+            Assert.Empty(summaries);
+
+            string log = logger.GetLog();
+            Assert.Contains("The profile 'WRONG' that you have provided returned 0 benchmarks.", log);
+            Assert.Contains("Available profiles:", log);
+            Assert.Contains("first", log);
+            Assert.Contains("second", log);
+            // the profile is not a benchmark name, so we must not suggest benchmark names
+            Assert.DoesNotContain("Please remember that the filter is applied to full benchmark name", log);
+        }
+
+        [Fact]
+        public void WhenUserProvidesBothProfileAndFilterBothAreApplied()
+        {
+            var logger = new OutputLogger(Output);
+            var configWithNamedJobs = ManualConfig.CreateEmpty().AddLogger(logger)
+                .AddJob(Job.Dry.WithId("first"))
+                .AddJob(Job.Dry.WithId("second"));
+
+            // the profile exists, but the filter matches nothing, so this is a wrong *filter*, not a wrong profile
+            var summaries = BenchmarkSwitcher
+                .FromTypes([typeof(ClassB)])
+                .Run(["--filter", "WRONG", "--profile", "first"], configWithNamedJobs);
+
+            Assert.Empty(summaries);
+            Assert.Contains("The filter 'WRONG' that you have provided returned 0 benchmarks.", logger.GetLog());
+        }
+
         [Fact]
         public void WhenUserCreatesStaticBenchmarkMethodWeDisplayAnError_FromTypes()
         {
@@ -416,6 +501,14 @@ public class JustBenchmark
         public void Method() { }
     }
 
+    [SimpleJob(RunStrategy.ColdStart, launchCount: 1, warmupCount: 1, iterationCount: 1, id: "firstAttributeJob")]
+    [SimpleJob(RunStrategy.ColdStart, launchCount: 1, warmupCount: 2, iterationCount: 1, id: "secondAttributeJob")]
+    public class WithTwoNamedJobAttributes
+    {
+        [Benchmark]
+        public void Method() { }
+    }
+
     public class MockExporter : ExporterBase
     {
         public bool exported = false;

tests/BenchmarkDotNet.Tests/ConfigParserTests.cs

@@ -10,6 +10,7 @@
 using BenchmarkDotNet.Exporters.Json;
 using BenchmarkDotNet.Exporters.OpenMetrics;
 using BenchmarkDotNet.Exporters.Xml;
+using BenchmarkDotNet.Filters;
 using BenchmarkDotNet.Jobs;
 using BenchmarkDotNet.Loggers;
 using BenchmarkDotNet.Portability;
@@ -823,6 +824,40 @@ public void UserCanSpecifyWasmMainJsTemplate()
             Assert.Equal("dummyFile.js", runtime.MainJsTemplate?.Name);
         }
 
+        [Fact]
+        public void NoProfileFilterIsAddedWhenProfileIsNotSpecified()
+        {
+            var config = ConfigParser.Parse(["--filter", "*"], new OutputLogger(Output)).config;
+
+            Assert.NotNull(config);
+            Assert.Empty(config.GetFilters().OfType<JobIdFilter>());
+        }
+
+        [Theory]
+        [InlineData("--profile", "net8")]
+        [InlineData("--profile", "net8", "net9")]
+        [InlineData("--PROFILE", "net8")] // case insensitive
+        public void UserCanSpecifyProfiles(params string[] args)
+        {
+            var config = ConfigParser.Parse(args, new OutputLogger(Output)).config;
+
+            Assert.NotNull(config);
+            Assert.Single(config.GetFilters().OfType<JobIdFilter>());
+        }
+
+        [Fact]
+        public void ProfileFilterIsNotUnionedWithTheOtherFilters()
+        {
+            // --profile must narrow down what --filter has selected, so it must be a filter of its own
+            var config = ConfigParser.Parse(["--filter", "*", "--profile", "net8"], new OutputLogger(Output)).config;
+
+            Assert.NotNull(config);
+            var filters = config.GetFilters().ToArray();
+            Assert.Equal(2, filters.Length);
+            Assert.Single(filters.OfType<GlobFilter>());
+            Assert.Single(filters.OfType<JobIdFilter>());
+        }
+
         [Theory]
         [InlineData("--filter abc", "--filter *")]
         [InlineData("-f abc", "--filter *")]