From a0cadb9fefa7ace40ba7bd1fc8d94595fbe03fde Mon Sep 17 00:00:00 2001 From: filzrev <103790468+filzrev@users.noreply.github.com> Date: Thu, 4 Dec 2025 17:18:17 +0900 Subject: [PATCH 1/5] chore: update samples project --- samples/BenchmarkDotNet.Samples/Program.cs | 76 +++++++++++++++++-- .../Properties/launchSettings.json | 34 +++++++++ 2 files changed, 105 insertions(+), 5 deletions(-) create mode 100644 samples/BenchmarkDotNet.Samples/Properties/launchSettings.json diff --git a/samples/BenchmarkDotNet.Samples/Program.cs b/samples/BenchmarkDotNet.Samples/Program.cs index 71daa668ba..3e0304d318 100644 --- a/samples/BenchmarkDotNet.Samples/Program.cs +++ b/samples/BenchmarkDotNet.Samples/Program.cs @@ -1,9 +1,75 @@ -using BenchmarkDotNet.Running; +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Engines; +using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Loggers; +using BenchmarkDotNet.Reports; +using BenchmarkDotNet.Running; +using BenchmarkDotNet.Toolchains.InProcess.Emit; +using System; +using System.Collections.Generic; +using System.Linq; -namespace BenchmarkDotNet.Samples +namespace BenchmarkDotNet.Samples; + +public class Program { - public class Program + public static int Main(string[] args) { - public static void Main(string[] args) => BenchmarkSwitcher.FromAssembly(typeof(Program).Assembly).Run(args); +#if DEBUG + ConsoleLogger.Default.WriteLineWarning("Benchmark is executed with DEBUG configuration."); + ConsoleLogger.Default.WriteLine(); +#endif + + if (args.Length != 0) + { + ConsoleLogger.Default.WriteLine($"Start benchmarks with args: {string.Join(" ", args)}"); + ConsoleLogger.Default.WriteLine(); + } + + IConfig config; +#if DEBUG + config = GetDebugConfig(); +#else + config = null; // `DefaultConfig.Instance` is used. +#endif + + var summaries = BenchmarkSwitcher.FromAssembly(typeof(Program).Assembly) + .Run(args, config) + .ToArray(); + + if (summaries.HasError()) + return 1; + + return 0; + } + + private static ManualConfig GetDebugConfig() + { + return DefaultConfig.Instance + .WithOptions(ConfigOptions.DisableOptimizationsValidator) + .WithOptions(ConfigOptions.StopOnFirstError) + .AddJob( + Job.Default + .WithId("WithDebugConfiguration") + .WithToolchain(InProcessEmitToolchain.Instance) + .WithStrategy(RunStrategy.Monitoring) + ); + } +} + +file static class ExtensionMethods +{ + public static bool HasError(this Summary[] summaries) + { + if (summaries.Length == 0) + { + var hashSet = new HashSet(["--help", "--list", "--info", "--version"]); + return !Environment.GetCommandLineArgs().Any(hashSet.Contains); + } + + if (summaries.Any(x => x.HasCriticalValidationErrors)) + return true; + + return summaries.Any(x => x.Reports.Any(r => !r.Success)); } -} \ No newline at end of file +} diff --git a/samples/BenchmarkDotNet.Samples/Properties/launchSettings.json b/samples/BenchmarkDotNet.Samples/Properties/launchSettings.json new file mode 100644 index 0000000000..23c12a0677 --- /dev/null +++ b/samples/BenchmarkDotNet.Samples/Properties/launchSettings.json @@ -0,0 +1,34 @@ +{ + "profiles": { + "Default": { + "commandName": "Project", + "commandLineArgs": "", + "workingDirectory": ".", + "environmentVariables": { + } + }, + "Run IntroBasic Benchmarks": { + "commandName": "Project", + "commandLineArgs": "--filter *IntroBasic*", + "workingDirectory": ".", + "environmentVariables": { + } + }, + "--list": { + "commandName": "Project", + "commandLineArgs": "--list" + }, + "--info": { + "commandName": "Project", + "commandLineArgs": "--info" + }, + "--help": { + "commandName": "Project", + "commandLineArgs": "--help" + }, + "--version": { + "commandName": "Project", + "commandLineArgs": "--version" + } + } +} From 8898ee55cef6d46b1a87ca8c2abd762facee6243 Mon Sep 17 00:00:00 2001 From: filzrev <103790468+filzrev@users.noreply.github.com> Date: Fri, 5 Dec 2025 18:20:00 +0900 Subject: [PATCH 2/5] chore-add-support-DebugBuildConfig --- samples/BenchmarkDotNet.Samples/Program.cs | 46 +++++++++++-------- .../Properties/launchSettings.json | 1 + 2 files changed, 28 insertions(+), 19 deletions(-) diff --git a/samples/BenchmarkDotNet.Samples/Program.cs b/samples/BenchmarkDotNet.Samples/Program.cs index 3e0304d318..375b9d399d 100644 --- a/samples/BenchmarkDotNet.Samples/Program.cs +++ b/samples/BenchmarkDotNet.Samples/Program.cs @@ -1,10 +1,11 @@ -using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Diagnosers; using BenchmarkDotNet.Engines; -using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Exporters; using BenchmarkDotNet.Loggers; using BenchmarkDotNet.Reports; using BenchmarkDotNet.Running; -using BenchmarkDotNet.Toolchains.InProcess.Emit; using System; using System.Collections.Generic; using System.Linq; @@ -26,12 +27,7 @@ public static int Main(string[] args) ConsoleLogger.Default.WriteLine(); } - IConfig config; -#if DEBUG - config = GetDebugConfig(); -#else - config = null; // `DefaultConfig.Instance` is used. -#endif + IConfig? config = GetConfig(ref args); var summaries = BenchmarkSwitcher.FromAssembly(typeof(Program).Assembly) .Run(args, config) @@ -43,17 +39,29 @@ public static int Main(string[] args) return 0; } - private static ManualConfig GetDebugConfig() + private static IConfig? GetConfig(ref string[] args) { - return DefaultConfig.Instance - .WithOptions(ConfigOptions.DisableOptimizationsValidator) - .WithOptions(ConfigOptions.StopOnFirstError) - .AddJob( - Job.Default - .WithId("WithDebugConfiguration") - .WithToolchain(InProcessEmitToolchain.Instance) - .WithStrategy(RunStrategy.Monitoring) - ); +#if !DEBUG + return null; // `DefaultConfig.Instance` is used. +#else + bool isInProcess = args.Contains("--inProcess"); + if (isInProcess) + args = args.Where(x => x != "--inProcess").ToArray(); + + DebugConfig config = isInProcess + ? new DebugInProcessConfig() + : new DebugBuildConfig(); + + return config.AddAnalyser(DefaultConfig.Instance.GetAnalysers().ToArray()) + .AddDiagnoser( + MemoryDiagnoser.Default, + new ExceptionDiagnoser(new ExceptionDiagnoserConfig(displayExceptionsIfZeroValue: false)), + new ThreadingDiagnoser(new ThreadingDiagnoserConfig(displayCompletedWorkItemCountWhenZero: false, displayLockContentionWhenZero: false)) + ) + .AddExporter(MarkdownExporter.Default) + .AddValidator(DefaultConfig.Instance.GetValidators().ToArray()) + .WithArtifactsPath(DefaultConfig.Instance.ArtifactsPath); +#endif } } diff --git a/samples/BenchmarkDotNet.Samples/Properties/launchSettings.json b/samples/BenchmarkDotNet.Samples/Properties/launchSettings.json index 23c12a0677..1a94bb98c5 100644 --- a/samples/BenchmarkDotNet.Samples/Properties/launchSettings.json +++ b/samples/BenchmarkDotNet.Samples/Properties/launchSettings.json @@ -10,6 +10,7 @@ "Run IntroBasic Benchmarks": { "commandName": "Project", "commandLineArgs": "--filter *IntroBasic*", + // "commandLineArgs": "--filter *IntroBasic* --inProcess --strategy Monitoring", // Use this setting for in-process debug "workingDirectory": ".", "environmentVariables": { } From c6a757d5147b47a61961daf67b03d1518632c6f5 Mon Sep 17 00:00:00 2001 From: filzrev <103790468+filzrev@users.noreply.github.com> Date: Sun, 7 Dec 2025 10:14:27 +0900 Subject: [PATCH 3/5] chore: fix ThreadingDiagnoser error when running benchmark on .NET Framework --- samples/BenchmarkDotNet.Samples/Program.cs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/samples/BenchmarkDotNet.Samples/Program.cs b/samples/BenchmarkDotNet.Samples/Program.cs index 375b9d399d..ccbedc3e1c 100644 --- a/samples/BenchmarkDotNet.Samples/Program.cs +++ b/samples/BenchmarkDotNet.Samples/Program.cs @@ -55,8 +55,10 @@ public static int Main(string[] args) return config.AddAnalyser(DefaultConfig.Instance.GetAnalysers().ToArray()) .AddDiagnoser( MemoryDiagnoser.Default, - new ExceptionDiagnoser(new ExceptionDiagnoserConfig(displayExceptionsIfZeroValue: false)), - new ThreadingDiagnoser(new ThreadingDiagnoserConfig(displayCompletedWorkItemCountWhenZero: false, displayLockContentionWhenZero: false)) +#if NET30_OR_GREATER + new ThreadingDiagnoser(new ThreadingDiagnoserConfig(displayCompletedWorkItemCountWhenZero: false, displayLockContentionWhenZero: false)), +#endif + new ExceptionDiagnoser(new ExceptionDiagnoserConfig(displayExceptionsIfZeroValue: false)) ) .AddExporter(MarkdownExporter.Default) .AddValidator(DefaultConfig.Instance.GetValidators().ToArray()) From 04e87b78a90b674bb56e6d4e2e000db27c37b076 Mon Sep 17 00:00:00 2001 From: filzrev <103790468+filzrev@users.noreply.github.com> Date: Mon, 15 Dec 2025 18:18:00 +0900 Subject: [PATCH 4/5] chore: fix wrong #if directive --- samples/BenchmarkDotNet.Samples/Program.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/BenchmarkDotNet.Samples/Program.cs b/samples/BenchmarkDotNet.Samples/Program.cs index ccbedc3e1c..217a8acd1b 100644 --- a/samples/BenchmarkDotNet.Samples/Program.cs +++ b/samples/BenchmarkDotNet.Samples/Program.cs @@ -55,7 +55,7 @@ public static int Main(string[] args) return config.AddAnalyser(DefaultConfig.Instance.GetAnalysers().ToArray()) .AddDiagnoser( MemoryDiagnoser.Default, -#if NET30_OR_GREATER +#if NETCOREAPP3_0_OR_GREATER new ThreadingDiagnoser(new ThreadingDiagnoserConfig(displayCompletedWorkItemCountWhenZero: false, displayLockContentionWhenZero: false)), #endif new ExceptionDiagnoser(new ExceptionDiagnoserConfig(displayExceptionsIfZeroValue: false)) From dfb94a1b3c83a3c784a7fbf002eb0a9f8fd23b02 Mon Sep 17 00:00:00 2001 From: filzrev <103790468+filzrev@users.noreply.github.com> Date: Wed, 17 Dec 2025 00:59:46 +0900 Subject: [PATCH 5/5] chore: reflect review comments and cleanup codes --- samples/BenchmarkDotNet.Samples/Program.cs | 41 ++++--------------- .../Properties/launchSettings.json | 9 ++-- .../Extensions/ReportExtensions.cs | 15 +++++++ 3 files changed, 28 insertions(+), 37 deletions(-) diff --git a/samples/BenchmarkDotNet.Samples/Program.cs b/samples/BenchmarkDotNet.Samples/Program.cs index 217a8acd1b..04b800d78f 100644 --- a/samples/BenchmarkDotNet.Samples/Program.cs +++ b/samples/BenchmarkDotNet.Samples/Program.cs @@ -1,13 +1,10 @@ -using BenchmarkDotNet.Attributes; -using BenchmarkDotNet.Configs; -using BenchmarkDotNet.Diagnosers; +using BenchmarkDotNet.Configs; using BenchmarkDotNet.Engines; using BenchmarkDotNet.Exporters; +using BenchmarkDotNet.Extensions; using BenchmarkDotNet.Loggers; -using BenchmarkDotNet.Reports; using BenchmarkDotNet.Running; using System; -using System.Collections.Generic; using System.Linq; namespace BenchmarkDotNet.Samples; @@ -49,37 +46,13 @@ public static int Main(string[] args) args = args.Where(x => x != "--inProcess").ToArray(); DebugConfig config = isInProcess - ? new DebugInProcessConfig() - : new DebugBuildConfig(); + ? new DebugInProcessConfig() + : new DebugBuildConfig(); return config.AddAnalyser(DefaultConfig.Instance.GetAnalysers().ToArray()) - .AddDiagnoser( - MemoryDiagnoser.Default, -#if NETCOREAPP3_0_OR_GREATER - new ThreadingDiagnoser(new ThreadingDiagnoserConfig(displayCompletedWorkItemCountWhenZero: false, displayLockContentionWhenZero: false)), + .AddExporter(MarkdownExporter.Default) + .AddValidator(DefaultConfig.Instance.GetValidators().ToArray()) + .WithArtifactsPath(DefaultConfig.Instance.ArtifactsPath); #endif - new ExceptionDiagnoser(new ExceptionDiagnoserConfig(displayExceptionsIfZeroValue: false)) - ) - .AddExporter(MarkdownExporter.Default) - .AddValidator(DefaultConfig.Instance.GetValidators().ToArray()) - .WithArtifactsPath(DefaultConfig.Instance.ArtifactsPath); -#endif - } -} - -file static class ExtensionMethods -{ - public static bool HasError(this Summary[] summaries) - { - if (summaries.Length == 0) - { - var hashSet = new HashSet(["--help", "--list", "--info", "--version"]); - return !Environment.GetCommandLineArgs().Any(hashSet.Contains); - } - - if (summaries.Any(x => x.HasCriticalValidationErrors)) - return true; - - return summaries.Any(x => x.Reports.Any(r => !r.Success)); } } diff --git a/samples/BenchmarkDotNet.Samples/Properties/launchSettings.json b/samples/BenchmarkDotNet.Samples/Properties/launchSettings.json index 1a94bb98c5..43ce1fbd99 100644 --- a/samples/BenchmarkDotNet.Samples/Properties/launchSettings.json +++ b/samples/BenchmarkDotNet.Samples/Properties/launchSettings.json @@ -3,15 +3,18 @@ "Default": { "commandName": "Project", "commandLineArgs": "", - "workingDirectory": ".", "environmentVariables": { } }, "Run IntroBasic Benchmarks": { "commandName": "Project", "commandLineArgs": "--filter *IntroBasic*", - // "commandLineArgs": "--filter *IntroBasic* --inProcess --strategy Monitoring", // Use this setting for in-process debug - "workingDirectory": ".", + "environmentVariables": { + } + }, + "Run IntroBasic Benchmarks with --inProcess": { + "commandName": "Project", + "commandLineArgs": "--filter *IntroBasic* --inProcess", "environmentVariables": { } }, diff --git a/src/BenchmarkDotNet/Extensions/ReportExtensions.cs b/src/BenchmarkDotNet/Extensions/ReportExtensions.cs index b90646f01f..0bdb4e0b86 100644 --- a/src/BenchmarkDotNet/Extensions/ReportExtensions.cs +++ b/src/BenchmarkDotNet/Extensions/ReportExtensions.cs @@ -34,5 +34,20 @@ public static Statistics GetStatistics(this IReadOnlyCollection run public static Statistics GetStatistics(this IEnumerable runs) => GetStatistics(runs.ToList()); + + public static bool HasError(this IEnumerable summaries) + { + if (summaries.Count() == 0) + { + // When following argument specified. BenchmarkDotNet show information only. + var knownArguments = new HashSet(["--help", "--list", "--info", "--version"]); + return !Environment.GetCommandLineArgs().Any(knownArguments.Contains); + } + + if (summaries.Any(x => x.HasCriticalValidationErrors)) + return true; + + return summaries.Any(x => x.Reports.Any(r => !r.Success)); + } } }