[build] Add windows-11-arm and ubuntu-24.04-arm to CI run-tests.yaml

.github/workflows/run-tests.yaml

@@ -13,7 +13,10 @@ concurrency:
 jobs:
 
   test-windows-core:
-    runs-on: windows-latest
+    strategy:
+      matrix:
+        os: [windows-latest, windows-11-arm]
+    runs-on: ${{ matrix.os }}
     steps:
       - name: Disable Windows Defender
         run: Set-MpPreference -DisableRealtimeMonitoring $true
@@ -31,11 +34,14 @@ jobs:
         uses: actions/upload-artifact@v4
         if: always()
         with:
-          name: test-windows-core-trx-${{ github.run_id }}
+          name: test-windows-core-trx-${{ github.run_id }}-${{ matrix.os }}
           path: "**/*.trx"
 
   test-windows-full:
-    runs-on: windows-latest
+    strategy:
+      matrix:
+        os: [windows-latest, windows-11-arm]
+    runs-on: ${{ matrix.os }}
     steps:
       - name: Disable Windows Defender
         run: Set-MpPreference -DisableRealtimeMonitoring $true
@@ -59,11 +65,14 @@ jobs:
         uses: actions/upload-artifact@v4
         if: always()
         with:
-          name: test-windows-full-trx-${{ github.run_id }}
+          name: test-windows-full-trx-${{ github.run_id }}-${{ matrix.os }}
           path: "**/*.trx"
 
   test-linux:
-    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        os: [ubuntu-latest, ubuntu-24.04-arm]
+    runs-on: ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v4
       # Set up the environment
@@ -96,7 +105,7 @@ jobs:
         uses: actions/upload-artifact@v4
         if: always()
         with:
-          name: test-linux-trx-${{ github.run_id }}
+          name: test-linux-trx-${{ github.run_id }}-${{ matrix.os }}
           path: "**/*.trx"
 
   test-macos:

src/BenchmarkDotNet/Disassemblers/DisassemblyDiagnoser.cs

@@ -119,7 +119,13 @@ public IEnumerable<ValidationError> Validate(ValidationParameters validationPara
             var currentPlatform = RuntimeInformation.GetCurrentPlatform();
             if (!(currentPlatform is Platform.X64 or Platform.X86 or Platform.Arm64))
             {
-                yield return new ValidationError(true, $"{currentPlatform} is not supported");
+                yield return new ValidationError(true, $"DisassemblyDiagnoser does not support {currentPlatform}");
+                yield break;
+            }
+
+            if (currentPlatform == Platform.Arm64 && OsDetector.IsWindows())
+            {
+                yield return new ValidationError(true, $"DisassemblyDiagnoser does not support Arm on Windows");
                 yield break;
             }
 

tests/BenchmarkDotNet.IntegrationTests/DisassemblyDiagnoserTests.cs

@@ -13,6 +13,7 @@
 using BenchmarkDotNet.Jobs;
 using BenchmarkDotNet.Portability;
 using BenchmarkDotNet.Tests.Loggers;
+using BenchmarkDotNet.Tests.XUnit;
 using BenchmarkDotNet.Toolchains;
 using BenchmarkDotNet.Toolchains.CsProj;
 using BenchmarkDotNet.Toolchains.InProcess.Emit;
@@ -84,7 +85,7 @@ public void Recursive()
             [MethodImpl(MethodImplOptions.NoInlining)] public void Benchmark(bool justAnOverload) { } // we need to test overloads (#562)
         }
 
-        [Theory]
+        [TheoryEnvSpecific("Not supported on Windows+Arm", EnvRequirement.NonWindowsArm)]
         [MemberData(nameof(GetAllJits), DisableDiscoveryEnumeration = true)]
         [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)]
         public void CanDisassembleAllMethodCalls(Jit jit, Platform platform, IToolchain toolchain)
@@ -104,7 +105,7 @@ public void CanDisassembleAllMethodCalls(Jit jit, Platform platform, IToolchain
             AssertDisassemblyResult(result, $"{nameof(WithCalls.Recursive)}()");
         }
 
-        [Theory]
+        [TheoryEnvSpecific("Not supported on Windows+Arm", EnvRequirement.NonWindowsArm)]
         [MemberData(nameof(GetAllJits), DisableDiscoveryEnumeration = true)]
         [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)]
         public void CanDisassembleAllMethodCallsUsingFilters(Jit jit, Platform platform, IToolchain toolchain)
@@ -130,7 +131,7 @@ public void CanDisassembleAllMethodCallsUsingFilters(Jit jit, Platform platform,
             public T Create() => new T();
         }
 
-        [Theory]
+        [TheoryEnvSpecific("Not supported on Windows+Arm", EnvRequirement.NonWindowsArm)]
         [MemberData(nameof(GetAllJits), DisableDiscoveryEnumeration = true)]
         [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)]
         public void CanDisassembleGenericTypes(Jit jit, Platform platform, IToolchain toolchain)
@@ -151,7 +152,7 @@ public class WithInlineable
             [Benchmark] public void JustReturn() { }
         }
 
-        [Theory]
+        [TheoryEnvSpecific("Not supported on Windows+Arm", EnvRequirement.NonWindowsArm)]
         [MemberData(nameof(GetAllJits), DisableDiscoveryEnumeration = true)]
         [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)]
         public void CanDisassembleInlinableBenchmarks(Jit jit, Platform platform, IToolchain toolchain)

tests/BenchmarkDotNet.IntegrationTests/FailingProcessSpawnTests.cs

@@ -23,6 +23,12 @@ public void NoHangs()
                 _ => Platform.X64
             };
 
+            if (wrongPlatform == Platform.X64 && RuntimeInformation.IsFullFramework)
+            {
+                // It seems full Framework on Arm ignores the platform and simply runs the native platform, causing this test to fail.
+                return;
+            }
+
             var invalidPlatformJob = Job.Dry.WithPlatform(wrongPlatform);
             var config = CreateSimpleConfig(job: invalidPlatformJob);
 

tests/BenchmarkDotNet.IntegrationTests/LargeAddressAwareTest.cs

@@ -5,6 +5,7 @@
 using BenchmarkDotNet.Configs;
 using BenchmarkDotNet.Environments;
 using BenchmarkDotNet.Jobs;
+using BenchmarkDotNet.Portability;
 using BenchmarkDotNet.Running;
 using BenchmarkDotNet.Tests.Loggers;
 using BenchmarkDotNet.Tests.XUnit;
@@ -19,35 +20,57 @@ public class LargeAddressAwareTest
 
         public LargeAddressAwareTest(ITestOutputHelper outputHelper) => output = outputHelper;
 
-        [FactEnvSpecific("CLR is a valid job only on Windows", EnvRequirement.WindowsOnly)]
-        public void BenchmarkCanAllocateMoreThan2Gb()
+        [Fact]
+        public void BenchmarkCanAllocateMoreThan2Gb_Core()
         {
-            var summary = BenchmarkRunner
-                .Run<NeedsMoreThan2GB>(
-                    ManualConfig.CreateEmpty()
-                        .AddJob(Job.Dry.WithRuntime(CoreRuntime.Core80).WithPlatform(Platform.X64).WithId("Core"))
-                        .AddJob(Job.Dry.WithRuntime(ClrRuntime.Net462).WithPlatform(Platform.X86).WithGcServer(false).WithLargeAddressAware().WithId("Framework"))
-                        .AddColumnProvider(DefaultColumnProviders.Instance)
-                        .AddLogger(new OutputLogger(output)));
+            var platform = RuntimeInformation.GetCurrentPlatform();
+            var config = ManualConfig.CreateEmpty();
+            // Running 32-bit benchmarks with .Net Core requires passing the path to 32-bit SDK,
+            // which makes this test more complex than it's worth in CI, so we only test 64-bit.
+            config.AddJob(Job.Dry.WithRuntime(CoreRuntime.Core80).WithPlatform(platform).WithId(platform.ToString()));
+            config.AddColumnProvider(DefaultColumnProviders.Instance)
+                  .AddLogger(new OutputLogger(output));
+
+            var summary = BenchmarkRunner.Run<NeedsMoreThan2GB>(config);
 
             Assert.True(summary.Reports
                 .All(report => report.ExecuteResults
                 .All(executeResult => executeResult.FoundExecutable)));
 
             Assert.True(summary.Reports.All(report => report.AllMeasurements.Any()));
+            Assert.True(summary.Reports.All(report => report.ExecuteResults.Any()));
+            Assert.Equal(1, summary.Reports.Count(report => report.BenchmarkCase.Job.Environment.Runtime is CoreRuntime));
 
-            Assert.True(summary.Reports
-                .Single(report => report.BenchmarkCase.Job.Environment.Runtime is ClrRuntime)
-                .ExecuteResults
-                .Any());
+            Assert.Contains(".NET 8.0", summary.AllRuntimes);
+        }
+
+        [FactEnvSpecific("Framework is only on Windows", EnvRequirement.WindowsOnly)]
+        public void BenchmarkCanAllocateMoreThan2Gb_Framework()
+        {
+            var platform = RuntimeInformation.GetCurrentPlatform();
+            var config = ManualConfig.CreateEmpty();
+            // Net481 officially only supports x86, x64, and Arm64.
+            config.AddJob(Job.Dry.WithRuntime(ClrRuntime.Net481).WithPlatform(platform).WithGcServer(false).WithLargeAddressAware().WithId(platform.ToString()));
+            int jobCount = 1;
+            if (platform == Platform.X64)
+            {
+                ++jobCount;
+                config.AddJob(Job.Dry.WithRuntime(ClrRuntime.Net462).WithPlatform(Platform.X86).WithGcServer(false).WithLargeAddressAware().WithId("X86"));
+            }
+            config.AddColumnProvider(DefaultColumnProviders.Instance)
+                  .AddLogger(new OutputLogger(output));
+
+            var summary = BenchmarkRunner.Run<NeedsMoreThan2GB>(config);
 
             Assert.True(summary.Reports
-                .Single(report => report.BenchmarkCase.Job.Environment.Runtime is CoreRuntime)
-                .ExecuteResults
-                .Any());
+                .All(report => report.ExecuteResults
+                .All(executeResult => executeResult.FoundExecutable)));
+
+            Assert.True(summary.Reports.All(report => report.AllMeasurements.Any()));
+            Assert.True(summary.Reports.All(report => report.ExecuteResults.Any()));
+            Assert.Equal(jobCount, summary.Reports.Count(report => report.BenchmarkCase.Job.Environment.Runtime is ClrRuntime));
 
             Assert.Contains(".NET Framework", summary.AllRuntimes);
-            Assert.Contains(".NET 8.0", summary.AllRuntimes);
         }
     }
 

tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs

@@ -80,7 +80,7 @@ public void MemoryDiagnoserSupportsNativeAOT()
             MemoryDiagnoserIsAccurate(NativeAotToolchain.Net80);
         }
 
-        [FactEnvSpecific("We don't want to test MonoVM twice (for .NET Framework 4.6.2 and .NET 8.0)", EnvRequirement.DotNetCoreOnly)]
+        [FactEnvSpecific("We don't want to test MonoVM twice (for .NET Framework 4.6.2 and .NET 8.0), and it's not supported on Windows+Arm", [EnvRequirement.DotNetCoreOnly, EnvRequirement.NonWindowsArm])]
         public void MemoryDiagnoserSupportsModernMono()
         {
             MemoryDiagnoserIsAccurate(MonoToolchain.Mono80);

tests/BenchmarkDotNet.IntegrationTests/MonoTests.cs

@@ -14,7 +14,7 @@ public class MonoTests : BenchmarkTestExecutor
     {
         public MonoTests(ITestOutputHelper output) : base(output) { }
 
-        [FactEnvSpecific("UseMonoRuntime option is available in .NET Core only starting from .NET 6", EnvRequirement.DotNetCoreOnly)]
+        [FactEnvSpecific("UseMonoRuntime option is available in .NET Core only starting from .NET 6, and it's not supported on Windows+Arm", [EnvRequirement.DotNetCoreOnly, EnvRequirement.NonWindowsArm])]
         public void Mono80IsSupported()
         {
             var logger = new OutputLogger(Output);

tests/BenchmarkDotNet.IntegrationTests/MultipleRuntimesTest.cs

@@ -29,7 +29,7 @@ public void SingleBenchmarkCanBeExecutedForMultipleRuntimes()
             var summary = BenchmarkRunner
                 .Run<C>(
                     ManualConfig.CreateEmpty()
-                        .AddJob(Job.Dry.WithRuntime(CoreRuntime.Core80).WithPlatform(Platform.X64).WithId("Core"))
+                        .AddJob(Job.Dry.WithRuntime(CoreRuntime.Core80).WithId("Core"))
                         .AddJob(Job.Dry.WithRuntime(ClrRuntime.Net462).WithId("Framework"))
                         .AddColumnProvider(DefaultColumnProviders.Instance)
                         .AddLogger(new OutputLogger(output)));

tests/BenchmarkDotNet.IntegrationTests/WasmTests.cs

@@ -3,6 +3,7 @@
 using System.Linq;
 using BenchmarkDotNet.Attributes;
 using BenchmarkDotNet.Configs;
+using BenchmarkDotNet.Detectors;
 using BenchmarkDotNet.Environments;
 using BenchmarkDotNet.IntegrationTests.Diagnosers;
 using BenchmarkDotNet.Jobs;
@@ -46,12 +47,24 @@ private ManualConfig GetConfig()
         [FactEnvSpecific("WASM is only supported on Unix", EnvRequirement.NonWindows)]
         public void WasmIsSupported()
         {
+            // Test fails on Linux non-x64.
+            if (OsDetector.IsLinux() && RuntimeInformation.GetCurrentPlatform() != Platform.X64)
+            {
+                return;
+            }
+
             CanExecute<WasmBenchmark>(GetConfig());
         }
 
         [FactEnvSpecific("WASM is only supported on Unix", EnvRequirement.NonWindows)]
         public void WasmSupportsInProcessDiagnosers()
         {
+            // Test fails on Linux non-x64.
+            if (OsDetector.IsLinux() && RuntimeInformation.GetCurrentPlatform() != Platform.X64)
+            {
+                return;
+            }
+
             var diagnoser = new MockInProcessDiagnoser1(BenchmarkDotNet.Diagnosers.RunMode.NoOverhead);
             var config = GetConfig().AddDiagnoser(diagnoser);
 

tests/BenchmarkDotNet.Tests/XUnit/EnvRequirement.cs

@@ -4,6 +4,7 @@ public enum EnvRequirement
 {
     WindowsOnly,
     NonWindows,
+    NonWindowsArm,
     NonLinux,
     FullFrameworkOnly,
     NonFullFramework,

tests/BenchmarkDotNet.Tests/XUnit/EnvRequirementChecker.cs

@@ -1,9 +1,8 @@
+using BenchmarkDotNet.Environments;
 using System;
 using System.Linq;
 using System.Runtime.InteropServices;
 using System.Security.Principal;
-using BenchmarkDotNet.Jobs;
-using JetBrains.Annotations;
 using BdnRuntimeInformation = BenchmarkDotNet.Portability.RuntimeInformation;
 
 namespace BenchmarkDotNet.Tests.XUnit;
@@ -16,6 +15,7 @@ public static class EnvRequirementChecker
     {
         EnvRequirement.WindowsOnly => RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? null : "Windows-only test",
         EnvRequirement.NonWindows => !RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? null : "Non-Windows test",
+        EnvRequirement.NonWindowsArm => !RuntimeInformation.IsOSPlatform(OSPlatform.Windows) || !IsArm() ? null : "Non-Windows+Arm test",
         EnvRequirement.NonLinux => !RuntimeInformation.IsOSPlatform(OSPlatform.Linux) ? null : "Non-Linux test",
         EnvRequirement.FullFrameworkOnly => BdnRuntimeInformation.IsFullFramework ? null : "Full .NET Framework-only test",
         EnvRequirement.NonFullFramework => !BdnRuntimeInformation.IsFullFramework ? null : "Non-Full .NET Framework test",
@@ -34,5 +34,6 @@ private static bool IsPrivilegedProcess()
 #endif
     }
 
-    private static bool IsRuntime(RuntimeMoniker moniker) => BdnRuntimeInformation.GetCurrentRuntime().RuntimeMoniker == moniker;
+    private static bool IsArm()
+        => BdnRuntimeInformation.GetCurrentPlatform() is Platform.Arm64 or Platform.Arm or Platform.Armv6;
 }