diff --git a/.github/workflows/run-tests.yaml b/.github/workflows/run-tests.yaml index a0b3ee4d30..556b84a6cd 100644 --- a/.github/workflows/run-tests.yaml +++ b/.github/workflows/run-tests.yaml @@ -13,7 +13,10 @@ concurrency: jobs: test-windows-core: - runs-on: windows-latest + strategy: + matrix: + os: [windows-latest, windows-11-arm] + runs-on: ${{ matrix.os }} steps: - name: Disable Windows Defender run: Set-MpPreference -DisableRealtimeMonitoring $true @@ -31,11 +34,14 @@ jobs: uses: actions/upload-artifact@v4 if: always() with: - name: test-windows-core-trx-${{ github.run_id }} + name: test-windows-core-trx-${{ github.run_id }}-${{ matrix.os }} path: "**/*.trx" test-windows-full: - runs-on: windows-latest + strategy: + matrix: + os: [windows-latest, windows-11-arm] + runs-on: ${{ matrix.os }} steps: - name: Disable Windows Defender run: Set-MpPreference -DisableRealtimeMonitoring $true @@ -59,11 +65,14 @@ jobs: uses: actions/upload-artifact@v4 if: always() with: - name: test-windows-full-trx-${{ github.run_id }} + name: test-windows-full-trx-${{ github.run_id }}-${{ matrix.os }} path: "**/*.trx" test-linux: - runs-on: ubuntu-latest + strategy: + matrix: + os: [ubuntu-latest, ubuntu-24.04-arm] + runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v4 # Set up the environment @@ -96,7 +105,7 @@ jobs: uses: actions/upload-artifact@v4 if: always() with: - name: test-linux-trx-${{ github.run_id }} + name: test-linux-trx-${{ github.run_id }}-${{ matrix.os }} path: "**/*.trx" test-macos: diff --git a/src/BenchmarkDotNet/Disassemblers/DisassemblyDiagnoser.cs b/src/BenchmarkDotNet/Disassemblers/DisassemblyDiagnoser.cs index 4cd2cd1262..2f8ae1ae50 100644 --- a/src/BenchmarkDotNet/Disassemblers/DisassemblyDiagnoser.cs +++ b/src/BenchmarkDotNet/Disassemblers/DisassemblyDiagnoser.cs @@ -119,7 +119,13 @@ public IEnumerable Validate(ValidationParameters validationPara var currentPlatform = RuntimeInformation.GetCurrentPlatform(); if (!(currentPlatform is Platform.X64 or Platform.X86 or Platform.Arm64)) { - yield return new ValidationError(true, $"{currentPlatform} is not supported"); + yield return new ValidationError(true, $"DisassemblyDiagnoser does not support {currentPlatform}"); + yield break; + } + + if (currentPlatform == Platform.Arm64 && OsDetector.IsWindows()) + { + yield return new ValidationError(true, $"DisassemblyDiagnoser does not support Arm on Windows"); yield break; } diff --git a/tests/BenchmarkDotNet.IntegrationTests/DisassemblyDiagnoserTests.cs b/tests/BenchmarkDotNet.IntegrationTests/DisassemblyDiagnoserTests.cs index e0f483abe1..6f2c05608b 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/DisassemblyDiagnoserTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/DisassemblyDiagnoserTests.cs @@ -13,6 +13,7 @@ using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Portability; using BenchmarkDotNet.Tests.Loggers; +using BenchmarkDotNet.Tests.XUnit; using BenchmarkDotNet.Toolchains; using BenchmarkDotNet.Toolchains.CsProj; using BenchmarkDotNet.Toolchains.InProcess.Emit; @@ -84,7 +85,7 @@ public void Recursive() [MethodImpl(MethodImplOptions.NoInlining)] public void Benchmark(bool justAnOverload) { } // we need to test overloads (#562) } - [Theory] + [TheoryEnvSpecific("Not supported on Windows+Arm", EnvRequirement.NonWindowsArm)] [MemberData(nameof(GetAllJits), DisableDiscoveryEnumeration = true)] [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)] public void CanDisassembleAllMethodCalls(Jit jit, Platform platform, IToolchain toolchain) @@ -104,7 +105,7 @@ public void CanDisassembleAllMethodCalls(Jit jit, Platform platform, IToolchain AssertDisassemblyResult(result, $"{nameof(WithCalls.Recursive)}()"); } - [Theory] + [TheoryEnvSpecific("Not supported on Windows+Arm", EnvRequirement.NonWindowsArm)] [MemberData(nameof(GetAllJits), DisableDiscoveryEnumeration = true)] [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)] public void CanDisassembleAllMethodCallsUsingFilters(Jit jit, Platform platform, IToolchain toolchain) @@ -130,7 +131,7 @@ public void CanDisassembleAllMethodCallsUsingFilters(Jit jit, Platform platform, public T Create() => new T(); } - [Theory] + [TheoryEnvSpecific("Not supported on Windows+Arm", EnvRequirement.NonWindowsArm)] [MemberData(nameof(GetAllJits), DisableDiscoveryEnumeration = true)] [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)] public void CanDisassembleGenericTypes(Jit jit, Platform platform, IToolchain toolchain) @@ -151,7 +152,7 @@ public class WithInlineable [Benchmark] public void JustReturn() { } } - [Theory] + [TheoryEnvSpecific("Not supported on Windows+Arm", EnvRequirement.NonWindowsArm)] [MemberData(nameof(GetAllJits), DisableDiscoveryEnumeration = true)] [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)] public void CanDisassembleInlinableBenchmarks(Jit jit, Platform platform, IToolchain toolchain) diff --git a/tests/BenchmarkDotNet.IntegrationTests/FailingProcessSpawnTests.cs b/tests/BenchmarkDotNet.IntegrationTests/FailingProcessSpawnTests.cs index adf7eb7900..17c5186d78 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/FailingProcessSpawnTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/FailingProcessSpawnTests.cs @@ -23,6 +23,12 @@ public void NoHangs() _ => Platform.X64 }; + if (wrongPlatform == Platform.X64 && RuntimeInformation.IsFullFramework) + { + // It seems full Framework on Arm ignores the platform and simply runs the native platform, causing this test to fail. + return; + } + var invalidPlatformJob = Job.Dry.WithPlatform(wrongPlatform); var config = CreateSimpleConfig(job: invalidPlatformJob); diff --git a/tests/BenchmarkDotNet.IntegrationTests/LargeAddressAwareTest.cs b/tests/BenchmarkDotNet.IntegrationTests/LargeAddressAwareTest.cs index 087d146822..2fb27498cd 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/LargeAddressAwareTest.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/LargeAddressAwareTest.cs @@ -5,6 +5,7 @@ using BenchmarkDotNet.Configs; using BenchmarkDotNet.Environments; using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Portability; using BenchmarkDotNet.Running; using BenchmarkDotNet.Tests.Loggers; using BenchmarkDotNet.Tests.XUnit; @@ -19,35 +20,57 @@ public class LargeAddressAwareTest public LargeAddressAwareTest(ITestOutputHelper outputHelper) => output = outputHelper; - [FactEnvSpecific("CLR is a valid job only on Windows", EnvRequirement.WindowsOnly)] - public void BenchmarkCanAllocateMoreThan2Gb() + [Fact] + public void BenchmarkCanAllocateMoreThan2Gb_Core() { - var summary = BenchmarkRunner - .Run( - ManualConfig.CreateEmpty() - .AddJob(Job.Dry.WithRuntime(CoreRuntime.Core80).WithPlatform(Platform.X64).WithId("Core")) - .AddJob(Job.Dry.WithRuntime(ClrRuntime.Net462).WithPlatform(Platform.X86).WithGcServer(false).WithLargeAddressAware().WithId("Framework")) - .AddColumnProvider(DefaultColumnProviders.Instance) - .AddLogger(new OutputLogger(output))); + var platform = RuntimeInformation.GetCurrentPlatform(); + var config = ManualConfig.CreateEmpty(); + // Running 32-bit benchmarks with .Net Core requires passing the path to 32-bit SDK, + // which makes this test more complex than it's worth in CI, so we only test 64-bit. + config.AddJob(Job.Dry.WithRuntime(CoreRuntime.Core80).WithPlatform(platform).WithId(platform.ToString())); + config.AddColumnProvider(DefaultColumnProviders.Instance) + .AddLogger(new OutputLogger(output)); + + var summary = BenchmarkRunner.Run(config); Assert.True(summary.Reports .All(report => report.ExecuteResults .All(executeResult => executeResult.FoundExecutable))); Assert.True(summary.Reports.All(report => report.AllMeasurements.Any())); + Assert.True(summary.Reports.All(report => report.ExecuteResults.Any())); + Assert.Equal(1, summary.Reports.Count(report => report.BenchmarkCase.Job.Environment.Runtime is CoreRuntime)); - Assert.True(summary.Reports - .Single(report => report.BenchmarkCase.Job.Environment.Runtime is ClrRuntime) - .ExecuteResults - .Any()); + Assert.Contains(".NET 8.0", summary.AllRuntimes); + } + + [FactEnvSpecific("Framework is only on Windows", EnvRequirement.WindowsOnly)] + public void BenchmarkCanAllocateMoreThan2Gb_Framework() + { + var platform = RuntimeInformation.GetCurrentPlatform(); + var config = ManualConfig.CreateEmpty(); + // Net481 officially only supports x86, x64, and Arm64. + config.AddJob(Job.Dry.WithRuntime(ClrRuntime.Net481).WithPlatform(platform).WithGcServer(false).WithLargeAddressAware().WithId(platform.ToString())); + int jobCount = 1; + if (platform == Platform.X64) + { + ++jobCount; + config.AddJob(Job.Dry.WithRuntime(ClrRuntime.Net462).WithPlatform(Platform.X86).WithGcServer(false).WithLargeAddressAware().WithId("X86")); + } + config.AddColumnProvider(DefaultColumnProviders.Instance) + .AddLogger(new OutputLogger(output)); + + var summary = BenchmarkRunner.Run(config); Assert.True(summary.Reports - .Single(report => report.BenchmarkCase.Job.Environment.Runtime is CoreRuntime) - .ExecuteResults - .Any()); + .All(report => report.ExecuteResults + .All(executeResult => executeResult.FoundExecutable))); + + Assert.True(summary.Reports.All(report => report.AllMeasurements.Any())); + Assert.True(summary.Reports.All(report => report.ExecuteResults.Any())); + Assert.Equal(jobCount, summary.Reports.Count(report => report.BenchmarkCase.Job.Environment.Runtime is ClrRuntime)); Assert.Contains(".NET Framework", summary.AllRuntimes); - Assert.Contains(".NET 8.0", summary.AllRuntimes); } } diff --git a/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs b/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs index 47c0e8a8e5..a486fe46b4 100755 --- a/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs @@ -80,7 +80,7 @@ public void MemoryDiagnoserSupportsNativeAOT() MemoryDiagnoserIsAccurate(NativeAotToolchain.Net80); } - [FactEnvSpecific("We don't want to test MonoVM twice (for .NET Framework 4.6.2 and .NET 8.0)", EnvRequirement.DotNetCoreOnly)] + [FactEnvSpecific("We don't want to test MonoVM twice (for .NET Framework 4.6.2 and .NET 8.0), and it's not supported on Windows+Arm", [EnvRequirement.DotNetCoreOnly, EnvRequirement.NonWindowsArm])] public void MemoryDiagnoserSupportsModernMono() { MemoryDiagnoserIsAccurate(MonoToolchain.Mono80); diff --git a/tests/BenchmarkDotNet.IntegrationTests/MonoTests.cs b/tests/BenchmarkDotNet.IntegrationTests/MonoTests.cs index d08e9166b7..fadfb38b91 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/MonoTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/MonoTests.cs @@ -14,7 +14,7 @@ public class MonoTests : BenchmarkTestExecutor { public MonoTests(ITestOutputHelper output) : base(output) { } - [FactEnvSpecific("UseMonoRuntime option is available in .NET Core only starting from .NET 6", EnvRequirement.DotNetCoreOnly)] + [FactEnvSpecific("UseMonoRuntime option is available in .NET Core only starting from .NET 6, and it's not supported on Windows+Arm", [EnvRequirement.DotNetCoreOnly, EnvRequirement.NonWindowsArm])] public void Mono80IsSupported() { var logger = new OutputLogger(Output); diff --git a/tests/BenchmarkDotNet.IntegrationTests/MultipleRuntimesTest.cs b/tests/BenchmarkDotNet.IntegrationTests/MultipleRuntimesTest.cs index 50a929bd83..7d9bd61532 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/MultipleRuntimesTest.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/MultipleRuntimesTest.cs @@ -29,7 +29,7 @@ public void SingleBenchmarkCanBeExecutedForMultipleRuntimes() var summary = BenchmarkRunner .Run( ManualConfig.CreateEmpty() - .AddJob(Job.Dry.WithRuntime(CoreRuntime.Core80).WithPlatform(Platform.X64).WithId("Core")) + .AddJob(Job.Dry.WithRuntime(CoreRuntime.Core80).WithId("Core")) .AddJob(Job.Dry.WithRuntime(ClrRuntime.Net462).WithId("Framework")) .AddColumnProvider(DefaultColumnProviders.Instance) .AddLogger(new OutputLogger(output))); diff --git a/tests/BenchmarkDotNet.IntegrationTests/WasmTests.cs b/tests/BenchmarkDotNet.IntegrationTests/WasmTests.cs index 844151b05a..fb22a91e36 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/WasmTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/WasmTests.cs @@ -3,6 +3,7 @@ using System.Linq; using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Environments; using BenchmarkDotNet.IntegrationTests.Diagnosers; using BenchmarkDotNet.Jobs; @@ -46,12 +47,24 @@ private ManualConfig GetConfig() [FactEnvSpecific("WASM is only supported on Unix", EnvRequirement.NonWindows)] public void WasmIsSupported() { + // Test fails on Linux non-x64. + if (OsDetector.IsLinux() && RuntimeInformation.GetCurrentPlatform() != Platform.X64) + { + return; + } + CanExecute(GetConfig()); } [FactEnvSpecific("WASM is only supported on Unix", EnvRequirement.NonWindows)] public void WasmSupportsInProcessDiagnosers() { + // Test fails on Linux non-x64. + if (OsDetector.IsLinux() && RuntimeInformation.GetCurrentPlatform() != Platform.X64) + { + return; + } + var diagnoser = new MockInProcessDiagnoser1(BenchmarkDotNet.Diagnosers.RunMode.NoOverhead); var config = GetConfig().AddDiagnoser(diagnoser); diff --git a/tests/BenchmarkDotNet.Tests/XUnit/EnvRequirement.cs b/tests/BenchmarkDotNet.Tests/XUnit/EnvRequirement.cs index 79bfb5a1ae..e9e546bd1e 100644 --- a/tests/BenchmarkDotNet.Tests/XUnit/EnvRequirement.cs +++ b/tests/BenchmarkDotNet.Tests/XUnit/EnvRequirement.cs @@ -4,6 +4,7 @@ public enum EnvRequirement { WindowsOnly, NonWindows, + NonWindowsArm, NonLinux, FullFrameworkOnly, NonFullFramework, diff --git a/tests/BenchmarkDotNet.Tests/XUnit/EnvRequirementChecker.cs b/tests/BenchmarkDotNet.Tests/XUnit/EnvRequirementChecker.cs index 22f131455d..f00a7bc0da 100644 --- a/tests/BenchmarkDotNet.Tests/XUnit/EnvRequirementChecker.cs +++ b/tests/BenchmarkDotNet.Tests/XUnit/EnvRequirementChecker.cs @@ -1,9 +1,8 @@ +using BenchmarkDotNet.Environments; using System; using System.Linq; using System.Runtime.InteropServices; using System.Security.Principal; -using BenchmarkDotNet.Jobs; -using JetBrains.Annotations; using BdnRuntimeInformation = BenchmarkDotNet.Portability.RuntimeInformation; namespace BenchmarkDotNet.Tests.XUnit; @@ -16,6 +15,7 @@ public static class EnvRequirementChecker { EnvRequirement.WindowsOnly => RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? null : "Windows-only test", EnvRequirement.NonWindows => !RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? null : "Non-Windows test", + EnvRequirement.NonWindowsArm => !RuntimeInformation.IsOSPlatform(OSPlatform.Windows) || !IsArm() ? null : "Non-Windows+Arm test", EnvRequirement.NonLinux => !RuntimeInformation.IsOSPlatform(OSPlatform.Linux) ? null : "Non-Linux test", EnvRequirement.FullFrameworkOnly => BdnRuntimeInformation.IsFullFramework ? null : "Full .NET Framework-only test", EnvRequirement.NonFullFramework => !BdnRuntimeInformation.IsFullFramework ? null : "Non-Full .NET Framework test", @@ -34,5 +34,6 @@ private static bool IsPrivilegedProcess() #endif } - private static bool IsRuntime(RuntimeMoniker moniker) => BdnRuntimeInformation.GetCurrentRuntime().RuntimeMoniker == moniker; + private static bool IsArm() + => BdnRuntimeInformation.GetCurrentPlatform() is Platform.Arm64 or Platform.Arm or Platform.Armv6; } \ No newline at end of file