From 75a8b6d0967cd8f6c420e9fb64592e04fafd3eb7 Mon Sep 17 00:00:00 2001 From: nietras Date: Sun, 8 Jun 2025 14:58:06 +0200 Subject: [PATCH 1/9] [build] Add windows-11-arm to CI run-tests.yaml --- .github/workflows/run-tests.yaml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/run-tests.yaml b/.github/workflows/run-tests.yaml index d06efb331e..8a6e86a114 100644 --- a/.github/workflows/run-tests.yaml +++ b/.github/workflows/run-tests.yaml @@ -12,7 +12,10 @@ concurrency: jobs: test-windows-core: - runs-on: windows-latest + strategy: + matrix: + os: [windows-latest, windows-11-arm] + runs-on: ${{ matrix.os }} steps: - name: Disable Windows Defender run: Set-MpPreference -DisableRealtimeMonitoring $true @@ -34,7 +37,10 @@ jobs: path: "**/*.trx" test-windows-full: - runs-on: windows-latest + strategy: + matrix: + os: [windows-latest, windows-11-arm] + runs-on: ${{ matrix.os }} steps: - name: Disable Windows Defender run: Set-MpPreference -DisableRealtimeMonitoring $true From 5bd15fa6f37903249645a08020d9e9887d26c26a Mon Sep 17 00:00:00 2001 From: nietras Date: Sun, 8 Jun 2025 15:02:51 +0200 Subject: [PATCH 2/9] [build] Add ubuntu-24.04-arm to CI run-tests.yaml --- .github/workflows/run-tests.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/run-tests.yaml b/.github/workflows/run-tests.yaml index d06efb331e..b8ee2695b5 100644 --- a/.github/workflows/run-tests.yaml +++ b/.github/workflows/run-tests.yaml @@ -56,7 +56,10 @@ jobs: path: "**/*.trx" test-linux: - runs-on: ubuntu-latest + strategy: + matrix: + os: [ubuntu-latest, ubuntu-24.04-arm] + runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v4 # Set up the environment From 19a5ddb2a8a62b572c4fbaff797abf08e87c5262 Mon Sep 17 00:00:00 2001 From: nietras Date: Sun, 8 Jun 2025 20:54:31 +0200 Subject: [PATCH 3/9] add os to upload name --- .github/workflows/run-tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run-tests.yaml b/.github/workflows/run-tests.yaml index b8ee2695b5..7bed89779a 100644 --- a/.github/workflows/run-tests.yaml +++ b/.github/workflows/run-tests.yaml @@ -90,7 +90,7 @@ jobs: uses: actions/upload-artifact@v4 if: always() with: - name: test-linux-trx-${{ github.run_id }} + name: test-linux-trx-${{ github.run_id }}-${{ matrix.os }} path: "**/*.trx" test-macos: From 6a2f05d45a93ba71d6970e672deacebc7d7efd63 Mon Sep 17 00:00:00 2001 From: nietras Date: Sun, 8 Jun 2025 20:55:17 +0200 Subject: [PATCH 4/9] add os to upload name --- .github/workflows/run-tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run-tests.yaml b/.github/workflows/run-tests.yaml index 8a6e86a114..91351783b5 100644 --- a/.github/workflows/run-tests.yaml +++ b/.github/workflows/run-tests.yaml @@ -58,7 +58,7 @@ jobs: uses: actions/upload-artifact@v4 if: always() with: - name: test-windows-full-trx-${{ github.run_id }} + name: test-windows-full-trx-${{ github.run_id }}-${{ matrix.os }} path: "**/*.trx" test-linux: From c80dd8af0e55c23fa6eb3b5d5b7b0b9e934dd8c5 Mon Sep 17 00:00:00 2001 From: nietras Date: Sun, 8 Jun 2025 21:54:12 +0200 Subject: [PATCH 5/9] add os to core name --- .github/workflows/run-tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run-tests.yaml b/.github/workflows/run-tests.yaml index 91351783b5..a0ca83b81c 100644 --- a/.github/workflows/run-tests.yaml +++ b/.github/workflows/run-tests.yaml @@ -33,7 +33,7 @@ jobs: uses: actions/upload-artifact@v4 if: always() with: - name: test-windows-core-trx-${{ github.run_id }} + name: test-windows-core-trx-${{ github.run_id }}-${{ matrix.os }} path: "**/*.trx" test-windows-full: From 60e1c65c3f4be04a6ba17edd1818dd83897e82f5 Mon Sep 17 00:00:00 2001 From: nietras Date: Mon, 9 Jun 2025 12:16:57 +0200 Subject: [PATCH 6/9] SingleBenchmarkCanBeExecutedForMultipleRuntimes remove Platform.X64 --- tests/BenchmarkDotNet.IntegrationTests/MultipleRuntimesTest.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/BenchmarkDotNet.IntegrationTests/MultipleRuntimesTest.cs b/tests/BenchmarkDotNet.IntegrationTests/MultipleRuntimesTest.cs index 50a929bd83..7d9bd61532 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/MultipleRuntimesTest.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/MultipleRuntimesTest.cs @@ -29,7 +29,7 @@ public void SingleBenchmarkCanBeExecutedForMultipleRuntimes() var summary = BenchmarkRunner .Run( ManualConfig.CreateEmpty() - .AddJob(Job.Dry.WithRuntime(CoreRuntime.Core80).WithPlatform(Platform.X64).WithId("Core")) + .AddJob(Job.Dry.WithRuntime(CoreRuntime.Core80).WithId("Core")) .AddJob(Job.Dry.WithRuntime(ClrRuntime.Net462).WithId("Framework")) .AddColumnProvider(DefaultColumnProviders.Instance) .AddLogger(new OutputLogger(output))); From 67d1e271c718b8208aeee15cbf7924d782112d3a Mon Sep 17 00:00:00 2001 From: nietras Date: Mon, 9 Jun 2025 12:24:42 +0200 Subject: [PATCH 7/9] BenchmarkCanAllocateMoreThan2Gb try use platform for switching --- .../LargeAddressAwareTest.cs | 22 +++++++++++++------ 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/tests/BenchmarkDotNet.IntegrationTests/LargeAddressAwareTest.cs b/tests/BenchmarkDotNet.IntegrationTests/LargeAddressAwareTest.cs index 087d146822..a76da3bb04 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/LargeAddressAwareTest.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/LargeAddressAwareTest.cs @@ -5,6 +5,7 @@ using BenchmarkDotNet.Configs; using BenchmarkDotNet.Environments; using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Portability; using BenchmarkDotNet.Running; using BenchmarkDotNet.Tests.Loggers; using BenchmarkDotNet.Tests.XUnit; @@ -22,13 +23,20 @@ public class LargeAddressAwareTest [FactEnvSpecific("CLR is a valid job only on Windows", EnvRequirement.WindowsOnly)] public void BenchmarkCanAllocateMoreThan2Gb() { - var summary = BenchmarkRunner - .Run( - ManualConfig.CreateEmpty() - .AddJob(Job.Dry.WithRuntime(CoreRuntime.Core80).WithPlatform(Platform.X64).WithId("Core")) - .AddJob(Job.Dry.WithRuntime(ClrRuntime.Net462).WithPlatform(Platform.X86).WithGcServer(false).WithLargeAddressAware().WithId("Framework")) - .AddColumnProvider(DefaultColumnProviders.Instance) - .AddLogger(new OutputLogger(output))); + var platform = RuntimeInformation.GetCurrentPlatform(); + var config = ManualConfig.CreateEmpty(); + if (platform == Platform.X64 || platform == Platform.Arm64) + { + config.AddJob(Job.Dry.WithRuntime(CoreRuntime.Core80).WithPlatform(platform).WithId("Core")); + } + if (platform == Platform.X64 || platform == Platform.X86) + { + config.AddJob(Job.Dry.WithRuntime(ClrRuntime.Net462).WithPlatform(Platform.X86).WithGcServer(false).WithLargeAddressAware().WithId("Framework")); + } + config.AddColumnProvider(DefaultColumnProviders.Instance) + .AddLogger(new OutputLogger(output)); + + var summary = BenchmarkRunner.Run(config); Assert.True(summary.Reports .All(report => report.ExecuteResults From 1b3a1a7650e28f147f03aea75ef8a76f32dbe726 Mon Sep 17 00:00:00 2001 From: Tim Cassell Date: Sat, 14 Jun 2025 02:57:41 -0400 Subject: [PATCH 8/9] Split LargeAddressAwareTests. Ignore NoHangs test on full Framework on non-x86. --- .../FailingProcessSpawnTests.cs | 6 +++ .../LargeAddressAwareTest.cs | 51 ++++++++++++------- 2 files changed, 39 insertions(+), 18 deletions(-) diff --git a/tests/BenchmarkDotNet.IntegrationTests/FailingProcessSpawnTests.cs b/tests/BenchmarkDotNet.IntegrationTests/FailingProcessSpawnTests.cs index adf7eb7900..17c5186d78 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/FailingProcessSpawnTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/FailingProcessSpawnTests.cs @@ -23,6 +23,12 @@ public void NoHangs() _ => Platform.X64 }; + if (wrongPlatform == Platform.X64 && RuntimeInformation.IsFullFramework) + { + // It seems full Framework on Arm ignores the platform and simply runs the native platform, causing this test to fail. + return; + } + var invalidPlatformJob = Job.Dry.WithPlatform(wrongPlatform); var config = CreateSimpleConfig(job: invalidPlatformJob); diff --git a/tests/BenchmarkDotNet.IntegrationTests/LargeAddressAwareTest.cs b/tests/BenchmarkDotNet.IntegrationTests/LargeAddressAwareTest.cs index a76da3bb04..2fb27498cd 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/LargeAddressAwareTest.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/LargeAddressAwareTest.cs @@ -20,19 +20,14 @@ public class LargeAddressAwareTest public LargeAddressAwareTest(ITestOutputHelper outputHelper) => output = outputHelper; - [FactEnvSpecific("CLR is a valid job only on Windows", EnvRequirement.WindowsOnly)] - public void BenchmarkCanAllocateMoreThan2Gb() + [Fact] + public void BenchmarkCanAllocateMoreThan2Gb_Core() { var platform = RuntimeInformation.GetCurrentPlatform(); var config = ManualConfig.CreateEmpty(); - if (platform == Platform.X64 || platform == Platform.Arm64) - { - config.AddJob(Job.Dry.WithRuntime(CoreRuntime.Core80).WithPlatform(platform).WithId("Core")); - } - if (platform == Platform.X64 || platform == Platform.X86) - { - config.AddJob(Job.Dry.WithRuntime(ClrRuntime.Net462).WithPlatform(Platform.X86).WithGcServer(false).WithLargeAddressAware().WithId("Framework")); - } + // Running 32-bit benchmarks with .Net Core requires passing the path to 32-bit SDK, + // which makes this test more complex than it's worth in CI, so we only test 64-bit. + config.AddJob(Job.Dry.WithRuntime(CoreRuntime.Core80).WithPlatform(platform).WithId(platform.ToString())); config.AddColumnProvider(DefaultColumnProviders.Instance) .AddLogger(new OutputLogger(output)); @@ -43,19 +38,39 @@ public void BenchmarkCanAllocateMoreThan2Gb() .All(executeResult => executeResult.FoundExecutable))); Assert.True(summary.Reports.All(report => report.AllMeasurements.Any())); + Assert.True(summary.Reports.All(report => report.ExecuteResults.Any())); + Assert.Equal(1, summary.Reports.Count(report => report.BenchmarkCase.Job.Environment.Runtime is CoreRuntime)); - Assert.True(summary.Reports - .Single(report => report.BenchmarkCase.Job.Environment.Runtime is ClrRuntime) - .ExecuteResults - .Any()); + Assert.Contains(".NET 8.0", summary.AllRuntimes); + } + + [FactEnvSpecific("Framework is only on Windows", EnvRequirement.WindowsOnly)] + public void BenchmarkCanAllocateMoreThan2Gb_Framework() + { + var platform = RuntimeInformation.GetCurrentPlatform(); + var config = ManualConfig.CreateEmpty(); + // Net481 officially only supports x86, x64, and Arm64. + config.AddJob(Job.Dry.WithRuntime(ClrRuntime.Net481).WithPlatform(platform).WithGcServer(false).WithLargeAddressAware().WithId(platform.ToString())); + int jobCount = 1; + if (platform == Platform.X64) + { + ++jobCount; + config.AddJob(Job.Dry.WithRuntime(ClrRuntime.Net462).WithPlatform(Platform.X86).WithGcServer(false).WithLargeAddressAware().WithId("X86")); + } + config.AddColumnProvider(DefaultColumnProviders.Instance) + .AddLogger(new OutputLogger(output)); + + var summary = BenchmarkRunner.Run(config); Assert.True(summary.Reports - .Single(report => report.BenchmarkCase.Job.Environment.Runtime is CoreRuntime) - .ExecuteResults - .Any()); + .All(report => report.ExecuteResults + .All(executeResult => executeResult.FoundExecutable))); + + Assert.True(summary.Reports.All(report => report.AllMeasurements.Any())); + Assert.True(summary.Reports.All(report => report.ExecuteResults.Any())); + Assert.Equal(jobCount, summary.Reports.Count(report => report.BenchmarkCase.Job.Environment.Runtime is ClrRuntime)); Assert.Contains(".NET Framework", summary.AllRuntimes); - Assert.Contains(".NET 8.0", summary.AllRuntimes); } } From 7134a1fe2896eb63644aff44ee6c97011c344d23 Mon Sep 17 00:00:00 2001 From: Tim Cassell Date: Sat, 14 Jun 2025 04:02:44 -0400 Subject: [PATCH 9/9] Skip WasmIsSupported test on Linux Arm. --- tests/BenchmarkDotNet.IntegrationTests/WasmTests.cs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/BenchmarkDotNet.IntegrationTests/WasmTests.cs b/tests/BenchmarkDotNet.IntegrationTests/WasmTests.cs index 6d3519a9c5..c54ca702a9 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/WasmTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/WasmTests.cs @@ -2,6 +2,7 @@ using System.IO; using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Environments; using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Portability; @@ -26,6 +27,12 @@ public class WasmTests(ITestOutputHelper output) : BenchmarkTestExecutor(output) [FactEnvSpecific("WASM is only supported on Unix", EnvRequirement.NonWindows)] public void WasmIsSupported() { + // Test fails on Linux non-x64. + if (OsDetector.IsLinux() && RuntimeInformation.GetCurrentPlatform() != Platform.X64) + { + return; + } + var dotnetVersion = "net8.0"; var logger = new OutputLogger(Output); var netCoreAppSettings = new NetCoreAppSettings(dotnetVersion, null, "Wasm");