diff --git a/.github/workflows/run-tests.yaml b/.github/workflows/run-tests.yaml index d06efb331e..b6169d4cbf 100644 --- a/.github/workflows/run-tests.yaml +++ b/.github/workflows/run-tests.yaml @@ -12,7 +12,10 @@ concurrency: jobs: test-windows-core: - runs-on: windows-latest + strategy: + matrix: + os: [windows-latest, windows-11-arm] + runs-on: ${{ matrix.os }} steps: - name: Disable Windows Defender run: Set-MpPreference -DisableRealtimeMonitoring $true @@ -30,11 +33,14 @@ jobs: uses: actions/upload-artifact@v4 if: always() with: - name: test-windows-core-trx-${{ github.run_id }} + name: test-windows-core-trx-${{ github.run_id }}-${{ matrix.os }} path: "**/*.trx" test-windows-full: - runs-on: windows-latest + strategy: + matrix: + os: [windows-latest, windows-11-arm] + runs-on: ${{ matrix.os }} steps: - name: Disable Windows Defender run: Set-MpPreference -DisableRealtimeMonitoring $true @@ -52,11 +58,14 @@ jobs: uses: actions/upload-artifact@v4 if: always() with: - name: test-windows-full-trx-${{ github.run_id }} + name: test-windows-full-trx-${{ github.run_id }}-${{ matrix.os }} path: "**/*.trx" test-linux: - runs-on: ubuntu-latest + strategy: + matrix: + os: [ubuntu-latest, ubuntu-24.04-arm] + runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v4 # Set up the environment @@ -87,7 +96,7 @@ jobs: uses: actions/upload-artifact@v4 if: always() with: - name: test-linux-trx-${{ github.run_id }} + name: test-linux-trx-${{ github.run_id }}-${{ matrix.os }} path: "**/*.trx" test-macos: diff --git a/tests/BenchmarkDotNet.IntegrationTests/FailingProcessSpawnTests.cs b/tests/BenchmarkDotNet.IntegrationTests/FailingProcessSpawnTests.cs index adf7eb7900..17c5186d78 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/FailingProcessSpawnTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/FailingProcessSpawnTests.cs @@ -23,6 +23,12 @@ public void NoHangs() _ => Platform.X64 }; + if (wrongPlatform == Platform.X64 && RuntimeInformation.IsFullFramework) + { + // It seems full Framework on Arm ignores the platform and simply runs the native platform, causing this test to fail. + return; + } + var invalidPlatformJob = Job.Dry.WithPlatform(wrongPlatform); var config = CreateSimpleConfig(job: invalidPlatformJob); diff --git a/tests/BenchmarkDotNet.IntegrationTests/LargeAddressAwareTest.cs b/tests/BenchmarkDotNet.IntegrationTests/LargeAddressAwareTest.cs index 087d146822..2fb27498cd 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/LargeAddressAwareTest.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/LargeAddressAwareTest.cs @@ -5,6 +5,7 @@ using BenchmarkDotNet.Configs; using BenchmarkDotNet.Environments; using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Portability; using BenchmarkDotNet.Running; using BenchmarkDotNet.Tests.Loggers; using BenchmarkDotNet.Tests.XUnit; @@ -19,35 +20,57 @@ public class LargeAddressAwareTest public LargeAddressAwareTest(ITestOutputHelper outputHelper) => output = outputHelper; - [FactEnvSpecific("CLR is a valid job only on Windows", EnvRequirement.WindowsOnly)] - public void BenchmarkCanAllocateMoreThan2Gb() + [Fact] + public void BenchmarkCanAllocateMoreThan2Gb_Core() { - var summary = BenchmarkRunner - .Run( - ManualConfig.CreateEmpty() - .AddJob(Job.Dry.WithRuntime(CoreRuntime.Core80).WithPlatform(Platform.X64).WithId("Core")) - .AddJob(Job.Dry.WithRuntime(ClrRuntime.Net462).WithPlatform(Platform.X86).WithGcServer(false).WithLargeAddressAware().WithId("Framework")) - .AddColumnProvider(DefaultColumnProviders.Instance) - .AddLogger(new OutputLogger(output))); + var platform = RuntimeInformation.GetCurrentPlatform(); + var config = ManualConfig.CreateEmpty(); + // Running 32-bit benchmarks with .Net Core requires passing the path to 32-bit SDK, + // which makes this test more complex than it's worth in CI, so we only test 64-bit. + config.AddJob(Job.Dry.WithRuntime(CoreRuntime.Core80).WithPlatform(platform).WithId(platform.ToString())); + config.AddColumnProvider(DefaultColumnProviders.Instance) + .AddLogger(new OutputLogger(output)); + + var summary = BenchmarkRunner.Run(config); Assert.True(summary.Reports .All(report => report.ExecuteResults .All(executeResult => executeResult.FoundExecutable))); Assert.True(summary.Reports.All(report => report.AllMeasurements.Any())); + Assert.True(summary.Reports.All(report => report.ExecuteResults.Any())); + Assert.Equal(1, summary.Reports.Count(report => report.BenchmarkCase.Job.Environment.Runtime is CoreRuntime)); - Assert.True(summary.Reports - .Single(report => report.BenchmarkCase.Job.Environment.Runtime is ClrRuntime) - .ExecuteResults - .Any()); + Assert.Contains(".NET 8.0", summary.AllRuntimes); + } + + [FactEnvSpecific("Framework is only on Windows", EnvRequirement.WindowsOnly)] + public void BenchmarkCanAllocateMoreThan2Gb_Framework() + { + var platform = RuntimeInformation.GetCurrentPlatform(); + var config = ManualConfig.CreateEmpty(); + // Net481 officially only supports x86, x64, and Arm64. + config.AddJob(Job.Dry.WithRuntime(ClrRuntime.Net481).WithPlatform(platform).WithGcServer(false).WithLargeAddressAware().WithId(platform.ToString())); + int jobCount = 1; + if (platform == Platform.X64) + { + ++jobCount; + config.AddJob(Job.Dry.WithRuntime(ClrRuntime.Net462).WithPlatform(Platform.X86).WithGcServer(false).WithLargeAddressAware().WithId("X86")); + } + config.AddColumnProvider(DefaultColumnProviders.Instance) + .AddLogger(new OutputLogger(output)); + + var summary = BenchmarkRunner.Run(config); Assert.True(summary.Reports - .Single(report => report.BenchmarkCase.Job.Environment.Runtime is CoreRuntime) - .ExecuteResults - .Any()); + .All(report => report.ExecuteResults + .All(executeResult => executeResult.FoundExecutable))); + + Assert.True(summary.Reports.All(report => report.AllMeasurements.Any())); + Assert.True(summary.Reports.All(report => report.ExecuteResults.Any())); + Assert.Equal(jobCount, summary.Reports.Count(report => report.BenchmarkCase.Job.Environment.Runtime is ClrRuntime)); Assert.Contains(".NET Framework", summary.AllRuntimes); - Assert.Contains(".NET 8.0", summary.AllRuntimes); } } diff --git a/tests/BenchmarkDotNet.IntegrationTests/MultipleRuntimesTest.cs b/tests/BenchmarkDotNet.IntegrationTests/MultipleRuntimesTest.cs index 50a929bd83..7d9bd61532 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/MultipleRuntimesTest.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/MultipleRuntimesTest.cs @@ -29,7 +29,7 @@ public void SingleBenchmarkCanBeExecutedForMultipleRuntimes() var summary = BenchmarkRunner .Run( ManualConfig.CreateEmpty() - .AddJob(Job.Dry.WithRuntime(CoreRuntime.Core80).WithPlatform(Platform.X64).WithId("Core")) + .AddJob(Job.Dry.WithRuntime(CoreRuntime.Core80).WithId("Core")) .AddJob(Job.Dry.WithRuntime(ClrRuntime.Net462).WithId("Framework")) .AddColumnProvider(DefaultColumnProviders.Instance) .AddLogger(new OutputLogger(output))); diff --git a/tests/BenchmarkDotNet.IntegrationTests/WasmTests.cs b/tests/BenchmarkDotNet.IntegrationTests/WasmTests.cs index 6d3519a9c5..c54ca702a9 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/WasmTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/WasmTests.cs @@ -2,6 +2,7 @@ using System.IO; using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Environments; using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Portability; @@ -26,6 +27,12 @@ public class WasmTests(ITestOutputHelper output) : BenchmarkTestExecutor(output) [FactEnvSpecific("WASM is only supported on Unix", EnvRequirement.NonWindows)] public void WasmIsSupported() { + // Test fails on Linux non-x64. + if (OsDetector.IsLinux() && RuntimeInformation.GetCurrentPlatform() != Platform.X64) + { + return; + } + var dotnetVersion = "net8.0"; var logger = new OutputLogger(Output); var netCoreAppSettings = new NetCoreAppSettings(dotnetVersion, null, "Wasm");