diff --git a/src/BenchmarkDotNet/Configs/ImmutableConfig.cs b/src/BenchmarkDotNet/Configs/ImmutableConfig.cs index 0043ecbe4b..06407f1b62 100644 --- a/src/BenchmarkDotNet/Configs/ImmutableConfig.cs +++ b/src/BenchmarkDotNet/Configs/ImmutableConfig.cs @@ -120,6 +120,8 @@ internal ImmutableConfig( internal bool HasPerfCollectProfiler() => diagnosers.OfType().Any(); + internal bool HasDisassemblyDiagnoser() => diagnosers.OfType().Any(); + public bool HasExtraStatsDiagnoser() => HasMemoryDiagnoser() || HasThreadingDiagnoser() || HasExceptionDiagnoser(); public IDiagnoser? GetCompositeDiagnoser(BenchmarkCase benchmarkCase, RunMode runMode) diff --git a/src/BenchmarkDotNet/Engines/Engine.cs b/src/BenchmarkDotNet/Engines/Engine.cs index 1fc1b96dfd..b36f9d8ad3 100644 --- a/src/BenchmarkDotNet/Engines/Engine.cs +++ b/src/BenchmarkDotNet/Engines/Engine.cs @@ -1,14 +1,12 @@ using System; using System.Collections.Generic; -using System.Diagnostics; -using System.Globalization; using System.Linq; using System.Runtime.CompilerServices; using System.Threading; using BenchmarkDotNet.Characteristics; using BenchmarkDotNet.Environments; +using BenchmarkDotNet.Helpers; using BenchmarkDotNet.Jobs; -using BenchmarkDotNet.Mathematics; using BenchmarkDotNet.Portability; using BenchmarkDotNet.Reports; using JetBrains.Annotations; @@ -19,64 +17,48 @@ namespace BenchmarkDotNet.Engines [UsedImplicitly] public class Engine : IEngine { - [PublicAPI] public IHost Host { get; } - [PublicAPI] public Action WorkloadAction { get; } - [PublicAPI] public Action Dummy1Action { get; } - [PublicAPI] public Action Dummy2Action { get; } - [PublicAPI] public Action Dummy3Action { get; } - [PublicAPI] public Action OverheadAction { get; } - [PublicAPI] public Job TargetJob { get; } - [PublicAPI] public long OperationsPerInvoke { get; } - [PublicAPI] public Action GlobalSetupAction { get; } - [PublicAPI] public Action GlobalCleanupAction { get; } - [PublicAPI] public Action IterationSetupAction { get; } - [PublicAPI] public Action IterationCleanupAction { get; } - [PublicAPI] public IResolver Resolver { get; } - [PublicAPI] public CultureInfo CultureInfo { get; } - [PublicAPI] public string BenchmarkName { get; } + internal EngineParameters Parameters { get; } private IClock Clock { get; } private bool ForceGcCleanups { get; } - private int UnrollFactor { get; } - private RunStrategy Strategy { get; } - private bool EvaluateOverhead { get; } private bool MemoryRandomization { get; } - private readonly List jittingMeasurements = new(10); - private readonly bool includeExtraStats; private readonly Random random; - internal Engine( - IHost host, - IResolver resolver, - Action dummy1Action, Action dummy2Action, Action dummy3Action, Action overheadAction, Action workloadAction, Job targetJob, - Action globalSetupAction, Action globalCleanupAction, Action iterationSetupAction, Action iterationCleanupAction, long operationsPerInvoke, - bool includeExtraStats, string benchmarkName) + private IHost Host => Parameters.Host; + private Job TargetJob => Parameters.TargetJob; + private IResolver Resolver => Parameters.Resolver; + + internal Engine(EngineParameters engineParameters) { + if (engineParameters == null) throw new ArgumentNullException(nameof(engineParameters)); - Host = host; - OverheadAction = overheadAction; - Dummy1Action = dummy1Action; - Dummy2Action = dummy2Action; - Dummy3Action = dummy3Action; - WorkloadAction = workloadAction; - TargetJob = targetJob; - GlobalSetupAction = globalSetupAction; - GlobalCleanupAction = globalCleanupAction; - IterationSetupAction = iterationSetupAction; - IterationCleanupAction = iterationCleanupAction; - OperationsPerInvoke = operationsPerInvoke; - this.includeExtraStats = includeExtraStats; - BenchmarkName = benchmarkName; - - Resolver = resolver; - - Clock = targetJob.ResolveValue(InfrastructureMode.ClockCharacteristic, Resolver); - ForceGcCleanups = targetJob.ResolveValue(GcMode.ForceCharacteristic, Resolver); - UnrollFactor = targetJob.ResolveValue(RunMode.UnrollFactorCharacteristic, Resolver); - Strategy = targetJob.ResolveValue(RunMode.RunStrategyCharacteristic, Resolver); - EvaluateOverhead = targetJob.ResolveValue(AccuracyMode.EvaluateOverheadCharacteristic, Resolver); - MemoryRandomization = targetJob.ResolveValue(RunMode.MemoryRandomizationCharacteristic, Resolver); + // EngineParameters properties are mutable, so we copy/freeze them all. + var job = engineParameters.TargetJob ?? throw new ArgumentNullException(nameof(EngineParameters.TargetJob)); + Parameters = new() + { + WorkloadActionNoUnroll = engineParameters.WorkloadActionNoUnroll ?? throw new ArgumentNullException(nameof(EngineParameters.WorkloadActionNoUnroll)), + WorkloadActionUnroll = engineParameters.WorkloadActionUnroll ?? throw new ArgumentNullException(nameof(EngineParameters.WorkloadActionUnroll)), + Dummy1Action = engineParameters.Dummy1Action ?? throw new ArgumentNullException(nameof(EngineParameters.Dummy1Action)), + Dummy2Action = engineParameters.Dummy2Action ?? throw new ArgumentNullException(nameof(EngineParameters.Dummy2Action)), + Dummy3Action = engineParameters.Dummy3Action ?? throw new ArgumentNullException(nameof(EngineParameters.Dummy3Action)), + OverheadActionNoUnroll = engineParameters.OverheadActionNoUnroll ?? throw new ArgumentNullException(nameof(EngineParameters.OverheadActionNoUnroll)), + OverheadActionUnroll = engineParameters.OverheadActionUnroll ?? throw new ArgumentNullException(nameof(EngineParameters.OverheadActionUnroll)), + GlobalSetupAction = engineParameters.GlobalSetupAction ?? throw new ArgumentNullException(nameof(EngineParameters.GlobalSetupAction)), + GlobalCleanupAction = engineParameters.GlobalCleanupAction ?? throw new ArgumentNullException(nameof(EngineParameters.GlobalCleanupAction)), + IterationSetupAction = engineParameters.IterationSetupAction ?? throw new ArgumentNullException(nameof(EngineParameters.IterationSetupAction)), + IterationCleanupAction = engineParameters.IterationCleanupAction ?? throw new ArgumentNullException(nameof(EngineParameters.IterationCleanupAction)), + TargetJob = new Job(job).Freeze(), + BenchmarkName = engineParameters.BenchmarkName, + MeasureExtraStats = engineParameters.MeasureExtraStats, + Host = engineParameters.Host, + OperationsPerInvoke = engineParameters.OperationsPerInvoke, + Resolver = engineParameters.Resolver + }; + + Clock = TargetJob.ResolveValue(InfrastructureMode.ClockCharacteristic, Resolver); + ForceGcCleanups = TargetJob.ResolveValue(GcMode.ForceCharacteristic, Resolver); + MemoryRandomization = TargetJob.ResolveValue(RunMode.MemoryRandomizationCharacteristic, Resolver); random = new Random(12345); // we are using constant seed to try to get repeatable results } @@ -85,7 +67,7 @@ public void Dispose() { try { - GlobalCleanupAction?.Invoke(); + Parameters.GlobalCleanupAction.Invoke(); } catch (Exception e) { @@ -103,16 +85,14 @@ public void Dispose() public RunResults Run() { var measurements = new List(); - measurements.AddRange(jittingMeasurements); - - long invokeCount = TargetJob.ResolveValue(RunMode.InvocationCountCharacteristic, Resolver, 1); if (EngineEventSource.Log.IsEnabled()) - EngineEventSource.Log.BenchmarkStart(BenchmarkName); + EngineEventSource.Log.BenchmarkStart(Parameters.BenchmarkName); + IterationData extraStatsIterationData = default; // Enumerate the stages and run iterations in a loop to ensure each benchmark invocation is called with a constant stack size. // #1120 - foreach (var stage in EngineStage.EnumerateStages(this, Strategy, EvaluateOverhead)) + foreach (var stage in EngineStage.EnumerateStages(Parameters)) { if (stage.Stage == IterationStage.Actual && stage.Mode == IterationMode.Workload) { @@ -120,17 +100,19 @@ public RunResults Run() } var stageMeasurements = stage.GetMeasurementList(); - // 1-based iterationIndex - int iterationIndex = 1; - while (stage.GetShouldRunIteration(stageMeasurements, ref invokeCount)) + while (stage.GetShouldRunIteration(stageMeasurements, out var iterationData)) { - var measurement = RunIteration(new IterationData(stage.Mode, stage.Stage, iterationIndex, invokeCount, UnrollFactor)); - stageMeasurements.Add(measurement); - ++iterationIndex; + var measurement = RunIteration(iterationData); + if (iterationData.mode != IterationMode.Dummy) + { + stageMeasurements.Add(measurement); + // Actual Workload is always the last stage, so we use the same data to run extra stats. + extraStatsIterationData = iterationData; + } } measurements.AddRange(stageMeasurements); - WriteLine(); + Host.WriteLine(); if (stage.Stage == IterationStage.Actual && stage.Mode == IterationMode.Workload) { @@ -138,12 +120,12 @@ public RunResults Run() } } - (GcStats workGcHasDone, ThreadingStats threadingStats, double exceptionFrequency) = includeExtraStats - ? GetExtraStats(new IterationData(IterationMode.Workload, IterationStage.Actual, 0, invokeCount, UnrollFactor)) - : (GcStats.Empty, ThreadingStats.Empty, 0); + (GcStats workGcHasDone, ThreadingStats threadingStats, double exceptionFrequency) = Parameters.MeasureExtraStats + ? GetExtraStats(extraStatsIterationData) + : default; if (EngineEventSource.Log.IsEnabled()) - EngineEventSource.Log.BenchmarkStop(BenchmarkName); + EngineEventSource.Log.BenchmarkStop(Parameters.BenchmarkName); var outlierMode = TargetJob.ResolveValue(AccuracyMode.OutlierModeCharacteristic, Resolver); @@ -151,36 +133,32 @@ public RunResults Run() } [MethodImpl(CodeGenHelper.AggressiveOptimizationOption)] - public Measurement RunIteration(IterationData data) + private Measurement RunIteration(IterationData data) { // Initialization - long invokeCount = data.InvokeCount; - int unrollFactor = data.UnrollFactor; + long invokeCount = data.invokeCount; + int unrollFactor = data.unrollFactor; if (invokeCount % unrollFactor != 0) throw new ArgumentOutOfRangeException(nameof(data), $"InvokeCount({invokeCount}) should be a multiple of UnrollFactor({unrollFactor})."); - long totalOperations = invokeCount * OperationsPerInvoke; - bool isOverhead = data.IterationMode == IterationMode.Overhead; - bool randomizeMemory = !isOverhead && MemoryRandomization; - var action = isOverhead ? OverheadAction : WorkloadAction; + long totalOperations = invokeCount * Parameters.OperationsPerInvoke; + bool randomizeMemory = data.mode == IterationMode.Workload && MemoryRandomization; - if (!isOverhead) - IterationSetupAction(); + data.setupAction(); GcCollect(); - if (EngineEventSource.Log.IsEnabled()) - EngineEventSource.Log.IterationStart(data.IterationMode, data.IterationStage, totalOperations); + if (EngineEventSource.Log.IsEnabled() && data.mode != IterationMode.Dummy) + EngineEventSource.Log.IterationStart(data.mode, data.stage, totalOperations); var clockSpan = randomizeMemory - ? MeasureWithRandomStack(action, invokeCount / unrollFactor) - : Measure(action, invokeCount / unrollFactor); + ? MeasureWithRandomStack(data.workloadAction, invokeCount / unrollFactor) + : Measure(data.workloadAction, invokeCount / unrollFactor); - if (EngineEventSource.Log.IsEnabled()) - EngineEventSource.Log.IterationStop(data.IterationMode, data.IterationStage, totalOperations); + if (EngineEventSource.Log.IsEnabled() && data.mode != IterationMode.Dummy) + EngineEventSource.Log.IterationStop(data.mode, data.stage, totalOperations); - if (!isOverhead) - IterationCleanupAction(); + data.cleanupAction(); if (randomizeMemory) RandomizeManagedHeapMemory(); @@ -188,11 +166,11 @@ public Measurement RunIteration(IterationData data) GcCollect(); // Results - var measurement = new Measurement(0, data.IterationMode, data.IterationStage, data.Index, totalOperations, clockSpan.GetNanoseconds()); - WriteLine(measurement.ToString()); - if (measurement.IterationStage == IterationStage.Jitting) - jittingMeasurements.Add(measurement); - + var measurement = new Measurement(0, data.mode, data.stage, data.index, totalOperations, clockSpan.GetNanoseconds()); + if (data.mode != IterationMode.Dummy) + { + Host.WriteLine(measurement.ToString()); + } return measurement; } @@ -208,7 +186,7 @@ private unsafe ClockSpan MeasureWithRandomStack(Action action, long invoke return clockSpan; } - [MethodImpl(MethodImplOptions.NoInlining)] + [MethodImpl(MethodImplOptions.NoInlining | CodeGenHelper.AggressiveOptimizationOption)] private unsafe void Consume(byte* _) { } [MethodImpl(MethodImplOptions.NoInlining | CodeGenHelper.AggressiveOptimizationOption)] @@ -219,45 +197,39 @@ private ClockSpan Measure(Action action, long invokeCount) return clock.GetElapsed(); } + [MethodImpl(CodeGenHelper.AggressiveOptimizationOption)] private (GcStats, ThreadingStats, double) GetExtraStats(IterationData data) { // Warm up the measurement functions before starting the actual measurement. DeadCodeEliminationHelper.KeepAliveWithoutBoxing(GcStats.ReadInitial()); DeadCodeEliminationHelper.KeepAliveWithoutBoxing(GcStats.ReadFinal()); - IterationSetupAction(); // we run iteration setup first, so even if it allocates, it is not included in the results + data.setupAction(); // we run iteration setup first, so even if it allocates, it is not included in the results var initialThreadingStats = ThreadingStats.ReadInitial(); // this method might allocate var exceptionsStats = new ExceptionsStats(); // allocates exceptionsStats.StartListening(); // this method might allocate -#if !NET7_0_OR_GREATER - if (RuntimeInformation.IsNetCore && Environment.Version.Major is >= 3 and <= 6 && RuntimeInformation.IsTieredJitEnabled) - { - // #1542 - // We put the current thread to sleep so tiered jit can kick in, compile its stuff, - // and NOT allocate anything on the background thread when we are measuring allocations. - // This is only an issue on netcoreapp3.0 to net6.0. Tiered jit allocations were "fixed" in net7.0 - // (maybe not completely eliminated forever, but at least reduced to a point where measurements are much more stable), - // and netcoreapp2.X uses only GetAllocatedBytesForCurrentThread which doesn't capture the tiered jit allocations. - Thread.Sleep(TimeSpan.FromMilliseconds(500)); - } -#endif - // GC collect before measuring allocations. ForceGcCollect(); + + // #1542 + // If the jit is tiered, we put the current thread to sleep so it can kick in, compile its stuff, + // and NOT allocate anything on the background thread when we are measuring allocations. + SleepHelper.SleepIfPositive(JitInfo.BackgroundCompilationDelay); + GcStats gcStats; using (FinalizerBlocker.MaybeStart()) { - gcStats = MeasureWithGc(data.InvokeCount / data.UnrollFactor); + gcStats = MeasureWithGc(data.workloadAction, data.invokeCount / data.unrollFactor); } exceptionsStats.Stop(); // this method might (de)allocate var finalThreadingStats = ThreadingStats.ReadFinal(); - IterationCleanupAction(); // we run iteration cleanup after collecting GC stats + data.cleanupAction(); // we run iteration cleanup after collecting GC stats - var totalOperationsCount = data.InvokeCount * OperationsPerInvoke; + var totalOperationsCount = data.invokeCount * Parameters.OperationsPerInvoke; return (gcStats.WithTotalOperations(totalOperationsCount), (finalThreadingStats - initialThreadingStats).WithTotalOperations(totalOperationsCount), exceptionsStats.ExceptionsCount / (double)totalOperationsCount); @@ -265,25 +237,26 @@ private ClockSpan Measure(Action action, long invokeCount) // Isolate the allocation measurement and skip tier0 jit to make sure we don't get any unexpected allocations. [MethodImpl(MethodImplOptions.NoInlining | CodeGenHelper.AggressiveOptimizationOption)] - private GcStats MeasureWithGc(long invokeCount) + private GcStats MeasureWithGc(Action action, long invokeCount) { var initialGcStats = GcStats.ReadInitial(); - WorkloadAction(invokeCount); + action(invokeCount); var finalGcStats = GcStats.ReadFinal(); return finalGcStats - initialGcStats; } + [MethodImpl(CodeGenHelper.AggressiveOptimizationOption)] private void RandomizeManagedHeapMemory() { // invoke global cleanup before global setup - GlobalCleanupAction?.Invoke(); + Parameters.GlobalCleanupAction.Invoke(); var gen0object = new byte[random.Next(32)]; var lohObject = new byte[85 * 1024 + random.Next(32)]; // we expect the key allocations to happen in global setup (not ctor) // so we call it while keeping the random-size objects alive - GlobalSetupAction?.Invoke(); + Parameters.GlobalSetupAction.Invoke(); GC.KeepAlive(gen0object); GC.KeepAlive(lohObject); @@ -291,6 +264,7 @@ private void RandomizeManagedHeapMemory() // we don't enforce GC.Collects here as engine does it later anyway } + [MethodImpl(CodeGenHelper.AggressiveOptimizationOption)] private void GcCollect() { if (!ForceGcCleanups) @@ -299,6 +273,7 @@ private void GcCollect() ForceGcCollect(); } + [MethodImpl(CodeGenHelper.AggressiveOptimizationOption)] internal static void ForceGcCollect() { GC.Collect(); @@ -306,10 +281,6 @@ internal static void ForceGcCollect() GC.Collect(); } - public void WriteLine(string text) => Host.WriteLine(text); - - public void WriteLine() => Host.WriteLine(); - [UsedImplicitly] public static class Signals { @@ -354,6 +325,7 @@ private sealed class Impl private readonly object hangLock = new(); private readonly ManualResetEventSlim enteredFinalizerEvent = new(false); + [MethodImpl(CodeGenHelper.AggressiveOptimizationOption)] ~Impl() { lock (hangLock) @@ -363,7 +335,7 @@ private sealed class Impl } } - [MethodImpl(MethodImplOptions.NoInlining)] + [MethodImpl(MethodImplOptions.NoInlining | CodeGenHelper.AggressiveOptimizationOption)] internal static (object hangLock, ManualResetEventSlim enteredFinalizerEvent) CreateWeakly() { var impl = new Impl(); @@ -371,6 +343,7 @@ internal static (object hangLock, ManualResetEventSlim enteredFinalizerEvent) Cr } } + [MethodImpl(CodeGenHelper.AggressiveOptimizationOption)] internal static FinalizerBlocker MaybeStart() { if (Environment.GetEnvironmentVariable(UnitTestBlockFinalizerEnvKey) != UnitTestBlockFinalizerEnvValue) @@ -387,6 +360,7 @@ internal static FinalizerBlocker MaybeStart() return new FinalizerBlocker(hangLock); } + [MethodImpl(CodeGenHelper.AggressiveOptimizationOption)] public void Dispose() { if (hangLock is not null) diff --git a/src/BenchmarkDotNet/Engines/EngineActualStage.cs b/src/BenchmarkDotNet/Engines/EngineActualStage.cs index 173f4e3432..bd4242bda0 100644 --- a/src/BenchmarkDotNet/Engines/EngineActualStage.cs +++ b/src/BenchmarkDotNet/Engines/EngineActualStage.cs @@ -1,6 +1,5 @@ using System; using System.Collections.Generic; -using BenchmarkDotNet.Characteristics; using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Mathematics; using BenchmarkDotNet.Reports; @@ -9,22 +8,31 @@ namespace BenchmarkDotNet.Engines { - internal abstract class EngineActualStage(IterationMode iterationMode) : EngineStage(IterationStage.Actual, iterationMode) + internal abstract class EngineActualStage(IterationMode iterationMode, long invokeCount, int unrollFactor, EngineParameters parameters) : EngineStage(IterationStage.Actual, iterationMode, parameters) { internal const int MaxOverheadIterationCount = 20; - internal static EngineActualStage GetOverhead(IEngine engine) - => new EngineActualStageAuto(engine.TargetJob, engine.Resolver, IterationMode.Overhead); + internal readonly long invokeCount = invokeCount; + internal readonly int unrollFactor = unrollFactor; - internal static EngineActualStage GetWorkload(IEngine engine, RunStrategy strategy) + internal static EngineActualStage GetOverhead(long invokeCount, int unrollFactor, EngineParameters parameters) + => new EngineActualStageAuto(IterationMode.Overhead, invokeCount, unrollFactor, parameters); + + internal static EngineActualStage GetWorkload(RunStrategy strategy, long invokeCount, int unrollFactor, EngineParameters parameters) { - var targetJob = engine.TargetJob; + var targetJob = parameters.TargetJob; int? iterationCount = targetJob.ResolveValueAsNullable(RunMode.IterationCountCharacteristic); const int DefaultWorkloadCount = 10; return iterationCount == null && strategy != RunStrategy.Monitoring - ? new EngineActualStageAuto(targetJob, engine.Resolver, IterationMode.Workload) - : new EngineActualStageSpecific(iterationCount ?? DefaultWorkloadCount, IterationMode.Workload); + ? new EngineActualStageAuto(IterationMode.Workload, invokeCount, unrollFactor, parameters) + : new EngineActualStageSpecific(iterationCount ?? DefaultWorkloadCount, IterationMode.Workload, invokeCount, unrollFactor, parameters); } + + protected IterationData GetIterationData() + => new(Mode, Stage, ++iterationIndex, invokeCount, unrollFactor, parameters.IterationSetupAction, parameters.IterationCleanupAction, + Mode == IterationMode.Workload + ? unrollFactor == 1 ? parameters.WorkloadActionNoUnroll : parameters.WorkloadActionUnroll + : unrollFactor == 1 ? parameters.OverheadActionNoUnroll : parameters.OverheadActionUnroll); } internal sealed class EngineActualStageAuto : EngineActualStage @@ -37,22 +45,23 @@ internal sealed class EngineActualStageAuto : EngineActualStage private readonly List measurementsForStatistics; private int iterationCounter = 0; - public EngineActualStageAuto(Job targetJob, IResolver resolver, IterationMode iterationMode) : base(iterationMode) + public EngineActualStageAuto(IterationMode iterationMode, long invokeCount, int unrollFactor, EngineParameters parameters) : base(iterationMode, invokeCount, unrollFactor, parameters) { - maxRelativeError = targetJob.ResolveValue(AccuracyMode.MaxRelativeErrorCharacteristic, resolver); - maxAbsoluteError = targetJob.ResolveValueAsNullable(AccuracyMode.MaxAbsoluteErrorCharacteristic); - outlierMode = targetJob.ResolveValue(AccuracyMode.OutlierModeCharacteristic, resolver); - minIterationCount = targetJob.ResolveValue(RunMode.MinIterationCountCharacteristic, resolver); - maxIterationCount = targetJob.ResolveValue(RunMode.MaxIterationCountCharacteristic, resolver); + maxRelativeError = parameters.TargetJob.ResolveValue(AccuracyMode.MaxRelativeErrorCharacteristic, parameters.Resolver); + maxAbsoluteError = parameters.TargetJob.ResolveValueAsNullable(AccuracyMode.MaxAbsoluteErrorCharacteristic); + outlierMode = parameters.TargetJob.ResolveValue(AccuracyMode.OutlierModeCharacteristic, parameters.Resolver); + minIterationCount = parameters.TargetJob.ResolveValue(RunMode.MinIterationCountCharacteristic, parameters.Resolver); + maxIterationCount = parameters.TargetJob.ResolveValue(RunMode.MaxIterationCountCharacteristic, parameters.Resolver); measurementsForStatistics = GetMeasurementList(); } internal override List GetMeasurementList() => new(maxIterationCount); - internal override bool GetShouldRunIteration(List measurements, ref long invokeCount) + internal override bool GetShouldRunIteration(List measurements, out IterationData iterationData) { if (measurements.Count == 0) { + iterationData = GetIterationData(); return true; } @@ -72,25 +81,36 @@ internal override bool GetShouldRunIteration(List measurements, ref if (iterationCounter >= minIterationCount && actualError < maxError) { + iterationData = default; return false; } if (iterationCounter >= maxIterationCount || isOverhead && iterationCounter >= MaxOverheadIterationCount) { + iterationData = default; return false; } + iterationData = GetIterationData(); return true; } } - internal sealed class EngineActualStageSpecific(int maxIterationCount, IterationMode iterationMode) : EngineActualStage(iterationMode) + internal sealed class EngineActualStageSpecific(int maxIterationCount, IterationMode iterationMode, long invokeCount, int unrollFactor, EngineParameters parameters) + : EngineActualStage(iterationMode, invokeCount, unrollFactor, parameters) { - private int iterationCount = 0; - internal override List GetMeasurementList() => new(maxIterationCount); - internal override bool GetShouldRunIteration(List measurements, ref long invokeCount) - => ++iterationCount <= maxIterationCount; + internal override bool GetShouldRunIteration(List measurements, out IterationData iterationData) + { + if (iterationIndex < maxIterationCount) + { + iterationData = GetIterationData(); + return true; + } + + iterationData = default; + return false; + } } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Engines/EngineFactory.cs b/src/BenchmarkDotNet/Engines/EngineFactory.cs index 0588218522..2e5c7774b0 100644 --- a/src/BenchmarkDotNet/Engines/EngineFactory.cs +++ b/src/BenchmarkDotNet/Engines/EngineFactory.cs @@ -1,130 +1,15 @@ -using System; -using BenchmarkDotNet.Jobs; -using Perfolizer.Horology; - namespace BenchmarkDotNet.Engines { public class EngineFactory : IEngineFactory { public IEngine CreateReadyToRun(EngineParameters engineParameters) { - if (engineParameters.WorkloadActionNoUnroll == null) - throw new ArgumentNullException(nameof(engineParameters.WorkloadActionNoUnroll)); - if (engineParameters.WorkloadActionUnroll == null) - throw new ArgumentNullException(nameof(engineParameters.WorkloadActionUnroll)); - if (engineParameters.Dummy1Action == null) - throw new ArgumentNullException(nameof(engineParameters.Dummy1Action)); - if (engineParameters.Dummy2Action == null) - throw new ArgumentNullException(nameof(engineParameters.Dummy2Action)); - if (engineParameters.Dummy3Action == null) - throw new ArgumentNullException(nameof(engineParameters.Dummy3Action)); - if (engineParameters.OverheadActionNoUnroll == null) - throw new ArgumentNullException(nameof(engineParameters.OverheadActionNoUnroll)); - if (engineParameters.OverheadActionUnroll == null) - throw new ArgumentNullException(nameof(engineParameters.OverheadActionUnroll)); - if (engineParameters.TargetJob == null) - throw new ArgumentNullException(nameof(engineParameters.TargetJob)); - - engineParameters.GlobalSetupAction?.Invoke(); // whatever the settings are, we MUST call global setup here, the global cleanup is part of Engine's Dispose - - if (!engineParameters.NeedsJitting) // just create the engine, do NOT jit - return CreateMultiActionEngine(engineParameters); - - int jitIndex = 0; - - if (engineParameters.HasInvocationCount || engineParameters.HasUnrollFactor) // it's a job with explicit configuration, just create the engine and jit it - { - var warmedUpMultiActionEngine = CreateMultiActionEngine(engineParameters); - - DeadCodeEliminationHelper.KeepAliveWithoutBoxing(Jit(warmedUpMultiActionEngine, ++jitIndex, invokeCount: engineParameters.UnrollFactor, unrollFactor: engineParameters.UnrollFactor)); - - return warmedUpMultiActionEngine; - } - - var singleActionEngine = CreateSingleActionEngine(engineParameters); - var singleInvocationTime = Jit(singleActionEngine, ++jitIndex, invokeCount: 1, unrollFactor: 1); - double timesPerIteration = engineParameters.IterationTime / singleInvocationTime; // how many times can we run given benchmark per iteration - - if ((timesPerIteration < 1.5) && (singleInvocationTime < TimeInterval.FromSeconds(10.0))) - { - // if the Jitting took more than IterationTime/1.5 but still less than 10s (a magic number based on observations of reported bugs) - // we call it one more time to see if Jitting itself has not dominated the first invocation - // if it did, it should NOT be a single invocation engine (see #837, #1337, #1338, and #1780) - singleInvocationTime = Jit(singleActionEngine, ++jitIndex, invokeCount: 1, unrollFactor: 1); - timesPerIteration = engineParameters.IterationTime / singleInvocationTime; - } - - // executing once takes longer than iteration time => long running benchmark, needs no pilot and no overhead - // Or executing twice would put us well past the iteration time ==> needs no pilot and no overhead - if (timesPerIteration < 1.5) - return singleActionEngine; - - int defaultUnrollFactor = Job.Default.ResolveValue(RunMode.UnrollFactorCharacteristic, EngineParameters.DefaultResolver); - int roundedUpTimesPerIteration = (int)Math.Ceiling(timesPerIteration); - - if (roundedUpTimesPerIteration < defaultUnrollFactor) // if we run it defaultUnrollFactor times per iteration, it's going to take longer than IterationTime - { - var needsPilot = engineParameters.TargetJob - .WithUnrollFactor(1) // we don't want to use unroll factor! - .WithMinInvokeCount(2) // the minimum is 2 (not the default 4 which can be too much and not 1 which we already know is not enough) - .WithEvaluateOverhead(false); // it's something very time consuming, it overhead is too small compared to total time - - return CreateEngine(engineParameters, needsPilot, engineParameters.OverheadActionNoUnroll, engineParameters.WorkloadActionNoUnroll); - } - - var multiActionEngine = CreateMultiActionEngine(engineParameters); - - DeadCodeEliminationHelper.KeepAliveWithoutBoxing(Jit(multiActionEngine, ++jitIndex, invokeCount: defaultUnrollFactor, unrollFactor: defaultUnrollFactor)); - - return multiActionEngine; - } - - /// the time it took to run the benchmark - private static TimeInterval Jit(Engine engine, int jitIndex, int invokeCount, int unrollFactor) - { - engine.Dummy1Action.Invoke(); - - DeadCodeEliminationHelper.KeepAliveWithoutBoxing(engine.RunIteration(new IterationData(IterationMode.Overhead, IterationStage.Jitting, jitIndex, invokeCount, unrollFactor))); // don't forget to JIT idle + var engine = new Engine(engineParameters); - engine.Dummy2Action.Invoke(); + // TODO: Move GlobalSetup/Cleanup to Engine.Run. + engine.Parameters.GlobalSetupAction.Invoke(); // whatever the settings are, we MUST call global setup here, the global cleanup is part of Engine's Dispose - var result = engine.RunIteration(new IterationData(IterationMode.Workload, IterationStage.Jitting, jitIndex, invokeCount, unrollFactor)); - - engine.Dummy3Action.Invoke(); - - engine.WriteLine(); - - return TimeInterval.FromNanoseconds(result.Nanoseconds); + return engine; } - - private static Engine CreateMultiActionEngine(EngineParameters engineParameters) - => CreateEngine(engineParameters, engineParameters.TargetJob, engineParameters.OverheadActionUnroll, engineParameters.WorkloadActionUnroll); - - private static Engine CreateSingleActionEngine(EngineParameters engineParameters) - => CreateEngine(engineParameters, - engineParameters.TargetJob - .WithInvocationCount(1).WithUnrollFactor(1) // run the benchmark exactly once per iteration - .WithEvaluateOverhead(false), // it's something very time consuming, it overhead is too small compared to total time - // todo: consider if we should set the warmup count to 2 - engineParameters.OverheadActionNoUnroll, - engineParameters.WorkloadActionNoUnroll); - - private static Engine CreateEngine(EngineParameters engineParameters, Job job, Action idle, Action main) - => new Engine( - engineParameters.Host, - EngineParameters.DefaultResolver, - engineParameters.Dummy1Action, - engineParameters.Dummy2Action, - engineParameters.Dummy3Action, - idle, - main, - job, - engineParameters.GlobalSetupAction, - engineParameters.GlobalCleanupAction, - engineParameters.IterationSetupAction, - engineParameters.IterationCleanupAction, - engineParameters.OperationsPerInvoke, - engineParameters.MeasureExtraStats, - engineParameters.BenchmarkName); } } diff --git a/src/BenchmarkDotNet/Engines/EngineJitStage.cs b/src/BenchmarkDotNet/Engines/EngineJitStage.cs new file mode 100644 index 0000000000..8bf626328c --- /dev/null +++ b/src/BenchmarkDotNet/Engines/EngineJitStage.cs @@ -0,0 +1,189 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using BenchmarkDotNet.Helpers; +using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Portability; +using BenchmarkDotNet.Reports; +using Perfolizer.Horology; + +namespace BenchmarkDotNet.Engines +{ + internal abstract class EngineJitStage(EngineParameters parameters) : EngineStage(IterationStage.Jitting, IterationMode.Workload, parameters) + { + protected readonly Action dummy1Action= _ => parameters.Dummy1Action(); + protected readonly Action dummy2Action= _ => parameters.Dummy2Action(); + protected readonly Action dummy3Action = _ => parameters.Dummy3Action(); + + protected IterationData GetDummyIterationData(Action dummyAction) + => new(IterationMode.Dummy, IterationStage.Jitting, iterationIndex, 1, 1, () => { }, () => { }, dummyAction); + } + + // We do our best to encourage the jit to fully promote methods to tier1, but tiered jit relies on heuristics, + // and we purposefully don't spend too much time in this stage, so we can't guarantee it. + // This should succeed for 99%+ of microbenchmarks. For any sufficiently short benchmarks where this fails, + // the following stages (Pilot and Warmup) will likely take it the rest of the way. Long-running benchmarks may never fully reach tier1. + internal sealed class EngineFirstJitStage : EngineJitStage + { + // It is not worth spending a long time in jit stage for macro-benchmarks. + private static readonly TimeInterval MaxTieringTime = TimeInterval.FromSeconds(10); + + // Jit call counting delay is only for when the app starts up. We don't need to wait for every benchmark if multiple benchmarks are ran in-process. + private static TimeSpan s_tieredDelay = JitInfo.TieredDelay; + + internal bool didStopEarly = false; + internal Measurement lastMeasurement; + + private readonly IEnumerator enumerator; + private readonly bool evaluateOverhead; + + internal EngineFirstJitStage(bool evaluateOverhead, EngineParameters parameters) : base(parameters) + { + enumerator = EnumerateIterations(); + this.evaluateOverhead = evaluateOverhead; + } + + internal override List GetMeasurementList() => new(GetMaxMeasurementCount()); + + private int GetMaxMeasurementCount() + { + int count = JitInfo.IsTiered + ? JitInfo.MaxTierPromotions * JitInfo.TieredCallCountThreshold + 2 + : 1; + if (evaluateOverhead) + { + count *= 2; + } + return count; + } + + internal override bool GetShouldRunIteration(List measurements, out IterationData iterationData) + { + if (measurements.Count > 0) + { + var measurement = measurements[measurements.Count - 1]; + if (measurement.IterationMode == IterationMode.Workload) + { + lastMeasurement = measurement; + } + } + if (enumerator.MoveNext()) + { + iterationData = enumerator.Current; + return true; + } + enumerator.Dispose(); + iterationData = default; + return false; + } + + private IEnumerator EnumerateIterations() + { + ++iterationIndex; + if (evaluateOverhead) + { + yield return GetDummyIterationData(dummy1Action); + yield return GetOverheadIterationData(1); + } + yield return GetDummyIterationData(dummy2Action); + yield return GetWorkloadIterationData(1); + yield return GetDummyIterationData(dummy3Action); + + // If the jit is not tiered, we're done. + if (!JitInfo.IsTiered) + { + yield break; + } + + // Wait enough time for jit call counting to begin. + SleepHelper.SleepIfPositive(s_tieredDelay); + // Don't make the next jit stage wait if it's ran in the same process. + s_tieredDelay = TimeSpan.Zero; + + // Attempt to promote methods to tier1, but don't spend too much time in jit stage. + StartedClock startedClock = parameters.TargetJob.ResolveValue(InfrastructureMode.ClockCharacteristic, parameters.Resolver).Start(); + + int remainingTiers = JitInfo.MaxTierPromotions; + int lastInvokeCount = 1; + while (remainingTiers > 0) + { + --remainingTiers; + int remainingCalls = JitInfo.TieredCallCountThreshold; + while (remainingCalls > 0) + { + // If we can run one batch of calls within the time limit (based on the last measurement), do that instead of multiple single-invocation iterations. + var remainingTimeLimit = MaxTieringTime.ToNanoseconds() - startedClock.GetElapsed().GetNanoseconds(); + var lastMeasurementSingleInvocationTime = lastMeasurement.Nanoseconds / lastInvokeCount; + int allowedCallsWithinTimeLimit = (int) Math.Floor(remainingTimeLimit / lastMeasurementSingleInvocationTime); + int invokeCount = allowedCallsWithinTimeLimit > 0 + ? Math.Min(remainingCalls, allowedCallsWithinTimeLimit) + : 1; + lastInvokeCount = invokeCount; + + remainingCalls -= invokeCount; + ++iterationIndex; + if (evaluateOverhead) + { + yield return GetOverheadIterationData(invokeCount); + } + yield return GetWorkloadIterationData(invokeCount); + + if ((remainingTiers + remainingCalls) > 0 + && startedClock.GetElapsed().GetTimeValue() >= MaxTieringTime) + { + didStopEarly = true; + yield break; + } + } + + SleepHelper.SleepIfPositive(JitInfo.BackgroundCompilationDelay); + } + + // Empirical evidence shows that the first call after the method is tiered up may take longer, + // so we run an extra iteration to ensure the next stage gets a stable measurement. + ++iterationIndex; + if (evaluateOverhead) + { + yield return GetOverheadIterationData(1); + } + yield return GetWorkloadIterationData(1); + } + + private IterationData GetOverheadIterationData(long invokeCount) + => new(IterationMode.Overhead, IterationStage.Jitting, iterationIndex, invokeCount, 1, () => { }, () => { }, parameters.OverheadActionNoUnroll); + + private IterationData GetWorkloadIterationData(long invokeCount) + => new(IterationMode.Workload, IterationStage.Jitting, iterationIndex, invokeCount, 1, parameters.IterationSetupAction, parameters.IterationCleanupAction, parameters.WorkloadActionNoUnroll); + } + + internal sealed class EngineSecondJitStage : EngineJitStage + { + private readonly int unrollFactor; + private readonly bool evaluateOverhead; + + public EngineSecondJitStage(int unrollFactor, bool evaluateOverhead, EngineParameters parameters) : base(parameters) + { + this.unrollFactor = unrollFactor; + this.evaluateOverhead = evaluateOverhead; + iterationIndex = evaluateOverhead ? 0 : 2; + } + + internal override List GetMeasurementList() => new(evaluateOverhead ? 2 : 1); + + // The benchmark method has already been jitted via *NoUnroll, we only need to jit the *Unroll methods here, which aren't tiered. + internal override bool GetShouldRunIteration(List measurements, out IterationData iterationData) + { + iterationData = ++iterationIndex switch + { + 1 => GetDummyIterationData(dummy1Action), + 2 => new(IterationMode.Overhead, IterationStage.Jitting, 1, unrollFactor, unrollFactor, () => { }, () => { }, parameters.OverheadActionUnroll), + 3 => GetDummyIterationData(dummy2Action), + // IterationSetup/Cleanup are only used for *NoUnroll benchmarks + 4 => new(IterationMode.Workload, IterationStage.Jitting, 1, unrollFactor, unrollFactor, () => { }, () => { }, parameters.WorkloadActionUnroll), + 5 => GetDummyIterationData(dummy3Action), + _ => default + }; + return iterationIndex <= 5; + } + } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Engines/EngineParameters.cs b/src/BenchmarkDotNet/Engines/EngineParameters.cs index ec61582529..681b7e9250 100644 --- a/src/BenchmarkDotNet/Engines/EngineParameters.cs +++ b/src/BenchmarkDotNet/Engines/EngineParameters.cs @@ -2,8 +2,6 @@ using BenchmarkDotNet.Characteristics; using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Running; -using JetBrains.Annotations; -using Perfolizer.Horology; namespace BenchmarkDotNet.Engines { @@ -11,6 +9,7 @@ public class EngineParameters { public static readonly IResolver DefaultResolver = new CompositeResolver(BenchmarkRunnerClean.DefaultResolver, EngineResolver.Instance); + public IResolver Resolver { get; set; } = DefaultResolver; public IHost Host { get; set; } public Action WorkloadActionNoUnroll { get; set; } public Action WorkloadActionUnroll { get; set; } @@ -26,17 +25,6 @@ public class EngineParameters public Action IterationSetupAction { get; set; } public Action IterationCleanupAction { get; set; } public bool MeasureExtraStats { get; set; } - - [PublicAPI] public string BenchmarkName { get; set; } - - public bool NeedsJitting => TargetJob.ResolveValue(RunMode.RunStrategyCharacteristic, DefaultResolver).NeedsJitting(); - - public bool HasInvocationCount => TargetJob.HasValue(RunMode.InvocationCountCharacteristic); - - public bool HasUnrollFactor => TargetJob.HasValue(RunMode.UnrollFactorCharacteristic); - - public int UnrollFactor => TargetJob.ResolveValue(RunMode.UnrollFactorCharacteristic, DefaultResolver); - - public TimeInterval IterationTime => TargetJob.ResolveValue(RunMode.IterationTimeCharacteristic, DefaultResolver); + public string BenchmarkName { get; set; } } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Engines/EnginePilotStage.cs b/src/BenchmarkDotNet/Engines/EnginePilotStage.cs index 43b543d5c1..0324eb12ad 100644 --- a/src/BenchmarkDotNet/Engines/EnginePilotStage.cs +++ b/src/BenchmarkDotNet/Engines/EnginePilotStage.cs @@ -1,6 +1,5 @@ using System; using System.Collections.Generic; -using BenchmarkDotNet.Characteristics; using BenchmarkDotNet.Environments; using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Reports; @@ -9,40 +8,92 @@ namespace BenchmarkDotNet.Engines { // TODO: use clockResolution - internal abstract class EnginePilotStage(Job targetJob, IResolver resolver) : EngineStage(IterationStage.Pilot, IterationMode.Workload) + internal abstract class EnginePilotStage(long invokeCount, int unrollFactor, int minInvokeCount, EngineParameters parameters) : EngineStage(IterationStage.Pilot, IterationMode.Workload, parameters) { internal const long MaxInvokeCount = (long.MaxValue / 2 + 1) / 2; - protected readonly int unrollFactor = targetJob.ResolveValue(RunMode.UnrollFactorCharacteristic, resolver); - protected readonly int minInvokeCount = targetJob.ResolveValue(AccuracyMode.MinInvokeCountCharacteristic, resolver); + internal long invokeCount = invokeCount; + internal int unrollFactor = unrollFactor; + internal int minInvokeCount = minInvokeCount; + + internal override List GetMeasurementList() => []; + + internal static EnginePilotStage GetStage(long invokeCount, int unrollFactor, int minInvokeCount, EngineParameters parameters) + // Here we want to guess "perfect" amount of invocation + => parameters.TargetJob.HasValue(RunMode.IterationTimeCharacteristic) + ? new EnginePilotStageSpecific(invokeCount, unrollFactor, minInvokeCount, parameters) + : new EnginePilotStageAuto(invokeCount, unrollFactor, minInvokeCount, parameters); protected long Autocorrect(long count) => (count + unrollFactor - 1) / unrollFactor * unrollFactor; - internal static EnginePilotStage GetStage(IEngine engine) - { - var targetJob = engine.TargetJob; - // If InvocationCount is specified, pilot stage should be skipped - return targetJob.HasValue(RunMode.InvocationCountCharacteristic) ? null - // Here we want to guess "perfect" amount of invocation - : targetJob.HasValue(RunMode.IterationTimeCharacteristic) ? new EnginePilotStageSpecific(targetJob, engine.Resolver) - : new EnginePilotStageAuto(targetJob, engine.Resolver); - } + protected IterationData GetIterationData() + => new(Mode, Stage, ++iterationIndex, invokeCount, unrollFactor, parameters.IterationSetupAction, parameters.IterationCleanupAction, + unrollFactor == 1 ? parameters.WorkloadActionNoUnroll : parameters.WorkloadActionUnroll); } - internal sealed class EnginePilotStageAuto(Job targetJob, IResolver resolver) : EnginePilotStage(targetJob, resolver) + internal sealed class EnginePilotStageInitial(long invokeCount, int unrollFactor, int minInvokeCount, EngineParameters parameters) : EnginePilotStage(invokeCount, unrollFactor, minInvokeCount, parameters) { - private readonly TimeInterval minIterationTime = targetJob.ResolveValue(AccuracyMode.MinIterationTimeCharacteristic, resolver); - private readonly double maxRelativeError = targetJob.ResolveValue(AccuracyMode.MaxRelativeErrorCharacteristic, resolver); - private readonly TimeInterval? maxAbsoluteError = targetJob.ResolveValueAsNullable(AccuracyMode.MaxAbsoluteErrorCharacteristic); - private readonly double resolution = targetJob.ResolveValue(InfrastructureMode.ClockCharacteristic, resolver).GetResolution().Nanoseconds; + internal bool evaluateOverhead = true; + internal bool needsFurtherPilot = true; internal override List GetMeasurementList() => []; - internal override bool GetShouldRunIteration(List measurements, ref long invokeCount) + internal override bool GetShouldRunIteration(List measurements, out IterationData iterationData) + { + if (measurements.Count == 0) + { + iterationData = new(Mode, Stage, ++iterationIndex, 1, 1, parameters.IterationSetupAction, parameters.IterationCleanupAction, parameters.WorkloadActionNoUnroll); + return true; + } + + CorrectValues(measurements[measurements.Count - 1]); + iterationData = default; + return false; + } + + internal void CorrectValues(Measurement measurement) + { + var iterationTime = parameters.TargetJob.ResolveValue(RunMode.IterationTimeCharacteristic, parameters.Resolver); + var singleInvokeNanoseconds = measurement.Nanoseconds * parameters.OperationsPerInvoke / measurement.Operations; + double timesPerIteration = iterationTime.Nanoseconds / singleInvokeNanoseconds; // how many times can we run given benchmark per iteration + // Executing once takes longer than iteration time -> long running benchmark, + // or executing twice would put us well past the iteration time. + if (timesPerIteration < 1.5) + { + invokeCount = 1; + unrollFactor = 1; + // It's very time consuming, overhead is too small compared to total time. + evaluateOverhead = false; + needsFurtherPilot = false; + return; + } + + int roundedUpTimesPerIteration = (int) Math.Ceiling(timesPerIteration); + // If we run it unrollFactor times per iteration, it's going to take longer than IterationTime. + if (roundedUpTimesPerIteration < unrollFactor) + { + unrollFactor = 1; + // The minimum is 2 (not the default 4 which can be too much and not 1 which we already know is not enough). + minInvokeCount = 2; + // It's very time consuming, overhead is too small compared to total time. + evaluateOverhead = false; + } + } + } + + internal sealed class EnginePilotStageAuto(long invokeCount, int unrollFactor, int minInvokeCount, EngineParameters parameters) : EnginePilotStage(invokeCount, unrollFactor, minInvokeCount, parameters) + { + private readonly TimeInterval minIterationTime = parameters.TargetJob.ResolveValue(AccuracyMode.MinIterationTimeCharacteristic, parameters.Resolver); + private readonly double maxRelativeError = parameters.TargetJob.ResolveValue(AccuracyMode.MaxRelativeErrorCharacteristic, parameters.Resolver); + private readonly TimeInterval? maxAbsoluteError = parameters.TargetJob.ResolveValueAsNullable(AccuracyMode.MaxAbsoluteErrorCharacteristic); + private readonly double resolution = parameters.TargetJob.ResolveValue(InfrastructureMode.ClockCharacteristic, parameters.Resolver).GetResolution().Nanoseconds; + + internal override bool GetShouldRunIteration(List measurements, out IterationData iterationData) { if (measurements.Count == 0) { invokeCount = Autocorrect(minInvokeCount); + iterationData = GetIterationData(); return true; } @@ -58,6 +109,7 @@ internal override bool GetShouldRunIteration(List measurements, ref bool isFinished = operationError < operationMaxError && iterationTime >= minIterationTime.Nanoseconds; if (isFinished || invokeCount >= MaxInvokeCount) { + iterationData = default; return false; } @@ -70,25 +122,23 @@ internal override bool GetShouldRunIteration(List measurements, ref invokeCount *= 2; } + iterationData = GetIterationData(); return true; } } - internal sealed class EnginePilotStageSpecific(Job targetJob, IResolver resolver) : EnginePilotStage(targetJob, resolver) + internal sealed class EnginePilotStageSpecific(long invokeCount, int unrollFactor, int minInvokeCount, EngineParameters parameters) : EnginePilotStage(invokeCount, unrollFactor, minInvokeCount, parameters) { - private const int MinInvokeCount = 4; - - private readonly double targetIterationTime = targetJob.ResolveValue(RunMode.IterationTimeCharacteristic, resolver).ToNanoseconds(); + private readonly double targetIterationTime = parameters.TargetJob.ResolveValue(RunMode.IterationTimeCharacteristic, parameters.Resolver).ToNanoseconds(); private int _downCount = 0; // Amount of iterations where newInvokeCount < invokeCount - internal override List GetMeasurementList() => []; - - internal override bool GetShouldRunIteration(List measurements, ref long invokeCount) + internal override bool GetShouldRunIteration(List measurements, out IterationData iterationData) { if (measurements.Count == 0) { - invokeCount = Autocorrect(MinInvokeCount); + invokeCount = Autocorrect(minInvokeCount); + iterationData = GetIterationData(); return true; } @@ -101,12 +151,16 @@ internal override bool GetShouldRunIteration(List measurements, ref _downCount++; } - if (Math.Abs(newInvokeCount - invokeCount) <= 1 || _downCount >= 3) + long diff = newInvokeCount - invokeCount; + if (_downCount >= 3 + || (diff != long.MinValue && Math.Abs(diff) <= 1)) { + iterationData = default; return false; } invokeCount = newInvokeCount; + iterationData = GetIterationData(); return true; } } diff --git a/src/BenchmarkDotNet/Engines/EngineStage.cs b/src/BenchmarkDotNet/Engines/EngineStage.cs index 82b6bc5fb3..8bd031112f 100644 --- a/src/BenchmarkDotNet/Engines/EngineStage.cs +++ b/src/BenchmarkDotNet/Engines/EngineStage.cs @@ -1,43 +1,91 @@ using System.Collections.Generic; using System.Runtime.CompilerServices; +using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Portability; using BenchmarkDotNet.Reports; namespace BenchmarkDotNet.Engines { - internal abstract class EngineStage(IterationStage stage, IterationMode mode) + internal abstract class EngineStage(IterationStage stage, IterationMode mode, EngineParameters parameters) { internal readonly IterationStage Stage = stage; internal readonly IterationMode Mode = mode; + protected readonly EngineParameters parameters = parameters; + protected int iterationIndex; internal abstract List GetMeasurementList(); - internal abstract bool GetShouldRunIteration(List measurements, ref long invokeCount); + internal abstract bool GetShouldRunIteration(List measurements, out IterationData iterationData); [MethodImpl(MethodImplOptions.NoInlining)] - internal static IEnumerable EnumerateStages(IEngine engine, RunStrategy strategy, bool evaluateOverhead) + internal static IEnumerable EnumerateStages(EngineParameters parameters) { - // It might be possible to add the jitting stage to this, but it's done in EngineFactory.CreateReadyToRun for now. + var strategy = parameters.TargetJob.ResolveValue(RunMode.RunStrategyCharacteristic, parameters.Resolver); + var invokeCount = parameters.TargetJob.ResolveValue(RunMode.InvocationCountCharacteristic, parameters.Resolver, 1); + var unrollFactor = parameters.TargetJob.ResolveValue(RunMode.UnrollFactorCharacteristic, parameters.Resolver); if (strategy != RunStrategy.ColdStart) { if (strategy != RunStrategy.Monitoring) { - var pilotStage = EnginePilotStage.GetStage(engine); - if (pilotStage != null) + // If InvocationCount is specified, pilot stage should be skipped + bool skipPilotStage = parameters.TargetJob.HasValue(RunMode.InvocationCountCharacteristic); + bool evaluateOverhead = parameters.TargetJob.ResolveValue(AccuracyMode.EvaluateOverheadCharacteristic, parameters.Resolver); + int minInvokeCount = parameters.TargetJob.ResolveValue(AccuracyMode.MinInvokeCountCharacteristic, parameters.Resolver); + + // AOT technically doesn't have a JIT, but we run jit stage regardless because of static constructors. #2004 + var jitStage = new EngineFirstJitStage(evaluateOverhead, parameters); + yield return jitStage; + + bool hasUnrollFactor = parameters.TargetJob.HasValue(RunMode.UnrollFactorCharacteristic); + if (!hasUnrollFactor && !skipPilotStage) + { + // Initial pilot stage adjusts unrollFactor from a single invocation. + var pilotStage = new EnginePilotStageInitial(invokeCount, unrollFactor, minInvokeCount, parameters); + // If the jit invocation was too time consuming, just correct the values without running another invocation. + if (jitStage.didStopEarly) + { + pilotStage.CorrectValues(jitStage.lastMeasurement); + } + else + { + yield return pilotStage; + } + + invokeCount = pilotStage.invokeCount; + unrollFactor = pilotStage.unrollFactor; + minInvokeCount = pilotStage.minInvokeCount; + evaluateOverhead &= pilotStage.evaluateOverhead; + skipPilotStage = !pilotStage.needsFurtherPilot; + } + + // The first jit stage only jitted *NoUnroll methods, now we need to jit *Unroll methods if they're going to be used. + // TODO: This stage can be removed after we refactor the engine/codegen to pass the clock into the delegates. + if (!RuntimeInformation.IsAot && unrollFactor != 1) { + yield return new EngineSecondJitStage(unrollFactor, evaluateOverhead, parameters); + } + + if (!skipPilotStage) + { + var pilotStage = EnginePilotStage.GetStage(invokeCount, unrollFactor, minInvokeCount, parameters); yield return pilotStage; + + invokeCount = pilotStage.invokeCount; } if (evaluateOverhead) { - yield return EngineWarmupStage.GetOverhead(); - yield return EngineActualStage.GetOverhead(engine); + yield return EngineWarmupStage.GetOverhead(invokeCount, unrollFactor, parameters); + yield return EngineActualStage.GetOverhead(invokeCount, unrollFactor, parameters); } } - yield return EngineWarmupStage.GetWorkload(engine, strategy); + yield return EngineWarmupStage.GetWorkload(strategy, invokeCount, unrollFactor, parameters); + + // TODO: restart pilot/warmup stages if some heuristic determines it's necessary (#2787, #1210). } - yield return EngineActualStage.GetWorkload(engine, strategy); + yield return EngineActualStage.GetWorkload(strategy, invokeCount, unrollFactor, parameters); } } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Engines/EngineWarmupStage.cs b/src/BenchmarkDotNet/Engines/EngineWarmupStage.cs index 822ea69101..8561309587 100644 --- a/src/BenchmarkDotNet/Engines/EngineWarmupStage.cs +++ b/src/BenchmarkDotNet/Engines/EngineWarmupStage.cs @@ -5,30 +5,37 @@ namespace BenchmarkDotNet.Engines { - internal abstract class EngineWarmupStage(IterationMode iterationMode) : EngineStage(IterationStage.Warmup, iterationMode) + internal abstract class EngineWarmupStage(IterationMode iterationMode, long invokeCount, int unrollFactor, EngineParameters parameters) : EngineStage(IterationStage.Warmup, iterationMode, parameters) { private const int MinOverheadIterationCount = 4; internal const int MaxOverheadIterationCount = 10; - internal static EngineWarmupStage GetOverhead() - => new EngineWarmupStageAuto(IterationMode.Overhead, MinOverheadIterationCount, MaxOverheadIterationCount); + internal static EngineWarmupStage GetOverhead(long invokeCount, int unrollFactor, EngineParameters parameters) + => new EngineWarmupStageAuto(IterationMode.Overhead, MinOverheadIterationCount, MaxOverheadIterationCount, invokeCount, unrollFactor, parameters); - internal static EngineWarmupStage GetWorkload(IEngine engine, RunStrategy runStrategy) + internal static EngineWarmupStage GetWorkload(RunStrategy runStrategy, long invokeCount, int unrollFactor, EngineParameters parameters) { - var job = engine.TargetJob; + var job = parameters.TargetJob; var count = job.ResolveValueAsNullable(RunMode.WarmupCountCharacteristic); if (count.HasValue && count.Value != EngineResolver.ForceAutoWarmup || runStrategy == RunStrategy.Monitoring) { - return new EngineWarmupStageSpecific(count ?? 0, IterationMode.Workload); + return new EngineWarmupStageSpecific(count ?? 0, IterationMode.Workload, invokeCount, unrollFactor, parameters); } - int minIterationCount = job.ResolveValue(RunMode.MinWarmupIterationCountCharacteristic, engine.Resolver); - int maxIterationCount = job.ResolveValue(RunMode.MaxWarmupIterationCountCharacteristic, engine.Resolver); - return new EngineWarmupStageAuto(IterationMode.Workload, minIterationCount, maxIterationCount); + int minIterationCount = job.ResolveValue(RunMode.MinWarmupIterationCountCharacteristic, parameters.Resolver); + int maxIterationCount = job.ResolveValue(RunMode.MaxWarmupIterationCountCharacteristic, parameters.Resolver); + return new EngineWarmupStageAuto(IterationMode.Workload, minIterationCount, maxIterationCount, invokeCount, unrollFactor, parameters); } + + protected IterationData GetIterationData() + => new(Mode, Stage, ++iterationIndex, invokeCount, unrollFactor, parameters.IterationSetupAction, parameters.IterationCleanupAction, + Mode == IterationMode.Workload + ? unrollFactor == 1 ? parameters.WorkloadActionNoUnroll : parameters.WorkloadActionUnroll + : unrollFactor == 1 ? parameters.OverheadActionNoUnroll: parameters.OverheadActionUnroll); } - internal sealed class EngineWarmupStageAuto(IterationMode iterationMode, int minIterationCount, int maxIterationCount) : EngineWarmupStage(iterationMode) + internal sealed class EngineWarmupStageAuto(IterationMode iterationMode, int minIterationCount, int maxIterationCount, long invokeCount, int unrollFactor, EngineParameters parameters) + : EngineWarmupStage(iterationMode, invokeCount, unrollFactor, parameters) { private const int MinFluctuationCount = 4; @@ -37,16 +44,18 @@ internal sealed class EngineWarmupStageAuto(IterationMode iterationMode, int min internal override List GetMeasurementList() => new(maxIterationCount); - internal override bool GetShouldRunIteration(List measurements, ref long invokeCount) + internal override bool GetShouldRunIteration(List measurements, out IterationData iterationData) { int n = measurements.Count; if (n >= maxIterationCount) { + iterationData = default; return false; } if (n < minIterationCount) { + iterationData = GetIterationData(); return true; } @@ -62,17 +71,26 @@ internal override bool GetShouldRunIteration(List measurements, ref } } + iterationData = GetIterationData(); return fluctuationCount < MinFluctuationCount; } } - internal sealed class EngineWarmupStageSpecific(int maxIterationCount, IterationMode iterationMode) : EngineWarmupStage(iterationMode) + internal sealed class EngineWarmupStageSpecific(int maxIterationCount, IterationMode iterationMode, long invokeCount, int unrollFactor, EngineParameters parameters) + : EngineWarmupStage(iterationMode, invokeCount, unrollFactor, parameters) { - private int iterationCount = 0; - internal override List GetMeasurementList() => new(maxIterationCount); - internal override bool GetShouldRunIteration(List measurements, ref long invokeCount) - => ++iterationCount <= maxIterationCount; + internal override bool GetShouldRunIteration(List measurements, out IterationData iterationData) + { + if (iterationIndex < maxIterationCount) + { + iterationData = GetIterationData(); + return true; + } + + iterationData = default; + return false; + } } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Engines/IEngine.cs b/src/BenchmarkDotNet/Engines/IEngine.cs index e35b870d8b..daba844023 100644 --- a/src/BenchmarkDotNet/Engines/IEngine.cs +++ b/src/BenchmarkDotNet/Engines/IEngine.cs @@ -1,36 +1,9 @@ using System; -using System.Diagnostics.CodeAnalysis; -using BenchmarkDotNet.Characteristics; -using BenchmarkDotNet.Jobs; -using BenchmarkDotNet.Reports; namespace BenchmarkDotNet.Engines { - [SuppressMessage("ReSharper", "UnusedMemberInSuper.Global")] public interface IEngine : IDisposable { - IHost Host { get; } - - void WriteLine(); - - void WriteLine(string line); - - Job TargetJob { get; } - - long OperationsPerInvoke { get; } - - Action? GlobalSetupAction { get; } - - Action? GlobalCleanupAction { get; } - - Action WorkloadAction { get; } - - Action OverheadAction { get; } - - IResolver Resolver { get; } - - Measurement RunIteration(IterationData data); - RunResults Run(); } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Engines/IterationData.cs b/src/BenchmarkDotNet/Engines/IterationData.cs index 2d73e58b68..f95f686922 100644 --- a/src/BenchmarkDotNet/Engines/IterationData.cs +++ b/src/BenchmarkDotNet/Engines/IterationData.cs @@ -1,20 +1,17 @@ -namespace BenchmarkDotNet.Engines +using System; + +namespace BenchmarkDotNet.Engines { - public struct IterationData + internal readonly struct IterationData(IterationMode iterationMode, IterationStage iterationStage, int index, long invokeCount, int unrollFactor, + Action setupAction, Action cleanupAction, Action workloadAction) { - public IterationMode IterationMode { get; } - public IterationStage IterationStage { get; } - public int Index { get; } - public long InvokeCount { get; } - public int UnrollFactor { get; } - - public IterationData(IterationMode iterationMode, IterationStage iterationStage, int index, long invokeCount, int unrollFactor) - { - IterationMode = iterationMode; - IterationStage = iterationStage; - Index = index; - InvokeCount = invokeCount; - UnrollFactor = unrollFactor; - } + public readonly IterationMode mode = iterationMode; + public readonly IterationStage stage = iterationStage; + public readonly int index = index; + public readonly long invokeCount = invokeCount; + public readonly int unrollFactor = unrollFactor; + public readonly Action setupAction = setupAction; + public readonly Action cleanupAction = cleanupAction; + public readonly Action workloadAction = workloadAction; } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Engines/IterationMode.cs b/src/BenchmarkDotNet/Engines/IterationMode.cs index a65b7faeb0..a58e646c2d 100644 --- a/src/BenchmarkDotNet/Engines/IterationMode.cs +++ b/src/BenchmarkDotNet/Engines/IterationMode.cs @@ -2,10 +2,9 @@ { public enum IterationMode { + Unknown, Overhead, - Workload, - - Unknown + Dummy, } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Engines/RunStrategyExtensions.cs b/src/BenchmarkDotNet/Engines/RunStrategyExtensions.cs deleted file mode 100644 index 1642436699..0000000000 --- a/src/BenchmarkDotNet/Engines/RunStrategyExtensions.cs +++ /dev/null @@ -1,7 +0,0 @@ -namespace BenchmarkDotNet.Engines -{ - public static class RunStrategyExtensions - { - public static bool NeedsJitting(this RunStrategy runStrategy) => runStrategy == RunStrategy.Throughput; - } -} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Environments/BenchmarkEnvironmentInfo.cs b/src/BenchmarkDotNet/Environments/BenchmarkEnvironmentInfo.cs index f2e2f34e26..18a4b006e9 100644 --- a/src/BenchmarkDotNet/Environments/BenchmarkEnvironmentInfo.cs +++ b/src/BenchmarkDotNet/Environments/BenchmarkEnvironmentInfo.cs @@ -35,8 +35,8 @@ protected BenchmarkEnvironmentInfo() Architecture = RuntimeInformation.GetArchitecture(); RuntimeVersion = RuntimeInformation.GetRuntimeVersion(); Configuration = RuntimeInformation.GetConfiguration(); - HasRyuJit = RuntimeInformation.HasRyuJit(); - JitInfo = RuntimeInformation.GetJitInfo(); + HasRyuJit = Portability.JitInfo.IsRyuJit; + JitInfo = Portability.JitInfo.GetInfo(); HardwareIntrinsicsShort = HardwareIntrinsics.GetShortInfo(); IsServerGC = GCSettings.IsServerGC; IsConcurrentGC = GCSettings.LatencyMode != GCLatencyMode.Batch; @@ -73,7 +73,7 @@ internal string GetRuntimeInfo() public static IEnumerable Validate(Job job) { - if (job.Environment.Jit == Jit.RyuJit && !RuntimeInformation.HasRyuJit()) + if (job.Environment.Jit == Jit.RyuJit && !Portability.JitInfo.IsRyuJit) yield return new ValidationError(true, "RyuJIT is requested but it is not available in current environment"); var currentRuntime = RuntimeInformation.GetCurrentRuntime(); if (job.Environment.Jit == Jit.LegacyJit && !(currentRuntime is ClrRuntime)) diff --git a/src/BenchmarkDotNet/Environments/EnvironmentResolver.cs b/src/BenchmarkDotNet/Environments/EnvironmentResolver.cs index 75762bf4bd..13369cc11f 100644 --- a/src/BenchmarkDotNet/Environments/EnvironmentResolver.cs +++ b/src/BenchmarkDotNet/Environments/EnvironmentResolver.cs @@ -17,7 +17,7 @@ private EnvironmentResolver() { Register(EnvironmentMode.PlatformCharacteristic, RuntimeInformation.GetCurrentPlatform); Register(EnvironmentMode.RuntimeCharacteristic, RuntimeInformation.GetCurrentRuntime); - Register(EnvironmentMode.JitCharacteristic, RuntimeInformation.GetCurrentJit); + Register(EnvironmentMode.JitCharacteristic, JitInfo.GetCurrentJit); Register(EnvironmentMode.AffinityCharacteristic, RuntimeInformation.GetCurrentAffinity); Register(EnvironmentMode.EnvironmentVariablesCharacteristic, Array.Empty); Register(EnvironmentMode.PowerPlanModeCharacteristic, () => PowerManagementApplier.Map(PowerPlan.HighPerformance)); diff --git a/src/BenchmarkDotNet/Extensions/ProcessExtensions.cs b/src/BenchmarkDotNet/Extensions/ProcessExtensions.cs index 4d0854ba03..d02db8027f 100644 --- a/src/BenchmarkDotNet/Extensions/ProcessExtensions.cs +++ b/src/BenchmarkDotNet/Extensions/ProcessExtensions.cs @@ -150,6 +150,11 @@ internal static void SetEnvironmentVariables(this ProcessStartInfo start, Benchm // disable ReSharper's Dynamic Program Analysis (see https://github.com/dotnet/BenchmarkDotNet/issues/1871 for details) start.EnvironmentVariables["JETBRAINS_DPA_AGENT_ENABLE"] = "0"; + // CallCountingDelayMs=0 breaks DisassemblyDiagnoser, so we only set it if the job doesn't need disassembly. https://github.com/dotnet/runtime/issues/117339 + if (!benchmarkCase.Config.HasDisassemblyDiagnoser()) + { + SetClrEnvironmentVariables(start, JitInfo.EnvCallCountingDelayMs, "0"); + } if (!benchmarkCase.Job.HasValue(EnvironmentMode.EnvironmentVariablesCharacteristic)) return; diff --git a/src/BenchmarkDotNet/Helpers/SleepHelper.cs b/src/BenchmarkDotNet/Helpers/SleepHelper.cs new file mode 100644 index 0000000000..93521f7fa2 --- /dev/null +++ b/src/BenchmarkDotNet/Helpers/SleepHelper.cs @@ -0,0 +1,16 @@ +using System; +using System.Threading; + +namespace BenchmarkDotNet.Helpers +{ + internal static class SleepHelper + { + public static void SleepIfPositive(TimeSpan timeSpan) + { + if (timeSpan > TimeSpan.Zero) + { + Thread.Sleep(timeSpan); + } + } + } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Portability/JitInfo.cs b/src/BenchmarkDotNet/Portability/JitInfo.cs new file mode 100644 index 0000000000..edfab165a4 --- /dev/null +++ b/src/BenchmarkDotNet/Portability/JitInfo.cs @@ -0,0 +1,230 @@ +using BenchmarkDotNet.Environments; +using System; +using System.Diagnostics; +using static BenchmarkDotNet.Portability.RuntimeInformation; + +namespace BenchmarkDotNet.Portability +{ + // Implementation is based on article https://medium.com/@meriffa/net-core-concepts-tiered-compilation-10f7da3a29c7 + // documentation https://learn.microsoft.com/en-us/dotnet/core/runtime-config/compilation + // and source https://github.com/dotnet/runtime/blob/3fb6fbb3efaa7f2ae5e76bf235615d7f70005201/src/coreclr/vm/eeconfig.cpp + // https://github.com/dotnet/runtime/blob/3fb6fbb3efaa7f2ae5e76bf235615d7f70005201/src/coreclr/jit/jitconfigvalues.h + internal static class JitInfo + { + public const string EnvCallCountingDelayMs = "TC_CallCountingDelayMs"; + + private const string EnvMinOpts = "JITMinOpts"; + private const string EnvTieredCompilation = "TieredCompilation"; + private const string EnvQuickJit = "TC_QuickJit"; + private const string EnvPGO = "TieredPGO"; + private const string EnvCallCountThreshold = "TC_CallCountThreshold"; + private const string EnvAggressiveTiering = "TC_AggressiveTiering"; + private const string EnvOSR = "TC_OnStackReplacement"; + + private const string KnobTieredCompilation = "System.Runtime.TieredCompilation"; + private const string KnobQuickJit = "System.Runtime.TieredCompilation.QuickJit"; + private const string KnobPGO = "System.Runtime.TieredPGO"; + private const string KnobCallCountThreshold = "System.Runtime.TieredCompilation.CallCountThreshold"; + private const string KnobCallCountingDelayMs = "System.Runtime.TieredCompilation.CallCountingDelayMs"; + + // .Net 5 and older uses COMPlus_ prefix, + // .Net 6+ uses DOTNET_ prefix, but still supports legacy COMPlus_. + private static bool IsEnvVarEnabled(string name) + => Environment.GetEnvironmentVariable($"COMPlus_{name}") == "1" + || Environment.GetEnvironmentVariable($"DOTNET_{name}") == "1"; + + private static bool IsEnvVarDisabled(string name) + => Environment.GetEnvironmentVariable($"COMPlus_{name}") == "0" + || Environment.GetEnvironmentVariable($"DOTNET_{name}") == "0"; + + private static bool TryParseEnvVar(string name, out int value) + => int.TryParse(Environment.GetEnvironmentVariable($"COMPlus_{name}"), out value) + || int.TryParse(Environment.GetEnvironmentVariable($"DOTNET_{name}"), out value); + + private static bool IsKnobEnabled(string name) + => AppContext.TryGetSwitch(name, out bool isEnabled) && isEnabled; + + private static bool IsKnobDisabled(string name) + => AppContext.TryGetSwitch(name, out bool isEnabled) && !isEnabled; + + private static bool TryParseKnob(string name, out int value) + => int.TryParse(AppContext.GetData(name) as string, out value); + + private static bool IsEnabled(string envName, string knobName) + => IsEnvVarEnabled(envName) || IsKnobEnabled(knobName); + + private static bool IsDisabled(string envName, string knobName) + => IsEnvVarDisabled(envName) || IsKnobDisabled(knobName); + + /// + /// Is tiered JIT enabled? + /// + public static readonly bool IsTiered = + IsNetCore + // JITMinOpts disables tiered compilation (all methods are effectively tier0 instead of tier1). + && !IsEnvVarEnabled(EnvMinOpts) + && ((CoreRuntime.TryGetVersion(out var version) && version.Major >= 3) + // Enabled by default in netcoreapp3.0+, check if it's disabled. + ? !IsDisabled(EnvTieredCompilation, KnobTieredCompilation) + // Disabled by default in netcoreapp2.X, check if it's enabled. + : IsEnabled(EnvTieredCompilation, KnobTieredCompilation)); + + /// + /// The maximum numbers of jit tiers that a method may be promoted through. This is the maximum number of jit tiers - 1. + /// + public static readonly int MaxTierPromotions = GetMaxTierPromotions(); + + private static int GetMaxTierPromotions() + { + if (!IsTiered) + { + return 0; + } + // Tier1 + int maxPromotions = 1; + if (GetIsDPGO()) + { + // Tier0 instrumented + ++maxPromotions; + } + if (GetIsOSR()) + { + // On-stack-replacement *shouldn't* interfere with promotion velocity, but there is a bug where OSR may cause a method to be tier0 instrumented twice. + // https://github.com/dotnet/runtime/issues/117787#issuecomment-3090771091 + ++maxPromotions; + } + return maxPromotions; + + static bool GetIsDPGO() => + // Added experimentally in .Net 6. + Environment.Version.Major >= 6 + // Disabled if QuickJit is disabled in .Net 7+. + && (Environment.Version.Major < 7 || !IsDisabled(EnvQuickJit, KnobQuickJit)) + && (Environment.Version.Major >= 8 + // Enabled by default in .Net 8, check if it's disabled. + ? !IsDisabled(EnvPGO, KnobPGO) + // Disabled by default in earlier versions, check if it's enabled. + : IsEnabled(EnvPGO, KnobPGO)); + + static bool GetIsOSR() => + // Added experimentally in .Net 5. + Environment.Version.Major >= 5 + && (Environment.Version.Major >= 7 + // Enabled by default in .Net 7, check if it's disabled. + ? !IsEnvVarDisabled(EnvOSR) + // Disabled by default in earlier versions, check if it's enabled. + : IsEnvVarEnabled(EnvOSR)); + } + + /// + /// The number of times a method must be called before it will be eligible for the next JIT tier. + /// + public static readonly int TieredCallCountThreshold = GetTieredCallCountThreshold(); + + private static int GetTieredCallCountThreshold() + { + if (!IsTiered) + { + return 0; + } + // AggressiveTiering was added in .Net 5. + if (Environment.Version.Major >= 5 && IsEnvVarEnabled(EnvAggressiveTiering)) + { + return 1; + } + if (TryParseEnvVar(EnvCallCountThreshold, out int callCountThreshold)) + { + return callCountThreshold; + } + // CallCountThreshold was added as a knob in .Net 8. + if (Environment.Version.Major >= 8 && TryParseKnob(KnobCallCountThreshold, out callCountThreshold)) + { + return callCountThreshold; + } + // Default 30 if it's not configured. + return 30; + } + + /// + /// How long to wait to ensure tiered JIT call counting has begun. + /// + public static readonly TimeSpan TieredDelay = GetTieredDelay(); + + private static TimeSpan GetTieredDelay() + { + if (!IsTiered) + { + return TimeSpan.Zero; + } + // AggressiveTiering was added in .Net 5. + if (Environment.Version.Major >= 5 && IsEnvVarEnabled(EnvAggressiveTiering)) + { + return TimeSpan.Zero; + } + if (TryParseEnvVar(EnvCallCountingDelayMs, out int callCountDelay)) + { + return TimeSpan.FromMilliseconds(callCountDelay); + } + // CallCountingDelayMs was added as a knob in .Net 8. + if (Environment.Version.Major >= 8 && TryParseKnob(KnobCallCountingDelayMs, out callCountDelay)) + { + return TimeSpan.FromMilliseconds(callCountDelay); + } + // Default 100 if it's not configured. + return TimeSpan.FromMilliseconds(100); + } + + /// + /// How long to wait for the JIT to have completed tiered compilation in the background. + /// + public static readonly TimeSpan BackgroundCompilationDelay = + IsTiered + // It's impossible for us to know exactly how long to wait without hooking into JIT notifications (which we can't do in-process). + // 100ms should be enough most of the time, but we bump it up to 250ms for higher confidence. + // When https://github.com/dotnet/runtime/issues/101868 is resolved, if AggressiveTiering is enabled, we can skip the wait time and return TimeSpan.Zero. + ? TimeSpan.FromMilliseconds(250) + : TimeSpan.Zero; + + public static readonly bool IsRyuJit = GetIsRyuJit(); + + private static bool GetIsRyuJit() + { + if (IsNetCore) // CoreCLR supports only RyuJIT. + return true; + if (IsMono || !IsFullFramework) // If it's not Core or Framework, it's not RyuJIT. + return false; + if (!Is64BitPlatform()) // Framework supports RyuJIT only in 64-bit process. + return false; + + // https://stackoverflow.com/a/31534544 + foreach (ProcessModule module in Process.GetCurrentProcess().Modules) + { + // clrjit.dll -> RyuJit + // compatjit.dll -> Legacy Jit + if (module.ModuleName == "clrjit.dll") + { + return true; + } + } + return false; + } + + public static Jit GetCurrentJit() => IsRyuJit ? Jit.RyuJit : Jit.LegacyJit; + + public static string GetInfo() + { + if (IsNativeAOT) + return "NativeAOT"; + if (IsAot) + return "AOT"; + if (IsMono || IsWasm) + return ""; // There is no helpful information about JIT on Mono + if (IsRyuJit) + return "RyuJIT"; + if (IsFullFramework) + return "LegacyJIT"; + + return Unknown; + } + } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Portability/RuntimeInformation.cs b/src/BenchmarkDotNet/Portability/RuntimeInformation.cs index 9b8348a125..df362c367e 100644 --- a/src/BenchmarkDotNet/Portability/RuntimeInformation.cs +++ b/src/BenchmarkDotNet/Portability/RuntimeInformation.cs @@ -80,19 +80,6 @@ public static bool IsNativeAOT && IsAot && !IsWasm && !IsMono; // Wasm and MonoAOTLLVM are also AOT - - public static readonly bool IsTieredJitEnabled = - IsNetCore - && (Environment.Version.Major < 3 - // Disabled by default in netcoreapp2.X, check if it's enabled. - ? Environment.GetEnvironmentVariable("COMPlus_TieredCompilation") == "1" - || Environment.GetEnvironmentVariable("DOTNET_TieredCompilation") == "1" - || (AppContext.TryGetSwitch("System.Runtime.TieredCompilation", out bool isEnabled) && isEnabled) - // Enabled by default in netcoreapp3.0+, check if it's disabled. - : Environment.GetEnvironmentVariable("COMPlus_TieredCompilation") != "0" - && Environment.GetEnvironmentVariable("DOTNET_TieredCompilation") != "0" - && (!AppContext.TryGetSwitch("System.Runtime.TieredCompilation", out isEnabled) || isEnabled)); - public static readonly bool IsRunningInContainer = string.Equals(Environment.GetEnvironmentVariable("DOTNET_RUNNING_IN_CONTAINER"), "true"); internal static string GetArchitecture() => GetCurrentPlatform().ToString(); @@ -243,36 +230,6 @@ public static Platform GetCurrentPlatform() public static bool Is64BitPlatform() => IntPtr.Size == 8; - internal static bool HasRyuJit() - { - if (IsMono) - return false; - if (IsNetCore) - return true; - - return Is64BitPlatform() - && GetConfiguration() != DebugConfigurationName - && !new JitHelper().IsMsX64(); - } - - internal static Jit GetCurrentJit() => HasRyuJit() ? Jit.RyuJit : Jit.LegacyJit; - - internal static string GetJitInfo() - { - if (IsNativeAOT) - return "NativeAOT"; - if (IsAot) - return "AOT"; - if (IsMono || IsWasm) - return ""; // There is no helpful information about JIT on Mono - if (IsNetCore || HasRyuJit()) // CoreCLR supports only RyuJIT - return "RyuJIT"; - if (IsFullFramework) - return "LegacyJIT"; - - return Unknown; - } - internal static IntPtr GetCurrentAffinity() => Process.GetCurrentProcess().TryGetAffinity() ?? default; internal static string GetConfiguration() @@ -285,39 +242,6 @@ internal static string GetConfiguration() return isDebug.Value ? DebugConfigurationName : ReleaseConfigurationName; } - // See http://aakinshin.net/en/blog/dotnet/jit-version-determining-in-runtime/ - private class JitHelper - { - [SuppressMessage("IDE0052", "IDE0052")] - [SuppressMessage("IDE0079", "IDE0079")] - [SuppressMessage("ReSharper", "NotAccessedField.Local")] - private int bar; - - public bool IsMsX64(int step = 1) - { - int value = 0; - for (int i = 0; i < step; i++) - { - bar = i + 10; - for (int j = 0; j < 2 * step; j += step) - value = j + 10; - } - return value == 20 + step; - } - } - - private class JitModule - { - public string Name { get; } - public string Version { get; } - - public JitModule(string name, string version) - { - Name = name; - Version = version; - } - } - internal static ICollection GetAntivirusProducts() { var products = new List(); diff --git a/tests/BenchmarkDotNet.IntegrationTests/CustomEngineTests.cs b/tests/BenchmarkDotNet.IntegrationTests/CustomEngineTests.cs index f6d0a10f83..646773e850 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/CustomEngineTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/CustomEngineTests.cs @@ -7,7 +7,6 @@ using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Engines; using BenchmarkDotNet.Reports; -using BenchmarkDotNet.Characteristics; using Perfolizer.Mathematics.OutlierDetection; namespace BenchmarkDotNet.IntegrationTests @@ -85,18 +84,8 @@ public RunResults Run() public void Dispose() => GlobalCleanupAction?.Invoke(); - public IHost Host { get; } - public void WriteLine() { } - public void WriteLine(string line) { } - public Job TargetJob { get; } - public long OperationsPerInvoke { get; } public Action GlobalSetupAction { get; set; } public Action GlobalCleanupAction { get; set; } - public Action WorkloadAction { get; } - public Action OverheadAction { get; } - public IResolver Resolver { get; } - - public Measurement RunIteration(IterationData data) { throw new NotImplementedException(); } } } } \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Engine/EngineActualStageTests.cs b/tests/BenchmarkDotNet.Tests/Engine/EngineActualStageTests.cs index 97078eb953..d8d1433c17 100644 --- a/tests/BenchmarkDotNet.Tests/Engine/EngineActualStageTests.cs +++ b/tests/BenchmarkDotNet.Tests/Engine/EngineActualStageTests.cs @@ -25,10 +25,10 @@ public EngineActualStageTests(ITestOutputHelper output) public void AutoTest_SteadyState() => AutoTest(data => TimeInterval.Second, MinIterationCount); [Fact] - public void AutoTest_InfiniteIncrease() => AutoTest(data => TimeInterval.Second * data.Index, MaxIterationCount); + public void AutoTest_InfiniteIncrease() => AutoTest(data => TimeInterval.Second * data.index, MaxIterationCount); [Fact] - public void AutoTest_InfiniteIncreaseOverhead() => AutoTest(data => TimeInterval.Second * data.Index, MaxOverheadIterationCount, + public void AutoTest_InfiniteIncreaseOverhead() => AutoTest(data => TimeInterval.Second * data.index, MaxOverheadIterationCount, iterationMode: IterationMode.Overhead); private void AutoTest(Func measure, int min, int max = -1, IterationMode iterationMode = IterationMode.Workload) @@ -38,9 +38,9 @@ private void AutoTest(Func measure, int min, int ma var job = Job.Default; var engine = new MockEngine(output, job, measure); var stage = iterationMode == IterationMode.Overhead - ? EngineActualStage.GetOverhead(engine) - : EngineActualStage.GetWorkload(engine, RunStrategy.Throughput); - var (_, measurements) = engine.Run(stage); + ? EngineActualStage.GetOverhead(1, 1, engine.Parameters) + : EngineActualStage.GetWorkload(RunStrategy.Throughput, 1, 1, engine.Parameters); + var measurements = engine.Run(stage); int count = measurements.Count; output.WriteLine($"MeasurementCount = {count} (Min= {min}, Max = {max})"); Assert.InRange(count, min, max); diff --git a/tests/BenchmarkDotNet.Tests/Engine/EngineFactoryTests.cs b/tests/BenchmarkDotNet.Tests/Engine/EngineFactoryTests.cs index fadfbe6f8a..c84f3a25a0 100644 --- a/tests/BenchmarkDotNet.Tests/Engine/EngineFactoryTests.cs +++ b/tests/BenchmarkDotNet.Tests/Engine/EngineFactoryTests.cs @@ -1,10 +1,8 @@ using System; using System.Collections.Generic; -using System.Threading; -using BenchmarkDotNet.Characteristics; using BenchmarkDotNet.Engines; using BenchmarkDotNet.Jobs; -using BenchmarkDotNet.Running; +using BenchmarkDotNet.Reports; using JetBrains.Annotations; using Perfolizer.Horology; using Xunit; @@ -13,40 +11,16 @@ namespace BenchmarkDotNet.Tests.Engine { public class EngineFactoryTests { - private int timesBenchmarkCalled = 0, timesOverheadCalled = 0; private int timesGlobalSetupCalled = 0, timesGlobalCleanupCalled = 0, timesIterationSetupCalled = 0, timesIterationCleanupCalled = 0; - private TimeSpan IterationTime => TimeSpan.FromMilliseconds(EngineResolver.Instance.Resolve(Job.Default, RunMode.IterationTimeCharacteristic).ToMilliseconds()); - - private IResolver DefaultResolver => BenchmarkRunnerClean.DefaultResolver; + private static TimeSpan IterationTime => TimeSpan.FromMilliseconds(EngineResolver.Instance.Resolve(Job.Default, RunMode.IterationTimeCharacteristic).ToMilliseconds()); + private static TimeInterval IterationTimeInternal => TimeInterval.FromMilliseconds(IterationTime.Milliseconds); private void GlobalSetup() => timesGlobalSetupCalled++; private void IterationSetup() => timesIterationSetupCalled++; private void IterationCleanup() => timesIterationCleanupCalled++; private void GlobalCleanup() => timesGlobalCleanupCalled++; - private void Throwing(long _) => throw new InvalidOperationException("must NOT be called"); - - private void VeryTimeConsumingSingle(long _) - { - timesBenchmarkCalled++; - Thread.Sleep(IterationTime); - } - - private void TimeConsumingOnlyForTheFirstCall(long _) - { - if (timesBenchmarkCalled++ == 0) - { - Thread.Sleep(IterationTime); - } - } - - private void InstantNoUnroll(long invocationCount) => timesBenchmarkCalled += (int) invocationCount; - private void InstantUnroll(long _) => timesBenchmarkCalled += 16; - - private void OverheadNoUnroll(long invocationCount) => timesOverheadCalled += (int) invocationCount; - private void OverheadUnroll(long _) => timesOverheadCalled += 16; - private static readonly Dictionary JobsWhichDontRequireJitting = new Dictionary { { "Dry", Job.Dry }, @@ -62,14 +36,20 @@ private void TimeConsumingOnlyForTheFirstCall(long _) public void ForJobsThatDontRequireJittingOnlyGlobalSetupIsCalled(string jobName) { var job = JobsWhichDontRequireJitting[jobName]; - var engineParameters = CreateEngineParameters(mainNoUnroll: Throwing, mainUnroll: Throwing, job: job); + var engineParameters = CreateEngineParameters(job); - var engine = new EngineFactory().CreateReadyToRun(engineParameters); + var engine = (Engines.Engine) new EngineFactory().CreateReadyToRun(engineParameters); + bool didRunStages = false; + foreach (var stage in EngineStage.EnumerateStages(engine.Parameters)) + { + Assert.True(stage is not EngineJitStage); + didRunStages = true; + break; + } + Assert.True(didRunStages); Assert.Equal(1, timesGlobalSetupCalled); Assert.Equal(0, timesIterationSetupCalled); - Assert.Equal(0, timesBenchmarkCalled); - Assert.Equal(0, timesOverheadCalled); Assert.Equal(0, timesIterationCleanupCalled); Assert.Equal(0, timesGlobalCleanupCalled); @@ -81,22 +61,33 @@ public void ForJobsThatDontRequireJittingOnlyGlobalSetupIsCalled(string jobName) [Fact] public void ForDefaultSettingsVeryTimeConsumingBenchmarksAreExecutedOncePerIterationWithoutOverheadDeduction() { - var engineParameters = CreateEngineParameters(mainNoUnroll: VeryTimeConsumingSingle, mainUnroll: Throwing, job: Job.Default); + var engineParameters = CreateEngineParameters(Job.Default); - var engine = new EngineFactory().CreateReadyToRun(engineParameters); + var engine = (Engines.Engine) new EngineFactory().CreateReadyToRun(engineParameters); + bool didRunActualStage = false; + foreach (var stage in EngineStage.EnumerateStages(engine.Parameters)) + { + Assert.NotEqual(IterationMode.Overhead, stage.Mode); - Assert.Equal(1, timesGlobalSetupCalled); - Assert.Equal(2, timesIterationSetupCalled); // 2x for Target - Assert.Equal(2, timesBenchmarkCalled); - Assert.Equal(2, timesOverheadCalled); - Assert.Equal(2, timesIterationCleanupCalled); // 2x for Target - Assert.Equal(0, timesGlobalCleanupCalled); // cleanup is called as part of dispose + var stageMeasurements = stage.GetMeasurementList(); + while (stage.GetShouldRunIteration(stageMeasurements, out var iterationData)) + { + var measurement = new Measurement(0, iterationData.mode, iterationData.stage, iterationData.index, 1, IterationTimeInternal.Nanoseconds); + stageMeasurements.Add(measurement); + } - Assert.Equal(1, engine.TargetJob.Run.InvocationCount); // call the benchmark once per iteration - Assert.Equal(1, engine.TargetJob.Run.UnrollFactor); // no unroll factor + if (stage is EngineActualStage { Mode: IterationMode.Workload } actualStage) + { + Assert.Equal(1, actualStage.invokeCount); + Assert.Equal(1, actualStage.unrollFactor); + didRunActualStage = true; + break; + } + } - Assert.True(engine.TargetJob.Run.HasValue(AccuracyMode.EvaluateOverheadCharacteristic)); // is set to false in explicit way - Assert.False(engine.TargetJob.Accuracy.EvaluateOverhead); // don't evaluate overhead in that case + Assert.True(didRunActualStage); + Assert.Equal(1, timesGlobalSetupCalled); + Assert.Equal(0, timesGlobalCleanupCalled); // cleanup is called as part of dispose engine.Dispose(); // cleanup is called as part of dispose @@ -104,84 +95,78 @@ public void ForDefaultSettingsVeryTimeConsumingBenchmarksAreExecutedOncePerItera } [Theory] - [InlineData(120)] // 120 ms as in the bug report - [InlineData(250)] // 250 ms as configured in dotnet/performance repo - [InlineData(EngineResolver.DefaultIterationTime)] // 500 ms - the default BDN setting - public void BenchmarksThatRunLongerThanIterationTimeOnlyDuringFirstInvocationAreNotInvokedOncePerIteration(int iterationTime) + [InlineData(120, 120)] // 120 ms as in the bug report + [InlineData(250, 250)] // 250 ms as configured in dotnet/performance repo + [InlineData(EngineResolver.DefaultIterationTime, EngineResolver.DefaultIterationTime)] // 500 ms - the default BDN setting + [InlineData(EngineResolver.DefaultIterationTime, 20000)] // 20 seconds - twice the cutoff threshold of the old jit stage heuristic #2004 + public void BenchmarksThatRunLongerThanIterationTimeOnlyDuringFirstInvocationAreNotInvokedOncePerIteration(int iterationTime, int callTime) { - var engineParameters = CreateEngineParameters( - mainNoUnroll: TimeConsumingOnlyForTheFirstCall, - mainUnroll: InstantUnroll, - job: Job.Default.WithIterationTime(TimeInterval.FromMilliseconds(iterationTime))); + var callTimeInterval = TimeInterval.FromMilliseconds(callTime); + var engineParameters = CreateEngineParameters(Job.Default.WithIterationTime(TimeInterval.FromMilliseconds(iterationTime))); + + var engine = (Engines.Engine) new EngineFactory().CreateReadyToRun(engineParameters); + bool didRunActualStage = false; + foreach (var stage in EngineStage.EnumerateStages(engine.Parameters)) + { + var stageMeasurements = stage.GetMeasurementList(); + while (stage.GetShouldRunIteration(stageMeasurements, out var iterationData)) + { + var measurement = new Measurement(0, iterationData.mode, iterationData.stage, iterationData.index, 1, callTimeInterval.Nanoseconds); + stageMeasurements.Add(measurement); + callTimeInterval = TimeInterval.FromNanoseconds(1); + } - var engine = new EngineFactory().CreateReadyToRun(engineParameters); + if (stage is EngineActualStage { Mode: IterationMode.Workload } actualStage) + { + Assert.NotEqual(1, actualStage.invokeCount * actualStage.unrollFactor); + didRunActualStage = true; + break; + } + } + Assert.True(didRunActualStage); Assert.Equal(1, timesGlobalSetupCalled); - // the factory should call the benchmark: - // 1st time with unroll factor to JIT the code - // one more to check that the Jitting has not dominated the reported time - // and one more time to JIT the 16 unroll factor case as it turned out that Jitting has dominated the time - Assert.Equal(1 + 1 + 1, timesIterationSetupCalled); - Assert.Equal(1 + 1 + 16, timesBenchmarkCalled); - Assert.Equal(1 + 1 + 16, timesOverheadCalled); - Assert.Equal(1 + 1 + 1, timesIterationCleanupCalled); // 2x for Target Assert.Equal(0, timesGlobalCleanupCalled); // cleanup is called as part of dispose - Assert.False(engine.TargetJob.Run.HasValue(RunMode.InvocationCountCharacteristic)); // we need pilot stage - - Assert.False(engine.TargetJob.Run.HasValue(AccuracyMode.EvaluateOverheadCharacteristic)); - engine.Dispose(); // cleanup is called as part of dispose Assert.Equal(1, timesGlobalCleanupCalled); } [Fact] - public void ForJobsWithExplicitUnrollFactorTheGlobalSetupIsCalledAndMultiActionCodeGetsJitted() - => AssertGlobalSetupWasCalledAndMultiActionGotJitted(Job.Default.WithUnrollFactor(16)); + public void ForJobsWithExplicitUnrollFactorTheGlobalSetupAndUnrollAreCalled() + => AssertGlobalSetupWasCalledAndUnrollWasRan(Job.Default.WithUnrollFactor(16)); [Fact] - public void ForJobsThatDontRequirePilotTheGlobalSetupIsCalledAndMultiActionCodeGetsJitted() - => AssertGlobalSetupWasCalledAndMultiActionGotJitted(Job.Default.WithInvocationCount(100)); - - private void AssertGlobalSetupWasCalledAndMultiActionGotJitted(Job job) - { - var engineParameters = CreateEngineParameters(mainNoUnroll: Throwing, mainUnroll: InstantUnroll, job: job); - - var engine = new EngineFactory().CreateReadyToRun(engineParameters); - - Assert.Equal(1, timesGlobalSetupCalled); - Assert.Equal(1, timesIterationSetupCalled); - Assert.Equal(16, timesBenchmarkCalled); - Assert.Equal(16, timesOverheadCalled); - Assert.Equal(1, timesIterationCleanupCalled); - Assert.Equal(0, timesGlobalCleanupCalled); - - Assert.False(engine.TargetJob.Run.HasValue(AccuracyMode.EvaluateOverheadCharacteristic)); // remains untouched - - engine.Dispose(); - - Assert.Equal(1, timesGlobalCleanupCalled); - } + public void ForJobsThatDontRequirePilotTheGlobalSetupAndUnrollAreCalled() + => AssertGlobalSetupWasCalledAndUnrollWasRan(Job.Default.WithInvocationCount(100)); [Fact] public void NonVeryTimeConsumingBenchmarksAreExecutedMoreThanOncePerIterationWithUnrollFactorForDefaultSettings() + => AssertGlobalSetupWasCalledAndUnrollWasRan(Job.Default); + + private void AssertGlobalSetupWasCalledAndUnrollWasRan(Job job) { - var engineParameters = CreateEngineParameters(mainNoUnroll: InstantNoUnroll, mainUnroll: InstantUnroll, job: Job.Default); + var engineParameters = CreateEngineParameters(job); + + var engine = (Engines.Engine) new EngineFactory().CreateReadyToRun(engineParameters); + bool didRunUnroll = false; + foreach (var stage in EngineStage.EnumerateStages(engine.Parameters)) + { + var stageMeasurements = stage.GetMeasurementList(); + while (stage.GetShouldRunIteration(stageMeasurements, out var iterationData)) + { + didRunUnroll |= iterationData.unrollFactor > 1; + var measurement = new Measurement(0, iterationData.mode, iterationData.stage, iterationData.index, 1, 1); + stageMeasurements.Add(measurement); + } + } - var engine = new EngineFactory().CreateReadyToRun(engineParameters); + Assert.True(didRunUnroll); Assert.Equal(1, timesGlobalSetupCalled); - Assert.Equal(1 + 1, timesIterationSetupCalled); // once for single and & once for 16 - Assert.Equal(1 + 16, timesBenchmarkCalled); - Assert.Equal(1 + 16, timesOverheadCalled); - Assert.Equal(1 + 1, timesIterationCleanupCalled); // once for single and & once for 16 Assert.Equal(0, timesGlobalCleanupCalled); - Assert.False(engine.TargetJob.Run.HasValue(AccuracyMode.EvaluateOverheadCharacteristic)); // remains untouched - - Assert.False(engine.TargetJob.Run.HasValue(RunMode.InvocationCountCharacteristic)); - engine.Dispose(); Assert.Equal(1, timesGlobalCleanupCalled); @@ -190,56 +175,47 @@ public void NonVeryTimeConsumingBenchmarksAreExecutedMoreThanOncePerIterationWit [Fact] public void MediumTimeConsumingBenchmarksShouldStartPilotFrom2AndIncrementItWithEveryStep() { - var unrollFactor = Job.Default.ResolveValue(RunMode.UnrollFactorCharacteristic, DefaultResolver); - const int times = 5; // how many times we should invoke the benchmark per iteration - var mediumTime = TimeSpan.FromMilliseconds(IterationTime.TotalMilliseconds / times); + var mediumTime = TimeInterval.FromMilliseconds(IterationTime.TotalMilliseconds / times); + + var engineParameters = CreateEngineParameters(Job.Default); - void MediumNoUnroll(long invocationCount) + var engine = (Engines.Engine) new EngineFactory().CreateReadyToRun(engineParameters); + bool didRunPilotStage = false; + foreach (var stage in EngineStage.EnumerateStages(engine.Parameters)) { - for (int i = 0; i < invocationCount; i++) + var stageMeasurements = stage.GetMeasurementList(); + while (stage.GetShouldRunIteration(stageMeasurements, out var iterationData)) { - timesBenchmarkCalled++; - - Thread.Sleep(mediumTime); + var measurement = new Measurement(0, iterationData.mode, iterationData.stage, iterationData.index, 1, mediumTime.Nanoseconds); + stageMeasurements.Add(measurement); } - } - void MediumUnroll(long _) - { - timesBenchmarkCalled += unrollFactor; - - for (int i = 0; i < unrollFactor; i++) // the real unroll factor obviously does not use loop ;) - Thread.Sleep(mediumTime); + if (stage is EnginePilotStageInitial pilotStage) + { + Assert.Equal(1, pilotStage.unrollFactor); + // We start from two (we know that 1 is not enough, the default is 4 so we need to override it). + Assert.Equal(2, pilotStage.minInvokeCount); + Assert.True(pilotStage.needsFurtherPilot); + Assert.False(pilotStage.evaluateOverhead); + + didRunPilotStage = true; + break; + } } - var engineParameters = CreateEngineParameters(mainNoUnroll: MediumNoUnroll, mainUnroll: MediumUnroll, job: Job.Default); - - var engine = new EngineFactory().CreateReadyToRun(engineParameters); - + Assert.True(didRunPilotStage); Assert.Equal(1, timesGlobalSetupCalled); - Assert.Equal(1, timesIterationSetupCalled); - Assert.Equal(1, timesBenchmarkCalled); // we run it just once and we know how many times it should be invoked - Assert.Equal(1, timesOverheadCalled); - Assert.Equal(1, timesIterationCleanupCalled); Assert.Equal(0, timesGlobalCleanupCalled); - Assert.False(engine.TargetJob.Run.HasValue(RunMode.InvocationCountCharacteristic)); // we need to run the pilot! - Assert.Equal(1, engine.TargetJob.Run.UnrollFactor); // no unroll factor! - Assert.Equal(2, - engine.TargetJob.Accuracy.MinInvokeCount); // we start from two (we know that 1 is not enough, the default is 4 so we need to override it) - - Assert.True(engine.TargetJob.Run.HasValue(AccuracyMode.EvaluateOverheadCharacteristic)); // is set to false in explicit way - Assert.False(engine.TargetJob.Accuracy.EvaluateOverhead); // don't evaluate overhead in that case - engine.Dispose(); Assert.Equal(1, timesGlobalCleanupCalled); } - private EngineParameters CreateEngineParameters(Action mainNoUnroll, Action mainUnroll, Job job) - => new EngineParameters + private EngineParameters CreateEngineParameters(Job job) + => new() { Dummy1Action = () => { }, Dummy2Action = () => { }, @@ -247,12 +223,12 @@ private EngineParameters CreateEngineParameters(Action mainNoUnroll, Actio GlobalSetupAction = GlobalSetup, GlobalCleanupAction = GlobalCleanup, Host = new NoAcknowledgementConsoleHost(), - OverheadActionUnroll = OverheadUnroll, - OverheadActionNoUnroll = OverheadNoUnroll, + OverheadActionUnroll = _ => { }, + OverheadActionNoUnroll = _ => { }, IterationCleanupAction = IterationCleanup, IterationSetupAction = IterationSetup, - WorkloadActionUnroll = mainUnroll, - WorkloadActionNoUnroll = mainNoUnroll, + WorkloadActionUnroll = _ => { }, + WorkloadActionNoUnroll = _ => { }, TargetJob = job }; } diff --git a/tests/BenchmarkDotNet.Tests/Engine/EnginePilotStageTests.cs b/tests/BenchmarkDotNet.Tests/Engine/EnginePilotStageTests.cs index fd3f4887ff..bfec18b9a7 100644 --- a/tests/BenchmarkDotNet.Tests/Engine/EnginePilotStageTests.cs +++ b/tests/BenchmarkDotNet.Tests/Engine/EnginePilotStageTests.cs @@ -48,8 +48,10 @@ private void AutoTest(Frequency clockFrequency, TimeInterval operationTime, doub Infrastructure = { Clock = new MockClock(clockFrequency) }, Accuracy = { MaxRelativeError = maxRelativeError } }.Freeze(); - var engine = new MockEngine(output, job, data => data.InvokeCount * operationTime); - var (invokeCount, _) = engine.Run(EnginePilotStage.GetStage(engine)); + var engine = new MockEngine(output, job, data => data.invokeCount * operationTime); + var pilotStage = EnginePilotStage.GetStage(1, 1, 1, engine.Parameters); + engine.Run(pilotStage); + var invokeCount = pilotStage.invokeCount; output.WriteLine($"InvokeCount = {invokeCount} (Min= {minInvokeCount}, Max = {MaxPossibleInvokeCount})"); Assert.InRange(invokeCount, minInvokeCount, MaxPossibleInvokeCount); } @@ -61,8 +63,10 @@ private void SpecificTest(TimeInterval iterationTime, TimeInterval operationTime Infrastructure = { Clock = new MockClock(Frequency.MHz) }, Run = { IterationTime = iterationTime } }.Freeze(); - var engine = new MockEngine(output, job, data => data.InvokeCount * operationTime); - var (invokeCount, _) = engine.Run(EnginePilotStage.GetStage(engine)); + var engine = new MockEngine(output, job, data => data.invokeCount * operationTime); + var pilotStage = EnginePilotStage.GetStage(1, 1, 1, engine.Parameters); + engine.Run(pilotStage); + var invokeCount = pilotStage.invokeCount; output.WriteLine($"InvokeCount = {invokeCount} (Min= {minInvokeCount}, Max = {maxInvokeCount})"); Assert.InRange(invokeCount, minInvokeCount, maxInvokeCount); } diff --git a/tests/BenchmarkDotNet.Tests/Engine/EngineWarmupStageTests.cs b/tests/BenchmarkDotNet.Tests/Engine/EngineWarmupStageTests.cs index b9c465b87b..213d3996b2 100644 --- a/tests/BenchmarkDotNet.Tests/Engine/EngineWarmupStageTests.cs +++ b/tests/BenchmarkDotNet.Tests/Engine/EngineWarmupStageTests.cs @@ -34,25 +34,25 @@ public void AutoTest_SteadyState() [Fact] public void AutoTest_InfiniteIncrease() { - AutoTest(data => TimeInterval.Millisecond * data.Index, MaxIterationCount); + AutoTest(data => TimeInterval.Millisecond * data.index, MaxIterationCount); } [Fact] public void AutoTest_Alternation() { - AutoTest(data => TimeInterval.Millisecond * (data.Index % 2), MinIterationCount, MaxIterationCount); + AutoTest(data => TimeInterval.Millisecond * (data.index % 2), MinIterationCount, MaxIterationCount); } [Fact] public void AutoTest_TenSteps() { - AutoTest(data => TimeInterval.Millisecond * Math.Max(0, 10 - data.Index), 10, MaxIterationCount); + AutoTest(data => TimeInterval.Millisecond * Math.Max(0, 10 - data.index), 10, MaxIterationCount); } [Fact] public void AutoTest_WithoutSteadyStateOverhead() { - AutoTest(data => TimeInterval.Millisecond * data.Index, MaxOverheadIterationCount, mode: IterationMode.Overhead); + AutoTest(data => TimeInterval.Millisecond * data.index, MaxOverheadIterationCount, mode: IterationMode.Overhead); } [Fact] @@ -70,7 +70,7 @@ public void MinAndMaxWarmupCountAttributesCanForceAutoWarmup() Assert.Equal(2, mergedJob.Run.MinWarmupIterationCount); Assert.Equal(4, mergedJob.Run.MaxWarmupIterationCount); - AutoTest(data => TimeInterval.Millisecond * (data.Index % 2), 2, 4, job: mergedJob); + AutoTest(data => TimeInterval.Millisecond * (data.index % 2), 2, 4, job: mergedJob); } [MinWarmupCount(2, forceAutoWarmup: true)] @@ -85,11 +85,12 @@ private void AutoTest(Func measure, int min, int ma { if (max == -1) max = min; - var engine = new MockEngine(output, job ?? Job.Default, measure); + job ??= Job.Default; + var engine = new MockEngine(output, job, measure); var stage = mode == IterationMode.Overhead - ? EngineWarmupStage.GetOverhead() - : EngineWarmupStage.GetWorkload(engine, RunStrategy.Throughput); - var (_, measurements) = engine.Run(stage); + ? EngineWarmupStage.GetOverhead(1, 1, engine.Parameters) + : EngineWarmupStage.GetWorkload(RunStrategy.Throughput, 1, 1, engine.Parameters); + var measurements = engine.Run(stage); int count = measurements.Count; output.WriteLine($"MeasurementCount = {count} (Min= {min}, Max = {max})"); Assert.InRange(count, min, max); diff --git a/tests/BenchmarkDotNet.Tests/Mocks/MockEngine.cs b/tests/BenchmarkDotNet.Tests/Mocks/MockEngine.cs index 7b17baa098..2085d11e6c 100644 --- a/tests/BenchmarkDotNet.Tests/Mocks/MockEngine.cs +++ b/tests/BenchmarkDotNet.Tests/Mocks/MockEngine.cs @@ -16,58 +16,50 @@ public class MockEngine : IEngine private readonly ITestOutputHelper output; private readonly Func measure; - // ReSharper disable once NotNullMemberIsNotInitialized - public MockEngine(ITestOutputHelper output, Job job, Func measure) + public EngineParameters Parameters { get; } + + internal MockEngine(ITestOutputHelper output, Job job, Func measure) { this.output = output; this.measure = measure; - TargetJob = job; + Parameters = new EngineParameters + { + TargetJob = job, + Dummy1Action = () => { }, + Dummy2Action = () => { }, + Dummy3Action = () => { }, + WorkloadActionUnroll = _ => { }, + WorkloadActionNoUnroll = _ => { }, + OverheadActionUnroll = _ => { }, + OverheadActionNoUnroll = _ => { }, + GlobalSetupAction = () => { }, + GlobalCleanupAction = () => { }, + IterationSetupAction = () => { }, + IterationCleanupAction = () => { }, + }; } - public void Dispose() => GlobalSetupAction?.Invoke(); - - [UsedImplicitly] - public IHost Host { get; } - - public Job TargetJob { get; } - public long OperationsPerInvoke { get; } = 1; - - [UsedImplicitly] - public Action GlobalSetupAction { get; set; } - - [UsedImplicitly] - public Action GlobalCleanupAction { get; set; } - - [UsedImplicitly] - public bool IsDiagnoserAttached { get; set; } - - public Action WorkloadAction { get; } = _ => { }; - public Action OverheadAction { get; } = _ => { }; - - [UsedImplicitly] - public IEngineFactory Factory => null; + public void Dispose() { } - public Measurement RunIteration(IterationData data) + private Measurement RunIteration(IterationData data) { double nanoseconds = measure(data).Nanoseconds; - var measurement = new Measurement(1, data.IterationMode, data.IterationStage, data.Index, data.InvokeCount * OperationsPerInvoke, nanoseconds); + var measurement = new Measurement(1, data.mode, data.stage, data.index, data.invokeCount, nanoseconds); WriteLine(measurement.ToString()); return measurement; } public RunResults Run() => default; - internal (long invokeCount, List measurements) Run(EngineStage stage, long invokeCount = 0) + internal List Run(EngineStage stage) { var measurements = stage.GetMeasurementList(); - int iterationIndex = 1; - while (stage.GetShouldRunIteration(measurements, ref invokeCount)) + while (stage.GetShouldRunIteration(measurements, out var iterationData)) { - var measurement = RunIteration(new IterationData(stage.Mode, stage.Stage, iterationIndex, invokeCount, 1)); + var measurement = RunIteration(iterationData); measurements.Add(measurement); - ++iterationIndex; } - return (invokeCount, measurements); + return measurements; } public void WriteLine() => output.WriteLine("");