Skip to content

Commit 3d49dce

Browse files
Benchmark tests (#180)
* introduce TestUtils lib * save * build before test * fix compile * fix ci action * implement benchmark * implement benchmark action * implement benchmark * fix benchmark tool * fix merge * try new benchmark * only net6 and net7 * json report * fix path * fix ci * add benchmark ui URL * fix ci --------- Co-authored-by: Meir Blachman <[email protected]>
1 parent 57bb1ca commit 3d49dce

23 files changed

+338
-147
lines changed

.github/workflows/benchmark.yml

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
name: Benchmark tests
2+
on: [pull_request]
3+
4+
permissions:
5+
contents: write
6+
deployments: write
7+
8+
jobs:
9+
benchmark:
10+
name: Performance regression check
11+
runs-on: ubuntu-latest
12+
steps:
13+
- uses: actions/checkout@v3
14+
- name: Setup .NET
15+
uses: actions/setup-dotnet@v3
16+
with:
17+
dotnet-version: '6.x'
18+
- name: Run benchmark
19+
run: cd src/FluentAssertions.Analyzers.BenchmarkTests && dotnet run -c Release --exporters json --filter '*'
20+
21+
- name: Store benchmark result
22+
uses: rhysd/github-action-benchmark@v1
23+
with:
24+
name: FluentAssertions.Analyzers Benchmark
25+
tool: 'benchmarkdotnet'
26+
output-file-path: src/FluentAssertions.Analyzers.BenchmarkTests/BenchmarkDotNet.Artifacts/results/FluentAssertions.Analyzers.BenchmarkTests.FluentAssertionsBenchmarks-report.json
27+
github-token: ${{ secrets.GITHUB_TOKEN }}
28+
auto-push: true
29+
alert-threshold: '200%'
30+
comment-on-alert: true
31+
fail-on-alert: true

.github/workflows/ci.yml

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,4 +21,6 @@ jobs:
2121
uses: actions/setup-dotnet@v3
2222
with:
2323
dotnet-version: 6.0.x
24-
- run: dotnet test --configuration ${{ matrix.config }} --filter 'TestCategory=Completed' /p:CollectCoverage=true /p:CoverletOutputFormat=cobertura
24+
- run: dotnet build
25+
- run: dotnet test --configuration Release --filter 'TestCategory=Completed' /p:CollectCoverage=true /p:CoverletOutputFormat=cobertura
26+
- run: dotnet pack src/FluentAssertions.Analyzers/FluentAssertions.Analyzers.csproj

FluentAssertions.Analyzers.sln

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
Microsoft Visual Studio Solution File, Format Version 12.00
1+
Microsoft Visual Studio Solution File, Format Version 12.00
22
# Visual Studio 15
33
VisualStudioVersion = 15.0.27004.2002
44
MinimumVisualStudioVersion = 10.0.40219.1
@@ -15,6 +15,10 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "SolutionItems", "SolutionIt
1515
EndProject
1616
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FluentAssertions.Analyzers", "src\FluentAssertions.Analyzers\FluentAssertions.Analyzers.csproj", "{3BA672F7-00D8-4E77-89A0-D46DD99D35AA}"
1717
EndProject
18+
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "FluentAssertions.Analyzers.TestUtils", "src\FluentAssertions.Analyzers.TestUtils\FluentAssertions.Analyzers.TestUtils.csproj", "{BD9FC8CC-C23D-4ECC-A926-4BE35C78D338}"
19+
EndProject
20+
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "FluentAssertions.Analyzers.BenchmarkTests", "src\FluentAssertions.Analyzers.BenchmarkTests\FluentAssertions.Analyzers.BenchmarkTests.csproj", "{FE6D8A05-1383-4BCD-AD65-2EF741E48F44}"
21+
EndProject
1822
Global
1923
GlobalSection(SolutionConfigurationPlatforms) = preSolution
2024
Debug|Any CPU = Debug|Any CPU
@@ -29,6 +33,14 @@ Global
2933
{3BA672F7-00D8-4E77-89A0-D46DD99D35AA}.Debug|Any CPU.Build.0 = Debug|Any CPU
3034
{3BA672F7-00D8-4E77-89A0-D46DD99D35AA}.Release|Any CPU.ActiveCfg = Release|Any CPU
3135
{3BA672F7-00D8-4E77-89A0-D46DD99D35AA}.Release|Any CPU.Build.0 = Release|Any CPU
36+
{BD9FC8CC-C23D-4ECC-A926-4BE35C78D338}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
37+
{BD9FC8CC-C23D-4ECC-A926-4BE35C78D338}.Debug|Any CPU.Build.0 = Debug|Any CPU
38+
{BD9FC8CC-C23D-4ECC-A926-4BE35C78D338}.Release|Any CPU.ActiveCfg = Release|Any CPU
39+
{BD9FC8CC-C23D-4ECC-A926-4BE35C78D338}.Release|Any CPU.Build.0 = Release|Any CPU
40+
{FE6D8A05-1383-4BCD-AD65-2EF741E48F44}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
41+
{FE6D8A05-1383-4BCD-AD65-2EF741E48F44}.Debug|Any CPU.Build.0 = Debug|Any CPU
42+
{FE6D8A05-1383-4BCD-AD65-2EF741E48F44}.Release|Any CPU.ActiveCfg = Release|Any CPU
43+
{FE6D8A05-1383-4BCD-AD65-2EF741E48F44}.Release|Any CPU.Build.0 = Release|Any CPU
3244
EndGlobalSection
3345
GlobalSection(SolutionProperties) = preSolution
3446
HideSolutionNode = FALSE

README.md

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,10 @@ dotnet build
4242
dotnet test --configuration Release --filter 'TestCategory=Completed'
4343
```
4444

45+
### Benchmarks
46+
47+
https://fluentassertions.github.io/fluentassertions.analyzers/dev/bench/
48+
4549
## Example Usages
4650
- https://github.com/SonarSource/sonar-dotnet/pull/2072
4751
- https://github.com/microsoft/component-detection/pull/634
Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
<Project Sdk="Microsoft.NET.Sdk">
2+
3+
<PropertyGroup>
4+
<TargetFramework>net6.0</TargetFramework>
5+
<OutputType>Exe</OutputType>
6+
<IsPackable>false</IsPackable>
7+
</PropertyGroup>
8+
9+
<ItemGroup>
10+
<PackageReference Include="BenchmarkDotNet" Version="0.13.5" />
11+
</ItemGroup>
12+
13+
<ItemGroup>
14+
<ProjectReference Include="..\FluentAssertions.Analyzers.TestUtils\FluentAssertions.Analyzers.TestUtils.csproj" />
15+
</ItemGroup>
16+
17+
</Project>
Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
using BenchmarkDotNet.Attributes;
2+
using BenchmarkDotNet.Jobs;
3+
using Microsoft.CodeAnalysis;
4+
using Microsoft.CodeAnalysis.Diagnostics;
5+
using System.Collections.Immutable;
6+
using FluentAssertions.Analyzers.TestUtils;
7+
using System.Threading.Tasks;
8+
using System;
9+
using System.Collections.Generic;
10+
11+
namespace FluentAssertions.Analyzers.BenchmarkTests
12+
{
13+
[SimpleJob(RuntimeMoniker.Net60, baseline: true)]
14+
[SimpleJob(RuntimeMoniker.Net70)]
15+
[JsonExporter]
16+
public class FluentAssertionsBenchmarks
17+
{
18+
private CompilationWithAnalyzers MinimalCompiliation;
19+
20+
[GlobalSetup]
21+
public async Task GlobalSetup()
22+
{
23+
MinimalCompiliation = await CreateCompilationFromAsync(GenerateCode.ObjectStatement("actual.Should().Equals(expected);"));
24+
}
25+
26+
[Benchmark]
27+
public Task MinimalCompilation()
28+
{
29+
return MinimalCompiliation.GetAnalyzerDiagnosticsAsync();
30+
}
31+
32+
private async Task<CompilationWithAnalyzers> CreateCompilationFromAsync(params string[] sources)
33+
{
34+
var project = CsProjectGenerator.CreateProject(sources);
35+
var compilation = await project.GetCompilationAsync();
36+
37+
if (compilation is null)
38+
{
39+
throw new InvalidOperationException("Compilation is null");
40+
}
41+
42+
return compilation.WithOptions(compilation.Options.WithSpecificDiagnosticOptions(new Dictionary<string, ReportDiagnostic>
43+
{
44+
["CS1701"] = ReportDiagnostic.Suppress, // Binding redirects
45+
["CS1702"] = ReportDiagnostic.Suppress,
46+
["CS1705"] = ReportDiagnostic.Suppress,
47+
["CS8019"] = ReportDiagnostic.Suppress // TODO: Unnecessary using directive
48+
})).WithAnalyzers(CodeAnalyzersUtils.GetAllAnalyzers().ToImmutableArray());
49+
}
50+
}
51+
}
Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
using BenchmarkDotNet.Running;
2+
3+
namespace FluentAssertions.Analyzers.BenchmarkTests
4+
{
5+
public class Program
6+
{
7+
public static void Main()
8+
=> BenchmarkDotNet.Running.BenchmarkRunner.Run<FluentAssertionsBenchmarks>();
9+
}
10+
}
Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
using System;
2+
using System.Linq;
3+
using Microsoft.CodeAnalysis.Diagnostics;
4+
5+
namespace FluentAssertions.Analyzers.TestUtils
6+
{
7+
public class CodeAnalyzersUtils
8+
{
9+
private static readonly DiagnosticAnalyzer[] AllAnalyzers = CreateAllAnalyzers();
10+
11+
public static DiagnosticAnalyzer[] GetAllAnalyzers() => AllAnalyzers;
12+
13+
private static DiagnosticAnalyzer[] CreateAllAnalyzers()
14+
{
15+
var assembly = typeof(Constants).Assembly;
16+
var analyzersTypes = assembly.GetTypes()
17+
.Where(type => !type.IsAbstract && typeof(DiagnosticAnalyzer).IsAssignableFrom(type));
18+
var analyzers = analyzersTypes.Select(type => (DiagnosticAnalyzer)Activator.CreateInstance(type));
19+
20+
return analyzers.ToArray();
21+
}
22+
}
23+
}
Lines changed: 141 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,141 @@
1+
using System;
2+
using System.Collections.Immutable;
3+
using System.Linq;
4+
using System.Reflection;
5+
using FluentAssertions.Execution;
6+
using Microsoft.CodeAnalysis;
7+
using Microsoft.CodeAnalysis.CSharp;
8+
using Microsoft.CodeAnalysis.Text;
9+
10+
using XunitAssert = Xunit.Assert;
11+
using System.Net.Http;
12+
using System.Collections.Concurrent;
13+
using System.Collections.ObjectModel;
14+
15+
namespace FluentAssertions.Analyzers.TestUtils
16+
{
17+
public class CsProjectGenerator
18+
{
19+
static CsProjectGenerator()
20+
{
21+
References = new[]
22+
{
23+
typeof(object), // System.Private.CoreLib
24+
typeof(Console), // System
25+
typeof(Uri), // System.Private.Uri
26+
typeof(Enumerable), // System.Linq
27+
typeof(CSharpCompilation), // Microsoft.CodeAnalysis.CSharp
28+
typeof(Compilation), // Microsoft.CodeAnalysis
29+
typeof(AssertionScope), // FluentAssertions.Core
30+
typeof(AssertionExtensions), // FluentAssertions
31+
typeof(HttpRequestMessage), // System.Net.Http
32+
typeof(ImmutableArray), // System.Collections.Immutable
33+
typeof(ConcurrentBag<>), // System.Collections.Concurrent
34+
typeof(ReadOnlyDictionary<,>), // System.ObjectModel
35+
typeof(Microsoft.VisualStudio.TestTools.UnitTesting.Assert), // MsTest
36+
typeof(XunitAssert), // Xunit
37+
}.Select(type => type.GetTypeInfo().Assembly.Location)
38+
.Append(GetSystemAssemblyPathByName("System.Globalization.dll"))
39+
.Append(GetSystemAssemblyPathByName("System.Text.RegularExpressions.dll"))
40+
.Append(GetSystemAssemblyPathByName("System.Runtime.Extensions.dll"))
41+
.Append(GetSystemAssemblyPathByName("System.Data.Common.dll"))
42+
.Append(GetSystemAssemblyPathByName("System.Threading.Tasks.dll"))
43+
.Append(GetSystemAssemblyPathByName("System.Runtime.dll"))
44+
.Append(GetSystemAssemblyPathByName("System.Reflection.dll"))
45+
.Append(GetSystemAssemblyPathByName("System.Xml.dll"))
46+
.Append(GetSystemAssemblyPathByName("System.Xml.XDocument.dll"))
47+
.Append(GetSystemAssemblyPathByName("System.Private.Xml.Linq.dll"))
48+
.Append(GetSystemAssemblyPathByName("System.Linq.Expressions.dll"))
49+
.Append(GetSystemAssemblyPathByName("System.Collections.dll"))
50+
.Append(GetSystemAssemblyPathByName("netstandard.dll"))
51+
.Append(GetSystemAssemblyPathByName("System.Xml.ReaderWriter.dll"))
52+
.Append(GetSystemAssemblyPathByName("System.Private.Xml.dll"))
53+
.Select(location => (MetadataReference)MetadataReference.CreateFromFile(location))
54+
.ToImmutableArray();
55+
56+
string GetSystemAssemblyPathByName(string assemblyName)
57+
{
58+
var root = System.IO.Path.GetDirectoryName(typeof(object).Assembly.Location);
59+
return System.IO.Path.Combine(root, assemblyName);
60+
}
61+
}
62+
// based on http://code.fitness/post/2017/02/using-csharpscript-with-netstandard.html
63+
public static string GetSystemAssemblyPathByName(string assemblyName)
64+
{
65+
var root = System.IO.Path.GetDirectoryName(typeof(object).Assembly.Location);
66+
return System.IO.Path.Combine(root, assemblyName);
67+
}
68+
69+
private static readonly ImmutableArray<MetadataReference> References;
70+
71+
private static readonly string DefaultFilePathPrefix = "Test";
72+
private static readonly string CSharpDefaultFileExt = "cs";
73+
private static readonly string VisualBasicDefaultExt = "vb";
74+
private static readonly string TestProjectName = "TestProject";
75+
76+
/// <summary>
77+
/// Given an array of strings as sources and a language, turn them into a project and return the documents and spans of it.
78+
/// </summary>
79+
/// <param name="sources">Classes in the form of strings</param>
80+
/// <param name="language">The language the source code is in</param>
81+
/// <returns>A Tuple containing the Documents produced from the sources and their TextSpans if relevant</returns>
82+
public static Document[] GetDocuments(string[] sources, string language)
83+
{
84+
if (language != LanguageNames.CSharp && language != LanguageNames.VisualBasic)
85+
{
86+
throw new ArgumentException("Unsupported Language");
87+
}
88+
89+
var project = CreateProject(sources, language);
90+
var documents = project.Documents.ToArray();
91+
92+
if (sources.Length != documents.Length)
93+
{
94+
throw new SystemException("Amount of sources did not match amount of Documents created");
95+
}
96+
97+
return documents;
98+
}
99+
100+
/// <summary>
101+
/// Create a Document from a string through creating a project that contains it.
102+
/// </summary>
103+
/// <param name="source">Classes in the form of a string</param>
104+
/// <param name="language">The language the source code is in</param>
105+
/// <returns>A Document created from the source string</returns>
106+
public static Document CreateDocument(string source, string language = LanguageNames.CSharp)
107+
{
108+
return CreateProject(new[] { source }, language).Documents.First();
109+
}
110+
111+
/// <summary>
112+
/// Create a project using the inputted strings as sources.
113+
/// </summary>
114+
/// <param name="sources">Classes in the form of strings</param>
115+
/// <param name="language">The language the source code is in</param>
116+
/// <returns>A Project created out of the Documents created from the source strings</returns>
117+
public static Project CreateProject(string[] sources, string language = LanguageNames.CSharp)
118+
{
119+
string fileNamePrefix = DefaultFilePathPrefix;
120+
string fileExt = language == LanguageNames.CSharp ? CSharpDefaultFileExt : VisualBasicDefaultExt;
121+
122+
var projectId = ProjectId.CreateNewId(debugName: TestProjectName);
123+
124+
var solution = new AdhocWorkspace()
125+
.CurrentSolution
126+
.AddProject(projectId, TestProjectName, TestProjectName, language)
127+
.AddMetadataReferences(projectId, References);
128+
129+
int count = 0;
130+
foreach (var source in sources)
131+
{
132+
var newFileName = fileNamePrefix + count + "." + fileExt;
133+
var documentId = DocumentId.CreateNewId(projectId, debugName: newFileName);
134+
solution = solution.AddDocument(documentId, newFileName, SourceText.From(source));
135+
count++;
136+
}
137+
return solution.GetProject(projectId);
138+
}
139+
}
140+
}
141+
Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
<Project Sdk="Microsoft.NET.Sdk">
2+
3+
<PropertyGroup>
4+
<TargetFramework>net6.0</TargetFramework>
5+
6+
<IsPackable>false</IsPackable>
7+
</PropertyGroup>
8+
9+
<ItemGroup>
10+
<PackageReference Include="FluentAssertions" Version="6.1.0" />
11+
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.3.0" />
12+
<PackageReference Include="MSTest.TestAdapter" Version="2.0.0" />
13+
<PackageReference Include="MSTest.TestFramework" Version="2.0.0" />
14+
<PackageReference Include="xunit.assert" Version="2.4.2" />
15+
</ItemGroup>
16+
17+
<ItemGroup>
18+
<ProjectReference Include="..\FluentAssertions.Analyzers\FluentAssertions.Analyzers.csproj" />
19+
</ItemGroup>
20+
21+
</Project>

0 commit comments

Comments
 (0)