Create a high-performance ORC reader

.gitignore

@@ -251,3 +251,6 @@ paket-files/
 .idea/
 *.sln.iml
 launchSettings.json
+
+# Benchamarks
+**/BenchmarkDotNet.Artifacts/results

ApacheOrcDotNet.sln

@@ -1,7 +1,7 @@
 
 Microsoft Visual Studio Solution File, Format Version 12.00
-# Visual Studio Version 16
-VisualStudioVersion = 16.0.31129.286
+# Visual Studio Version 17
+VisualStudioVersion = 17.1.32319.34
 MinimumVisualStudioVersion = 10.0.40219.1
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{58463841-CD45-4229-9E70-A0B74E70ADD7}"
 	ProjectSection(SolutionItems) = preProject
@@ -29,6 +29,16 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ApacheOrcDotNet.WriterTest.
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ApacheOrcDotNet.ReaderTest.App", "src\ApacheOrcDotNet.ReaderTest.App\ApacheOrcDotNet.ReaderTest.App.csproj", "{F3C9D56A-913A-453D-9C42-B9F467748413}"
 EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ApacheOrcDotNet.OptimizedReader", "src\ApacheOrcDotNet.OptimizedReader\ApacheOrcDotNet.OptimizedReader.csproj", "{6C94BFE7-2452-42FF-A17A-4E090F2F8DBD}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ApacheOrcDotNet.OptimizedReader.Test", "test\ApacheOrcDotNet.OptimizedReader.Test\ApacheOrcDotNet.OptimizedReader.Test.csproj", "{AE3C0606-2210-40ED-AE93-67DBD06461BE}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ApacheOrcDotNet.OptimizedReaderTest.App", "src\ApacheOrcDotNet.OptimizedReaderTest.App\ApacheOrcDotNet.OptimizedReaderTest.App.csproj", "{E8C6F47D-51EE-4488-AF63-2898C01A6515}"
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "perf", "perf", "{23793296-3A05-4668-8967-364E6F73EE78}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MicroBenchmarks", "perf\MicroBenchmarks\MicroBenchmarks.csproj", "{F03F6A0D-A431-4A8A-9ED3-FF4690051713}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -51,6 +61,22 @@ Global
 		{F3C9D56A-913A-453D-9C42-B9F467748413}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{F3C9D56A-913A-453D-9C42-B9F467748413}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{F3C9D56A-913A-453D-9C42-B9F467748413}.Release|Any CPU.Build.0 = Release|Any CPU
+		{6C94BFE7-2452-42FF-A17A-4E090F2F8DBD}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{6C94BFE7-2452-42FF-A17A-4E090F2F8DBD}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{6C94BFE7-2452-42FF-A17A-4E090F2F8DBD}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{6C94BFE7-2452-42FF-A17A-4E090F2F8DBD}.Release|Any CPU.Build.0 = Release|Any CPU
+		{AE3C0606-2210-40ED-AE93-67DBD06461BE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{AE3C0606-2210-40ED-AE93-67DBD06461BE}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{AE3C0606-2210-40ED-AE93-67DBD06461BE}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{AE3C0606-2210-40ED-AE93-67DBD06461BE}.Release|Any CPU.Build.0 = Release|Any CPU
+		{E8C6F47D-51EE-4488-AF63-2898C01A6515}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{E8C6F47D-51EE-4488-AF63-2898C01A6515}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{E8C6F47D-51EE-4488-AF63-2898C01A6515}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{E8C6F47D-51EE-4488-AF63-2898C01A6515}.Release|Any CPU.Build.0 = Release|Any CPU
+		{F03F6A0D-A431-4A8A-9ED3-FF4690051713}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{F03F6A0D-A431-4A8A-9ED3-FF4690051713}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{F03F6A0D-A431-4A8A-9ED3-FF4690051713}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{F03F6A0D-A431-4A8A-9ED3-FF4690051713}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -60,6 +86,10 @@ Global
 		{D43761A7-6270-4C43-A2CE-2C244117F30A} = {40D373E1-A120-41E0-9D3D-1BACC719BB38}
 		{8FC4EFCA-35E9-4965-A2F8-52974F29E4DA} = {58463841-CD45-4229-9E70-A0B74E70ADD7}
 		{F3C9D56A-913A-453D-9C42-B9F467748413} = {58463841-CD45-4229-9E70-A0B74E70ADD7}
+		{6C94BFE7-2452-42FF-A17A-4E090F2F8DBD} = {58463841-CD45-4229-9E70-A0B74E70ADD7}
+		{AE3C0606-2210-40ED-AE93-67DBD06461BE} = {40D373E1-A120-41E0-9D3D-1BACC719BB38}
+		{E8C6F47D-51EE-4488-AF63-2898C01A6515} = {58463841-CD45-4229-9E70-A0B74E70ADD7}
+		{F03F6A0D-A431-4A8A-9ED3-FF4690051713} = {23793296-3A05-4668-8967-364E6F73EE78}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {BAAC5D9A-83A0-432F-9987-90C49734F761}

perf/MicroBenchmarks/BenchmarkDotNet.Artifacts/results/MicroBenchmarks.SwitchStatements-report-github.md

@@ -0,0 +1,14 @@
+``` ini
+
+BenchmarkDotNet=v0.13.1, OS=Windows 10.0.19044.1706 (21H2)
+Intel Core i7-10750H CPU 2.60GHz, 1 CPU, 12 logical and 6 physical cores
+.NET SDK=6.0.300
+  [Host]     : .NET 5.0.17 (5.0.1722.21314), X64 RyuJIT
+  DefaultJob : .NET 5.0.17 (5.0.1722.21314), X64 RyuJIT
+
+
+```
+|          Method |     Mean |     Error |    StdDev | Rank | Allocated |
+|---------------- |---------:|----------:|----------:|-----:|----------:|
+|        Standard | 1.770 ns | 0.0102 ns | 0.0090 ns |    2 |         - |
+| PatternMatching | 1.688 ns | 0.0584 ns | 0.0738 ns |    1 |         - |

perf/MicroBenchmarks/BenchmarkDotNet.Artifacts/results/MicroBenchmarks.SwitchStatements-report.csv

@@ -0,0 +1,3 @@
+Method;Job;AnalyzeLaunchVariance;EvaluateOverhead;MaxAbsoluteError;MaxRelativeError;MinInvokeCount;MinIterationTime;OutlierMode;Affinity;EnvironmentVariables;Jit;Platform;PowerPlanMode;Runtime;AllowVeryLargeObjects;Concurrent;CpuGroups;Force;HeapAffinitizeMask;HeapCount;NoAffinitize;RetainVm;Server;Arguments;BuildConfiguration;Clock;EngineFactory;NuGetReferences;Toolchain;IsMutator;InvocationCount;IterationCount;IterationTime;LaunchCount;MaxIterationCount;MaxWarmupIterationCount;MemoryRandomization;MinIterationCount;MinWarmupIterationCount;RunStrategy;UnrollFactor;WarmupCount;Mean;Error;StdDev;Rank;Allocated
+Standard;DefaultJob;False;Default;Default;Default;Default;Default;Default;111111111111;Empty;RyuJit;X64;8c5e7fda-e8bf-4a96-9a85-a6e23a8c635c;.NET 5.0;False;True;False;True;Default;Default;False;False;False;Default;Default;Default;Default;Default;Default;Default;1;Default;Default;Default;Default;Default;Default;Default;Default;Default;16;Default;1.770 ns;0.0102 ns;0.0090 ns;2;0 B
+PatternMatching;DefaultJob;False;Default;Default;Default;Default;Default;Default;111111111111;Empty;RyuJit;X64;8c5e7fda-e8bf-4a96-9a85-a6e23a8c635c;.NET 5.0;False;True;False;True;Default;Default;False;False;False;Default;Default;Default;Default;Default;Default;Default;1;Default;Default;Default;Default;Default;Default;Default;Default;Default;16;Default;1.688 ns;0.0584 ns;0.0738 ns;1;0 B

perf/MicroBenchmarks/BenchmarkDotNet.Artifacts/results/MicroBenchmarks.SwitchStatements-report.html

@@ -0,0 +1,31 @@
+<!DOCTYPE html>
+<html lang='en'>
+<head>
+<meta charset='utf-8' />
+<title>MicroBenchmarks.SwitchStatements-20220517-160420</title>
+
+<style type="text/css">
+	table { border-collapse: collapse; display: block; width: 100%; overflow: auto; }
+	td, th { padding: 6px 13px; border: 1px solid #ddd; text-align: right; }
+	tr { background-color: #fff; border-top: 1px solid #ccc; }
+	tr:nth-child(even) { background: #f8f8f8; }
+</style>
+</head>
+<body>
+<pre><code>
+BenchmarkDotNet=v0.13.1, OS=Windows 10.0.19044.1706 (21H2)
+Intel Core i7-10750H CPU 2.60GHz, 1 CPU, 12 logical and 6 physical cores
+.NET SDK=6.0.300
+  [Host]     : .NET 5.0.17 (5.0.1722.21314), X64 RyuJIT
+  DefaultJob : .NET 5.0.17 (5.0.1722.21314), X64 RyuJIT
+</code></pre>
+<pre><code></code></pre>
+
+<table>
+<thead><tr><th>   Method</th><th>Mean</th><th>Error</th><th>StdDev</th><th>Rank</th><th>Allocated</th>
+</tr>
+</thead><tbody><tr><td>Standard</td><td>1.770 ns</td><td>0.0102 ns</td><td>0.0090 ns</td><td>2</td><td>-</td>
+</tr><tr><td>PatternMatching</td><td>1.688 ns</td><td>0.0584 ns</td><td>0.0738 ns</td><td>1</td><td>-</td>
+</tr></tbody></table>
+</body>
+</html>

perf/MicroBenchmarks/BufferFillersBenchmarks.cs

@@ -0,0 +1,33 @@
+using BenchmarkDotNet.Attributes;
+using System;
+
+/*
+|         Method |        Mean |     Error |    StdDev | Rank | Allocated |
+|--------------- |------------:|----------:|----------:|-----:|----------:|
+|   FillWithLoop | 29,797.8 ns | 481.98 ns | 427.27 ns |    2 |         - |
+| FillWithMethod |    902.8 ns |  17.79 ns |  28.73 ns |    1 |         - |
+ */
+
+namespace MicroBenchmarks
+{
+    [RankColumn]
+    [MemoryDiagnoser]
+    public class BufferFillersBenchmarks
+    {
+        private const int size = 65_536;
+        private readonly byte[] _buffer = new byte[size];
+
+        [Benchmark]
+        public void FillWithLoop()
+        {
+            for (int i = 0; i < size; i++)
+                _buffer[i] = 255;
+        }
+
+        [Benchmark]
+        public void FillWithMethod()
+        {
+            _buffer.AsSpan().Fill(255);
+        }
+    }
+}

perf/MicroBenchmarks/BufferReaderBenchmarks.cs

@@ -0,0 +1,56 @@
+using ApacheOrcDotNet.OptimizedReader;
+using BenchmarkDotNet.Attributes;
+using System.IO;
+
+namespace MicroBenchmarks
+{
+    [RankColumn]
+    [MemoryDiagnoser]
+    public class BufferReaderBenchmarks
+    {
+        private readonly byte[] _readBuffer1 = new byte[3];
+        private readonly byte[] _readBuffer2 = new byte[3];
+        private readonly string _testFilePath = @"BenchmarkDotNet.SampleData/optimized_reader_test_file.orc";
+        private readonly byte[] _testBuffer;
+
+        public BufferReaderBenchmarks()
+        {
+            using (var fileStream = File.OpenRead(_testFilePath))
+            {
+                _testBuffer = new byte[fileStream.Length];
+                fileStream.Read(_testBuffer);
+            }
+        }
+
+        [Benchmark]
+        public void TryRead()
+        {
+            var reader = new BufferReader(_testBuffer);
+            while (reader.TryRead(out _)) { }
+        }
+
+        [Benchmark]
+        public void TryCopyTo()
+        {
+            var reader = new BufferReader(_testBuffer);
+            while (reader.TryReadTo(_readBuffer1)) { }
+        }
+
+        [Benchmark]
+        public void TryRead3Bytes()
+        {
+            var reader = new BufferReader(_testBuffer);
+
+            _ = reader.TryRead(out _);
+            _ = reader.TryRead(out _);
+            _ = reader.TryRead(out _);
+        }
+
+        [Benchmark]
+        public void TryCopyTo3Bytes()
+        {
+            var reader = new BufferReader(_testBuffer);
+            _ = reader.TryReadTo(_readBuffer2);
+        }
+    }
+}

perf/MicroBenchmarks/ComparisonBenchmarks.cs

@@ -0,0 +1,43 @@
+using ApacheOrcDotNet.OptimizedReader.Infrastructure;
+using BenchmarkDotNet.Attributes;
+using System;
+using System.Collections.Generic;
+
+/*
+|                              Method |      Mean |     Error |    StdDev | Rank | Allocated |
+|------------------------------------ |----------:|----------:|----------:|-----:|----------:|
+|                       ArrayEquality |  7.720 ns | 0.0755 ns | 0.0706 ns |    1 |         - |
+|    DictionaryContainsKeyAndEquality | 12.429 ns | 0.0690 ns | 0.0645 ns |    2 |         - |
+|               DictionaryTryGetValue | 13.052 ns | 0.0677 ns | 0.0633 ns |    3 |         - |
+ */
+
+namespace MicroBenchmarks
+{
+    [RankColumn]
+    [MemoryDiagnoser]
+    public class ComparisonBenchmarks
+    {
+        private const int size = 255;
+        private readonly StreamRange[] _buffer = new StreamRange[size];
+        private readonly Dictionary<int, StreamRange> _buffer2 = new();
+        private readonly StreamRange _testRange = new StreamRange(255, 255, 255);
+
+        public ComparisonBenchmarks()
+        {
+            for (int i = 0; i < size; i++)
+            {
+                _buffer[i] = new StreamRange(i, i, i);
+                _buffer2.Add(i, new StreamRange(i, i, i));
+            }
+        }
+
+        [Benchmark]
+        public bool ArrayEquality() => _buffer[128] == _testRange;
+
+        [Benchmark]
+        public bool DictionaryContainsKeyAndEquality() => _buffer2.ContainsKey(128) && _buffer[128] == _testRange;
+
+        [Benchmark]
+        public bool DictionaryTryGetValue() => _buffer2.TryGetValue(128, out var range) && range == _testRange;
+    }
+}

perf/MicroBenchmarks/LoopUnrollBenchmarks.cs

@@ -0,0 +1,55 @@
+using BenchmarkDotNet.Attributes;
+using System;
+
+namespace MicroBenchmarks
+{
+    [RankColumn]
+    [MemoryDiagnoser]
+    public class LoopUnrollBenchmarks
+    {
+        [Benchmark]
+        public void NormalLoop()
+        {
+            var numValuesRead = 0;
+            var numByteValuesRead = 1000;
+            Span<bool> outputValues = stackalloc bool[numByteValuesRead * 8];
+
+            for (int idx = 0; idx < numByteValuesRead; idx++)
+            {
+                var decodedByte = 0xff;
+
+                for (int bitIdx = 7; bitIdx >= 0; bitIdx--)
+                {
+                    outputValues[numValuesRead++] = (decodedByte & 1 << bitIdx) != 0;
+                }
+            }
+        }
+
+        [Benchmark]
+        public void UnrolledLoop()
+        {
+            var decodedByte = 0xff;
+            var numValuesRead = 0;
+            var numByteValuesRead = 1000;
+            Span<bool> outputValues = stackalloc bool[numByteValuesRead * 8];
+
+            for (int idx = 0; idx < numByteValuesRead; idx++)
+            {
+                outputValues = outputValues.Slice(8);
+                if (outputValues.Length < 8)
+                    break;
+
+                outputValues[0] = (decodedByte & 128) != 0;
+                outputValues[1] = (decodedByte & 64) != 0;
+                outputValues[2] = (decodedByte & 32) != 0;
+                outputValues[3] = (decodedByte & 16) != 0;
+                outputValues[4] = (decodedByte & 8) != 0;
+                outputValues[5] = (decodedByte & 4) != 0;
+                outputValues[6] = (decodedByte & 2) != 0;
+                outputValues[7] = (decodedByte & 1) != 0;
+
+                numValuesRead += 8;
+            }
+        }
+    }
+}

perf/MicroBenchmarks/MicroBenchmarks.csproj

@@ -0,0 +1,22 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net5.0</TargetFramework>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <Content Include="BenchmarkDotNet.SampleData\*.orc">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </Content>
+  </ItemGroup>
+
+  <ItemGroup>
+    <PackageReference Include="BenchmarkDotNet" Version="0.13.1" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\..\src\ApacheOrcDotNet.OptimizedReader\ApacheOrcDotNet.OptimizedReader.csproj" />
+  </ItemGroup>
+
+</Project>