From 2f8f3b63cfa83cae679d0afdd43ebfefc491d326 Mon Sep 17 00:00:00 2001 From: Daniel Crenna Date: Tue, 5 Aug 2025 01:40:30 -0400 Subject: [PATCH 1/7] Add convenience script to provision docker image and run tests --- run-tests.ps1 | 232 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 232 insertions(+) create mode 100644 run-tests.ps1 diff --git a/run-tests.ps1 b/run-tests.ps1 new file mode 100644 index 00000000..68fecaea --- /dev/null +++ b/run-tests.ps1 @@ -0,0 +1,232 @@ +# ClickHouse .NET Driver Test Runner Script +# This script configures the environment and runs tests against a Docker ClickHouse instance + +param( + [string]$Filter = "", + [string]$Framework = "", + [switch]$IntegrationTests = $false, + [switch]$Verbose = $false, + [switch]$SkipDockerSetup = $false +) + +Write-Host "ClickHouse .NET Driver Test Runner" -ForegroundColor Cyan +Write-Host "===================================" -ForegroundColor Cyan + +# Function to check if a command exists +function Test-CommandExists { + param($Command) + try { + if (Get-Command $Command -ErrorAction Stop) { + return $true + } + } catch { + return $false + } +} + +# Function to wait for ClickHouse to be ready +function Wait-ForClickHouse { + param( + [string]$ContainerName = "clickhouse-test", + [int]$MaxAttempts = 30 + ) + + Write-Host "Waiting for ClickHouse to be ready..." -ForegroundColor Yellow + + for ($i = 1; $i -le $MaxAttempts; $i++) { + try { + # Try to execute a simple query + $result = docker exec $ContainerName clickhouse-client --query "SELECT 1" 2>&1 + if ($result -eq "1") { + Write-Host "ClickHouse is ready!" -ForegroundColor Green + return $true + } + } catch { + # Ignore errors during startup + } + + Write-Host "Attempt $i/$MaxAttempts - ClickHouse not ready yet..." -ForegroundColor Gray + Start-Sleep -Seconds 1 + } + + return $false +} + +if (-not $SkipDockerSetup) { + # Check if Docker is installed + Write-Host "`nChecking Docker installation..." -ForegroundColor Yellow + if (-not (Test-CommandExists "docker")) { + Write-Host "Docker is not installed!" -ForegroundColor Red + Write-Host "`nDocker Desktop can be downloaded from: https://www.docker.com/products/docker-desktop" -ForegroundColor Yellow + Write-Host "After installing Docker, please restart this script." -ForegroundColor Yellow + exit 1 + } + + # Check if Docker is running + Write-Host "Checking if Docker is running..." -ForegroundColor Yellow + try { + docker version | Out-Null + Write-Host "Docker is installed and running." -ForegroundColor Green + } catch { + Write-Host "Docker is installed but not running!" -ForegroundColor Red + Write-Host "Please start Docker Desktop and try again." -ForegroundColor Yellow + exit 1 + } + + # Define expected container configuration + $expectedContainerName = "clickhouse-test" + $expectedUsername = "default" + $expectedPassword = "test123" + $expectedHttpPort = 8123 + $expectedNativePort = 9000 + + # Check for any running ClickHouse container + Write-Host "`nChecking for ClickHouse containers..." -ForegroundColor Yellow + $runningContainers = docker ps --format "table {{.Names}}\t{{.Ports}}" | Select-Object -Skip 1 | Where-Object { $_ -match "clickhouse" } + + $needNewContainer = $true + + if ($runningContainers) { + Write-Host "Found running ClickHouse container(s):" -ForegroundColor Yellow + $runningContainers | ForEach-Object { Write-Host " $_" -ForegroundColor Gray } + + # Check if our expected container is running with correct ports + $expectedContainer = docker ps --filter "name=$expectedContainerName" --format "json" | ConvertFrom-Json + if ($expectedContainer) { + $ports = docker port $expectedContainerName 2>$null + if ($ports -match "${expectedHttpPort}/tcp" -and $ports -match "${expectedNativePort}/tcp") { + Write-Host "Container '$expectedContainerName' is running with correct ports." -ForegroundColor Green + $needNewContainer = $false + } else { + Write-Host "Container '$expectedContainerName' exists but has incorrect port mappings." -ForegroundColor Yellow + } + } + } + + if ($needNewContainer) { + Write-Host "`nSetting up ClickHouse container..." -ForegroundColor Yellow + + # Stop and remove existing test container if it exists + $existingContainer = docker ps -a --filter "name=$expectedContainerName" --format "json" | ConvertFrom-Json + if ($existingContainer) { + Write-Host "Removing existing container '$expectedContainerName'..." -ForegroundColor Yellow + docker stop $expectedContainerName 2>$null | Out-Null + docker rm $expectedContainerName 2>$null | Out-Null + } + + # Pull latest ClickHouse image + Write-Host "Pulling latest ClickHouse image..." -ForegroundColor Yellow + docker pull clickhouse/clickhouse-server:latest + + # Start new container with proper configuration + Write-Host "Starting new ClickHouse container..." -ForegroundColor Yellow + $containerId = docker run -d ` + --name $expectedContainerName ` + -p ${expectedHttpPort}:8123 ` + -p ${expectedNativePort}:9000 ` + -e CLICKHOUSE_DB=default ` + -e CLICKHOUSE_USER=$expectedUsername ` + -e CLICKHOUSE_PASSWORD=$expectedPassword ` + clickhouse/clickhouse-server:latest + + if ($LASTEXITCODE -eq 0) { + Write-Host "Container started successfully (ID: $($containerId.Substring(0, 12)))" -ForegroundColor Green + + # Wait for ClickHouse to be ready + if (-not (Wait-ForClickHouse -ContainerName $expectedContainerName)) { + Write-Host "ClickHouse failed to start properly!" -ForegroundColor Red + Write-Host "`nContainer logs:" -ForegroundColor Yellow + docker logs $expectedContainerName --tail 50 + exit 1 + } + } else { + Write-Host "Failed to start ClickHouse container!" -ForegroundColor Red + exit 1 + } + } + + # Verify connection + Write-Host "`nVerifying ClickHouse connection..." -ForegroundColor Yellow + try { + $testUrl = "http://localhost:${expectedHttpPort}/ping" + $response = Invoke-WebRequest -Uri $testUrl -Method Get -Headers @{ + "X-ClickHouse-User" = $expectedUsername + "X-ClickHouse-Key" = $expectedPassword + } -UseBasicParsing -ErrorAction Stop + + if ($response.StatusCode -eq 200) { + Write-Host "Successfully connected to ClickHouse!" -ForegroundColor Green + } + } catch { + Write-Host "Warning: Could not verify HTTP connection. Tests may fail." -ForegroundColor Yellow + Write-Host "Error: $_" -ForegroundColor Gray + } +} + +# Set environment variable for tests +$env:CLICKHOUSE_CONNECTION = "Host=localhost;Port=8123;Username=default;Password=test123;Database=default" +Write-Host "`nConnection string set: $env:CLICKHOUSE_CONNECTION" -ForegroundColor Green + +# Build the test command +$testProject = if ($IntegrationTests) { + "ClickHouse.Driver.IntegrationTests" +} else { + "ClickHouse.Driver.Tests" +} + +$testCommand = "dotnet test $testProject/" + +# Add framework if specified +if ($Framework) { + $testCommand += " --framework $Framework" +} + +# Add filter if specified +if ($Filter) { + $testCommand += " --filter `"$Filter`"" +} + +# Add verbosity +if ($Verbose) { + $testCommand += " -v normal" +} else { + $testCommand += " -v minimal" +} + +# Display test configuration +Write-Host "`nTest Configuration:" -ForegroundColor Cyan +Write-Host " Project: $testProject" -ForegroundColor White +if ($Framework) { Write-Host " Framework: $Framework" -ForegroundColor White } +if ($Filter) { Write-Host " Filter: $Filter" -ForegroundColor White } +Write-Host " Verbosity: $(if ($Verbose) { 'normal' } else { 'minimal' })" -ForegroundColor White + +Write-Host "`nRunning command: $testCommand" -ForegroundColor Yellow +Write-Host "`nStarting tests..." -ForegroundColor Green +Write-Host "=================" -ForegroundColor Green + +# Run the tests +try { + Invoke-Expression $testCommand + $exitCode = $LASTEXITCODE +} catch { + Write-Host "`nError running tests: $_" -ForegroundColor Red + exit 1 +} + +# Check test results +if ($exitCode -eq 0) { + Write-Host "`nTests completed successfully!" -ForegroundColor Green +} else { + Write-Host "`nTests failed with exit code: $exitCode" -ForegroundColor Red +} + +# Offer to view container logs if tests failed +if ($exitCode -ne 0) { + $viewLogs = Read-Host "`nWould you like to view ClickHouse container logs? (y/n)" + if ($viewLogs -eq 'y') { + Write-Host "`nLast 50 lines of ClickHouse logs:" -ForegroundColor Yellow + docker logs clickhouse-test --tail 50 + } +} + +exit $exitCode \ No newline at end of file From 492ad323ac707c7b706e1dbd01f9561a09ddc251 Mon Sep 17 00:00:00 2001 From: Daniel Crenna Date: Tue, 5 Aug 2025 01:51:56 -0400 Subject: [PATCH 2/7] Add convenience script to run benchmarks --- run-benchmarks.ps1 | 52 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 run-benchmarks.ps1 diff --git a/run-benchmarks.ps1 b/run-benchmarks.ps1 new file mode 100644 index 00000000..5aa09b94 --- /dev/null +++ b/run-benchmarks.ps1 @@ -0,0 +1,52 @@ +#!/usr/bin/env pwsh + +# Run ArrayPool benchmark for ClickHouseDataReader + +param( + [switch]$SkipBuild, + [switch]$Detailed, + [string]$Filter = "*RecyclableMemoryStreamBenchmark*", + [int]$WarmupCount = 3, + [int]$IterationCount = 5 +) + +if (-not $SkipBuild) { + Write-Host "Building ClickHouse.Driver.Benchmark in Release mode..." -ForegroundColor Green + dotnet build ClickHouse.Driver.Benchmark/ClickHouse.Driver.Benchmark.csproj --configuration Release + + if ($LASTEXITCODE -ne 0) { + Write-Host "Build failed!" -ForegroundColor Red + exit 1 + } +} else { + Write-Host "Skipping build (using existing binaries)..." -ForegroundColor Yellow +} + +Write-Host "`nRunning benchmark with filter: $Filter" -ForegroundColor Green +Write-Host "WarmupCount: $WarmupCount, IterationCount: $IterationCount" -ForegroundColor Cyan + +if ($Detailed) { + # Run with detailed memory diagnostics + Write-Host "Running with detailed diagnostics..." -ForegroundColor Yellow + dotnet run --project ClickHouse.Driver.Benchmark/ClickHouse.Driver.Benchmark.csproj ` + --configuration Release ` + --no-build ` + -- --filter "$Filter" ` + --warmupCount $WarmupCount ` + --iterationCount $IterationCount ` + --memory ` + --disasm ` + --profiler ETW +} else { + # Standard run with memory diagnostics + dotnet run --project ClickHouse.Driver.Benchmark/ClickHouse.Driver.Benchmark.csproj ` + --configuration Release ` + --no-build ` + -- --filter "$Filter" ` + --warmupCount $WarmupCount ` + --iterationCount $IterationCount ` + --memory +} + +Write-Host "`nBenchmark complete! Results are in BenchmarkDotNet.Artifacts folder" -ForegroundColor Green +Write-Host "You can find detailed results in: BenchmarkDotNet.Artifacts\results\" -ForegroundColor Cyan \ No newline at end of file From be276d7b209ff82ecc46e470825acd85b24176d6 Mon Sep 17 00:00:00 2001 From: Daniel Crenna Date: Tue, 5 Aug 2025 01:52:23 -0400 Subject: [PATCH 3/7] Update benchmark and integration test projects to run on .NET 9 --- ClickHouse.Driver.Benchmark/ClickHouse.Driver.Benchmark.csproj | 2 +- .../ClickHouse.Driver.IntegrationTests.csproj | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ClickHouse.Driver.Benchmark/ClickHouse.Driver.Benchmark.csproj b/ClickHouse.Driver.Benchmark/ClickHouse.Driver.Benchmark.csproj index a5b755e9..b36896fa 100644 --- a/ClickHouse.Driver.Benchmark/ClickHouse.Driver.Benchmark.csproj +++ b/ClickHouse.Driver.Benchmark/ClickHouse.Driver.Benchmark.csproj @@ -2,7 +2,7 @@ Exe - net8.0 + net9.0 true true snupkg diff --git a/ClickHouse.Driver.IntegrationTests/ClickHouse.Driver.IntegrationTests.csproj b/ClickHouse.Driver.IntegrationTests/ClickHouse.Driver.IntegrationTests.csproj index ed6c301d..118fa46e 100644 --- a/ClickHouse.Driver.IntegrationTests/ClickHouse.Driver.IntegrationTests.csproj +++ b/ClickHouse.Driver.IntegrationTests/ClickHouse.Driver.IntegrationTests.csproj @@ -1,7 +1,7 @@ - net8.0 + net9.0 latest enable enable From 91ac7d547cf05e49fe804d1c8d227064a3137bf1 Mon Sep 17 00:00:00 2001 From: Daniel Crenna Date: Tue, 5 Aug 2025 01:59:23 -0400 Subject: [PATCH 4/7] PERF: use RecyclableMemoryStream in ClickHouseDataReader Optimize memory usage for small to medium result sets by replacing BufferedStream with RecyclableMemoryStream in ClickHouseDataReader Every query allocated a new 512KB buffer via BufferedStream, regardless of actual response size, creating allocation and GC pressure for typical workloads. --- .../RecyclableMemoryStreamBenchmark.cs | 194 ++++++++++++++++ .../BufferedStreamClickHouseDataReader.cs | 217 ++++++++++++++++++ .../ADO/Readers/ClickHouseDataReader.cs | 32 ++- ClickHouse.Driver/Types/TypeConverter.cs | 1 + 4 files changed, 439 insertions(+), 5 deletions(-) create mode 100644 ClickHouse.Driver.Benchmark/RecyclableMemoryStreamBenchmark.cs create mode 100644 ClickHouse.Driver.Benchmark/References/BufferedStreamClickHouseDataReader.cs diff --git a/ClickHouse.Driver.Benchmark/RecyclableMemoryStreamBenchmark.cs b/ClickHouse.Driver.Benchmark/RecyclableMemoryStreamBenchmark.cs new file mode 100644 index 00000000..b3f894d5 --- /dev/null +++ b/ClickHouse.Driver.Benchmark/RecyclableMemoryStreamBenchmark.cs @@ -0,0 +1,194 @@ +using System; +using System.IO; +using System.Net; +using System.Net.Http; +using BenchmarkDotNet.Attributes; +using ClickHouse.Driver.ADO.Readers; +using ClickHouse.Driver.Benchmark.References; +using ClickHouse.Driver.Formats; + +namespace ClickHouse.Driver.Benchmark; + +[MemoryDiagnoser] +[SimpleJob(warmupCount: 3, iterationCount: 10)] +public class RecyclableMemoryStreamBenchmark +{ + private byte[] smallResponseData; // 10 KB + private byte[] mediumResponseData; // 500 KB + private byte[] largeResponseData; // 5 MB + private TypeSettings typeSettings; + + [GlobalSetup] + public void Setup() + { + typeSettings = new TypeSettings(); + + // Create mock responses of different sizes + smallResponseData = CreateMockBinaryResponse(100, 10); // 100 rows, 10 columns ~ 10KB + mediumResponseData = CreateMockBinaryResponse(5000, 10); // 5000 rows, 10 columns ~ 500KB + largeResponseData = CreateMockBinaryResponse(50000, 10); // 50000 rows, 10 columns ~ 5MB + } + + [Benchmark(Baseline = true)] + [Arguments(100)] // Small response + [Arguments(5000)] // Medium response + [Arguments(50000)] // Large response + public long ReadAllData_BufferedStream(int rowCount) + { + var data = rowCount switch + { + 100 => smallResponseData, + 5000 => mediumResponseData, + 50000 => largeResponseData, + _ => throw new ArgumentException() + }; + + using var reader = BufferedStreamClickHouseDataReader.FromHttpResponse( + CreateMockHttpResponse(data), typeSettings); + + long sum = 0; + var rowsRead = 0; + while (reader.Read() && rowsRead < rowCount) + { + for (var i = 0; i < reader.FieldCount; i++) + { + if (reader.GetFieldType(i) == typeof(int)) + sum += reader.GetInt32(i); + } + rowsRead++; + } + + return sum; + } + + [Benchmark] + [Arguments(100)] // Small response + [Arguments(5000)] // Medium response + [Arguments(50000)] // Large response + public long ReadAllData_RecyclableMemoryStream(int rowCount) + { + var data = rowCount switch + { + 100 => smallResponseData, + 5000 => mediumResponseData, + 50000 => largeResponseData, + _ => throw new ArgumentException() + }; + + using var reader = ClickHouseDataReader.FromHttpResponse( + CreateMockHttpResponse(data), typeSettings); + + long sum = 0; + var rowsRead = 0; + while (reader.Read() && rowsRead < rowCount) + { + for (var i = 0; i < reader.FieldCount; i++) + { + if (reader.GetFieldType(i) == typeof(int)) + sum += reader.GetInt32(i); + } + rowsRead++; + } + + return sum; + } + + [Benchmark(Baseline = true)] + public void MultipleSmallQueries_BufferedStream() + { + // Simulate multiple small queries - BufferedStream allocates new buffer each time + for (var i = 0; i < 100; i++) + { + using var reader = BufferedStreamClickHouseDataReader.FromHttpResponse( + CreateMockHttpResponse(smallResponseData), typeSettings); + + while (reader.Read()) + { + _ = reader.GetValue(0); + } + } + } + + [Benchmark] + public void MultipleSmallQueries_RecyclableStream() + { + // Simulate multiple small queries that would benefit from buffer pooling + for (var i = 0; i < 100; i++) + { + using var reader = ClickHouseDataReader.FromHttpResponse( + CreateMockHttpResponse(smallResponseData), typeSettings); + + while (reader.Read()) + { + _ = reader.GetValue(0); + } + } + } + + private static HttpResponseMessage CreateMockHttpResponse(byte[] data) + { + var content = new ByteArrayContent(data); + content.Headers.ContentType = new System.Net.Http.Headers.MediaTypeHeaderValue("application/octet-stream"); + + return new HttpResponseMessage(HttpStatusCode.OK) + { + Content = content + }; + } + + private static byte[] CreateMockBinaryResponse(int rows, int columns) + { + using var stream = new MemoryStream(); + using var writer = new ExtendedBinaryWriter(stream); + + writer.Write7BitEncodedInt(columns); + + for (var i = 0; i < columns; i++) + { + writer.Write($"column{i}"); + } + + for (var i = 0; i < columns; i++) + { + switch (i % 4) + { + case 0: + writer.Write("Int32"); + break; + case 1: + writer.Write("String"); + break; + case 2: + writer.Write("Float64"); + break; + case 3: + writer.Write("UInt64"); + break; + } + } + + for (var row = 0; row < rows; row++) + { + for (var col = 0; col < columns; col++) + { + switch (col % 4) + { + case 0: // Int32 + writer.Write(row); + break; + case 1: // String + writer.Write($"row{row}_col{col}"); + break; + case 2: // Float64 + writer.Write(row / 100.0); + break; + case 3: // UInt64 + writer.Write((ulong)row * 1000); + break; + } + } + } + + return stream.ToArray(); + } +} diff --git a/ClickHouse.Driver.Benchmark/References/BufferedStreamClickHouseDataReader.cs b/ClickHouse.Driver.Benchmark/References/BufferedStreamClickHouseDataReader.cs new file mode 100644 index 00000000..ec84fa43 --- /dev/null +++ b/ClickHouse.Driver.Benchmark/References/BufferedStreamClickHouseDataReader.cs @@ -0,0 +1,217 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.Data; +using System.Data.Common; +using System.Globalization; +using System.IO; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using ClickHouse.Driver.Formats; +using ClickHouse.Driver.Numerics; +using ClickHouse.Driver.Types; + +namespace ClickHouse.Driver.Benchmark.References; + +/// +/// Original ClickHouseDataReader implementation using BufferedStream for benchmarking comparison +/// +internal class BufferedStreamClickHouseDataReader : DbDataReader, IEnumerator, IEnumerable, IDataRecord +{ + private const int BufferSize = 512 * 1024; + + private readonly HttpResponseMessage httpResponse; + private readonly ExtendedBinaryReader reader; + + private BufferedStreamClickHouseDataReader(HttpResponseMessage httpResponse, ExtendedBinaryReader reader, string[] names, ClickHouseType[] types) + { + this.httpResponse = httpResponse ?? throw new ArgumentNullException(nameof(httpResponse)); + this.reader = reader ?? throw new ArgumentNullException(nameof(reader)); + RawTypes = types; + FieldNames = names; + CurrentRow = new object[FieldNames.Length]; + } + + internal static BufferedStreamClickHouseDataReader FromHttpResponse(HttpResponseMessage httpResponse, TypeSettings settings) + { + if (httpResponse is null) throw new ArgumentNullException(nameof(httpResponse)); + ExtendedBinaryReader reader = null; + try + { + var stream = new BufferedStream(httpResponse.Content.ReadAsStreamAsync().GetAwaiter().GetResult(), BufferSize); + reader = new ExtendedBinaryReader(stream); + var (names, types) = ReadHeaders(reader, settings); + return new BufferedStreamClickHouseDataReader(httpResponse, reader, names, types); + } + catch (Exception) + { + httpResponse?.Dispose(); + reader?.Dispose(); + throw; + } + } + + internal ClickHouseType GetClickHouseType(int ordinal) => RawTypes[ordinal]; + + public override object this[int ordinal] => GetValue(ordinal); + + public override object this[string name] => this[GetOrdinal(name)]; + + public override int Depth { get; } + + public override int FieldCount => RawTypes?.Length ?? throw new InvalidOperationException(); + + public override bool IsClosed => false; + + public sealed override bool HasRows => true; + + public override int RecordsAffected { get; } + + protected object[] CurrentRow { get; set; } + + protected string[] FieldNames { get; set; } + + private protected ClickHouseType[] RawTypes { get; set; } + + public override bool GetBoolean(int ordinal) => Convert.ToBoolean(GetValue(ordinal), CultureInfo.InvariantCulture); + + public override byte GetByte(int ordinal) => (byte)GetValue(ordinal); + + public override long GetBytes(int ordinal, long dataOffset, byte[] buffer, int bufferOffset, int length) => throw new NotImplementedException(); + + public override char GetChar(int ordinal) => (char)GetValue(ordinal); + + public override long GetChars(int ordinal, long dataOffset, char[] buffer, int bufferOffset, int length) => throw new NotImplementedException(); + + public override string GetDataTypeName(int ordinal) => GetClickHouseType(ordinal).ToString(); + + public override DateTime GetDateTime(int ordinal) => (DateTime)GetValue(ordinal); + + public override decimal GetDecimal(int ordinal) + { + var value = GetValue(ordinal); + return value is ClickHouseDecimal clickHouseDecimal ? clickHouseDecimal.ToDecimal(CultureInfo.InvariantCulture) : (decimal)value; + } + + public override double GetDouble(int ordinal) => (double)GetValue(ordinal); + + public override Type GetFieldType(int ordinal) + { + var rawType = RawTypes[ordinal]; + return rawType is NullableType nt ? nt.UnderlyingType.FrameworkType : rawType.FrameworkType; + } + + public override float GetFloat(int ordinal) => (float)GetValue(ordinal); + + public override Guid GetGuid(int ordinal) => (Guid)GetValue(ordinal); + + public override short GetInt16(int ordinal) => (short)GetValue(ordinal); + + public override int GetInt32(int ordinal) => (int)GetValue(ordinal); + + public override long GetInt64(int ordinal) => (long)GetValue(ordinal); + + public override string GetName(int ordinal) => FieldNames[ordinal]; + + public override int GetOrdinal(string name) + { + var index = Array.FindIndex(FieldNames, (fn) => fn == name); + if (index == -1) + { + throw new ArgumentException("Column does not exist", nameof(name)); + } + + return index; + } + + public override string GetString(int ordinal) => GetValue(ordinal)?.ToString(); + + public override object GetValue(int ordinal) => CurrentRow[ordinal]; + + public override int GetValues(object[] values) + { + if (CurrentRow == null) + { + throw new InvalidOperationException(); + } + + CurrentRow.CopyTo(values, 0); + return CurrentRow.Length; + } + + public override bool IsDBNull(int ordinal) + { + var value = GetValue(ordinal); + return value is DBNull || value is null; + } + + public override bool NextResult() => false; + + public override void Close() => Dispose(); + + public override T GetFieldValue(int ordinal) => (T)GetValue(ordinal); + + public override DataTable GetSchemaTable() => throw new NotImplementedException(); + + public override Task NextResultAsync(CancellationToken cancellationToken) => Task.FromResult(false); + + public override bool Read() + { + if (reader.PeekChar() == -1) + return false; + + var count = RawTypes.Length; + var data = CurrentRow; + for (var i = 0; i < count; i++) + { + var rawType = RawTypes[i]; + data[i] = rawType.Read(reader); + } + return true; + } + + protected override void Dispose(bool disposing) + { + if (disposing) + { + httpResponse?.Dispose(); + reader?.Dispose(); + } + } + + private static (string[], ClickHouseType[]) ReadHeaders(ExtendedBinaryReader reader, TypeSettings settings) + { + if (reader.PeekChar() == -1) + { + return ([], []); + } + var count = reader.Read7BitEncodedInt(); + var names = new string[count]; + var types = new ClickHouseType[count]; + + for (var i = 0; i < count; i++) + { + names[i] = reader.ReadString(); + } + + for (var i = 0; i < count; i++) + { + var chType = reader.ReadString(); + types[i] = TypeConverter.ParseClickHouseType(chType, settings); + } + return (names, types); + } + + public bool MoveNext() => Read(); + + public void Reset() => throw new NotSupportedException(); + + public override IEnumerator GetEnumerator() => this; + + IEnumerator IEnumerable.GetEnumerator() => this; + + public IDataReader Current => this; + + object IEnumerator.Current => this; +} diff --git a/ClickHouse.Driver/ADO/Readers/ClickHouseDataReader.cs b/ClickHouse.Driver/ADO/Readers/ClickHouseDataReader.cs index 6a534adc..f4e76bd6 100644 --- a/ClickHouse.Driver/ADO/Readers/ClickHouseDataReader.cs +++ b/ClickHouse.Driver/ADO/Readers/ClickHouseDataReader.cs @@ -16,21 +16,37 @@ using ClickHouse.Driver.Numerics; using ClickHouse.Driver.Types; using ClickHouse.Driver.Utility; +using Microsoft.IO; namespace ClickHouse.Driver.ADO.Readers; // TODO: implement IDbColumnSchemaGenerator public class ClickHouseDataReader : DbDataReader, IEnumerator, IEnumerable, IDataRecord { - private const int BufferSize = 512 * 1024; +#if NET6_0_OR_GREATER + private static readonly RecyclableMemoryStreamManager StreamManager = new(new RecyclableMemoryStreamManager.Options + { + BlockSize = 128 * 1024, // 128KiB blocks + LargeBufferMultiple = 1024 * 1024, // 1MiB + MaximumBufferSize = 128 * 1024 * 1024, // 128MiB + GenerateCallStacks = false, + AggressiveBufferReturn = true, + MaximumLargePoolFreeBytes = 256 * 1024 * 1024, // 256MiB + MaximumSmallPoolFreeBytes = 128 * 1024 * 1024, // 128MiB + }); +#else + private static readonly RecyclableMemoryStreamManager StreamManager = new RecyclableMemoryStreamManager(); +#endif private readonly HttpResponseMessage httpResponse; // Used to dispose at the end of reader private readonly ExtendedBinaryReader reader; + private readonly RecyclableMemoryStream recyclableStream; - private ClickHouseDataReader(HttpResponseMessage httpResponse, ExtendedBinaryReader reader, string[] names, ClickHouseType[] types) + private ClickHouseDataReader(HttpResponseMessage httpResponse, ExtendedBinaryReader reader, RecyclableMemoryStream recyclableStream, string[] names, ClickHouseType[] types) { this.httpResponse = httpResponse ?? throw new ArgumentNullException(nameof(httpResponse)); this.reader = reader ?? throw new ArgumentNullException(nameof(reader)); + this.recyclableStream = recyclableStream; RawTypes = types; FieldNames = names; CurrentRow = new object[FieldNames.Length]; @@ -40,17 +56,22 @@ internal static ClickHouseDataReader FromHttpResponse(HttpResponseMessage httpRe { if (httpResponse is null) throw new ArgumentNullException(nameof(httpResponse)); ExtendedBinaryReader reader = null; + RecyclableMemoryStream recyclableStream = null; try { - var stream = new BufferedStream(httpResponse.Content.ReadAsStreamAsync().GetAwaiter().GetResult(), BufferSize); - reader = new ExtendedBinaryReader(stream); // will dispose of stream + recyclableStream = StreamManager.GetStream("ClickHouseDataReader"); + httpResponse.Content.CopyToAsync(recyclableStream).GetAwaiter().GetResult(); + recyclableStream.Position = 0; + + reader = new ExtendedBinaryReader(recyclableStream); // will dispose of stream var (names, types) = ReadHeaders(reader, settings); - return new ClickHouseDataReader(httpResponse, reader, names, types); + return new ClickHouseDataReader(httpResponse, reader, recyclableStream, names, types); } catch (Exception) { httpResponse?.Dispose(); reader?.Dispose(); + recyclableStream?.Dispose(); throw; } } @@ -213,6 +234,7 @@ protected override void Dispose(bool disposing) { httpResponse?.Dispose(); reader?.Dispose(); + recyclableStream?.Dispose(); } } #pragma warning restore CA2215 // Dispose methods should call base class dispose diff --git a/ClickHouse.Driver/Types/TypeConverter.cs b/ClickHouse.Driver/Types/TypeConverter.cs index c51e3546..d98d8eb9 100644 --- a/ClickHouse.Driver/Types/TypeConverter.cs +++ b/ClickHouse.Driver/Types/TypeConverter.cs @@ -9,6 +9,7 @@ using NodaTime; [assembly: InternalsVisibleTo("ClickHouse.Driver.Tests")] // assembly-level tag to expose below classes to tests +[assembly: InternalsVisibleTo("ClickHouse.Driver.Benchmark")] // allow benchmarks to access internal types namespace ClickHouse.Driver.Types; From fee97ceaffa96db5eedb547a1d7b3e81dbd5c471 Mon Sep 17 00:00:00 2001 From: Daniel Crenna Date: Tue, 5 Aug 2025 10:07:23 -0400 Subject: [PATCH 5/7] Add Markdown exporter to benchmark --- ClickHouse.Driver.Benchmark/RecyclableMemoryStreamBenchmark.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/ClickHouse.Driver.Benchmark/RecyclableMemoryStreamBenchmark.cs b/ClickHouse.Driver.Benchmark/RecyclableMemoryStreamBenchmark.cs index b3f894d5..94f7c005 100644 --- a/ClickHouse.Driver.Benchmark/RecyclableMemoryStreamBenchmark.cs +++ b/ClickHouse.Driver.Benchmark/RecyclableMemoryStreamBenchmark.cs @@ -11,6 +11,7 @@ namespace ClickHouse.Driver.Benchmark; [MemoryDiagnoser] [SimpleJob(warmupCount: 3, iterationCount: 10)] +[MarkdownExporter] public class RecyclableMemoryStreamBenchmark { private byte[] smallResponseData; // 10 KB From 3ac80c0bf8e3c1ce4bb2a1e518be72f5f98b03e2 Mon Sep 17 00:00:00 2001 From: Daniel Crenna Date: Thu, 7 Aug 2025 12:33:39 -0400 Subject: [PATCH 6/7] Update ClickHouse.Driver/ADO/Readers/ClickHouseDataReader.cs Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- ClickHouse.Driver/ADO/Readers/ClickHouseDataReader.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ClickHouse.Driver/ADO/Readers/ClickHouseDataReader.cs b/ClickHouse.Driver/ADO/Readers/ClickHouseDataReader.cs index f4e76bd6..7f23cec4 100644 --- a/ClickHouse.Driver/ADO/Readers/ClickHouseDataReader.cs +++ b/ClickHouse.Driver/ADO/Readers/ClickHouseDataReader.cs @@ -63,7 +63,7 @@ internal static ClickHouseDataReader FromHttpResponse(HttpResponseMessage httpRe httpResponse.Content.CopyToAsync(recyclableStream).GetAwaiter().GetResult(); recyclableStream.Position = 0; - reader = new ExtendedBinaryReader(recyclableStream); // will dispose of stream + reader = new ExtendedBinaryReader(recyclableStream); // recyclableStream is disposed separately var (names, types) = ReadHeaders(reader, settings); return new ClickHouseDataReader(httpResponse, reader, recyclableStream, names, types); } From add819098ef436b7296250fe63f63d47f34cdbab Mon Sep 17 00:00:00 2001 From: Daniel Crenna Date: Thu, 7 Aug 2025 12:36:31 -0400 Subject: [PATCH 7/7] Fix warning from CoPilot about a magic string --- ClickHouse.Driver/ADO/Readers/ClickHouseDataReader.cs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/ClickHouse.Driver/ADO/Readers/ClickHouseDataReader.cs b/ClickHouse.Driver/ADO/Readers/ClickHouseDataReader.cs index f4e76bd6..54ff5a59 100644 --- a/ClickHouse.Driver/ADO/Readers/ClickHouseDataReader.cs +++ b/ClickHouse.Driver/ADO/Readers/ClickHouseDataReader.cs @@ -1,11 +1,9 @@ using System; using System.Collections; using System.Collections.Generic; -using System.Collections.ObjectModel; using System.Data; using System.Data.Common; using System.Globalization; -using System.IO; using System.Net; using System.Net.Http; using System.Numerics; @@ -26,7 +24,7 @@ public class ClickHouseDataReader : DbDataReader, IEnumerator, IEnu #if NET6_0_OR_GREATER private static readonly RecyclableMemoryStreamManager StreamManager = new(new RecyclableMemoryStreamManager.Options { - BlockSize = 128 * 1024, // 128KiB blocks + BlockSize = 128 * 1024, // 128KiB LargeBufferMultiple = 1024 * 1024, // 1MiB MaximumBufferSize = 128 * 1024 * 1024, // 128MiB GenerateCallStacks = false, @@ -59,7 +57,7 @@ internal static ClickHouseDataReader FromHttpResponse(HttpResponseMessage httpRe RecyclableMemoryStream recyclableStream = null; try { - recyclableStream = StreamManager.GetStream("ClickHouseDataReader"); + recyclableStream = StreamManager.GetStream(nameof(ClickHouseDataReader)); httpResponse.Content.CopyToAsync(recyclableStream).GetAwaiter().GetResult(); recyclableStream.Position = 0;