diff --git a/src/KurrentDB.Common/Utils/Empty.cs b/src/KurrentDB.Common/Utils/Empty.cs index 704bdc90064..1040db6fd50 100644 --- a/src/KurrentDB.Common/Utils/Empty.cs +++ b/src/KurrentDB.Common/Utils/Empty.cs @@ -6,12 +6,9 @@ namespace KurrentDB.Common.Utils; public static class Empty { - public static readonly byte[] ByteArray = new byte[0]; - public static readonly string[] StringArray = new string[0]; - public static readonly object[] ObjectArray = new object[0]; + public static byte[] ByteArray => []; - public static readonly Action Action = () => { }; - public static readonly object Result = new object(); - public static readonly string Xml = String.Empty; + public static readonly object Result = new(); + public static readonly string Xml = string.Empty; public static readonly string Json = "{}"; } diff --git a/src/KurrentDB.Core.Tests/Constants.cs b/src/KurrentDB.Core.Tests/Constants.cs deleted file mode 100644 index 65538558851..00000000000 --- a/src/KurrentDB.Core.Tests/Constants.cs +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright (c) Kurrent, Inc and/or licensed to Kurrent, Inc under one or more agreements. -// Kurrent, Inc licenses this file to you under the Kurrent License v1 (see LICENSE.md). - -using KurrentDB.Core.Settings; -using KurrentDB.Core.TransactionLog.Chunks; - -namespace KurrentDB.Core.Tests; - -public class Constants { - public const int PTableInitialReaderCount = ESConsts.PTableInitialReaderCount; - - public const int PTableMaxReaderCountDefault = 1 /* StorageWriter */ - + 1 /* StorageChaser */ - + 1 /* Projections */ - + TFChunkScavenger.MaxThreadCount /* Scavenging (1 per thread) */ - + 1 /* Subscription LinkTos resolving */ - + 4 /* Reader Threads Count */ - + 5 /* just in case reserve :) */; -} - diff --git a/src/KurrentDB.Core.Tests/Index/AutoMergeLevelTests/when_max_auto_merge_level_is_set.cs b/src/KurrentDB.Core.Tests/Index/AutoMergeLevelTests/when_max_auto_merge_level_is_set.cs index 632dea00841..10f741309a7 100644 --- a/src/KurrentDB.Core.Tests/Index/AutoMergeLevelTests/when_max_auto_merge_level_is_set.cs +++ b/src/KurrentDB.Core.Tests/Index/AutoMergeLevelTests/when_max_auto_merge_level_is_set.cs @@ -37,11 +37,11 @@ protected void AddTables(int count) { var first = _map; if (_result != null) first = _result.MergedMap; - var pTable = PTable.FromMemtable(memtable, GetTempFilePath(), Constants.PTableInitialReaderCount, Constants.PTableMaxReaderCountDefault, skipIndexVerify: _skipIndexVerify); + var pTable = PTable.FromMemtable(memtable, GetTempFilePath(), skipIndexVerify: _skipIndexVerify); _result = first.AddAndMergePTable(pTable, 10, 20, _fileNameProvider, _ptableVersion, 0, skipIndexVerify: _skipIndexVerify); for (int i = 3; i <= count * 2; i += 2) { - pTable = PTable.FromMemtable(memtable, GetTempFilePath(), Constants.PTableInitialReaderCount, Constants.PTableMaxReaderCountDefault, skipIndexVerify: _skipIndexVerify); + pTable = PTable.FromMemtable(memtable, GetTempFilePath(), skipIndexVerify: _skipIndexVerify); _result = _result.MergedMap.AddAndMergePTable( pTable, prepareCheckpoint: i * 10, diff --git a/src/KurrentDB.Core.Tests/Index/IndexMapTestFactory.cs b/src/KurrentDB.Core.Tests/Index/IndexMapTestFactory.cs index 0558723eedd..6871ae66f5b 100644 --- a/src/KurrentDB.Core.Tests/Index/IndexMapTestFactory.cs +++ b/src/KurrentDB.Core.Tests/Index/IndexMapTestFactory.cs @@ -13,9 +13,8 @@ public static IndexMap FromFile(string filename, int maxTablesPerLevel = 4, bool useBloomFilter = true, int lruCacheSize = 1_000_000, int threads = 1, - int maxAutoMergeLevel = int.MaxValue, - int pTableMaxReaderCount = Constants.PTableMaxReaderCountDefault) { + int maxAutoMergeLevel = int.MaxValue) { return IndexMap.FromFile(filename, maxTablesPerLevel, loadPTables, cacheDepth, skipIndexVerify, useBloomFilter, lruCacheSize, threads, - maxAutoMergeLevel, pTableMaxReaderCount); + maxAutoMergeLevel); } } diff --git a/src/KurrentDB.Core.Tests/Index/IndexV1/PTableReadScenario.cs b/src/KurrentDB.Core.Tests/Index/IndexV1/PTableReadScenario.cs index 53ef4edf04e..b6bd6463e69 100644 --- a/src/KurrentDB.Core.Tests/Index/IndexV1/PTableReadScenario.cs +++ b/src/KurrentDB.Core.Tests/Index/IndexV1/PTableReadScenario.cs @@ -28,7 +28,7 @@ public override async Task SetUp() { AddItemsForScenario(table); - PTable = PTable.FromMemtable(table, Filename, Constants.PTableInitialReaderCount, Constants.PTableMaxReaderCountDefault, cacheDepth: _midpointCacheDepth, + PTable = PTable.FromMemtable(table, Filename, cacheDepth: _midpointCacheDepth, skipIndexVerify: _skipIndexVerify); } diff --git a/src/KurrentDB.Core.Tests/Index/IndexV1/adding_four_items_to_empty_index_map_with_four_tables_per_level_causes_merge.cs b/src/KurrentDB.Core.Tests/Index/IndexV1/adding_four_items_to_empty_index_map_with_four_tables_per_level_causes_merge.cs index 37038af5a86..030fcc820f9 100644 --- a/src/KurrentDB.Core.Tests/Index/IndexV1/adding_four_items_to_empty_index_map_with_four_tables_per_level_causes_merge.cs +++ b/src/KurrentDB.Core.Tests/Index/IndexV1/adding_four_items_to_empty_index_map_with_four_tables_per_level_causes_merge.cs @@ -44,7 +44,7 @@ public override async Task TestFixtureSetUp() { memtable.Add(0, 1, 0); _result = _map.AddAndMergePTable( - PTable.FromMemtable(memtable, GetTempFilePath(), Constants.PTableInitialReaderCount, Constants.PTableMaxReaderCountDefault, skipIndexVerify: _skipIndexVerify), 1, 2, + PTable.FromMemtable(memtable, GetTempFilePath(), skipIndexVerify: _skipIndexVerify), 1, 2, new GuidFilenameProvider(PathName), _ptableVersion, 0, @@ -52,19 +52,19 @@ public override async Task TestFixtureSetUp() { _result.ToDelete.ForEach(x => x.MarkForDestruction()); _result = _result.MergedMap.AddAndMergePTable( - PTable.FromMemtable(memtable, GetTempFilePath(), Constants.PTableInitialReaderCount, Constants.PTableMaxReaderCountDefault, skipIndexVerify: _skipIndexVerify), 3, 4, + PTable.FromMemtable(memtable, GetTempFilePath(), skipIndexVerify: _skipIndexVerify), 3, 4, new GuidFilenameProvider(PathName), _ptableVersion, 0, skipIndexVerify: _skipIndexVerify); _result.ToDelete.ForEach(x => x.MarkForDestruction()); _result = _result.MergedMap.AddAndMergePTable( - PTable.FromMemtable(memtable, GetTempFilePath(), Constants.PTableInitialReaderCount, Constants.PTableMaxReaderCountDefault, skipIndexVerify: _skipIndexVerify), 4, 5, + PTable.FromMemtable(memtable, GetTempFilePath(), skipIndexVerify: _skipIndexVerify), 4, 5, new GuidFilenameProvider(PathName), _ptableVersion, 0, skipIndexVerify: _skipIndexVerify); _result.ToDelete.ForEach(x => x.MarkForDestruction()); _result = _result.MergedMap.AddAndMergePTable( - PTable.FromMemtable(memtable, GetTempFilePath(), Constants.PTableInitialReaderCount, Constants.PTableMaxReaderCountDefault, skipIndexVerify: _skipIndexVerify), 0, 1, + PTable.FromMemtable(memtable, GetTempFilePath(), skipIndexVerify: _skipIndexVerify), 0, 1, new FakeFilenameProvider(_mergeFile), _ptableVersion, 0, skipIndexVerify: _skipIndexVerify); _result.ToDelete.ForEach(x => x.MarkForDestruction()); diff --git a/src/KurrentDB.Core.Tests/Index/IndexV1/adding_four_items_to_empty_index_map_with_two_tables_per_level_causes_double_merge.cs b/src/KurrentDB.Core.Tests/Index/IndexV1/adding_four_items_to_empty_index_map_with_two_tables_per_level_causes_double_merge.cs index d4805f3cd75..83d8997238b 100644 --- a/src/KurrentDB.Core.Tests/Index/IndexV1/adding_four_items_to_empty_index_map_with_two_tables_per_level_causes_double_merge.cs +++ b/src/KurrentDB.Core.Tests/Index/IndexV1/adding_four_items_to_empty_index_map_with_two_tables_per_level_causes_double_merge.cs @@ -44,25 +44,25 @@ public override async Task TestFixtureSetUp() { memtable.Add(0, 1, 0); _result = _map.AddAndMergePTable( - PTable.FromMemtable(memtable, GetTempFilePath(), Constants.PTableInitialReaderCount, Constants.PTableMaxReaderCountDefault, skipIndexVerify: _skipIndexVerify), + PTable.FromMemtable(memtable, GetTempFilePath(), skipIndexVerify: _skipIndexVerify), 10, 20, new GuidFilenameProvider(PathName), _ptableVersion, 0, skipIndexVerify: _skipIndexVerify); _result.ToDelete.ForEach(x => x.MarkForDestruction()); _result = _result.MergedMap.AddAndMergePTable( - PTable.FromMemtable(memtable, GetTempFilePath(), Constants.PTableInitialReaderCount, Constants.PTableMaxReaderCountDefault, skipIndexVerify: _skipIndexVerify), + PTable.FromMemtable(memtable, GetTempFilePath(), skipIndexVerify: _skipIndexVerify), 20, 30, new GuidFilenameProvider(PathName), _ptableVersion, 0, skipIndexVerify: _skipIndexVerify); _result.ToDelete.ForEach(x => x.MarkForDestruction()); _result = _result.MergedMap.AddAndMergePTable( - PTable.FromMemtable(memtable, GetTempFilePath(), Constants.PTableInitialReaderCount, Constants.PTableMaxReaderCountDefault, skipIndexVerify: _skipIndexVerify), + PTable.FromMemtable(memtable, GetTempFilePath(), skipIndexVerify: _skipIndexVerify), 30, 40, new GuidFilenameProvider(PathName), _ptableVersion, 0, skipIndexVerify: _skipIndexVerify); _result.ToDelete.ForEach(x => x.MarkForDestruction()); _result = _result.MergedMap.AddAndMergePTable( - PTable.FromMemtable(memtable, GetTempFilePath(), Constants.PTableInitialReaderCount, Constants.PTableMaxReaderCountDefault, skipIndexVerify: _skipIndexVerify), + PTable.FromMemtable(memtable, GetTempFilePath(), skipIndexVerify: _skipIndexVerify), 50, 60, new FakeFilenameProvider(_mergeFile + ".firstmerge", _mergeFile), _ptableVersion, 0, skipIndexVerify: _skipIndexVerify); _result.ToDelete.ForEach(x => x.MarkForDestruction()); diff --git a/src/KurrentDB.Core.Tests/Index/IndexV1/adding_item_to_empty_index_map.cs b/src/KurrentDB.Core.Tests/Index/IndexV1/adding_item_to_empty_index_map.cs index e2deedbb709..be637e30bd2 100644 --- a/src/KurrentDB.Core.Tests/Index/IndexV1/adding_item_to_empty_index_map.cs +++ b/src/KurrentDB.Core.Tests/Index/IndexV1/adding_item_to_empty_index_map.cs @@ -41,7 +41,7 @@ public override async Task TestFixtureSetUp() { _map = IndexMapTestFactory.FromFile(_filename, maxAutoMergeLevel: _maxAutoMergeIndexLevel); var memtable = new HashListMemTable(_ptableVersion, maxSize: 10); memtable.Add(0, 1, 0); - var table = PTable.FromMemtable(memtable, _tablename, Constants.PTableInitialReaderCount, Constants.PTableMaxReaderCountDefault, skipIndexVerify: _skipIndexVerify); + var table = PTable.FromMemtable(memtable, _tablename, skipIndexVerify: _skipIndexVerify); _result = _map.AddAndMergePTable(table, 7, 11, new FakeFilenameProvider(_mergeFile), _ptableVersion, 0, skipIndexVerify: _skipIndexVerify); table.MarkForDestruction(); diff --git a/src/KurrentDB.Core.Tests/Index/IndexV1/adding_sixteen_items_to_empty_index_map_with_four_tables_per_level_causes_double_merge.cs b/src/KurrentDB.Core.Tests/Index/IndexV1/adding_sixteen_items_to_empty_index_map_with_four_tables_per_level_causes_double_merge.cs index c758f691750..36b1b3b9949 100644 --- a/src/KurrentDB.Core.Tests/Index/IndexV1/adding_sixteen_items_to_empty_index_map_with_four_tables_per_level_causes_double_merge.cs +++ b/src/KurrentDB.Core.Tests/Index/IndexV1/adding_sixteen_items_to_empty_index_map_with_four_tables_per_level_causes_double_merge.cs @@ -47,82 +47,82 @@ public override async Task TestFixtureSetUp() { memtable.Add(0, 1, 0); var guidFilename = new GuidFilenameProvider(PathName); _result = _map.AddAndMergePTable( - PTable.FromMemtable(memtable, GetTempFilePath(), Constants.PTableInitialReaderCount, Constants.PTableMaxReaderCountDefault, skipIndexVerify: _skipIndexVerify), 0, 0, + PTable.FromMemtable(memtable, GetTempFilePath(), skipIndexVerify: _skipIndexVerify), 0, 0, guidFilename, _ptableVersion, 0, skipIndexVerify: _skipIndexVerify); _result.ToDelete.ForEach(x => x.MarkForDestruction()); _result = _result.MergedMap.AddAndMergePTable( - PTable.FromMemtable(memtable, GetTempFilePath(), Constants.PTableInitialReaderCount, Constants.PTableMaxReaderCountDefault, skipIndexVerify: _skipIndexVerify), 0, 0, + PTable.FromMemtable(memtable, GetTempFilePath(), skipIndexVerify: _skipIndexVerify), 0, 0, guidFilename, _ptableVersion, 0, skipIndexVerify: _skipIndexVerify); _result.ToDelete.ForEach(x => x.MarkForDestruction()); _result = _result.MergedMap.AddAndMergePTable( - PTable.FromMemtable(memtable, GetTempFilePath(), Constants.PTableInitialReaderCount, Constants.PTableMaxReaderCountDefault, skipIndexVerify: _skipIndexVerify), 0, 0, + PTable.FromMemtable(memtable, GetTempFilePath(), skipIndexVerify: _skipIndexVerify), 0, 0, guidFilename, _ptableVersion, 0, skipIndexVerify: _skipIndexVerify); _result.ToDelete.ForEach(x => x.MarkForDestruction()); _result = _result.MergedMap.AddAndMergePTable( - PTable.FromMemtable(memtable, GetTempFilePath(), Constants.PTableInitialReaderCount, Constants.PTableMaxReaderCountDefault, skipIndexVerify: _skipIndexVerify), 0, 0, + PTable.FromMemtable(memtable, GetTempFilePath(), skipIndexVerify: _skipIndexVerify), 0, 0, guidFilename, _ptableVersion, 0, skipIndexVerify: _skipIndexVerify); _result.ToDelete.ForEach(x => x.MarkForDestruction()); _result = _result.MergedMap.AddAndMergePTable( - PTable.FromMemtable(memtable, GetTempFilePath(), Constants.PTableInitialReaderCount, Constants.PTableMaxReaderCountDefault, skipIndexVerify: _skipIndexVerify), 0, 0, + PTable.FromMemtable(memtable, GetTempFilePath(), skipIndexVerify: _skipIndexVerify), 0, 0, guidFilename, _ptableVersion, 0, skipIndexVerify: _skipIndexVerify); _result.ToDelete.ForEach(x => x.MarkForDestruction()); _result = _result.MergedMap.AddAndMergePTable( - PTable.FromMemtable(memtable, GetTempFilePath(), Constants.PTableInitialReaderCount, Constants.PTableMaxReaderCountDefault, skipIndexVerify: _skipIndexVerify), 0, 0, + PTable.FromMemtable(memtable, GetTempFilePath(), skipIndexVerify: _skipIndexVerify), 0, 0, guidFilename, _ptableVersion, 0, skipIndexVerify: _skipIndexVerify); _result.ToDelete.ForEach(x => x.MarkForDestruction()); _result = _result.MergedMap.AddAndMergePTable( - PTable.FromMemtable(memtable, GetTempFilePath(), Constants.PTableInitialReaderCount, Constants.PTableMaxReaderCountDefault, skipIndexVerify: _skipIndexVerify), 0, 0, + PTable.FromMemtable(memtable, GetTempFilePath(), skipIndexVerify: _skipIndexVerify), 0, 0, guidFilename, _ptableVersion, 0, skipIndexVerify: _skipIndexVerify); _result.ToDelete.ForEach(x => x.MarkForDestruction()); _result = _result.MergedMap.AddAndMergePTable( - PTable.FromMemtable(memtable, GetTempFilePath(), Constants.PTableInitialReaderCount, Constants.PTableMaxReaderCountDefault, skipIndexVerify: _skipIndexVerify), 0, 0, + PTable.FromMemtable(memtable, GetTempFilePath(), skipIndexVerify: _skipIndexVerify), 0, 0, guidFilename, _ptableVersion, 0, skipIndexVerify: _skipIndexVerify); _result.ToDelete.ForEach(x => x.MarkForDestruction()); _result = _result.MergedMap.AddAndMergePTable( - PTable.FromMemtable(memtable, GetTempFilePath(), Constants.PTableInitialReaderCount, Constants.PTableMaxReaderCountDefault, skipIndexVerify: _skipIndexVerify), 0, 0, + PTable.FromMemtable(memtable, GetTempFilePath(), skipIndexVerify: _skipIndexVerify), 0, 0, guidFilename, _ptableVersion, 0, skipIndexVerify: _skipIndexVerify); _result.ToDelete.ForEach(x => x.MarkForDestruction()); _result = _result.MergedMap.AddAndMergePTable( - PTable.FromMemtable(memtable, GetTempFilePath(), Constants.PTableInitialReaderCount, Constants.PTableMaxReaderCountDefault, skipIndexVerify: _skipIndexVerify), 0, 0, + PTable.FromMemtable(memtable, GetTempFilePath(), skipIndexVerify: _skipIndexVerify), 0, 0, guidFilename, _ptableVersion, 0, skipIndexVerify: _skipIndexVerify); _result.ToDelete.ForEach(x => x.MarkForDestruction()); _result = _result.MergedMap.AddAndMergePTable( - PTable.FromMemtable(memtable, GetTempFilePath(), Constants.PTableInitialReaderCount, Constants.PTableMaxReaderCountDefault, skipIndexVerify: _skipIndexVerify), 0, 0, + PTable.FromMemtable(memtable, GetTempFilePath(), skipIndexVerify: _skipIndexVerify), 0, 0, guidFilename, _ptableVersion, 0, skipIndexVerify: _skipIndexVerify); _result.ToDelete.ForEach(x => x.MarkForDestruction()); _result = _result.MergedMap.AddAndMergePTable( - PTable.FromMemtable(memtable, GetTempFilePath(), Constants.PTableInitialReaderCount, Constants.PTableMaxReaderCountDefault, skipIndexVerify: _skipIndexVerify), 0, 0, + PTable.FromMemtable(memtable, GetTempFilePath(), skipIndexVerify: _skipIndexVerify), 0, 0, guidFilename, _ptableVersion, 0, skipIndexVerify: _skipIndexVerify); _result.ToDelete.ForEach(x => x.MarkForDestruction()); _result = _result.MergedMap.AddAndMergePTable( - PTable.FromMemtable(memtable, GetTempFilePath(), Constants.PTableInitialReaderCount, Constants.PTableMaxReaderCountDefault, skipIndexVerify: _skipIndexVerify), 0, 0, + PTable.FromMemtable(memtable, GetTempFilePath(), skipIndexVerify: _skipIndexVerify), 0, 0, guidFilename, _ptableVersion, 0, skipIndexVerify: _skipIndexVerify); _result.ToDelete.ForEach(x => x.MarkForDestruction()); _result = _result.MergedMap.AddAndMergePTable( - PTable.FromMemtable(memtable, GetTempFilePath(), Constants.PTableInitialReaderCount, Constants.PTableMaxReaderCountDefault, skipIndexVerify: _skipIndexVerify), 0, 0, + PTable.FromMemtable(memtable, GetTempFilePath(), skipIndexVerify: _skipIndexVerify), 0, 0, guidFilename, _ptableVersion, 0, skipIndexVerify: _skipIndexVerify); _result.ToDelete.ForEach(x => x.MarkForDestruction()); _result = _result.MergedMap.AddAndMergePTable( - PTable.FromMemtable(memtable, GetTempFilePath(), Constants.PTableInitialReaderCount, Constants.PTableMaxReaderCountDefault, skipIndexVerify: _skipIndexVerify), 0, 0, + PTable.FromMemtable(memtable, GetTempFilePath(), skipIndexVerify: _skipIndexVerify), 0, 0, guidFilename, _ptableVersion, 0, skipIndexVerify: _skipIndexVerify); _result.ToDelete.ForEach(x => x.MarkForDestruction()); _result = _result.MergedMap.AddAndMergePTable( - PTable.FromMemtable(memtable, GetTempFilePath(), Constants.PTableInitialReaderCount, Constants.PTableMaxReaderCountDefault, skipIndexVerify: _skipIndexVerify), 1, 2, + PTable.FromMemtable(memtable, GetTempFilePath(), skipIndexVerify: _skipIndexVerify), 1, 2, new FakeFilenameProvider(_finalmergefile, _finalmergefile2), _ptableVersion, 0, skipIndexVerify: _skipIndexVerify); _result.ToDelete.ForEach(x => x.MarkForDestruction()); diff --git a/src/KurrentDB.Core.Tests/Index/IndexV1/adding_two_items_to_empty_index_map_with_two_tables_per_level_causes_merge.cs b/src/KurrentDB.Core.Tests/Index/IndexV1/adding_two_items_to_empty_index_map_with_two_tables_per_level_causes_merge.cs index 35aaa8270f9..20285d1768d 100644 --- a/src/KurrentDB.Core.Tests/Index/IndexV1/adding_two_items_to_empty_index_map_with_two_tables_per_level_causes_merge.cs +++ b/src/KurrentDB.Core.Tests/Index/IndexV1/adding_two_items_to_empty_index_map_with_two_tables_per_level_causes_merge.cs @@ -44,13 +44,13 @@ public override async Task TestFixtureSetUp() { memtable.Add(0, 1, 0); _result = _map.AddAndMergePTable( - PTable.FromMemtable(memtable, GetTempFilePath(), Constants.PTableInitialReaderCount, Constants.PTableMaxReaderCountDefault, skipIndexVerify: _skipIndexVerify), + PTable.FromMemtable(memtable, GetTempFilePath(), skipIndexVerify: _skipIndexVerify), 123, 321, new GuidFilenameProvider(PathName), _ptableVersion, 0, skipIndexVerify: _skipIndexVerify); _result.ToDelete.ForEach(x => x.MarkForDestruction()); _result = _result.MergedMap.AddAndMergePTable( - PTable.FromMemtable(memtable, GetTempFilePath(), Constants.PTableInitialReaderCount, Constants.PTableMaxReaderCountDefault, skipIndexVerify: _skipIndexVerify), + PTable.FromMemtable(memtable, GetTempFilePath(), skipIndexVerify: _skipIndexVerify), 100, 400, new FakeFilenameProvider(_mergeFile), _ptableVersion, 0, skipIndexVerify: _skipIndexVerify); diff --git a/src/KurrentDB.Core.Tests/Index/IndexV1/corrupt_index_should.cs b/src/KurrentDB.Core.Tests/Index/IndexV1/corrupt_index_should.cs index 3620e999154..898fe86b52b 100644 --- a/src/KurrentDB.Core.Tests/Index/IndexV1/corrupt_index_should.cs +++ b/src/KurrentDB.Core.Tests/Index/IndexV1/corrupt_index_should.cs @@ -23,7 +23,7 @@ private string ConstructPTable(byte version) { } string pTableFilename = GetTempFilePath(); - var pTable = PTable.FromMemtable(memTable, pTableFilename, Constants.PTableInitialReaderCount, Constants.PTableMaxReaderCountDefault, depth); + var pTable = PTable.FromMemtable(memTable, pTableFilename, depth); pTable.Dispose(); return pTableFilename; } @@ -184,7 +184,7 @@ public void throw_exception_if_index_entries_not_multiple_of_index_entry_size(by bool skipIndexVerify) { string ptableFileName = ConstructPTable(version); CorruptPTableFile(ptableFileName, version, "notMultipleIndexEntrySize"); - Assert.Throws(() => PTable.FromFile(ptableFileName, Constants.PTableInitialReaderCount, Constants.PTableMaxReaderCountDefault, depth, skipIndexVerify)); + Assert.Throws(() => PTable.FromFile(ptableFileName, depth, skipIndexVerify)); } [TestCase(PTableVersions.IndexV2, false)] @@ -197,7 +197,7 @@ public void throw_exception_if_midpoints_index_entries_not_in_descending_order(b bool skipIndexVerify) { string ptableFileName = ConstructPTable(version); CorruptPTableFile(ptableFileName, version, "zeroOutMiddleEntries"); - Assert.Throws(() => PTable.FromFile(ptableFileName, Constants.PTableInitialReaderCount, Constants.PTableMaxReaderCountDefault, depth, skipIndexVerify)); + Assert.Throws(() => PTable.FromFile(ptableFileName, depth, skipIndexVerify)); } @@ -207,7 +207,7 @@ public void throw_exception_if_midpoints_item_indexes_not_in_ascending_order(byt bool skipIndexVerify) { string ptableFileName = ConstructPTable(version); CorruptPTableFile(ptableFileName, version, "midpointItemIndexesNotAscendingOrder"); - Assert.Throws(() => PTable.FromFile(ptableFileName, Constants.PTableInitialReaderCount, Constants.PTableMaxReaderCountDefault, depth, skipIndexVerify)); + Assert.Throws(() => PTable.FromFile(ptableFileName, depth, skipIndexVerify)); } [TestCase(PTableVersions.IndexV2, true)] @@ -218,7 +218,7 @@ public void throw_exception_if_index_entries_not_descending_during_ptable_get_ra string ptableFileName = ConstructPTable(version); CorruptPTableFile(ptableFileName, version, "zeroOutMiddleEntries"); //loading with a depth of 1 should load only 2 midpoints (first and last index entry) - PTable pTable = PTable.FromFile(ptableFileName, Constants.PTableInitialReaderCount, Constants.PTableMaxReaderCountDefault, 1, skipIndexVerify); + PTable pTable = PTable.FromFile(ptableFileName, 1, skipIndexVerify); Assert.Throws(() => pTable.GetRange(GetOriginalHash(numIndexEntries / 4, version), 1, 1)); pTable.Dispose(); @@ -232,7 +232,7 @@ public void throw_exception_if_index_entries_not_descending_during_ptable_get_ra string ptableFileName = ConstructPTable(version); CorruptPTableFile(ptableFileName, version, "maxOutMiddleEntries"); //loading with a depth of 1 should load only 2 midpoints (first and last index entry) - PTable pTable = PTable.FromFile(ptableFileName, Constants.PTableInitialReaderCount, Constants.PTableMaxReaderCountDefault, 1, skipIndexVerify); + PTable pTable = PTable.FromFile(ptableFileName, 1, skipIndexVerify); Assert.Throws(() => pTable.GetRange(GetOriginalHash(numIndexEntries / 4, version), 1, 1)); pTable.Dispose(); @@ -246,7 +246,7 @@ public void throw_exception_if_index_entries_not_descending_during_ptable_get_la string ptableFileName = ConstructPTable(version); CorruptPTableFile(ptableFileName, version, "zeroOutMiddleEntries"); //loading with a depth of 1 should load only 2 midpoints (first and last index entry) - PTable pTable = PTable.FromFile(ptableFileName, Constants.PTableInitialReaderCount, Constants.PTableMaxReaderCountDefault, 1, skipIndexVerify); + PTable pTable = PTable.FromFile(ptableFileName, 1, skipIndexVerify); IndexEntry entry; Assert.Throws(() => // changed 2 to 4 here because the corruption actually removes the stream at /2, so it isn't in the bloom filter @@ -262,7 +262,7 @@ public void throw_exception_if_index_entries_not_descending_during_ptable_get_la string ptableFileName = ConstructPTable(version); CorruptPTableFile(ptableFileName, version, "maxOutMiddleEntries"); //loading with a depth of 1 should load only 2 midpoints (first and last index entry) - PTable pTable = PTable.FromFile(ptableFileName, Constants.PTableInitialReaderCount, Constants.PTableMaxReaderCountDefault, 1, skipIndexVerify); + PTable pTable = PTable.FromFile(ptableFileName, 1, skipIndexVerify); IndexEntry entry; Assert.Throws(() => pTable.TryGetLatestEntry(GetOriginalHash(numIndexEntries / 4, version), out entry)); @@ -277,7 +277,7 @@ public void throw_exception_if_index_entries_not_descending_during_ptable_get_ol string ptableFileName = ConstructPTable(version); CorruptPTableFile(ptableFileName, version, "zeroOutMiddleEntries"); //loading with a depth of 1 should load only 2 midpoints (first and last index entry) - PTable pTable = PTable.FromFile(ptableFileName, Constants.PTableInitialReaderCount, Constants.PTableMaxReaderCountDefault, 1, skipIndexVerify); + PTable pTable = PTable.FromFile(ptableFileName, 1, skipIndexVerify); IndexEntry entry; Assert.Throws(() => pTable.TryGetOldestEntry(GetOriginalHash(numIndexEntries / 4, version), out entry)); @@ -292,7 +292,7 @@ public void throw_exception_if_index_entries_not_descending_during_ptable_get_ol string ptableFileName = ConstructPTable(version); CorruptPTableFile(ptableFileName, version, "maxOutMiddleEntries"); //loading with a depth of 1 should load only 2 midpoints (first and last index entry) - PTable pTable = PTable.FromFile(ptableFileName, Constants.PTableInitialReaderCount, Constants.PTableMaxReaderCountDefault, 1, skipIndexVerify); + PTable pTable = PTable.FromFile(ptableFileName, 1, skipIndexVerify); IndexEntry entry; Assert.Throws(() => pTable.TryGetOldestEntry(GetOriginalHash(numIndexEntries / 4, version), out entry)); @@ -307,7 +307,7 @@ public void throw_exception_if_index_entries_not_descending_during_ptable_get_on string ptableFileName = ConstructPTable(version); CorruptPTableFile(ptableFileName, version, "zeroOutMiddleEntries"); //loading with a depth of 1 should load only 2 midpoints (first and last index entry) - PTable pTable = PTable.FromFile(ptableFileName, Constants.PTableInitialReaderCount, Constants.PTableMaxReaderCountDefault, 1, skipIndexVerify); + PTable pTable = PTable.FromFile(ptableFileName, 1, skipIndexVerify); long position; Assert.Throws(() => pTable.TryGetOneValue(GetOriginalHash(numIndexEntries / 4, version), 1, out position)); @@ -322,7 +322,7 @@ public void throw_exception_if_index_entries_not_descending_during_ptable_get_on string ptableFileName = ConstructPTable(version); CorruptPTableFile(ptableFileName, version, "maxOutMiddleEntries"); //loading with a depth of 1 should load only 2 midpoints (first and last index entry) - PTable pTable = PTable.FromFile(ptableFileName, Constants.PTableInitialReaderCount, Constants.PTableMaxReaderCountDefault, 1, skipIndexVerify); + PTable pTable = PTable.FromFile(ptableFileName, 1, skipIndexVerify); long position; Assert.Throws(() => pTable.TryGetOneValue(GetOriginalHash(numIndexEntries / 4, version), 1, out position)); @@ -334,7 +334,7 @@ public void throw_exception_if_index_entries_not_descending_during_ptable_get_on public void throw_exception_on_invalid_ptable_filenumber_in_footer(byte version, bool skipIndexVerify) { string ptableFileName = ConstructPTable(version); CorruptPTableFile(ptableFileName, version, "footerFileType"); - Assert.Throws(() => PTable.FromFile(ptableFileName, Constants.PTableInitialReaderCount, Constants.PTableMaxReaderCountDefault, depth, skipIndexVerify)); + Assert.Throws(() => PTable.FromFile(ptableFileName, depth, skipIndexVerify)); } [TestCase(PTableVersions.IndexV4, false)] @@ -342,7 +342,7 @@ public void throw_exception_on_invalid_ptable_filenumber_in_footer(byte version, public void throw_exception_on_header_footer_version_mismatch(byte version, bool skipIndexVerify) { string ptableFileName = ConstructPTable(version); CorruptPTableFile(ptableFileName, version, "footerVersion"); - Assert.Throws(() => PTable.FromFile(ptableFileName, Constants.PTableInitialReaderCount, Constants.PTableMaxReaderCountDefault, depth, skipIndexVerify)); + Assert.Throws(() => PTable.FromFile(ptableFileName, depth, skipIndexVerify)); } [TestCase(PTableVersions.IndexV4, false)] @@ -350,7 +350,7 @@ public void throw_exception_on_header_footer_version_mismatch(byte version, bool public void throw_exception_if_negative_index_entries_size(byte version, bool skipIndexVerify) { string ptableFileName = ConstructPTable(version); CorruptPTableFile(ptableFileName, version, "negativeIndexEntriesSize"); - Assert.Throws(() => PTable.FromFile(ptableFileName, Constants.PTableInitialReaderCount, Constants.PTableMaxReaderCountDefault, depth, skipIndexVerify)); + Assert.Throws(() => PTable.FromFile(ptableFileName, depth, skipIndexVerify)); } [TestCase(PTableVersions.IndexV4, false)] @@ -358,7 +358,7 @@ public void throw_exception_if_negative_index_entries_size(byte version, bool sk public void throw_exception_if_less_than_2_midpoints_cached(byte version, bool skipIndexVerify) { string ptableFileName = ConstructPTable(version); CorruptPTableFile(ptableFileName, version, "lessThan2Midpoints"); - Assert.Throws(() => PTable.FromFile(ptableFileName, Constants.PTableInitialReaderCount, Constants.PTableMaxReaderCountDefault, depth, skipIndexVerify)); + Assert.Throws(() => PTable.FromFile(ptableFileName, depth, skipIndexVerify)); } [TestCase(PTableVersions.IndexV4, false)] @@ -366,7 +366,7 @@ public void throw_exception_if_less_than_2_midpoints_cached(byte version, bool s public void throw_exception_if_more_midpoints_than_index_entries(byte version, bool skipIndexVerify) { string ptableFileName = ConstructPTable(version); CorruptPTableFile(ptableFileName, version, "moreMidpointsThanIndexEntries"); - Assert.Throws(() => PTable.FromFile(ptableFileName, Constants.PTableInitialReaderCount, Constants.PTableMaxReaderCountDefault, depth, skipIndexVerify)); + Assert.Throws(() => PTable.FromFile(ptableFileName, depth, skipIndexVerify)); } [TestCase(PTableVersions.IndexV4, false)] @@ -377,7 +377,7 @@ public void continue_without_corrupt_bloom_filter(byte version, bool corrupt) { if (corrupt) CorruptBloomFilter(PTable.GenBloomFilterFilename(ptableFileName)); - var table = PTable.FromFile(ptableFileName, Constants.PTableInitialReaderCount, Constants.PTableMaxReaderCountDefault, depth, skipIndexVerify: true); + var table = PTable.FromFile(ptableFileName, depth, skipIndexVerify: true); Assert.AreEqual(!corrupt, table.HasBloomFilter); } } diff --git a/src/KurrentDB.Core.Tests/Index/IndexV1/destroying_ptable.cs b/src/KurrentDB.Core.Tests/Index/IndexV1/destroying_ptable.cs index 10d6ab648b6..53a4d944d55 100644 --- a/src/KurrentDB.Core.Tests/Index/IndexV1/destroying_ptable.cs +++ b/src/KurrentDB.Core.Tests/Index/IndexV1/destroying_ptable.cs @@ -28,7 +28,7 @@ public void Setup() { var mtable = new HashListMemTable(_ptableVersion, maxSize: 10); mtable.Add(0x010100000000, 0x0001, 0x0001); mtable.Add(0x010500000000, 0x0001, 0x0002); - _table = PTable.FromMemtable(mtable, Filename, Constants.PTableInitialReaderCount, Constants.PTableMaxReaderCountDefault, + _table = PTable.FromMemtable(mtable, Filename, skipIndexVerify: _skipIndexVerify, useBloomFilter: true); _table.MarkForDestruction(); diff --git a/src/KurrentDB.Core.Tests/Index/IndexV1/index_map_should.cs b/src/KurrentDB.Core.Tests/Index/IndexV1/index_map_should.cs index e183945f5d9..23df4472519 100644 --- a/src/KurrentDB.Core.Tests/Index/IndexV1/index_map_should.cs +++ b/src/KurrentDB.Core.Tests/Index/IndexV1/index_map_should.cs @@ -38,7 +38,7 @@ public override async Task SetUp() { var memTable = new HashListMemTable(_ptableVersion, maxSize: 10); memTable.Add(0, 1, 2); - _ptable = PTable.FromMemtable(memTable, _ptableFileName, Constants.PTableInitialReaderCount, Constants.PTableMaxReaderCountDefault); + _ptable = PTable.FromMemtable(memTable, _ptableFileName); } [TearDown] diff --git a/src/KurrentDB.Core.Tests/Index/IndexV1/index_map_should_detect_corruption.cs b/src/KurrentDB.Core.Tests/Index/IndexV1/index_map_should_detect_corruption.cs index 75fbc5608ea..8b1f62a7cc6 100644 --- a/src/KurrentDB.Core.Tests/Index/IndexV1/index_map_should_detect_corruption.cs +++ b/src/KurrentDB.Core.Tests/Index/IndexV1/index_map_should_detect_corruption.cs @@ -41,7 +41,7 @@ public override async Task SetUp() { var memtable = new HashListMemTable(_ptableVersion, maxSize: 10); memtable.Add(0, 0, 0); memtable.Add(1, 1, 100); - _ptable = PTable.FromMemtable(memtable, _ptableFileName, Constants.PTableInitialReaderCount, Constants.PTableMaxReaderCountDefault, skipIndexVerify: _skipIndexVerify); + _ptable = PTable.FromMemtable(memtable, _ptableFileName, skipIndexVerify: _skipIndexVerify); indexMap = indexMap.AddAndMergePTable(_ptable, 0, 0, new GuidFilenameProvider(PathName), _ptableVersion, 0, skipIndexVerify: _skipIndexVerify).MergedMap; diff --git a/src/KurrentDB.Core.Tests/Index/IndexV1/opening_a_ptable_with_more_than_32bits_of_records.cs b/src/KurrentDB.Core.Tests/Index/IndexV1/opening_a_ptable_with_more_than_32bits_of_records.cs index 79f32557264..cb89718af84 100644 --- a/src/KurrentDB.Core.Tests/Index/IndexV1/opening_a_ptable_with_more_than_32bits_of_records.cs +++ b/src/KurrentDB.Core.Tests/Index/IndexV1/opening_a_ptable_with_more_than_32bits_of_records.cs @@ -36,7 +36,7 @@ public override async Task TestFixtureSetUp() { _size = _ptableCount * (long)_indexEntrySize + PTableHeader.Size + PTable.MD5Size; Console.WriteLine("Creating PTable at {0}. Size of PTable: {1}", Filename, _size); CreatePTableFile(Filename, _size, _indexEntrySize); - _ptable = PTable.FromFile(Filename, Constants.PTableInitialReaderCount, Constants.PTableMaxReaderCountDefault, 22, false); + _ptable = PTable.FromFile(Filename, 22, false); } public static void CreatePTableFile(string filename, long ptableSize, int indexEntrySize, int cacheDepth = 16) { diff --git a/src/KurrentDB.Core.Tests/Index/IndexV1/ptable_midpoint_cache_should.cs b/src/KurrentDB.Core.Tests/Index/IndexV1/ptable_midpoint_cache_should.cs index fda461a806b..dd3bd1c5a9f 100644 --- a/src/KurrentDB.Core.Tests/Index/IndexV1/ptable_midpoint_cache_should.cs +++ b/src/KurrentDB.Core.Tests/Index/IndexV1/ptable_midpoint_cache_should.cs @@ -50,7 +50,7 @@ private PTable ConstructPTable(string file, int count, Random rnd, int depth) { memTable.Add((uint)rnd.Next(), rnd.Next(0, 1 << 20), Math.Abs(rnd.Next() * rnd.Next())); } - var ptable = PTable.FromMemtable(memTable, file, Constants.PTableInitialReaderCount, Constants.PTableMaxReaderCountDefault, depth, skipIndexVerify: _skipIndexVerify); + var ptable = PTable.FromMemtable(memTable, file, depth, skipIndexVerify: _skipIndexVerify); return ptable; } diff --git a/src/KurrentDB.Core.Tests/Index/IndexV1/ptable_range_query_tests.cs b/src/KurrentDB.Core.Tests/Index/IndexV1/ptable_range_query_tests.cs index dfad8a6bed0..087e8b0458c 100644 --- a/src/KurrentDB.Core.Tests/Index/IndexV1/ptable_range_query_tests.cs +++ b/src/KurrentDB.Core.Tests/Index/IndexV1/ptable_range_query_tests.cs @@ -41,7 +41,7 @@ public override async Task TestFixtureSetUp() { table.Add(0x010300000000, 0x0001, 0xFFF1); table.Add(0x010300000000, 0x0003, 0xFFF3); table.Add(0x010300000000, 0x0005, 0xFFF5); - _ptable = PTable.FromMemtable(table, Filename, Constants.PTableInitialReaderCount, Constants.PTableMaxReaderCountDefault, cacheDepth: 0, + _ptable = PTable.FromMemtable(table, Filename, cacheDepth: 0, skipIndexVerify: _skipIndexVerify, useBloomFilter: _useBloomFilter, lruCacheSize: _lruCacheSize); diff --git a/src/KurrentDB.Core.Tests/Index/IndexV1/ptable_should.cs b/src/KurrentDB.Core.Tests/Index/IndexV1/ptable_should.cs index 17c1f4b2ee7..0ff98973838 100644 --- a/src/KurrentDB.Core.Tests/Index/IndexV1/ptable_should.cs +++ b/src/KurrentDB.Core.Tests/Index/IndexV1/ptable_should.cs @@ -30,7 +30,7 @@ public override async Task TestFixtureSetUp() { var table = new HashListMemTable(_ptableVersion, maxSize: 10); table.Add(0x010100000000, 0x0001, 0x0001); - _ptable = PTable.FromMemtable(table, Filename, Constants.PTableInitialReaderCount, Constants.PTableMaxReaderCountDefault, cacheDepth: 0, skipIndexVerify: _skipIndexVerify); + _ptable = PTable.FromMemtable(table, Filename, cacheDepth: 0, skipIndexVerify: _skipIndexVerify); } public override void TestFixtureTearDown() { diff --git a/src/KurrentDB.Core.Tests/Index/IndexV1/saving_index_with_single_item_to_a_file.cs b/src/KurrentDB.Core.Tests/Index/IndexV1/saving_index_with_single_item_to_a_file.cs index ca9abeca591..93102e0f487 100644 --- a/src/KurrentDB.Core.Tests/Index/IndexV1/saving_index_with_single_item_to_a_file.cs +++ b/src/KurrentDB.Core.Tests/Index/IndexV1/saving_index_with_single_item_to_a_file.cs @@ -38,7 +38,7 @@ public override async Task TestFixtureSetUp() { _map = IndexMapTestFactory.FromFile(_filename, maxAutoMergeLevel: _maxAutoMergeIndexLevel); var memtable = new HashListMemTable(_ptableVersion, maxSize: 10); memtable.Add(0, 2, 7); - var table = PTable.FromMemtable(memtable, _tablename, Constants.PTableInitialReaderCount, Constants.PTableMaxReaderCountDefault); + var table = PTable.FromMemtable(memtable, _tablename); _result = _map.AddAndMergePTable(table, 7, 11, new FakeFilenameProvider(_mergeFile), _ptableVersion, 0); _result.MergedMap.SaveToFile(_filename); diff --git a/src/KurrentDB.Core.Tests/Index/IndexV1/saving_index_with_six_items_to_a_file.cs b/src/KurrentDB.Core.Tests/Index/IndexV1/saving_index_with_six_items_to_a_file.cs index aa0c4344b4b..ec6ad108495 100644 --- a/src/KurrentDB.Core.Tests/Index/IndexV1/saving_index_with_six_items_to_a_file.cs +++ b/src/KurrentDB.Core.Tests/Index/IndexV1/saving_index_with_six_items_to_a_file.cs @@ -36,7 +36,7 @@ public override async Task SetUp() { _map = IndexMapTestFactory.FromFile(_filename, maxTablesPerLevel: 4); var memtable = new HashListMemTable(_ptableVersion, maxSize: 10); memtable.Add(0, 2, 123); - var table = PTable.FromMemtable(memtable, _tablename, Constants.PTableInitialReaderCount, Constants.PTableMaxReaderCountDefault); + var table = PTable.FromMemtable(memtable, _tablename); _result = _map.AddAndMergePTable(table, 0, 0, new FakeFilenameProvider(_mergeFile), _ptableVersion, 0); _result = _result.MergedMap.AddAndMergePTable(table, 0, 0, diff --git a/src/KurrentDB.Core.Tests/Index/IndexV1/table_index_on_range_query.cs b/src/KurrentDB.Core.Tests/Index/IndexV1/table_index_on_range_query.cs index 56c8cdf3c9b..ab4092c27d5 100644 --- a/src/KurrentDB.Core.Tests/Index/IndexV1/table_index_on_range_query.cs +++ b/src/KurrentDB.Core.Tests/Index/IndexV1/table_index_on_range_query.cs @@ -38,7 +38,7 @@ public override async Task TestFixtureSetUp() { () => new HashListMemTable(version: _ptableVersion, maxSize: 40), new FakeTfReader(), _ptableVersion, - 5, Constants.PTableMaxReaderCountDefault, + 5, maxSizeForMemory: 5, skipIndexVerify: _skipIndexVerify); _tableIndex.Initialize(long.MaxValue); diff --git a/src/KurrentDB.Core.Tests/Index/IndexV1/table_index_on_try_get_one_value_query.cs b/src/KurrentDB.Core.Tests/Index/IndexV1/table_index_on_try_get_one_value_query.cs index d6d34405bee..c18c2c73e5c 100644 --- a/src/KurrentDB.Core.Tests/Index/IndexV1/table_index_on_try_get_one_value_query.cs +++ b/src/KurrentDB.Core.Tests/Index/IndexV1/table_index_on_try_get_one_value_query.cs @@ -41,7 +41,7 @@ public override async Task TestFixtureSetUp() { () => new HashListMemTable(_ptableVersion, maxSize: 10), fakeReader, _ptableVersion, - 5, Constants.PTableMaxReaderCountDefault, + 5, maxSizeForMemory: 5, skipIndexVerify: _skipIndexVerify); _tableIndex.Initialize(long.MaxValue); diff --git a/src/KurrentDB.Core.Tests/Index/IndexV1/table_index_should.cs b/src/KurrentDB.Core.Tests/Index/IndexV1/table_index_should.cs index 857d295bc7f..d6c7ad7d98c 100644 --- a/src/KurrentDB.Core.Tests/Index/IndexV1/table_index_should.cs +++ b/src/KurrentDB.Core.Tests/Index/IndexV1/table_index_should.cs @@ -35,7 +35,7 @@ public override async Task TestFixtureSetUp() { () => new HashListMemTable(_ptableVersion, maxSize: 20), new FakeTfReader(), _ptableVersion, - 5, Constants.PTableMaxReaderCountDefault, + 5, maxSizeForMemory: 10, skipIndexVerify: _skipIndexVerify); _tableIndex.Initialize(long.MaxValue); diff --git a/src/KurrentDB.Core.Tests/Index/IndexV1/table_index_with_corrupt_index_entries_should.cs b/src/KurrentDB.Core.Tests/Index/IndexV1/table_index_with_corrupt_index_entries_should.cs index fc4033e8fe1..43689d19cb9 100644 --- a/src/KurrentDB.Core.Tests/Index/IndexV1/table_index_with_corrupt_index_entries_should.cs +++ b/src/KurrentDB.Core.Tests/Index/IndexV1/table_index_with_corrupt_index_entries_should.cs @@ -29,7 +29,7 @@ public void ConstructTableIndexWithCorruptIndexEntries(byte version, bool skipIn () => new HashListMemTable(version, maxSize: NumIndexEntries), fakeReader, version, - int.MaxValue, Constants.PTableMaxReaderCountDefault, + int.MaxValue, maxSizeForMemory: NumIndexEntries, skipIndexVerify: skipIndexVerify); _tableIndex.Initialize(long.MaxValue); @@ -70,7 +70,7 @@ public void ConstructTableIndexWithCorruptIndexEntries(byte version, bool skipIn () => new HashListMemTable(version, maxSize: NumIndexEntries), fakeReader, version, - int.MaxValue, Constants.PTableMaxReaderCountDefault, + int.MaxValue, maxSizeForMemory: NumIndexEntries, skipIndexVerify: skipIndexVerify, indexCacheDepth: 8); diff --git a/src/KurrentDB.Core.Tests/Index/IndexV1/table_index_with_two_ptables_and_memtable_on_range_query.cs b/src/KurrentDB.Core.Tests/Index/IndexV1/table_index_with_two_ptables_and_memtable_on_range_query.cs index 0c2a7c97999..e13b7f2fe53 100644 --- a/src/KurrentDB.Core.Tests/Index/IndexV1/table_index_with_two_ptables_and_memtable_on_range_query.cs +++ b/src/KurrentDB.Core.Tests/Index/IndexV1/table_index_with_two_ptables_and_memtable_on_range_query.cs @@ -40,7 +40,7 @@ public override async Task TestFixtureSetUp() { () => new HashListMemTable(_ptableVersion, maxSize: 10), fakeReader, _ptableVersion, - 5, Constants.PTableMaxReaderCountDefault, + 5, maxSizeForMemory: 2, maxTablesPerLevel: 2, skipIndexVerify: _skipIndexVerify); diff --git a/src/KurrentDB.Core.Tests/Index/IndexV1/when_a_ptable_header_is_corrupt_on_disk.cs b/src/KurrentDB.Core.Tests/Index/IndexV1/when_a_ptable_header_is_corrupt_on_disk.cs index 71a269dee4f..89c9921e99b 100644 --- a/src/KurrentDB.Core.Tests/Index/IndexV1/when_a_ptable_header_is_corrupt_on_disk.cs +++ b/src/KurrentDB.Core.Tests/Index/IndexV1/when_a_ptable_header_is_corrupt_on_disk.cs @@ -29,7 +29,7 @@ public void Setup() { var mtable = new HashListMemTable(_ptableVersion, maxSize: 10); mtable.Add(0x010100000000, 0x0001, 0x0001); mtable.Add(0x010500000000, 0x0001, 0x0002); - _table = PTable.FromMemtable(mtable, _filename, Constants.PTableInitialReaderCount, Constants.PTableMaxReaderCountDefault); + _table = PTable.FromMemtable(mtable, _filename); _table.Dispose(); File.Copy(_filename, _copiedfilename); using (var f = new FileStream(_copiedfilename, FileMode.Open, FileAccess.ReadWrite, FileShare.ReadWrite)) { @@ -40,14 +40,14 @@ public void Setup() { [Test] public void the_hash_is_invalid() { - var exc = Assert.Throws(() => _table = PTable.FromFile(_copiedfilename, Constants.PTableInitialReaderCount, Constants.PTableMaxReaderCountDefault, 16, false)); + var exc = Assert.Throws(() => _table = PTable.FromFile(_copiedfilename, 16, false)); Assert.IsInstanceOf(exc.InnerException); } [Test] public void no_error_if_index_verification_disabled() { Assert.DoesNotThrow( - () => _table = PTable.FromFile(_copiedfilename, Constants.PTableInitialReaderCount, Constants.PTableMaxReaderCountDefault, 16, true) + () => _table = PTable.FromFile(_copiedfilename, 16, true) ); } diff --git a/src/KurrentDB.Core.Tests/Index/IndexV1/when_a_ptable_is_corrupt_on_disk.cs b/src/KurrentDB.Core.Tests/Index/IndexV1/when_a_ptable_is_corrupt_on_disk.cs index ccb78fd2062..fc5cef71fdb 100644 --- a/src/KurrentDB.Core.Tests/Index/IndexV1/when_a_ptable_is_corrupt_on_disk.cs +++ b/src/KurrentDB.Core.Tests/Index/IndexV1/when_a_ptable_is_corrupt_on_disk.cs @@ -31,7 +31,7 @@ public override async Task SetUp() { var mtable = new HashListMemTable(_ptableVersion, maxSize: 10); mtable.Add(0x010100000000, 0x0001, 0x0001); mtable.Add(0x010500000000, 0x0001, 0x0002); - _table = PTable.FromMemtable(mtable, _filename, Constants.PTableInitialReaderCount, Constants.PTableMaxReaderCountDefault); + _table = PTable.FromMemtable(mtable, _filename); _table.Dispose(); File.Copy(_filename, _copiedfilename); using (var f = new FileStream(_copiedfilename, FileMode.Open, FileAccess.ReadWrite, FileShare.ReadWrite)) { @@ -50,7 +50,7 @@ public override Task TearDown() { [Test] public void the_hash_is_invalid() { - var exc = Assert.Throws(() => PTable.FromFile(_copiedfilename, Constants.PTableInitialReaderCount, Constants.PTableMaxReaderCountDefault, 16, false)); + var exc = Assert.Throws(() => PTable.FromFile(_copiedfilename, 16, false)); Assert.IsInstanceOf(exc.InnerException); } } diff --git a/src/KurrentDB.Core.Tests/Index/IndexV1/when_a_ptable_is_loaded_from_disk.cs b/src/KurrentDB.Core.Tests/Index/IndexV1/when_a_ptable_is_loaded_from_disk.cs index feb68ff4f43..647109d8643 100644 --- a/src/KurrentDB.Core.Tests/Index/IndexV1/when_a_ptable_is_loaded_from_disk.cs +++ b/src/KurrentDB.Core.Tests/Index/IndexV1/when_a_ptable_is_loaded_from_disk.cs @@ -51,7 +51,7 @@ public override async Task SetUp() { mtable.Add(streamId, eventNumber, logPosition); } - _table = PTable.FromMemtable(mtable, _filename, Constants.PTableInitialReaderCount, Constants.PTableMaxReaderCountDefault, skipIndexVerify: _skipIndexVerify); + _table = PTable.FromMemtable(mtable, _filename, skipIndexVerify: _skipIndexVerify); _table.Dispose(); File.Copy(_filename, _copiedfilename); } @@ -59,8 +59,8 @@ public override async Task SetUp() { [Test] public void same_midpoints_are_loaded_when_enabling_or_disabling_index_verification() { for (int depth = 2; depth <= 20; depth++) { - var ptableWithMD5Verification = PTable.FromFile(_copiedfilename, Constants.PTableInitialReaderCount, Constants.PTableMaxReaderCountDefault, depth, false); - var ptableWithoutVerification = PTable.FromFile(_copiedfilename, Constants.PTableInitialReaderCount, Constants.PTableMaxReaderCountDefault, depth, true); + var ptableWithMD5Verification = PTable.FromFile(_copiedfilename, depth, false); + var ptableWithoutVerification = PTable.FromFile(_copiedfilename, depth, true); var midPoints1 = ptableWithMD5Verification.GetMidPoints(); var midPoints2 = ptableWithoutVerification.GetMidPoints(); diff --git a/src/KurrentDB.Core.Tests/Index/IndexV1/when_constructing_v1_ptable.cs b/src/KurrentDB.Core.Tests/Index/IndexV1/when_constructing_v1_ptable.cs index 8ee544e45a5..6609f606bbe 100644 --- a/src/KurrentDB.Core.Tests/Index/IndexV1/when_constructing_v1_ptable.cs +++ b/src/KurrentDB.Core.Tests/Index/IndexV1/when_constructing_v1_ptable.cs @@ -13,9 +13,7 @@ public void an_exception_is_thrown() { Assert.Throws(() => { using var table = PTable.FromMemtable( new HashListMemTable(PTableVersions.IndexV1, maxSize: 20), - GetTempFilePath(), - Constants.PTableInitialReaderCount, - Constants.PTableMaxReaderCountDefault); + GetTempFilePath()); }); } } diff --git a/src/KurrentDB.Core.Tests/Index/IndexV1/when_creating_ptable_from_memtable.cs b/src/KurrentDB.Core.Tests/Index/IndexV1/when_creating_ptable_from_memtable.cs index d3eae7b27e0..fb551f02539 100644 --- a/src/KurrentDB.Core.Tests/Index/IndexV1/when_creating_ptable_from_memtable.cs +++ b/src/KurrentDB.Core.Tests/Index/IndexV1/when_creating_ptable_from_memtable.cs @@ -27,21 +27,21 @@ public when_creating_ptable_from_memtable(byte version, bool skipIndexVerify) { [Test] public void null_file_throws_null_exception() { Assert.Throws(() => - PTable.FromMemtable(new HashListMemTable(_ptableVersion, maxSize: 10), null, Constants.PTableInitialReaderCount, Constants.PTableMaxReaderCountDefault, + PTable.FromMemtable(new HashListMemTable(_ptableVersion, maxSize: 10), null, skipIndexVerify: _skipIndexVerify)); } [Test] public void null_memtable_throws_null_exception() { Assert.Throws(() => - PTable.FromMemtable(null, "C:\\foo.txt", Constants.PTableInitialReaderCount, Constants.PTableMaxReaderCountDefault, skipIndexVerify: _skipIndexVerify)); + PTable.FromMemtable(null, "C:\\foo.txt", skipIndexVerify: _skipIndexVerify)); } [Test] public void wait_for_destroy_will_timeout() { var table = new HashListMemTable(_ptableVersion, maxSize: 10); table.Add(0x010100000000, 0x0001, 0x0001); - var ptable = PTable.FromMemtable(table, Filename, Constants.PTableInitialReaderCount, Constants.PTableMaxReaderCountDefault, skipIndexVerify: _skipIndexVerify); + var ptable = PTable.FromMemtable(table, Filename, skipIndexVerify: _skipIndexVerify); Assert.Throws(() => ptable.WaitForDisposal(1)); // tear down @@ -75,7 +75,7 @@ public void the_file_gets_created_and_can_be_read() { table.Add(0x010500000000, 0x0001, 0x0002); table.Add(0x010200000000, 0x0001, 0x0003); table.Add(0x010200000000, 0x0002, 0x0003); - using (var sstable = PTable.FromMemtable(table, Filename, Constants.PTableInitialReaderCount, Constants.PTableMaxReaderCountDefault, + using (var sstable = PTable.FromMemtable(table, Filename, skipIndexVerify: _skipIndexVerify, useBloomFilter: true)) { @@ -110,7 +110,7 @@ public void the_hash_of_file_is_valid() { table.Add(0x010200000000, 0x0001, 0x0003); table.Add(0x010200000000, 0x0002, 0x0003); Assert.DoesNotThrow(() => { - using (var sstable = PTable.FromMemtable(table, Filename, Constants.PTableInitialReaderCount, Constants.PTableMaxReaderCountDefault, skipIndexVerify: false)) { + using (var sstable = PTable.FromMemtable(table, Filename, skipIndexVerify: false)) { } }); } diff --git a/src/KurrentDB.Core.Tests/Index/IndexV1/when_merging_four_ptables.cs b/src/KurrentDB.Core.Tests/Index/IndexV1/when_merging_four_ptables.cs index addccfdc2e9..64772086cad 100644 --- a/src/KurrentDB.Core.Tests/Index/IndexV1/when_merging_four_ptables.cs +++ b/src/KurrentDB.Core.Tests/Index/IndexV1/when_merging_four_ptables.cs @@ -39,12 +39,12 @@ public override async Task TestFixtureSetUp() { table.Add((ulong)(0x010100000000 << (j + 1)), i + 1, i * j); } - _tables.Add(PTable.FromMemtable(table, _files[i], Constants.PTableInitialReaderCount, Constants.PTableMaxReaderCountDefault, skipIndexVerify: _skipIndexVerify)); + _tables.Add(PTable.FromMemtable(table, _files[i], skipIndexVerify: _skipIndexVerify)); } _files.Add(GetTempFilePath()); _newtable = PTable.MergeTo(_tables, _files[4], - _ptableVersion, Constants.PTableInitialReaderCount, Constants.PTableMaxReaderCountDefault, + _ptableVersion, skipIndexVerify: _skipIndexVerify); } diff --git a/src/KurrentDB.Core.Tests/Index/IndexV1/when_merging_ptables.cs b/src/KurrentDB.Core.Tests/Index/IndexV1/when_merging_ptables.cs index 86877f79c75..6bb4cb6d60a 100644 --- a/src/KurrentDB.Core.Tests/Index/IndexV1/when_merging_ptables.cs +++ b/src/KurrentDB.Core.Tests/Index/IndexV1/when_merging_ptables.cs @@ -36,19 +36,17 @@ public override async Task TestFixtureSetUp() { table.Add(0x0102, 0, 0x0102); table.Add(0x0103, 0, 0x0103); table.Add(0x0104, 0, 0x0104); - _tables.Add(PTable.FromMemtable(table, GetTempFilePath(), Constants.PTableInitialReaderCount, Constants.PTableMaxReaderCountDefault)); + _tables.Add(PTable.FromMemtable(table, GetTempFilePath())); table = new HashListMemTable(_ptableVersion, maxSize: 20); table.Add(0x0105, 0, 0x0105); table.Add(0x0106, 0, 0x0106); table.Add(0x0107, 0, 0x0107); table.Add(0x0108, 0, 0x0108); - _tables.Add(PTable.FromMemtable(table, GetTempFilePath(), Constants.PTableInitialReaderCount, Constants.PTableMaxReaderCountDefault)); + _tables.Add(PTable.FromMemtable(table, GetTempFilePath())); _newtable = PTable.MergeTo( tables: _tables, outputFile: GetTempFilePath(), version: PTableVersions.IndexV4, - initialReaders: Constants.PTableInitialReaderCount, - maxReaders: Constants.PTableMaxReaderCountDefault, skipIndexVerify: _skipIndexVerify, useBloomFilter: true); } diff --git a/src/KurrentDB.Core.Tests/Index/IndexV1/when_merging_ptables_with_entries_to_nonexisting_record.cs b/src/KurrentDB.Core.Tests/Index/IndexV1/when_merging_ptables_with_entries_to_nonexisting_record.cs index d0908059feb..3db61feab02 100644 --- a/src/KurrentDB.Core.Tests/Index/IndexV1/when_merging_ptables_with_entries_to_nonexisting_record.cs +++ b/src/KurrentDB.Core.Tests/Index/IndexV1/when_merging_ptables_with_entries_to_nonexisting_record.cs @@ -39,12 +39,12 @@ public override async Task TestFixtureSetUp() { table.Add((ulong)(0x010100000000 << i), j, i * 10 + j); } - _tables.Add(PTable.FromMemtable(table, _files[i], Constants.PTableInitialReaderCount, Constants.PTableMaxReaderCountDefault, skipIndexVerify: _skipIndexVerify)); + _tables.Add(PTable.FromMemtable(table, _files[i], skipIndexVerify: _skipIndexVerify)); } _files.Add(GetTempFilePath()); _newtable = PTable.MergeTo(_tables, _files[4], - _ptableVersion, Constants.PTableInitialReaderCount, Constants.PTableMaxReaderCountDefault, + _ptableVersion, skipIndexVerify: _skipIndexVerify); } diff --git a/src/KurrentDB.Core.Tests/Index/IndexV1/when_merging_two_ptables.cs b/src/KurrentDB.Core.Tests/Index/IndexV1/when_merging_two_ptables.cs index 53db70fa174..b1e24b3d17c 100644 --- a/src/KurrentDB.Core.Tests/Index/IndexV1/when_merging_two_ptables.cs +++ b/src/KurrentDB.Core.Tests/Index/IndexV1/when_merging_two_ptables.cs @@ -43,13 +43,13 @@ public override async Task TestFixtureSetUp() { table.Add((ulong)(0x010100000000 << (j + 1)), i + 1, i * j); } - _tables.Add(PTable.FromMemtable(table, _files[i], Constants.PTableInitialReaderCount, Constants.PTableMaxReaderCountDefault, + _tables.Add(PTable.FromMemtable(table, _files[i], skipIndexVerify: _skipIndexVerify, useBloomFilter: true)); } _files.Add(GetTempFilePath()); _newtable = PTable.MergeTo(_tables, _files[2], - _ptableVersion, Constants.PTableInitialReaderCount, Constants.PTableMaxReaderCountDefault, + _ptableVersion, skipIndexVerify: _skipIndexVerify, useBloomFilter: true); } diff --git a/src/KurrentDB.Core.Tests/Index/IndexV1/when_trying_to_get_latest_entry.cs b/src/KurrentDB.Core.Tests/Index/IndexV1/when_trying_to_get_latest_entry.cs index 3e4583c181a..fb35702cdc7 100644 --- a/src/KurrentDB.Core.Tests/Index/IndexV1/when_trying_to_get_latest_entry.cs +++ b/src/KurrentDB.Core.Tests/Index/IndexV1/when_trying_to_get_latest_entry.cs @@ -36,8 +36,6 @@ private PTable ConstructPTable(HashListMemTable memTable) { return PTable.FromMemtable( memTable, Filename, - Constants.PTableInitialReaderCount, - Constants.PTableMaxReaderCountDefault, skipIndexVerify: _skipIndexVerify, useBloomFilter: _useBloomFilter, lruCacheSize: _lruCacheSize); diff --git a/src/KurrentDB.Core.Tests/Index/IndexV1/when_trying_to_get_latest_entry_before_position.cs b/src/KurrentDB.Core.Tests/Index/IndexV1/when_trying_to_get_latest_entry_before_position.cs index 0720ddc0188..1a04f4f1d42 100644 --- a/src/KurrentDB.Core.Tests/Index/IndexV1/when_trying_to_get_latest_entry_before_position.cs +++ b/src/KurrentDB.Core.Tests/Index/IndexV1/when_trying_to_get_latest_entry_before_position.cs @@ -58,8 +58,6 @@ public override async Task SetUp() { _pTable = PTable.FromMemtable( table: _memTable, filename: Filename, - initialReaders: Constants.PTableInitialReaderCount, - maxReaders: Constants.PTableMaxReaderCountDefault, skipIndexVerify: _skipIndexVerify); } diff --git a/src/KurrentDB.Core.Tests/Index/IndexV1/when_trying_to_get_next_entry.cs b/src/KurrentDB.Core.Tests/Index/IndexV1/when_trying_to_get_next_entry.cs index b493725cd8f..812f3ab63d2 100644 --- a/src/KurrentDB.Core.Tests/Index/IndexV1/when_trying_to_get_next_entry.cs +++ b/src/KurrentDB.Core.Tests/Index/IndexV1/when_trying_to_get_next_entry.cs @@ -52,8 +52,6 @@ public override async Task SetUp() { _pTable = PTable.FromMemtable( table: _memTable, filename: Filename, - initialReaders: Constants.PTableInitialReaderCount, - maxReaders: Constants.PTableMaxReaderCountDefault, skipIndexVerify: _skipIndexVerify); } diff --git a/src/KurrentDB.Core.Tests/Index/IndexV1/when_trying_to_get_oldest_entry.cs b/src/KurrentDB.Core.Tests/Index/IndexV1/when_trying_to_get_oldest_entry.cs index 7097570d6de..76539b49298 100644 --- a/src/KurrentDB.Core.Tests/Index/IndexV1/when_trying_to_get_oldest_entry.cs +++ b/src/KurrentDB.Core.Tests/Index/IndexV1/when_trying_to_get_oldest_entry.cs @@ -36,8 +36,6 @@ private PTable ConstructPTable(HashListMemTable memTable) { return PTable.FromMemtable( memTable, Filename, - Constants.PTableInitialReaderCount, - Constants.PTableMaxReaderCountDefault, skipIndexVerify: _skipIndexVerify, useBloomFilter: _useBloomFilter, lruCacheSize: _lruCacheSize); diff --git a/src/KurrentDB.Core.Tests/Index/IndexV1/when_trying_to_get_previous_entry.cs b/src/KurrentDB.Core.Tests/Index/IndexV1/when_trying_to_get_previous_entry.cs index cf3cc84c7bb..474eedc674e 100644 --- a/src/KurrentDB.Core.Tests/Index/IndexV1/when_trying_to_get_previous_entry.cs +++ b/src/KurrentDB.Core.Tests/Index/IndexV1/when_trying_to_get_previous_entry.cs @@ -52,8 +52,6 @@ public override async Task SetUp() { _pTable = PTable.FromMemtable( table: _memTable, filename: Filename, - initialReaders: Constants.PTableInitialReaderCount, - maxReaders: Constants.PTableMaxReaderCountDefault, skipIndexVerify: _skipIndexVerify); } diff --git a/src/KurrentDB.Core.Tests/Index/IndexV3/when_upgrading_index_to_64bit_stream_version.cs b/src/KurrentDB.Core.Tests/Index/IndexV3/when_upgrading_index_to_64bit_stream_version.cs index f08f0bbc0ce..cd49859d450 100644 --- a/src/KurrentDB.Core.Tests/Index/IndexV3/when_upgrading_index_to_64bit_stream_version.cs +++ b/src/KurrentDB.Core.Tests/Index/IndexV3/when_upgrading_index_to_64bit_stream_version.cs @@ -37,7 +37,7 @@ public override async Task TestFixtureSetUp() { () => new HashListMemTable(PTableVersions.IndexV2, maxSize: 5), fakeReader, PTableVersions.IndexV2, - 5, Constants.PTableMaxReaderCountDefault, + 5, maxSizeForMemory: 5, maxTablesPerLevel: 2); _tableIndex.Initialize(long.MaxValue); @@ -54,7 +54,7 @@ public override async Task TestFixtureSetUp() { () => new HashListMemTable(_ptableVersion, maxSize: 5), fakeReader, _ptableVersion, - 5, Constants.PTableMaxReaderCountDefault, + 5, maxSizeForMemory: 5, maxTablesPerLevel: 2); _tableIndex.Initialize(long.MaxValue); diff --git a/src/KurrentDB.Core.Tests/Index/IndexV4/when_merging_ptables_vx_to_v4.cs b/src/KurrentDB.Core.Tests/Index/IndexV4/when_merging_ptables_vx_to_v4.cs index 6aedc250068..7ab575f063f 100644 --- a/src/KurrentDB.Core.Tests/Index/IndexV4/when_merging_ptables_vx_to_v4.cs +++ b/src/KurrentDB.Core.Tests/Index/IndexV4/when_merging_ptables_vx_to_v4.cs @@ -46,7 +46,7 @@ public override async Task TestFixtureSetUp() { table.Add(0x0104, 0, 0x0104); } - _tables.Add(PTable.FromMemtable(table, GetTempFilePath(), Constants.PTableInitialReaderCount, Constants.PTableMaxReaderCountDefault, skipIndexVerify: _skipIndexVerify)); + _tables.Add(PTable.FromMemtable(table, GetTempFilePath(), skipIndexVerify: _skipIndexVerify)); table = new HashListMemTable(_fromVersion, maxSize: 20); if (_fromVersion == PTableVersions.IndexV1) { @@ -61,10 +61,10 @@ public override async Task TestFixtureSetUp() { table.Add(0x0108, 0, 0x0108); } - _tables.Add(PTable.FromMemtable(table, GetTempFilePath(), Constants.PTableInitialReaderCount, Constants.PTableMaxReaderCountDefault, skipIndexVerify: _skipIndexVerify)); + _tables.Add(PTable.FromMemtable(table, GetTempFilePath(), skipIndexVerify: _skipIndexVerify)); _newtableFile = GetTempFilePath(); _newtable = PTable.MergeTo(_tables, _newtableFile, - PTableVersions.IndexV4, Constants.PTableInitialReaderCount, Constants.PTableMaxReaderCountDefault, + PTableVersions.IndexV4, skipIndexVerify: _skipIndexVerify); } @@ -96,16 +96,14 @@ public void midpoints_are_cached_in_ptable_footer() { var newTableFileCopy = GetTempFilePath(); File.Copy(_newtableFile, newTableFileCopy); using (var filestream = File.Open(newTableFileCopy, FileMode.Open, FileAccess.Read)) { - var footerSize = PTableFooter.GetSize(PTableVersions.IndexV4); + var footerSize = PTableFooter.Size; Assert.AreEqual(filestream.Length, PTableHeader.Size + numIndexEntries * PTable.IndexEntryV4Size + requiredMidpoints * PTable.IndexEntryV4Size + footerSize + PTable.MD5Size); - filestream.Seek( - PTableHeader.Size + numIndexEntries * PTable.IndexEntryV4Size + - requiredMidpoints * PTable.IndexEntryV4Size, SeekOrigin.Begin); + var footerOffset = PTableHeader.Size + numIndexEntries * PTable.IndexEntryV4Size + + requiredMidpoints * PTable.IndexEntryV4Size; - var ptableFooter = PTableFooter.FromStream(filestream); - Assert.AreEqual(FileType.PTableFile, ptableFooter.FileType); + var ptableFooter = PTableFooter.Parse(filestream.SafeFileHandle, footerOffset); Assert.AreEqual(PTableVersions.IndexV4, ptableFooter.Version); Assert.AreEqual(requiredMidpoints, ptableFooter.NumMidpointsCached); } @@ -167,24 +165,24 @@ public override async Task TestFixtureSetUp() { table.Add(0x010200000000, 0, 2); table.Add(0x010300000000, 0, 3); table.Add(0x010300000000, 1, 4); - _tables.Add(PTable.FromMemtable(table, GetTempFilePath(), Constants.PTableInitialReaderCount, Constants.PTableMaxReaderCountDefault, skipIndexVerify: _skipIndexVerify)); + _tables.Add(PTable.FromMemtable(table, GetTempFilePath(), skipIndexVerify: _skipIndexVerify)); table = new HashListMemTable(_fromVersion, maxSize: 20); table.Add(0x010100000000, 2, 5); table.Add(0x010200000000, 1, 6); table.Add(0x010200000000, 2, 7); table.Add(0x010400000000, 0, 8); table.Add(0x010400000000, 1, 9); - _tables.Add(PTable.FromMemtable(table, GetTempFilePath(), Constants.PTableInitialReaderCount, Constants.PTableMaxReaderCountDefault, skipIndexVerify: _skipIndexVerify)); + _tables.Add(PTable.FromMemtable(table, GetTempFilePath(), skipIndexVerify: _skipIndexVerify)); table = new HashListMemTable(_fromVersion, maxSize: 20); table.Add(0x010100000000, 1, 10); table.Add(0x010100000000, 2, 11); table.Add(0x010500000000, 1, 12); table.Add(0x010500000000, 2, 13); table.Add(0x010500000000, 3, 14); - _tables.Add(PTable.FromMemtable(table, GetTempFilePath(), Constants.PTableInitialReaderCount, Constants.PTableMaxReaderCountDefault, skipIndexVerify: _skipIndexVerify)); + _tables.Add(PTable.FromMemtable(table, GetTempFilePath(), skipIndexVerify: _skipIndexVerify)); _newtableFile = GetTempFilePath(); _newtable = PTable.MergeTo(_tables, _newtableFile, - PTableVersions.IndexV4, Constants.PTableInitialReaderCount, Constants.PTableMaxReaderCountDefault, skipIndexVerify: _skipIndexVerify); + PTableVersions.IndexV4, skipIndexVerify: _skipIndexVerify); } [OneTimeTearDown] @@ -215,16 +213,14 @@ public void midpoints_are_cached_in_ptable_footer() { var newTableFileCopy = GetTempFilePath(); File.Copy(_newtableFile, newTableFileCopy); using (var filestream = File.Open(newTableFileCopy, FileMode.Open, FileAccess.Read)) { - var footerSize = PTableFooter.GetSize(PTableVersions.IndexV4); + var footerSize = PTableFooter.Size; Assert.AreEqual(filestream.Length, PTableHeader.Size + numIndexEntries * PTable.IndexEntryV4Size + requiredMidpoints * PTable.IndexEntryV4Size + footerSize + PTable.MD5Size); - filestream.Seek( - PTableHeader.Size + numIndexEntries * PTable.IndexEntryV4Size + - requiredMidpoints * PTable.IndexEntryV4Size, SeekOrigin.Begin); + var footerOffset = PTableHeader.Size + numIndexEntries * PTable.IndexEntryV4Size + + requiredMidpoints * PTable.IndexEntryV4Size; - var ptableFooter = PTableFooter.FromStream(filestream); - Assert.AreEqual(FileType.PTableFile, ptableFooter.FileType); + var ptableFooter = PTableFooter.Parse(filestream.SafeFileHandle, footerOffset); Assert.AreEqual(PTableVersions.IndexV4, ptableFooter.Version); Assert.AreEqual(requiredMidpoints, ptableFooter.NumMidpointsCached); } diff --git a/src/KurrentDB.Core.Tests/Index/IndexVAny/when_opening_ptable_without_right_flag_in_header.cs b/src/KurrentDB.Core.Tests/Index/IndexVAny/when_opening_ptable_without_right_flag_in_header.cs index 2a7df6cee5f..0a0c1c2bcc0 100644 --- a/src/KurrentDB.Core.Tests/Index/IndexVAny/when_opening_ptable_without_right_flag_in_header.cs +++ b/src/KurrentDB.Core.Tests/Index/IndexVAny/when_opening_ptable_without_right_flag_in_header.cs @@ -23,7 +23,7 @@ public override async Task SetUp() { [Test] public void the_invalid_file_exception_is_thrown() { - var exc = Assert.Throws(() => PTable.FromFile(Filename, Constants.PTableInitialReaderCount, Constants.PTableMaxReaderCountDefault, 16, false)); + var exc = Assert.Throws(() => PTable.FromFile(Filename, 16, false)); Assert.IsInstanceOf(exc.InnerException); } } diff --git a/src/KurrentDB.Core.Tests/Index/IndexVAny/when_opening_v1_indexmap.cs b/src/KurrentDB.Core.Tests/Index/IndexVAny/when_opening_v1_indexmap.cs index 8aec2f590b1..a9f9a0f99eb 100644 --- a/src/KurrentDB.Core.Tests/Index/IndexVAny/when_opening_v1_indexmap.cs +++ b/src/KurrentDB.Core.Tests/Index/IndexVAny/when_opening_v1_indexmap.cs @@ -54,7 +54,7 @@ public override async Task TestFixtureSetUp() { await base.TestFixtureSetUp(); _filename = GetFilePathFor("indexfile"); - var empty = IndexMap.CreateEmpty(2, 4, Constants.PTableMaxReaderCountDefault); + var empty = IndexMap.CreateEmpty(2, 4); empty.SaveToFile(_filename); } diff --git a/src/KurrentDB.Core.Tests/Index/Scavenge/when_scavenging_a_table_index.cs b/src/KurrentDB.Core.Tests/Index/Scavenge/when_scavenging_a_table_index.cs index 962f32e3e6a..061774b8629 100644 --- a/src/KurrentDB.Core.Tests/Index/Scavenge/when_scavenging_a_table_index.cs +++ b/src/KurrentDB.Core.Tests/Index/Scavenge/when_scavenging_a_table_index.cs @@ -43,7 +43,7 @@ public override async Task TestFixtureSetUp() { () => new HashListMemTable(PTableVersions.IndexV4, maxSize: 5), fakeReader, PTableVersions.IndexV4, - 5, Constants.PTableMaxReaderCountDefault, + 5, maxSizeForMemory: 2, maxTablesPerLevel: 5, skipIndexVerify: _skipIndexVerify, useBloomFilter: _useBloomFilter); @@ -67,7 +67,7 @@ public override async Task TestFixtureSetUp() { () => new HashListMemTable(PTableVersions.IndexV4, maxSize: 5), fakeReader, PTableVersions.IndexV4, - 5, Constants.PTableMaxReaderCountDefault, + 5, maxSizeForMemory: 2, maxTablesPerLevel: 5, useBloomFilter: _useBloomFilter); diff --git a/src/KurrentDB.Core.Tests/Index/Scavenge/when_scavenging_a_table_index_and_another_table_is_completed_during.cs b/src/KurrentDB.Core.Tests/Index/Scavenge/when_scavenging_a_table_index_and_another_table_is_completed_during.cs index 77d723cad68..9137d8db4e2 100644 --- a/src/KurrentDB.Core.Tests/Index/Scavenge/when_scavenging_a_table_index_and_another_table_is_completed_during.cs +++ b/src/KurrentDB.Core.Tests/Index/Scavenge/when_scavenging_a_table_index_and_another_table_is_completed_during.cs @@ -50,7 +50,7 @@ public override async Task TestFixtureSetUp() { () => new HashListMemTable(PTableVersions.IndexV4, maxSize: 5), fakeReader, PTableVersions.IndexV4, - 5, Constants.PTableMaxReaderCountDefault, + 5, maxSizeForMemory: 2, maxTablesPerLevel: 5, useBloomFilter: _useBloomFilter); @@ -83,7 +83,7 @@ public override async Task TestFixtureSetUp() { () => new HashListMemTable(PTableVersions.IndexV4, maxSize: 5), fakeReader, PTableVersions.IndexV4, - 5, Constants.PTableMaxReaderCountDefault, + 5, maxSizeForMemory: 2, maxTablesPerLevel: 5, useBloomFilter: _useBloomFilter); diff --git a/src/KurrentDB.Core.Tests/Index/Scavenge/when_scavenging_a_table_index_cancelled_while_scavenging_table.cs b/src/KurrentDB.Core.Tests/Index/Scavenge/when_scavenging_a_table_index_cancelled_while_scavenging_table.cs index e9908d8cacd..2f41e742235 100644 --- a/src/KurrentDB.Core.Tests/Index/Scavenge/when_scavenging_a_table_index_cancelled_while_scavenging_table.cs +++ b/src/KurrentDB.Core.Tests/Index/Scavenge/when_scavenging_a_table_index_cancelled_while_scavenging_table.cs @@ -39,7 +39,7 @@ public override async Task TestFixtureSetUp() { () => new HashListMemTable(PTableVersions.IndexV4, maxSize: 5), fakeReader, PTableVersions.IndexV4, - 5, Constants.PTableMaxReaderCountDefault, + 5, maxSizeForMemory: 2, maxTablesPerLevel: 5); _tableIndex.Initialize(long.MaxValue); @@ -64,7 +64,7 @@ public override async Task TestFixtureSetUp() { () => new HashListMemTable(PTableVersions.IndexV4, maxSize: 5), fakeReader, PTableVersions.IndexV4, - 5, Constants.PTableMaxReaderCountDefault, + 5, maxSizeForMemory: 2, maxTablesPerLevel: 5); diff --git a/src/KurrentDB.Core.Tests/Index/Scavenge/when_scavenging_a_table_index_cancelled_while_waiting_for_lock.cs b/src/KurrentDB.Core.Tests/Index/Scavenge/when_scavenging_a_table_index_cancelled_while_waiting_for_lock.cs index 3f3489426d9..dfb68358647 100644 --- a/src/KurrentDB.Core.Tests/Index/Scavenge/when_scavenging_a_table_index_cancelled_while_waiting_for_lock.cs +++ b/src/KurrentDB.Core.Tests/Index/Scavenge/when_scavenging_a_table_index_cancelled_while_waiting_for_lock.cs @@ -33,7 +33,7 @@ public override async Task TestFixtureSetUp() { () => new HashListMemTable(PTableVersions.IndexV4, maxSize: 5), fakeReader, PTableVersions.IndexV4, - 5, Constants.PTableMaxReaderCountDefault, + 5, maxSizeForMemory: 2, maxTablesPerLevel: 5); _tableIndex.Initialize(long.MaxValue); @@ -60,7 +60,7 @@ public override async Task TestFixtureSetUp() { () => new HashListMemTable(PTableVersions.IndexV4, maxSize: 5), fakeReader, PTableVersions.IndexV4, - 5, Constants.PTableMaxReaderCountDefault, + 5, maxSizeForMemory: 2, maxTablesPerLevel: 5); diff --git a/src/KurrentDB.Core.Tests/Index/Scavenge/when_scavenging_a_table_index_fails.cs b/src/KurrentDB.Core.Tests/Index/Scavenge/when_scavenging_a_table_index_fails.cs index 649466dd5a4..b3c56ac18f5 100644 --- a/src/KurrentDB.Core.Tests/Index/Scavenge/when_scavenging_a_table_index_fails.cs +++ b/src/KurrentDB.Core.Tests/Index/Scavenge/when_scavenging_a_table_index_fails.cs @@ -40,7 +40,7 @@ public override async Task TestFixtureSetUp() { () => new HashListMemTable(PTableVersions.IndexV4, maxSize: 5), fakeReader /* throw an exception when the first PTable scavenge starts and tries to acquire a reader */, PTableVersions.IndexV4, - 5, Constants.PTableMaxReaderCountDefault, + 5, maxSizeForMemory: 2, maxTablesPerLevel: 5, useBloomFilter: _useBloomFilter); @@ -64,7 +64,7 @@ public override async Task TestFixtureSetUp() { () => new HashListMemTable(PTableVersions.IndexV4, maxSize: 5), fakeReader, PTableVersions.IndexV4, - 5, Constants.PTableMaxReaderCountDefault, + 5, maxSizeForMemory: 2, maxTablesPerLevel: 5, useBloomFilter: _useBloomFilter); diff --git a/src/KurrentDB.Core.Tests/Index/Scavenge/when_scavenging_an_index.cs b/src/KurrentDB.Core.Tests/Index/Scavenge/when_scavenging_an_index.cs index a968a75cf7a..f98a9c8d114 100644 --- a/src/KurrentDB.Core.Tests/Index/Scavenge/when_scavenging_an_index.cs +++ b/src/KurrentDB.Core.Tests/Index/Scavenge/when_scavenging_an_index.cs @@ -35,13 +35,12 @@ public override async Task TestFixtureSetUp() { table.Add(0x010200000000, 0, 2); table.Add(0x010300000000, 0, 3); table.Add(0x010300000000, 1, 4); - _oldTable = PTable.FromMemtable(table, GetTempFilePath(), Constants.PTableInitialReaderCount, Constants.PTableMaxReaderCountDefault); + _oldTable = PTable.FromMemtable(table, GetTempFilePath()); Func existsAt = x => x.Position % 2 == 0; (_newtable, _) = await PTable.Scavenged(_oldTable, GetTempFilePath(), PTableVersions.IndexV4, existsAt.ToAsync(), skipIndexVerify: _skipIndexVerify, - initialReaders: Constants.PTableInitialReaderCount, maxReaders: Constants.PTableMaxReaderCountDefault, useBloomFilter: true); } diff --git a/src/KurrentDB.Core.Tests/Index/Scavenge/when_scavenging_an_index_fails.cs b/src/KurrentDB.Core.Tests/Index/Scavenge/when_scavenging_an_index_fails.cs index 6196abe3aa2..f2bb3a3addd 100644 --- a/src/KurrentDB.Core.Tests/Index/Scavenge/when_scavenging_an_index_fails.cs +++ b/src/KurrentDB.Core.Tests/Index/Scavenge/when_scavenging_an_index_fails.cs @@ -24,14 +24,13 @@ public override async Task TestFixtureSetUp() { table.Add(0x010200000000, 0, 2); table.Add(0x010300000000, 0, 3); table.Add(0x010300000000, 1, 4); - _oldTable = PTable.FromMemtable(table, GetTempFilePath(), Constants.PTableInitialReaderCount, Constants.PTableMaxReaderCountDefault); + _oldTable = PTable.FromMemtable(table, GetTempFilePath()); Func existsAt = x => { throw new Exception("Expected exception"); }; _expectedOutputFile = GetTempFilePath(); var ex = Assert.ThrowsAsync(async () => await PTable.Scavenged(_oldTable, _expectedOutputFile, - PTableVersions.IndexV4, existsAt.ToAsync(), initialReaders: Constants.PTableInitialReaderCount, - maxReaders: Constants.PTableMaxReaderCountDefault, + PTableVersions.IndexV4, existsAt.ToAsync(), useBloomFilter: true)); Assert.AreEqual("Expected exception", ex?.Message); diff --git a/src/KurrentDB.Core.Tests/Index/Scavenge/when_scavenging_an_index_is_cancelled.cs b/src/KurrentDB.Core.Tests/Index/Scavenge/when_scavenging_an_index_is_cancelled.cs index 3d338eaf58a..14da014e88b 100644 --- a/src/KurrentDB.Core.Tests/Index/Scavenge/when_scavenging_an_index_is_cancelled.cs +++ b/src/KurrentDB.Core.Tests/Index/Scavenge/when_scavenging_an_index_is_cancelled.cs @@ -25,7 +25,7 @@ public override async Task TestFixtureSetUp() { table.Add(0x010200000000, 0, 2); table.Add(0x010300000000, 0, 3); table.Add(0x010300000000, 1, 4); - _oldTable = PTable.FromMemtable(table, GetTempFilePath(), Constants.PTableInitialReaderCount, Constants.PTableMaxReaderCountDefault); + _oldTable = PTable.FromMemtable(table, GetTempFilePath()); var cancellationTokenSource = new CancellationTokenSource(); Func existsAt = x => { @@ -36,7 +36,6 @@ public override async Task TestFixtureSetUp() { _expectedOutputFile = GetTempFilePath(); Assert.ThrowsAsync(async () => await PTable.Scavenged(_oldTable, _expectedOutputFile, PTableVersions.IndexV4, existsAt.ToAsync(), ct: cancellationTokenSource.Token, - initialReaders: Constants.PTableInitialReaderCount, maxReaders: Constants.PTableMaxReaderCountDefault, useBloomFilter: true)); } diff --git a/src/KurrentDB.Core.Tests/Index/Scavenge/when_scavenging_an_index_removes_nothing.cs b/src/KurrentDB.Core.Tests/Index/Scavenge/when_scavenging_an_index_removes_nothing.cs index d6d1bf7ab7e..375d9e9dcca 100644 --- a/src/KurrentDB.Core.Tests/Index/Scavenge/when_scavenging_an_index_removes_nothing.cs +++ b/src/KurrentDB.Core.Tests/Index/Scavenge/when_scavenging_an_index_removes_nothing.cs @@ -37,14 +37,13 @@ public override async Task TestFixtureSetUp() { table.Add(0x010200000000, 0, 2); table.Add(0x010300000000, 0, 3); table.Add(0x010300000000, 1, 4); - _oldTable = PTable.FromMemtable(table, GetTempFilePath(), Constants.PTableInitialReaderCount, Constants.PTableMaxReaderCountDefault); + _oldTable = PTable.FromMemtable(table, GetTempFilePath()); Func existsAt = x => true; _expectedOutputFile = GetTempFilePath(); (_newtable, var spaceSaved) = await PTable.Scavenged(_oldTable, _expectedOutputFile, - PTableVersions.IndexV4, existsAt.ToAsync(), skipIndexVerify: _skipIndexVerify, - initialReaders: Constants.PTableInitialReaderCount, maxReaders: Constants.PTableMaxReaderCountDefault); + PTableVersions.IndexV4, existsAt.ToAsync(), skipIndexVerify: _skipIndexVerify); } [OneTimeTearDown] diff --git a/src/KurrentDB.Core.Tests/Services/Storage/BuildingIndex/when_building_an_index_off_tfile_with_duplicate_events_in_a_stream.cs b/src/KurrentDB.Core.Tests/Services/Storage/BuildingIndex/when_building_an_index_off_tfile_with_duplicate_events_in_a_stream.cs index ef517f7bb87..3a1130258c3 100644 --- a/src/KurrentDB.Core.Tests/Services/Storage/BuildingIndex/when_building_an_index_off_tfile_with_duplicate_events_in_a_stream.cs +++ b/src/KurrentDB.Core.Tests/Services/Storage/BuildingIndex/when_building_an_index_off_tfile_with_duplicate_events_in_a_stream.cs @@ -143,7 +143,6 @@ public override async Task TestFixtureSetUp() { reader, IndexBitnessVersion, int.MaxValue, - Constants.PTableMaxReaderCountDefault, MaxEntriesInMemTable); _logFormat.StreamNamesProvider.SetTableIndex(_tableIndex); @@ -192,7 +191,6 @@ public override async Task TestFixtureSetUp() { reader, IndexBitnessVersion, int.MaxValue, - Constants.PTableMaxReaderCountDefault, MaxEntriesInMemTable); readIndex = new ReadIndex(new NoopPublisher(), diff --git a/src/KurrentDB.Core.Tests/Services/Storage/HashCollisions/with_hash_collisions.cs b/src/KurrentDB.Core.Tests/Services/Storage/HashCollisions/with_hash_collisions.cs index 7bf54d487bf..1fcec8ca4dd 100644 --- a/src/KurrentDB.Core.Tests/Services/Storage/HashCollisions/with_hash_collisions.cs +++ b/src/KurrentDB.Core.Tests/Services/Storage/HashCollisions/with_hash_collisions.cs @@ -62,7 +62,7 @@ public void Setup() { () => new HashListMemTable(PTableVersions.IndexV4, maxSize: _maxMemTableSize), _fakeReader, PTableVersions.IndexV4, - 5, Constants.PTableMaxReaderCountDefault, + 5, maxSizeForMemory: _maxMemTableSize, maxTablesPerLevel: 2); _logFormat.StreamNamesProvider.SetTableIndex(_tableIndex); @@ -319,7 +319,7 @@ protected override void when() { () => new HashListMemTable(PTableVersions.IndexV2, maxSize: _maxMemTableSize), _fakeReader, PTableVersions.IndexV2, - 5, Constants.PTableMaxReaderCountDefault, + 5, maxSizeForMemory: _maxMemTableSize, maxTablesPerLevel: 2); _tableIndex.Initialize(long.MaxValue); diff --git a/src/KurrentDB.Core.Tests/Services/Storage/ReadIndexTestScenario.cs b/src/KurrentDB.Core.Tests/Services/Storage/ReadIndexTestScenario.cs index fba6620d922..29473fa9291 100644 --- a/src/KurrentDB.Core.Tests/Services/Storage/ReadIndexTestScenario.cs +++ b/src/KurrentDB.Core.Tests/Services/Storage/ReadIndexTestScenario.cs @@ -115,7 +115,6 @@ public override async Task TestFixtureSetUp() { reader, IndexBitnessVersion, int.MaxValue, - Constants.PTableMaxReaderCountDefault, MaxEntriesInMemTable)); _logFormat.StreamNamesProvider.SetTableIndex(TableIndex); diff --git a/src/KurrentDB.Core.Tests/Services/Storage/RepeatableDbTestScenario.cs b/src/KurrentDB.Core.Tests/Services/Storage/RepeatableDbTestScenario.cs index 017f2964720..56279d35bf7 100644 --- a/src/KurrentDB.Core.Tests/Services/Storage/RepeatableDbTestScenario.cs +++ b/src/KurrentDB.Core.Tests/Services/Storage/RepeatableDbTestScenario.cs @@ -65,7 +65,6 @@ public async ValueTask CreateDb(Rec[] records, CancellationToken token = default reader, PTableVersions.IndexV3, int.MaxValue, - Constants.PTableMaxReaderCountDefault, MaxEntriesInMemTable); _logFormat.StreamNamesProvider.SetTableIndex(TableIndex); diff --git a/src/KurrentDB.Core.Tests/Services/Storage/SimpleDbTestScenario.cs b/src/KurrentDB.Core.Tests/Services/Storage/SimpleDbTestScenario.cs index 899c26a54db..2fe765e2ace 100644 --- a/src/KurrentDB.Core.Tests/Services/Storage/SimpleDbTestScenario.cs +++ b/src/KurrentDB.Core.Tests/Services/Storage/SimpleDbTestScenario.cs @@ -64,7 +64,6 @@ public override async Task TestFixtureSetUp() { reader, PTableVersions.IndexV2, int.MaxValue, - Constants.PTableMaxReaderCountDefault, MaxEntriesInMemTable); _logFormat.StreamNamesProvider.SetTableIndex(TableIndex); diff --git a/src/KurrentDB.Core.Tests/Services/Storage/Transactions/when_rebuilding_index_for_partially_persisted_transaction.cs b/src/KurrentDB.Core.Tests/Services/Storage/Transactions/when_rebuilding_index_for_partially_persisted_transaction.cs index f01f2bc20b5..2bf3bab0d2b 100644 --- a/src/KurrentDB.Core.Tests/Services/Storage/Transactions/when_rebuilding_index_for_partially_persisted_transaction.cs +++ b/src/KurrentDB.Core.Tests/Services/Storage/Transactions/when_rebuilding_index_for_partially_persisted_transaction.cs @@ -39,7 +39,7 @@ public override async Task TestFixtureSetUp() { () => new HashListMemTable(PTableVersions.IndexV2, maxSize: MaxEntriesInMemTable * 2), reader, PTableVersions.IndexV2, - 5, Constants.PTableMaxReaderCountDefault, + 5, MaxEntriesInMemTable); var readIndex = new ReadIndex(new NoopPublisher(), reader, diff --git a/src/KurrentDB.Core.Tests/TransactionLog/Scavenging/Helpers/ScavengeTestScenario.cs b/src/KurrentDB.Core.Tests/TransactionLog/Scavenging/Helpers/ScavengeTestScenario.cs index adb26cbec1b..7ddb8a90685 100644 --- a/src/KurrentDB.Core.Tests/TransactionLog/Scavenging/Helpers/ScavengeTestScenario.cs +++ b/src/KurrentDB.Core.Tests/TransactionLog/Scavenging/Helpers/ScavengeTestScenario.cs @@ -67,7 +67,7 @@ public override async Task TestFixtureSetUp() { () => new HashListMemTable(PTableVersions.IndexV3, maxSize: 200), reader, PTableVersions.IndexV3, - 5, Constants.PTableMaxReaderCountDefault, + 5, maxSizeForMemory: 100, maxTablesPerLevel: 2); _logFormat.StreamNamesProvider.SetTableIndex(tableIndex); diff --git a/src/KurrentDB.Core.Tests/TransactionLog/Truncation/TruncateAndReOpenDbScenario.cs b/src/KurrentDB.Core.Tests/TransactionLog/Truncation/TruncateAndReOpenDbScenario.cs index 66b650f32b0..526fd11c376 100644 --- a/src/KurrentDB.Core.Tests/TransactionLog/Truncation/TruncateAndReOpenDbScenario.cs +++ b/src/KurrentDB.Core.Tests/TransactionLog/Truncation/TruncateAndReOpenDbScenario.cs @@ -42,7 +42,6 @@ private async ValueTask ReOpenDb(CancellationToken token) { reader, PTableVersions.IndexV3, int.MaxValue, - Constants.PTableMaxReaderCountDefault, MaxEntriesInMemTable); _logFormat.StreamNamesProvider.SetTableIndex(TableIndex); var readIndex = new ReadIndex(new NoopPublisher(), diff --git a/src/KurrentDB.Core.XUnit.Tests/LogV2/LogV2StreamExistenceFilterInitializerTests.cs b/src/KurrentDB.Core.XUnit.Tests/LogV2/LogV2StreamExistenceFilterInitializerTests.cs index 4ae6681e2ac..cf7611d87d7 100644 --- a/src/KurrentDB.Core.XUnit.Tests/LogV2/LogV2StreamExistenceFilterInitializerTests.cs +++ b/src/KurrentDB.Core.XUnit.Tests/LogV2/LogV2StreamExistenceFilterInitializerTests.cs @@ -35,8 +35,7 @@ public LogV2StreamExistenceFilterInitializerTests() { maxSizeForMemory: 100_000, tfReader: _log, ptableVersion: PTableVersions.IndexV4, - maxAutoMergeIndexLevel: int.MaxValue, - pTableMaxReaderCount: 5); + maxAutoMergeIndexLevel: int.MaxValue); _tableIndex.Initialize(0); _sut = new LogV2StreamExistenceFilterInitializer( @@ -188,8 +187,7 @@ public async Task cannot_initialize_with_v1_indexes() { maxSize: 1_000_000 * 2), new FakeTfReader(), ptableVersion: PTableVersions.IndexV1, - maxAutoMergeIndexLevel: int.MaxValue, - pTableMaxReaderCount: 5); + maxAutoMergeIndexLevel: int.MaxValue); tableIndex.Initialize(0); var sut = new LogV2StreamExistenceFilterInitializer( diff --git a/src/KurrentDB.Core.XUnit.Tests/Scavenge/Infrastructure/Scenario.cs b/src/KurrentDB.Core.XUnit.Tests/Scavenge/Infrastructure/Scenario.cs index 50f6fb3fd6d..164db3dfee1 100644 --- a/src/KurrentDB.Core.XUnit.Tests/Scavenge/Infrastructure/Scenario.cs +++ b/src/KurrentDB.Core.XUnit.Tests/Scavenge/Infrastructure/Scenario.cs @@ -317,7 +317,6 @@ private async Task RunInternalAsync( reader, ptableVersion: PTableVersions.IndexV4, maxAutoMergeIndexLevel: int.MaxValue, - pTableMaxReaderCount: ESConsts.PTableInitialReaderCount, maxSizeForMemory: _skipIndexCheck ? 1_000_000 // we aren't going to check the index so no need to convert to ptables : 1, // convert everything to ptables immediately so we can check the index diff --git a/src/KurrentDB.Core/ClusterVNode.cs b/src/KurrentDB.Core/ClusterVNode.cs index ccc291309fb..0557c2846fb 100644 --- a/src/KurrentDB.Core/ClusterVNode.cs +++ b/src/KurrentDB.Core/ClusterVNode.cs @@ -327,8 +327,7 @@ public ClusterVNode(ClusterVNodeOptions options, httpEndPoint, options.Cluster.ReadOnlyReplica); var dbConfig = CreateDbConfig( - out var statsHelper, - out var readerThreadsCount); + out var statsHelper); var trackers = new Trackers(); var metricsConfiguration = MetricsConfiguration.Get(configuration); @@ -364,8 +363,7 @@ public ClusterVNode(ClusterVNodeOptions options, }); TFChunkDbConfig CreateDbConfig( - out SystemStatsHelper statsHelper, - out int readerThreadsCount) { + out SystemStatsHelper statsHelper) { ICheckpoint writerChk; ICheckpoint chaserChk; @@ -455,12 +453,6 @@ TFChunkDbConfig CreateDbConfig( : Timeout.Infinite; statsHelper = new SystemStatsHelper(Log, writerChk.AsReadOnly(), dbPath, statsCollectionPeriod); - var processorCount = Environment.ProcessorCount; - - readerThreadsCount = - ThreadCountCalculator.CalculateReaderThreadCount(options.Database.ReaderThreadsCount, - processorCount, isRunningInContainer); - return new TFChunkDbConfig(dbPath, options.Database.ChunkSize, cache, @@ -608,15 +600,11 @@ void StartSubsystems() { monitoringInnerBus.Subscribe(monitoring); var indexPath = options.Database.Index ?? Path.Combine(Db.Config.Path, ESConsts.DefaultIndexDirectoryName); - - var pTableMaxReaderCount = GetPTableMaxReaderCount(readerThreadsCount); var tfReader = new TFChunkReader(Db, Db.Config.WriterCheckpoint.AsReadOnly()); var logFormat = logFormatAbstractorFactory.Create(new() { InMemory = options.Database.MemDb, IndexDirectory = indexPath, - InitialReaderCount = ESConsts.PTableInitialReaderCount, - MaxReaderCount = pTableMaxReaderCount, StreamExistenceFilterSize = options.Database.StreamExistenceFilterSize, StreamExistenceFilterCheckpoint = Db.Config.StreamExistenceFilterCheckpoint, TFReader = tfReader, @@ -685,7 +673,6 @@ void StartSubsystems() { initializationThreads: options.Database.InitializationThreads, additionalReclaim: false, maxAutoMergeIndexLevel: options.Database.MaxAutoMergeIndexLevel, - pTableMaxReaderCount: pTableMaxReaderCount, statusTracker: trackers.IndexStatusTracker); logFormat.StreamNamesProvider.SetTableIndex(tableIndex); @@ -1676,19 +1663,6 @@ await Db.Open(!options.Database.SkipDbVerify, threads: options.Database.Initiali _mainBus.Subscribe(periodicLogging); } - static int GetPTableMaxReaderCount(int readerThreadsCount) { - var ptableMaxReaderCount = - 1 /* StorageWriter */ - + 1 /* StorageChaser */ - + 1 /* Projections */ - + TFChunkScavenger.MaxThreadCount /* Scavenging (1 per thread) */ - + 1 /* Redaction */ - + 1 /* Subscription LinkTos resolving */ - + readerThreadsCount - + 5 /* just in case reserve :) */; - return Math.Max(ptableMaxReaderCount, ESConsts.PTableInitialReaderCount); - } - private static void CreateStaticStreamInfoCache( int streamInfoCacheCapacity, out ILRUCache.EventNumberCached> streamLastEventNumberCache, diff --git a/src/KurrentDB.Core/Configuration/ClusterVNodeOptions.cs b/src/KurrentDB.Core/Configuration/ClusterVNodeOptions.cs index 19dcc445950..eaafaba57ec 100644 --- a/src/KurrentDB.Core/Configuration/ClusterVNodeOptions.cs +++ b/src/KurrentDB.Core/Configuration/ClusterVNodeOptions.cs @@ -427,6 +427,7 @@ public record DatabaseOptions { public int InitializationThreads { get; init; } = 1; [Description("The number of reader threads to use for processing reads. Set to '0' to scale automatically (Default)")] + [Deprecated("The ReaderThreadsCount parameter has been deprecated as of version 26.0.0 and currently has no effect. The workers automatically scale as necessary.")] public int ReaderThreadsCount { get; init; } = 0; [Description("During large Index Merge operations, writes may be slowed down. Set this to the maximum " + diff --git a/src/KurrentDB.Core/Index/IndexMap.cs b/src/KurrentDB.Core/Index/IndexMap.cs index 65053373e6e..318d93aba3b 100644 --- a/src/KurrentDB.Core/Index/IndexMap.cs +++ b/src/KurrentDB.Core/Index/IndexMap.cs @@ -30,10 +30,9 @@ public class IndexMap { private readonly List> _map; private readonly int _maxTablesPerLevel; private readonly int _maxTableLevelsForAutomaticMerge; - private readonly int _pTableMaxReaderCount; private IndexMap(int version, List> tables, long prepareCheckpoint, long commitCheckpoint, - int maxTablesPerLevel, int maxTableLevelsForAutomaticMerge, int pTableMaxReaderCount) { + int maxTablesPerLevel, int maxTableLevelsForAutomaticMerge) { Ensure.Nonnegative(version, "version"); if (prepareCheckpoint < -1) throw new ArgumentOutOfRangeException("prepareCheckpoint"); @@ -50,7 +49,6 @@ private IndexMap(int version, List> tables, long prepareCheckpoint, _map = CopyFrom(tables); _maxTablesPerLevel = maxTablesPerLevel; _maxTableLevelsForAutomaticMerge = maxTableLevelsForAutomaticMerge; - _pTableMaxReaderCount = pTableMaxReaderCount; VerifyStructure(); } @@ -118,9 +116,9 @@ from table in level select table.Filename; } - public static IndexMap CreateEmpty(int maxTablesPerLevel, int maxTableLevelsForAutomaticMerge, int pTableMaxReaderCount) { + public static IndexMap CreateEmpty(int maxTablesPerLevel, int maxTableLevelsForAutomaticMerge) { return new IndexMap(IndexMapVersion, new List>(), -1, -1, maxTablesPerLevel, - maxTableLevelsForAutomaticMerge, pTableMaxReaderCount); + maxTableLevelsForAutomaticMerge); } public static IndexMap FromFile( @@ -132,10 +130,9 @@ public static IndexMap FromFile( bool useBloomFilter, int lruCacheSize, int threads, - int maxAutoMergeLevel, - int pTableMaxReaderCount) { + int maxAutoMergeLevel) { if (!File.Exists(filename)) - return CreateEmpty(maxTablesPerLevel, maxAutoMergeLevel, pTableMaxReaderCount); + return CreateEmpty(maxTablesPerLevel, maxAutoMergeLevel); using (var f = File.OpenRead(filename)) { // calculate real MD5 hash except first 32 bytes which are string representation of stored hash @@ -164,7 +161,7 @@ public static IndexMap FromFile( //we are doing a logical upgrade of the version to have the new data, so we will change the version to match so that new files are saved with the right version version = IndexMapVersion; var tables = loadPTables - ? LoadPTables(reader, filename, checkpoints, cacheDepth, skipIndexVerify, useBloomFilter, lruCacheSize, threads, pTableMaxReaderCount) + ? LoadPTables(reader, filename, checkpoints, cacheDepth, skipIndexVerify, useBloomFilter, lruCacheSize, threads) : new List>(); if (!loadPTables && reader.ReadLine() != null) @@ -173,7 +170,7 @@ public static IndexMap FromFile( checkpoints)); return new IndexMap(version, tables, prepareCheckpoint, commitCheckpoint, maxTablesPerLevel, - maxAutoMergeLevel, pTableMaxReaderCount); + maxAutoMergeLevel); } } } @@ -255,8 +252,7 @@ private static IEnumerable GetAllLines(StreamReader reader) { private static List> LoadPTables(StreamReader reader, string indexmapFilename, TFPos checkpoints, int cacheDepth, bool skipIndexVerify, bool useBloomFilter, int lruCacheSize, - int threads, - int pTableMaxReaderCount) { + int threads) { var tables = new List>(); try { try { @@ -279,7 +275,8 @@ private static List> LoadPTables(StreamReader reader, string indexm var path = Path.GetDirectoryName(indexmapFilename); var ptablePath = Path.Combine(path, file); - ptable = PTable.FromFile(ptablePath, ESConsts.PTableInitialReaderCount, pTableMaxReaderCount, cacheDepth, skipIndexVerify, useBloomFilter, lruCacheSize); + ptable = PTable.FromFile(ptablePath, cacheDepth, skipIndexVerify, useBloomFilter, + lruCacheSize); lock (tables) { InsertTableToTables(tables, level, position, ptable); @@ -394,7 +391,7 @@ public AddResult AddPTable(PTable tableToAdd, AddTableToTables(tables, 0, tableToAdd); var indexMap = new IndexMap(Version, tables, prepareCheckpoint, commitCheckpoint, _maxTablesPerLevel, - _maxTableLevelsForAutomaticMerge, _pTableMaxReaderCount); + _maxTableLevelsForAutomaticMerge); var canMergeAny = false; var maxTableLevelsToMerge = Math.Min(tables.Count, _maxTableLevelsForAutomaticMerge); @@ -433,8 +430,6 @@ public MergeResult TryMergeOneLevel( tables[level], filename, version, - ESConsts.PTableInitialReaderCount, - _pTableMaxReaderCount, indexCacheDepth, skipIndexVerify, useBloomFilter, @@ -448,7 +443,7 @@ public MergeResult TryMergeOneLevel( } var indexMap = new IndexMap(Version, tables, PrepareCheckpoint, CommitCheckpoint, _maxTablesPerLevel, - _maxTableLevelsForAutomaticMerge, _pTableMaxReaderCount); + _maxTableLevelsForAutomaticMerge); return new MergeResult(indexMap, toDelete, hasMergedAny, canMergeAny); } @@ -476,8 +471,6 @@ public MergeResult TryManualMerge( tablesToMerge, filename, version, - ESConsts.PTableInitialReaderCount, - _pTableMaxReaderCount, indexCacheDepth, skipIndexVerify, useBloomFilter, @@ -492,7 +485,7 @@ public MergeResult TryManualMerge( toDelete.AddRange(tablesToMerge); var indexMap = new IndexMap(Version, tables, PrepareCheckpoint, CommitCheckpoint, _maxTablesPerLevel, - _maxTableLevelsForAutomaticMerge, _pTableMaxReaderCount); + _maxTableLevelsForAutomaticMerge); return new MergeResult(indexMap, toDelete, true, false); } @@ -517,8 +510,6 @@ public async ValueTask Scavenge(Guid toScavenge, CancellationTok filename, version, shouldKeep, - ESConsts.PTableInitialReaderCount, - _pTableMaxReaderCount, indexCacheDepth, skipIndexVerify, useBloomFilter, @@ -532,7 +523,7 @@ public async ValueTask Scavenge(Guid toScavenge, CancellationTok scavengedMap[level][i] = scavenged; var indexMap = new IndexMap(Version, scavengedMap, PrepareCheckpoint, CommitCheckpoint, - _maxTablesPerLevel, _maxTableLevelsForAutomaticMerge, _pTableMaxReaderCount); + _maxTablesPerLevel, _maxTableLevelsForAutomaticMerge); return ScavengeResult.Success(indexMap, oldTable, scavenged, spaceSaved, level, i); } diff --git a/src/KurrentDB.Core/Index/PTable.cs b/src/KurrentDB.Core/Index/PTable.cs index 9ea690a351d..fcc93faf37e 100644 --- a/src/KurrentDB.Core/Index/PTable.cs +++ b/src/KurrentDB.Core/Index/PTable.cs @@ -2,22 +2,27 @@ // Kurrent, Inc licenses this file to you under the Kurrent License v1 (see LICENSE.md). using System; +using System.Buffers.Binary; using System.Collections.Generic; using System.Diagnostics; using System.IO; -using System.Runtime.InteropServices; using System.Security.Cryptography; using System.Threading; using System.Threading.Tasks; +using DotNext; +using DotNext.Buffers; +using DotNext.Buffers.Text; +using DotNext.Diagnostics; +using DotNext.IO; +using DotNext.Threading; using KurrentDB.Common.Utils; using KurrentDB.Core.DataStructures; using KurrentDB.Core.DataStructures.ProbabilisticFilter; using KurrentDB.Core.Exceptions; using KurrentDB.Core.TransactionLog.Unbuffered; +using Microsoft.Win32.SafeHandles; using ILogger = Serilog.ILogger; -using MD5 = KurrentDB.Core.Hashing.MD5; using Range = KurrentDB.Core.Data.Range; -using RuntimeInformation = System.Runtime.RuntimeInformation; namespace KurrentDB.Core.Index; @@ -46,8 +51,8 @@ public partial class PTable : ISearchTable, IDisposable { public const int IndexKeyV3Size = sizeof(long) + sizeof(long); public const int IndexKeyV4Size = IndexKeyV3Size; public const int MD5Size = 16; - public const int DefaultBufferSize = 8192; - public const int DefaultSequentialBufferSize = 65536; + private const int DefaultBufferSize = 8192; + private const int DefaultSequentialBufferSize = 65536; private static readonly ILogger Log = Serilog.Log.ForContext(); public Guid Id { @@ -97,35 +102,32 @@ private static long GenBloomFilterSizeBytes(long entryCount) { private readonly LRUCache _lruConfirmedNotPresent; private readonly IndexEntryKey _minEntry, _maxEntry; - private readonly ObjectPool _workItems; + + // Handle lifetime is managed by AcquireFileHandle() and ReleaseFileHandle() methods + // and built-in reference counting mechanism provided by SafeHandle class + private readonly SafeFileHandle _handle; private readonly byte _version; private readonly int _indexEntrySize; private readonly int _indexKeySize; - private readonly ManualResetEventSlim _destroyEvent = new ManualResetEventSlim(false); + private readonly ManualResetEventSlim _destroyEvent = new(initialState: false); private volatile bool _deleteFile; private bool _disposed; + private Atomic.Boolean _cleanupBarrier; - public ReadOnlySpan GetMidPoints() { - if (_midpoints == null) - return ReadOnlySpan.Empty; - - return _midpoints.AsSpan(); - } + public ReadOnlySpan GetMidPoints() + => _midpoints is null ? [] : _midpoints.AsSpan(); private PTable(string filename, Guid id, - int initialReaders, - int maxReaders, int depth = 16, bool skipIndexVerify = false, bool useBloomFilter = true, int lruCacheSize = 1_000_000) { - Ensure.NotNullOrEmpty(filename, "filename"); - Ensure.NotEmptyGuid(id, "id"); - Ensure.Positive(maxReaders, "maxReaders"); - Ensure.Nonnegative(depth, "depth"); + ArgumentException.ThrowIfNullOrWhiteSpace(filename); + ArgumentOutOfRangeException.ThrowIfEqual(id, Guid.Empty); + ArgumentOutOfRangeException.ThrowIfNegative(depth); if (!File.Exists(filename)) throw new CorruptIndexException(new PTableNotFoundException(filename)); @@ -135,121 +137,87 @@ private PTable(string filename, Log.Debug("Loading " + (skipIndexVerify ? "" : "and Verification ") + "of PTable '{pTable}' started...", Path.GetFileName(Filename)); - var sw = Stopwatch.StartNew(); - _size = new FileInfo(_filename).Length; + var sw = new Timestamp(); + _handle = File.OpenHandle(filename, FileMode.Open, FileAccess.Read, FileShare.Read, FileOptions.RandomAccess); + _size = RandomAccess.GetLength(_handle); - Helper.EatException(_filename, static filename => { + Helper.EatException(_handle, static handle => { // this action will fail if the file is created on a Unix system that does not have permissions to make files read-only - File.SetAttributes(filename, FileAttributes.ReadOnly | FileAttributes.NotContentIndexed); + File.SetAttributes(handle, FileAttributes.ReadOnly | FileAttributes.NotContentIndexed); }); - _workItems = new ObjectPool(string.Format("PTable {0} work items", _id), - initialReaders, - maxReaders, - () => new WorkItem(filename, DefaultBufferSize), - workItem => workItem.Dispose(), - pool => OnAllWorkItemsDisposed()); - - var readerWorkItem = GetWorkItem(); try { - readerWorkItem.Stream.Seek(0, SeekOrigin.Begin); - var header = PTableHeader.FromStream(readerWorkItem.Stream); - - if (header.Version == PTableVersions.IndexV1) { - throw new CorruptIndexException(new UnsupportedFileVersionException( - _filename, header.Version, Version, - "Detected a V1 index file, which is no longer supported. " + - "The index will be backed up and rebuilt in a supported format. " + - "This may take a long time for large databases. " + - "You can also use a version of ESDB (>= 3.9.0 and < 24.10.0) to upgrade the " + - "indexes to a supported format by performing an index merge.")); - } - - if ((header.Version != PTableVersions.IndexV2) && - (header.Version != PTableVersions.IndexV3) && - (header.Version != PTableVersions.IndexV4)) - throw new CorruptIndexException(new UnsupportedFileVersionException(_filename, header.Version, Version)); - _version = header.Version; + var header = PTableHeader.Parse(_handle, fileOffset: 0L); + long indexEntriesTotalSize = _size - PTableHeader.Size - MD5Size; + switch (_version = header.Version) { + case PTableVersions.IndexV1: + throw new CorruptIndexException(new UnsupportedFileVersionException( + _filename, header.Version, Version, + "Detected a V1 index file, which is no longer supported. " + + "The index will be backed up and rebuilt in a supported format. " + + "This may take a long time for large databases. " + + "You can also use a version of ESDB (>= 3.9.0 and < 24.10.0) to upgrade the " + + "indexes to a supported format by performing an index merge.")); + case PTableVersions.IndexV2: + _indexEntrySize = IndexEntryV2Size; + _indexKeySize = IndexKeyV2Size; + break; + case PTableVersions.IndexV3: + _indexEntrySize = IndexEntryV3Size; + _indexKeySize = IndexKeyV3Size; + break; + case >= PTableVersions.IndexV4: + //read the PTable footer + var footer = PTableFooter.Parse(_handle, _size - MD5Size - PTableFooter.Size); + if (footer.Version != header.Version) + throw new CorruptIndexException( + $"PTable header/footer version mismatch: {header.Version}/{footer.Version}", + new InvalidFileException("Invalid PTable file.")); - if (_version == PTableVersions.IndexV1) { - _indexEntrySize = IndexEntryV1Size; - _indexKeySize = IndexKeyV1Size; - } - - if (_version == PTableVersions.IndexV2) { - _indexEntrySize = IndexEntryV2Size; - _indexKeySize = IndexKeyV2Size; - } - - if (_version == PTableVersions.IndexV3) { - _indexEntrySize = IndexEntryV3Size; - _indexKeySize = IndexKeyV3Size; - } - - if (_version >= PTableVersions.IndexV4) { - //read the PTable footer - var previousPosition = readerWorkItem.Stream.Position; - readerWorkItem.Stream.Seek(readerWorkItem.Stream.Length - MD5Size - PTableFooter.GetSize(_version), - SeekOrigin.Begin); - var footer = PTableFooter.FromStream(readerWorkItem.Stream); - if (footer.Version != header.Version) - throw new CorruptIndexException( - String.Format("PTable header/footer version mismatch: {0}/{1}", header.Version, - footer.Version), new InvalidFileException("Invalid PTable file.")); - - if (_version == PTableVersions.IndexV4) { _indexEntrySize = IndexEntryV4Size; _indexKeySize = IndexKeyV4Size; - } else - throw new InvalidOperationException("Unknown PTable version: " + _version); - _midpointsCached = footer.NumMidpointsCached; - _midpointsCacheSize = _midpointsCached * _indexEntrySize; - readerWorkItem.Stream.Seek(previousPosition, SeekOrigin.Begin); + _midpointsCached = footer.NumMidpointsCached; + _midpointsCacheSize = _midpointsCached * _indexEntrySize; + indexEntriesTotalSize = indexEntriesTotalSize - PTableFooter.Size - _midpointsCacheSize; + break; + default: + throw new CorruptIndexException( + new UnsupportedFileVersionException(_filename, header.Version, Version)); } - long indexEntriesTotalSize = (_size - PTableHeader.Size - _midpointsCacheSize - - PTableFooter.GetSize(_version) - MD5Size); - if (indexEntriesTotalSize < 0) { - throw new CorruptIndexException(String.Format( - "Total size of index entries < 0: {0}. _size: {1}, header size: {2}, _midpointsCacheSize: {3}, footer size: {4}, md5 size: {5}", - indexEntriesTotalSize, _size, PTableHeader.Size, _midpointsCacheSize, - PTableFooter.GetSize(_version), MD5Size)); - } else if (indexEntriesTotalSize % _indexEntrySize != 0) { - throw new CorruptIndexException(String.Format( - "Total size of index entries: {0} is not divisible by index entry size: {1}", - indexEntriesTotalSize, _indexEntrySize)); + throw new CorruptIndexException( + $"Total size of index entries < 0: {indexEntriesTotalSize}. _size: {_size}, header size: {PTableHeader.Size}, _midpointsCacheSize: {_midpointsCacheSize}, version: {_version}, md5 size: {MD5Size}"); + } else if (indexEntriesTotalSize % _indexEntrySize is not 0) { + throw new CorruptIndexException( + $"Total size of index entries: {indexEntriesTotalSize} is not divisible by index entry size: {_indexEntrySize}"); } _count = indexEntriesTotalSize / _indexEntrySize; - if (_version >= PTableVersions.IndexV4 && _count > 0 && _midpointsCached > 0 && _midpointsCached < 2) { + if (_version >= PTableVersions.IndexV4 && _count > 0 && _midpointsCached is > 0 and < 2) { //if there is at least 1 index entry with version>=4 and there are cached midpoints, there should always be at least 2 midpoints cached - throw new CorruptIndexException(String.Format( - "Less than 2 midpoints cached in PTable. Index entries: {0}, Midpoints cached: {1}", _count, - _midpointsCached)); + throw new CorruptIndexException( + $"Less than 2 midpoints cached in PTable. Index entries: {_count}, Midpoints cached: {_midpointsCached}"); } else if (_count >= 2 && _midpointsCached > _count) { //if there are at least 2 index entries, midpoints count should be at most the number of index entries - throw new CorruptIndexException(String.Format( - "More midpoints cached in PTable than index entries. Midpoints: {0} , Index entries: {1}", - _midpointsCached, _count)); + throw new CorruptIndexException( + $"More midpoints cached in PTable than index entries. Midpoints: {_midpointsCached} , Index entries: {_count}"); } - if (Count == 0) { + if (Count is 0) { _minEntry = new IndexEntryKey(ulong.MaxValue, long.MaxValue); _maxEntry = new IndexEntryKey(ulong.MinValue, long.MinValue); } else { - var minEntry = ReadEntry(_indexEntrySize, Count - 1, readerWorkItem, _version); + var minEntry = ReadEntry(_handle, _indexEntrySize, Count - 1, _version); _minEntry = new IndexEntryKey(minEntry.Stream, minEntry.Version); - var maxEntry = ReadEntry(_indexEntrySize, 0, readerWorkItem, _version); + var maxEntry = ReadEntry(_handle, _indexEntrySize, 0, _version); _maxEntry = new IndexEntryKey(maxEntry.Stream, maxEntry.Version); } } catch (Exception) { Dispose(); throw; - } finally { - ReturnWorkItem(readerWorkItem); } int calcdepth = 0; @@ -288,147 +256,135 @@ private PTable(string filename, ~PTable() => Dispose(false); - internal UnmanagedMemoryAppendOnlyList CacheMidpointsAndVerifyHash(int depth, bool skipIndexVerify) { - var buffer = new byte[4096]; - if (depth < 0 || depth > 30) - throw new ArgumentOutOfRangeException("depth"); + private UnmanagedMemoryAppendOnlyList CacheMidpointsAndVerifyHash(int depth, bool skipIndexVerify) { + if (depth is < 0 or > 30) + throw new ArgumentOutOfRangeException(nameof(depth)); + var count = Count; - if (count == 0 || depth == 0) + if (count is 0 || depth is 0) return null; if (skipIndexVerify) { Log.Debug("Disabling Verification of PTable"); } - Stream stream = null; - WorkItem workItem = null; - if (RuntimeInformation.IsUnix) { - workItem = GetWorkItem(); - stream = workItem.Stream; - } else { - stream = UnbufferedFileStream.Create(_filename, FileMode.Open, FileAccess.Read, FileShare.Read, - 4096, 4096, false, 4096); - } + var stream = OperatingSystem.IsWindows() + ? UnbufferedFileStream.Create(_filename, FileMode.Open, FileAccess.Read, FileShare.Read, + 4096, 4096, false, 4096) + : _handle.AsUnbufferedStream(FileAccess.Read); UnmanagedMemoryAppendOnlyList midpoints = null; + var md5 = IncrementalHash.CreateHash(HashAlgorithmName.MD5); + Span buffer = stackalloc byte[PTableHeader.Size]; + Debug.Assert(buffer.Length >= _indexEntrySize); + Debug.Assert(buffer.Length >= _indexKeySize); + + var tmpBuffer = new SpanOwner(DefaultBufferSize, exactSize: false); try { - using (var md5 = MD5.Create()) { - int midpointsCount; - try { - midpointsCount = (int)Math.Max(2L, Math.Min((long)1 << depth, count)); - midpoints = new UnmanagedMemoryAppendOnlyList(midpointsCount); - } catch (OutOfMemoryException exc) { - throw new PossibleToHandleOutOfMemoryException("Failed to allocate memory for Midpoint cache.", - exc); - } + int midpointsCount; + try { + midpointsCount = (int)Math.Max(2L, Math.Min((long)1 << depth, count)); + midpoints = new UnmanagedMemoryAppendOnlyList(midpointsCount); + } catch (OutOfMemoryException exc) { + throw new PossibleToHandleOutOfMemoryException("Failed to allocate memory for Midpoint cache.", + exc); + } - if (skipIndexVerify && (_version >= PTableVersions.IndexV4)) { + switch (skipIndexVerify) { + case true when _version >= PTableVersions.IndexV4: if (_midpointsCached == midpointsCount) { //index verification is disabled and cached midpoints with the same depth requested are available //so, we can load them directly from the PTable file Log.Debug("Loading {midpointsCached} cached midpoints from PTable", _midpointsCached); - long startOffset = stream.Length - MD5Size - PTableFooter.GetSize(_version) - - _midpointsCacheSize; + long startOffset = stream.Length - MD5Size - PTableFooter.Size - + _midpointsCacheSize; stream.Seek(startOffset, SeekOrigin.Begin); for (int k = 0; k < (int)_midpointsCached; k++) { - stream.Read(buffer, 0, _indexEntrySize); - IndexEntryKey key; - long index; - if (_version == PTableVersions.IndexV4) { - key = new IndexEntryKey(BitConverter.ToUInt64(buffer, 8), - BitConverter.ToInt64(buffer, 0)); - index = BitConverter.ToInt64(buffer, 8 + 8); - } else - throw new InvalidOperationException("Unknown PTable version: " + _version); + stream.ReadExactly(buffer.Slice(0, _indexEntrySize)); + var key = new IndexEntryKey(BinaryPrimitives.ReadUInt64LittleEndian(buffer.Slice(sizeof(long))), + BinaryPrimitives.ReadInt64LittleEndian(buffer)); + var index = BinaryPrimitives.ReadInt64LittleEndian( + buffer.Slice(sizeof(long) + sizeof(long))); midpoints.Add(new Midpoint(key, index)); if (k > 0) { if (midpoints[k].Key.GreaterThan(midpoints[k - 1].Key)) { - throw new CorruptIndexException(String.Format( - "Index entry key for midpoint {0} (stream: {1}, version: {2}) < index entry key for midpoint {3} (stream: {4}, version: {5})", - k - 1, midpoints[k - 1].Key.Stream, midpoints[k - 1].Key.Version, k, - midpoints[k].Key.Stream, midpoints[k].Key.Version)); + throw new CorruptIndexException( + $"Index entry key for midpoint {k - 1} (stream: {midpoints[k - 1].Key.Stream}, version: {midpoints[k - 1].Key.Version}) < index entry key for midpoint {k} (stream: {midpoints[k].Key.Stream}, version: {midpoints[k].Key.Version})"); } else if (midpoints[k - 1].ItemIndex > midpoints[k].ItemIndex) { - throw new CorruptIndexException(String.Format( - "Item index for midpoint {0} ({1}) > Item index for midpoint {2} ({3})", - k - 1, midpoints[k - 1].ItemIndex, k, midpoints[k].ItemIndex)); + throw new CorruptIndexException( + $"Item index for midpoint {k - 1} ({midpoints[k - 1].ItemIndex}) > Item index for midpoint {k} ({midpoints[k].ItemIndex})"); } } } return midpoints; - } else - Log.Debug( - "Skipping loading of cached midpoints from PTable due to count mismatch, cached midpoints: {midpointsCached} / required midpoints: {midpointsCount}", - _midpointsCached, midpointsCount); - } + } - if (!skipIndexVerify) { + Log.Debug( + "Skipping loading of cached midpoints from PTable due to count mismatch, cached midpoints: {midpointsCached} / required midpoints: {midpointsCount}", + _midpointsCached, midpointsCount); + break; + case false: stream.Seek(0, SeekOrigin.Begin); - stream.Read(buffer, 0, PTableHeader.Size); - md5.TransformBlock(buffer, 0, PTableHeader.Size, null, 0); - } - - long previousNextIndex = long.MinValue; - var previousKey = new IndexEntryKey(long.MaxValue, long.MaxValue); - for (int k = 0; k < midpointsCount; ++k) { - long nextIndex = GetMidpointIndex(k, count, midpointsCount); - if (previousNextIndex != nextIndex) { - if (!skipIndexVerify) { - ReadUntilWithMd5(PTableHeader.Size + _indexEntrySize * nextIndex, stream, md5); - stream.Read(buffer, 0, _indexKeySize); - md5.TransformBlock(buffer, 0, _indexKeySize, null, 0); - } else { - stream.Seek(PTableHeader.Size + _indexEntrySize * nextIndex, SeekOrigin.Begin); - stream.Read(buffer, 0, _indexKeySize); - } - - IndexEntryKey key; - if (_version == PTableVersions.IndexV1) { - key = new IndexEntryKey(BitConverter.ToUInt32(buffer, 4), - BitConverter.ToInt32(buffer, 0)); - } else if (_version == PTableVersions.IndexV2) { - key = new IndexEntryKey(BitConverter.ToUInt64(buffer, 4), - BitConverter.ToInt32(buffer, 0)); - } else { - key = new IndexEntryKey(BitConverter.ToUInt64(buffer, 8), - BitConverter.ToInt64(buffer, 0)); - } + stream.ReadExactly(buffer.Slice(0, PTableHeader.Size)); + md5.AppendData(buffer.Slice(0, PTableHeader.Size)); + break; + } - midpoints.Add(new Midpoint(key, nextIndex)); - previousNextIndex = nextIndex; - previousKey = key; + long previousNextIndex = long.MinValue; + var previousKey = new IndexEntryKey(long.MaxValue, long.MaxValue); + for (int k = 0; k < midpointsCount; ++k) { + long nextIndex = GetMidpointIndex(k, count, midpointsCount); + if (previousNextIndex != nextIndex) { + if (!skipIndexVerify) { + ReadUntilWithMd5(PTableHeader.Size + _indexEntrySize * nextIndex, stream, md5, tmpBuffer.Span); + stream.ReadExactly(buffer.Slice(0, _indexKeySize)); + md5.AppendData(buffer.Slice(0, _indexKeySize)); } else { - midpoints.Add(new Midpoint(previousKey, previousNextIndex)); + stream.Seek(PTableHeader.Size + _indexEntrySize * nextIndex, SeekOrigin.Begin); + stream.ReadExactly(buffer.Slice(0, _indexKeySize)); } - if (k > 0) { - if (midpoints[k].Key.GreaterThan(midpoints[k - 1].Key)) { - throw new CorruptIndexException(String.Format( - "Index entry key for midpoint {0} (stream: {1}, version: {2}) < index entry key for midpoint {3} (stream: {4}, version: {5})", - k - 1, midpoints[k - 1].Key.Stream, midpoints[k - 1].Key.Version, k, - midpoints[k].Key.Stream, midpoints[k].Key.Version)); - } else if (midpoints[k - 1].ItemIndex > midpoints[k].ItemIndex) { - throw new CorruptIndexException(String.Format( - "Item index for midpoint {0} ({1}) > Item index for midpoint {2} ({3})", k - 1, - midpoints[k - 1].ItemIndex, k, midpoints[k].ItemIndex)); - } - } + IndexEntryKey key = _version switch { + PTableVersions.IndexV1 => new IndexEntryKey( + BinaryPrimitives.ReadUInt32LittleEndian(buffer.Slice(sizeof(int))), + BinaryPrimitives.ReadInt32LittleEndian(buffer)), + PTableVersions.IndexV2 => new IndexEntryKey( + BinaryPrimitives.ReadUInt64LittleEndian(buffer.Slice(sizeof(int))), + BinaryPrimitives.ReadInt32LittleEndian(buffer)), + _ => new IndexEntryKey(BinaryPrimitives.ReadUInt64LittleEndian(buffer.Slice(sizeof(long))), + BinaryPrimitives.ReadInt64LittleEndian(buffer)) + }; + + midpoints.Add(new Midpoint(key, nextIndex)); + previousNextIndex = nextIndex; + previousKey = key; + } else { + midpoints.Add(new Midpoint(previousKey, previousNextIndex)); } - if (!skipIndexVerify) { - ReadUntilWithMd5(stream.Length - MD5Size, stream, md5); - //verify hash (should be at stream.length - MD5Size) - md5.TransformFinalBlock(Empty.ByteArray, 0, 0); - var fileHash = new byte[MD5Size]; - stream.Read(fileHash, 0, MD5Size); - ValidateHash(md5.Hash, fileHash); + if (k > 0) { + if (midpoints[k].Key.GreaterThan(midpoints[k - 1].Key)) { + throw new CorruptIndexException(String.Format( + "Index entry key for midpoint {0} (stream: {1}, version: {2}) < index entry key for midpoint {3} (stream: {4}, version: {5})", + k - 1, midpoints[k - 1].Key.Stream, midpoints[k - 1].Key.Version, k, + midpoints[k].Key.Stream, midpoints[k].Key.Version)); + } else if (midpoints[k - 1].ItemIndex > midpoints[k].ItemIndex) { + throw new CorruptIndexException(String.Format( + "Item index for midpoint {0} ({1}) > Item index for midpoint {2} ({3})", k - 1, + midpoints[k - 1].ItemIndex, k, midpoints[k].ItemIndex)); + } } + } - return midpoints; + if (!skipIndexVerify) { + ValidateHash(stream, md5, tmpBuffer.Span); } + + return midpoints; } catch (PossibleToHandleOutOfMemoryException) { midpoints?.Dispose(); throw; @@ -437,12 +393,30 @@ internal UnmanagedMemoryAppendOnlyList CacheMidpointsAndVerifyHash(int Dispose(); throw; } finally { - if (RuntimeInformation.IsUnix) { - if (workItem != null) - ReturnWorkItem(workItem); - } else { - stream?.Dispose(); + md5.Dispose(); + stream.Dispose(); + tmpBuffer.Dispose(); + } + } + + private void AcquireFileHandle() { + var acquired = false; + try { + _handle.DangerousAddRef(ref acquired); + } catch (ObjectDisposedException) { + if (_cleanupBarrier.FalseToTrue()) { + DeleteFileIfNeeded(); } + + throw new FileBeingDeletedException(); + } + } + + private void ReleaseFileHandle() { + _handle.DangerousRelease(); + + if (_handle.IsClosed && _cleanupBarrier.FalseToTrue()) { + DeleteFileIfNeeded(); } } @@ -473,56 +447,50 @@ private PersistentBloomFilter TryOpenBloomFilter() { } } - private readonly byte[] TmpReadBuf = new byte[DefaultBufferSize]; - - private void ReadUntilWithMd5(long nextPos, Stream fileStream, HashAlgorithm md5) { + private static void ReadUntilWithMd5(long nextPos, Stream fileStream, IncrementalHash md5, Span tmpBuffer) { long toRead = nextPos - fileStream.Position; if (toRead < 0) throw new Exception("should not do negative reads."); while (toRead > 0) { - var localReadCount = Math.Min(toRead, TmpReadBuf.Length); - int read = fileStream.Read(TmpReadBuf, 0, (int)localReadCount); - md5.TransformBlock(TmpReadBuf, 0, read, null, 0); + int read = fileStream.Read(tmpBuffer.TrimLength(int.CreateSaturating(toRead))); + md5.AppendData(tmpBuffer.Slice(0, read)); toRead -= read; } } - void ValidateHash(byte[] fromFile, byte[] computed) { - if (computed == null) - throw new CorruptIndexException(new HashValidationException("Calculated MD5 hash is null!")); - if (fromFile == null) - throw new CorruptIndexException(new HashValidationException("Read from file MD5 hash is null!")); + private static void ValidateHash(Stream stream, IncrementalHash actual, Span buffer) { + ReadUntilWithMd5(stream.Length - MD5Size, stream, actual, buffer); + //verify hash (should be at stream.length - MD5Size) + Span fileHash = stackalloc byte[MD5Size]; + stream.ReadExactly(fileHash); + ValidateHash(fileHash, actual); + } + + private static void ValidateHash(Span expected, IncrementalHash actual) { + Debug.Assert(actual is not null); - if (computed.Length != fromFile.Length) + Span actualHash = stackalloc byte[MD5Size]; + var bytesWritten = actual.GetCurrentHash(actualHash); + Debug.Assert(bytesWritten is MD5Size); + + // Perf: use hardware accelerated byte array comparison + if (!expected.SequenceEqual(actualHash)) { throw new CorruptIndexException( new HashValidationException( - string.Format( - "Hash sizes differ! FileHash({0}): {1}, hash({2}): {3}.", - computed.Length, - BitConverter.ToString(computed), - fromFile.Length, - BitConverter.ToString(fromFile)))); - - for (int i = 0; i < fromFile.Length; i++) { - if (fromFile[i] != computed[i]) - throw new CorruptIndexException( - new HashValidationException( - string.Format( - "Hashes are different! computed: {0}, hash: {1}.", - BitConverter.ToString(computed), - BitConverter.ToString(fromFile)))); + $"Hashes are different! computed: {Hex.EncodeToUtf16(actualHash)}, hash: {Hex.EncodeToUtf16(expected)}.")); } } public IEnumerable IterateAllInOrder() { - var workItem = GetWorkItem(); + AcquireFileHandle(); try { - workItem.Stream.Position = PTableHeader.Size; + long fileOffset = PTableHeader.Size; for (long i = 0, n = Count; i < n; i++) { - yield return ReadNextNoSeek(workItem, _version); + yield return ReadEntry(_handle, fileOffset, _version, out var bytesRead); + fileOffset += bytesRead; } } finally { - ReturnWorkItem(workItem); + ReleaseFileHandle(); } } @@ -548,7 +516,7 @@ private bool TryGetLatestEntryWithCache(StreamHash stream, out IndexEntry entry) return false; } - entry = ReadEntry(_indexEntrySize, value.LatestOffset, _version); + entry = ReadEntry(value.LatestOffset); return true; } @@ -570,7 +538,7 @@ private bool TryGetLatestEntryWithCache(StreamHash stream, out IndexEntry entry) if (!MightContainStream(streamHash)) return null; - var workItem = GetWorkItem(); + AcquireFileHandle(); try { var recordRange = LocateRecordRange(endKey, startKey, out var lowBoundsCheck, out var highBoundsCheck); @@ -582,7 +550,6 @@ private bool TryGetLatestEntryWithCache(StreamHash stream, out IndexEntry entry) recordRange, lowBoundsCheck, highBoundsCheck, - workItem, token); } catch (HashCollisionException) { // fall back to linear search if there's a hash collision @@ -593,11 +560,10 @@ private bool TryGetLatestEntryWithCache(StreamHash stream, out IndexEntry entry) recordRange, lowBoundsCheck, highBoundsCheck, - workItem, token); } } finally { - ReturnWorkItem(workItem); + ReleaseFileHandle(); } } @@ -610,14 +576,13 @@ private bool TryGetLatestEntryWithCache(StreamHash stream, out IndexEntry entry) Range recordRange, IndexEntryKey lowBoundsCheck, IndexEntryKey highBoundsCheck, - WorkItem workItem, CancellationToken token) { long maxBeforePosition = long.MinValue; IndexEntry maxEntry = default; for (var idx = recordRange.Lower; idx <= recordRange.Upper; idx++) { - var candidateEntry = ReadEntry(_indexEntrySize, idx, workItem, _version); + var candidateEntry = ReadEntry(idx); var candidateEntryKey = new IndexEntryKey(candidateEntry.Stream, candidateEntry.Version); if (candidateEntryKey.GreaterThan(lowBoundsCheck)) { @@ -652,7 +617,6 @@ await isForThisStream(candidateEntry, token)) { Range recordRange, IndexEntryKey lowBoundsCheck, IndexEntryKey highBoundsCheck, - WorkItem workItem, CancellationToken token) { var startKey = BuildKey(stream, 0); @@ -663,7 +627,7 @@ await isForThisStream(candidateEntry, token)) { while (low < high) { var mid = low + (high - low) / 2; - IndexEntry midpoint = ReadEntry(_indexEntrySize, mid, workItem, _version); + IndexEntry midpoint = ReadEntry(mid); var midpointKey = new IndexEntryKey(midpoint.Stream, midpoint.Version); if (midpointKey.GreaterThan(lowBoundsCheck)) { @@ -706,7 +670,7 @@ await isForThisStream(candidateEntry, token)) { } } - var candidateEntry = ReadEntry(_indexEntrySize, high, workItem, _version); + var candidateEntry = ReadEntry(high); // index entry is for a different hash if (candidateEntry.Stream != stream.Hash) @@ -750,16 +714,15 @@ private bool TrySearchForLatestEntry(StreamHash stream, long startNumber, long e return false; } - var workItem = GetWorkItem(); + AcquireFileHandle(); try { - var high = ChopForLatest(workItem, endKey); - var candEntry = ReadEntry(_indexEntrySize, high, workItem, _version); + var high = ChopForLatest(endKey); + var candEntry = ReadEntry(high); var candKey = new IndexEntryKey(candEntry.Stream, candEntry.Version); if (candKey.GreaterThan(endKey)) - throw new MaybeCorruptIndexException(string.Format( - "candEntry ({0}@{1}) > startKey {2}, stream {3}, startNum {4}, endNum {5}, PTable: {6}.", - candEntry.Stream, candEntry.Version, startKey, stream, startNumber, endNumber, Filename)); + throw new MaybeCorruptIndexException( + $"candEntry ({candEntry.Stream}@{candEntry.Version}) > startKey {startKey}, stream {stream}, startNum {startNumber}, endNum {endNumber}, PTable: {Filename}."); if (candKey.SmallerThan(startKey)) { offset = default; return false; @@ -769,7 +732,7 @@ private bool TrySearchForLatestEntry(StreamHash stream, long startNumber, long e offset = high; return true; } finally { - ReturnWorkItem(workItem); + ReleaseFileHandle(); } } @@ -785,7 +748,7 @@ private bool TryGetOldestEntryWithCache(StreamHash stream, out IndexEntry entry) return false; } - entry = ReadEntry(_indexEntrySize, value.OldestOffset, _version); + entry = ReadEntry(value.OldestOffset); return true; } @@ -831,10 +794,10 @@ private bool TrySearchForOldestEntry(StreamHash stream, long startNumber, long e return false; } - var workItem = GetWorkItem(); + AcquireFileHandle(); try { - var high = ChopForOldest(workItem, startKey); - var candEntry = ReadEntry(_indexEntrySize, high, workItem, _version); + var high = ChopForOldest(startKey); + var candEntry = ReadEntry(high); var candidateKey = new IndexEntryKey(candEntry.Stream, candEntry.Version); if (candidateKey.SmallerThan(startKey)) throw new MaybeCorruptIndexException(string.Format( @@ -849,13 +812,13 @@ private bool TrySearchForOldestEntry(StreamHash stream, long startNumber, long e offset = high; return true; } finally { - ReturnWorkItem(workItem); + ReleaseFileHandle(); } } public IReadOnlyList GetRange(ulong stream, long startNumber, long endNumber, int? limit = null) { - Ensure.Nonnegative(startNumber, "startNumber"); - Ensure.Nonnegative(endNumber, "endNumber"); + ArgumentOutOfRangeException.ThrowIfNegative(startNumber); + ArgumentOutOfRangeException.ThrowIfNegative(endNumber); return _lruCache is null ? GetRangeNoCache(GetHash(stream), startNumber, endNumber, limit) @@ -924,58 +887,71 @@ private long UpperMidpointBound(ReadOnlySpan midpoints, IndexEntryKey return r; } - private static void PositionAtEntry(int indexEntrySize, long indexNum, WorkItem workItem) { - workItem.Stream.Seek(indexEntrySize * indexNum + PTableHeader.Size, SeekOrigin.Begin); - } + private static long PositionAtEntry(int indexEntrySize, long indexNum) + => indexEntrySize * indexNum + PTableHeader.Size; - private static IndexEntry ReadEntry(int indexEntrySize, long indexNum, WorkItem workItem, int ptableVersion) { - long seekTo = indexEntrySize * indexNum + PTableHeader.Size; - workItem.Stream.Seek(seekTo, SeekOrigin.Begin); - return ReadNextNoSeek(workItem, ptableVersion); - } + private static IndexEntry ReadEntry(SafeFileHandle handle, int indexEntrySize, long indexNum, byte ptableVersion) + => ReadEntry(handle, PositionAtEntry(indexEntrySize, indexNum), ptableVersion, out _); - private IndexEntry ReadEntry(int indexEntrySize, long indexNum, int ptableVersion) { - var workItem = GetWorkItem(); + private IndexEntry ReadEntry(long indexNum) { + AcquireFileHandle(); try { - return ReadEntry(_indexEntrySize, indexNum, workItem, ptableVersion); + return ReadEntry(_handle, _indexEntrySize, indexNum, _version); } finally { - ReturnWorkItem(workItem); + ReleaseFileHandle(); } } - private static IndexEntry ReadNextNoSeek(WorkItem workItem, int ptableVersion) { - long version = (ptableVersion >= PTableVersions.IndexV3) - ? workItem.Reader.ReadInt64() - : workItem.Reader.ReadInt32(); - ulong stream = ptableVersion == PTableVersions.IndexV1 - ? workItem.Reader.ReadUInt32() - : workItem.Reader.ReadUInt64(); - long position = workItem.Reader.ReadInt64(); - return new IndexEntry(stream, version, position); - } - - private WorkItem GetWorkItem() { - try { - return _workItems.Get(); - } catch (ObjectPoolDisposingException) { - throw new FileBeingDeletedException(); - } catch (ObjectPoolMaxLimitReachedException) { - throw new Exception("Unable to acquire work item."); + private static IndexEntry ReadEntry(SafeFileHandle handle, long fileOffset, byte ptableVersion, out int bytesRead) { + // allocate buffer for the worst case + Span buffer = stackalloc byte[IndexEntryV4Size]; + var reader = new SpanReader(buffer); + long version; + ulong stream; + switch (ptableVersion) { + case PTableVersions.IndexV1: + buffer = buffer.Slice(0, IndexEntryV1Size); + RandomAccess.Read(handle, buffer, fileOffset); + version = reader.ReadLittleEndian(); + stream = reader.ReadLittleEndian(); + break; + case PTableVersions.IndexV2: + buffer = buffer.Slice(0, IndexEntryV2Size); + RandomAccess.Read(handle, buffer, fileOffset); + version = reader.ReadLittleEndian(); + stream = reader.ReadLittleEndian(); + break; + default: + // V3 or higher + RandomAccess.Read(handle, buffer, fileOffset); + version = reader.ReadLittleEndian(); + stream = reader.ReadLittleEndian(); + break; } + + var position = reader.ReadLittleEndian(); + bytesRead = reader.ConsumedCount; + + Debug.Assert(bytesRead == GetIndexEntrySize(ptableVersion)); + return new(stream, version, position); } - private void ReturnWorkItem(WorkItem workItem) { - _workItems.Return(workItem); + private void DisposeFileHandle() { + _handle.Dispose(); // it decrements the counter + + if (_handle.IsClosed && _cleanupBarrier.FalseToTrue()) { + DeleteFileIfNeeded(); + } } public void MarkForDestruction() { _deleteFile = true; - _workItems.MarkForDisposal(); + DisposeFileHandle(); } public void Dispose() { _deleteFile = false; - _workItems.MarkForDisposal(); + DisposeFileHandle(); } protected virtual void Dispose(bool disposing) { @@ -992,7 +968,7 @@ protected virtual void Dispose(bool disposing) { _disposed = true; } - private void OnAllWorkItemsDisposed() { + private void DeleteFileIfNeeded() { File.SetAttributes(_filename, FileAttributes.Normal); if (_deleteFile) { _bloomFilter?.Dispose(); @@ -1052,27 +1028,27 @@ private List PositionAndReadForward(StreamHash stream, long startNum if (startKey.GreaterThan(_maxEntry) || endKey.SmallerThan(_minEntry)) return result; - var workItem = GetWorkItem(); + AcquireFileHandle(); try { - var high = tableLatestOffset ?? ChopForLatest(workItem, endKey); - PositionAtEntry(_indexEntrySize, high, workItem); - result = ReadForward(workItem, high, startKey, endKey, limit); + var high = tableLatestOffset ?? ChopForLatest(endKey); + result = ReadForward(PositionAtEntry(_indexEntrySize, high), high, startKey, endKey, limit); return result; } catch (MaybeCorruptIndexException ex) { throw new MaybeCorruptIndexException( $"{ex.Message}. stream {stream}, startNum {startNumber}, endNum {endNumber}, PTable: {Filename}."); } finally { - ReturnWorkItem(workItem); + ReleaseFileHandle(); } } // forward here meaning forward in the file. towards the older records. - private List ReadForward(WorkItem workItem, long high, IndexEntryKey startKey, IndexEntryKey endKey, int? limit) { + private List ReadForward(long position, long high, IndexEntryKey startKey, IndexEntryKey endKey, int? limit) { var result = new List(); for (long i = high, n = Count; i < n; ++i) { - var entry = ReadNextNoSeek(workItem, _version); + var entry = ReadEntry(_handle, position, _version, out var bytesRead); + position += bytesRead; var candidateKey = new IndexEntryKey(entry.Stream, entry.Version); @@ -1091,13 +1067,13 @@ private List ReadForward(WorkItem workItem, long high, IndexEntryKey return result; } - private long ChopForLatest(WorkItem workItem, IndexEntryKey endKey) { + private long ChopForLatest(IndexEntryKey endKey) { var recordRange = LocateRecordRange(endKey, out var lowBoundsCheck, out var highBoundsCheck); long low = recordRange.Lower; long high = recordRange.Upper; while (low < high) { var mid = low + (high - low) / 2; - var midpoint = ReadEntry(_indexEntrySize, mid, workItem, _version); + var midpoint = ReadEntry(mid); var midpointKey = new IndexEntryKey(midpoint.Stream, midpoint.Version); if (midpointKey.GreaterThan(lowBoundsCheck)) { @@ -1122,23 +1098,21 @@ private long ChopForLatest(WorkItem workItem, IndexEntryKey endKey) { return high; } - private long ChopForOldest(WorkItem workItem, IndexEntryKey startKey) { + private long ChopForOldest(IndexEntryKey startKey) { var recordRange = LocateRecordRange(startKey, out var lowBoundsCheck, out var highBoundsCheck); long low = recordRange.Lower; long high = recordRange.Upper; while (low < high) { var mid = low + (high - low + 1) / 2; - var midpoint = ReadEntry(_indexEntrySize, mid, workItem, _version); + var midpoint = ReadEntry(mid); var midpointKey = new IndexEntryKey(midpoint.Stream, midpoint.Version); if (midpointKey.GreaterThan(lowBoundsCheck)) { - throw new MaybeCorruptIndexException(String.Format( - "Midpoint key (stream: {0}, version: {1}) > low bounds check key (stream: {2}, version: {3})", - midpointKey.Stream, midpointKey.Version, lowBoundsCheck.Stream, lowBoundsCheck.Version)); + throw new MaybeCorruptIndexException( + $"Midpoint key (stream: {midpointKey.Stream}, version: {midpointKey.Version}) > low bounds check key (stream: {lowBoundsCheck.Stream}, version: {lowBoundsCheck.Version})"); } else if (!midpointKey.GreaterEqualsThan(highBoundsCheck)) { - throw new MaybeCorruptIndexException(String.Format( - "Midpoint key (stream: {0}, version: {1}) < high bounds check key (stream: {2}, version: {3})", - midpointKey.Stream, midpointKey.Version, highBoundsCheck.Stream, highBoundsCheck.Version)); + throw new MaybeCorruptIndexException( + $"Midpoint key (stream: {midpointKey.Stream}, version: {midpointKey.Version}) < high bounds check key (stream: {highBoundsCheck.Stream}, version: {highBoundsCheck.Version})"); } if (midpointKey.SmallerThan(startKey)) { @@ -1222,27 +1196,23 @@ private bool MightContainStream(StreamHash stream) { // with a workitem checked out the ptable (and bloom filter specifically) // wont get disposed - var workItem = GetWorkItem(); + AcquireFileHandle(); try { - var streamHash = stream.Hash; - return _bloomFilter.MightContain(GetSpan(ref streamHash)); + return _bloomFilter.MightContain(Span.AsReadOnlyBytes(in stream.Hash)); } finally { - ReturnWorkItem(workItem); + ReleaseFileHandle(); } } - private static ReadOnlySpan GetSpan(ref ulong streamHash) => - MemoryMarshal.AsBytes(MemoryMarshal.CreateReadOnlySpan(ref streamHash, 1)); - // construct this struct with a 64 bit hash and it will convert it to a hash // for the specified table version public readonly struct StreamHash : IEquatable { + public readonly ulong Hash; + public StreamHash(byte version, ulong hash) { - Hash = version == PTableVersions.IndexV1 ? hash >> 32 : hash; + Hash = version is PTableVersions.IndexV1 ? hash >> 32 : hash; } - public ulong Hash { get; init; } - public override int GetHashCode() => Hash.GetHashCode(); @@ -1322,33 +1292,6 @@ public override string ToString() { return string.Format("Stream: {0}, Version: {1}", Stream, Version); } } - - private class WorkItem : IDisposable { - public readonly FileStream Stream; - public readonly BinaryReader Reader; - - public WorkItem(string filename, int bufferSize) { - Stream = new FileStream(filename, FileMode.Open, FileAccess.Read, FileShare.Read, bufferSize, - FileOptions.RandomAccess); - Reader = new BinaryReader(Stream); - } - - ~WorkItem() { - Dispose(false); - } - - public void Dispose() { - Dispose(true); - GC.SuppressFinalize(this); - } - - private void Dispose(bool disposing) { - if (disposing) { - Stream.Dispose(); - Reader.Dispose(); - } - } - } } internal class HashCollisionException : Exception { diff --git a/src/KurrentDB.Core/Index/PTableConstruction.cs b/src/KurrentDB.Core/Index/PTableConstruction.cs index 09e28584448..f7755d81b54 100644 --- a/src/KurrentDB.Core/Index/PTableConstruction.cs +++ b/src/KurrentDB.Core/Index/PTableConstruction.cs @@ -10,6 +10,7 @@ using System.Security.Cryptography; using System.Threading; using System.Threading.Tasks; +using DotNext; using KurrentDB.Common.Utils; using KurrentDB.Core.DataStructures; using KurrentDB.Core.DataStructures.ProbabilisticFilter; @@ -18,12 +19,12 @@ namespace KurrentDB.Core.Index; public partial class PTable { - public static PTable FromFile(string filename, int initialReaders, int maxReaders, + public static PTable FromFile(string filename, int cacheDepth, bool skipIndexVerify, bool useBloomFilter = true, int lruCacheSize = 1_000_000) { - return new PTable(filename, Guid.NewGuid(), initialReaders, maxReaders, + return new PTable(filename, Guid.NewGuid(), cacheDepth, skipIndexVerify, useBloomFilter, lruCacheSize); } @@ -54,7 +55,7 @@ public static PersistentBloomFilter ConstructBloomFilter( } } - public static PTable FromMemtable(IMemTable table, string filename, int initialReaders, int maxReaders, + public static PTable FromMemtable(IMemTable table, string filename, int cacheDepth = 16, bool skipIndexVerify = false, bool useBloomFilter = true, @@ -105,8 +106,7 @@ public static PTable FromMemtable(IMemTable table, string filename, int initialR // WRITE BLOOM FILTER ENTRY if (bloomFilter != null && rec.Stream != previousHash) { // we are creating a PTable of the same version as the Memtable. therefore the hash is the right format - var streamHash = rec.Stream; - bloomFilter.Add(GetSpan(ref streamHash)); + bloomFilter.Add(Span.AsReadOnlyBytes(in rec.Stream)); previousHash = rec.Stream; } @@ -149,15 +149,13 @@ public static PTable FromMemtable(IMemTable table, string filename, int initialR } Log.Debug("Dumped MemTable [{id}, {table} entries] in {elapsed}.", table.Id, table.Count, sw.Elapsed); - return new PTable(filename, table.Id, initialReaders, maxReaders, cacheDepth, skipIndexVerify, useBloomFilter, lruCacheSize); + return new PTable(filename, table.Id, cacheDepth, skipIndexVerify, useBloomFilter, lruCacheSize); } public static PTable MergeTo( IList tables, string outputFile, byte version, - int initialReaders, - int maxReaders, int cacheDepth = 16, bool skipIndexVerify = false, bool useBloomFilter = true, @@ -176,7 +174,7 @@ public static PTable MergeTo( var fileSizeUpToIndexEntries = GetFileSizeUpToIndexEntries(numIndexEntries, version); if (tables.Count == 2) return MergeTo2(tables, numIndexEntries, indexEntrySize, outputFile, - version, initialReaders, maxReaders, cacheDepth, skipIndexVerify, useBloomFilter, lruCacheSize); // special case + version, cacheDepth, skipIndexVerify, useBloomFilter, lruCacheSize); // special case Log.Debug("PTables merge started."); var watch = Stopwatch.StartNew(); @@ -227,8 +225,7 @@ public static PTable MergeTo( // WRITE BLOOM FILTER ENTRY if (bloomFilter != null && current.Stream != previousHash) { - var streamHash = current.Stream; - bloomFilter.Add(GetSpan(ref streamHash)); + bloomFilter.Add(Span.AsReadOnlyBytes(in current.Stream)); previousHash = current.Stream; } @@ -279,7 +276,7 @@ public static PTable MergeTo( Log.Debug( "PTables merge finished in {elapsed} ([{entryCount}] entries merged into {dumpedEntryCount}).", watch.Elapsed, string.Join(", ", tables.Select(x => x.Count)), dumpedEntryCount); - return new PTable(outputFile, Guid.NewGuid(), initialReaders, maxReaders, cacheDepth, skipIndexVerify, useBloomFilter, lruCacheSize); + return new PTable(outputFile, Guid.NewGuid(), cacheDepth, skipIndexVerify, useBloomFilter, lruCacheSize); } finally { foreach (var enumerableTable in enumerators) { enumerableTable.Dispose(); @@ -287,25 +284,16 @@ public static PTable MergeTo( } } - public static int GetIndexEntrySize(byte version) { - if (version == PTableVersions.IndexV1) { - return IndexEntryV1Size; - } - - if (version == PTableVersions.IndexV2) { - return IndexEntryV2Size; - } - - if (version == PTableVersions.IndexV3) { - return IndexEntryV3Size; - } - - return IndexEntryV4Size; - } + private static int GetIndexEntrySize(byte version) => version switch { + PTableVersions.IndexV1 => IndexEntryV1Size, + PTableVersions.IndexV2 => IndexEntryV2Size, + PTableVersions.IndexV3 => IndexEntryV3Size, + _ => IndexEntryV4Size + }; private static PTable MergeTo2(IList tables, long numIndexEntries, int indexEntrySize, string outputFile, - byte version, int initialReaders, int maxReaders, + byte version, int cacheDepth, bool skipIndexVerify, bool useBloomFilter, int lruCacheSize) { @@ -369,8 +357,7 @@ private static PTable MergeTo2(IList tables, long numIndexEntries, int i // WRITE BLOOM FILTER ENTRY if (bloomFilter != null && current.Stream != previousHash) { // upgradeHash has already ensured the hash is in the right format for the target - var streamHash = current.Stream; - bloomFilter.Add(GetSpan(ref streamHash)); + bloomFilter.Add(Span.AsReadOnlyBytes(in current.Stream)); previousHash = current.Stream; } @@ -415,7 +402,7 @@ private static PTable MergeTo2(IList tables, long numIndexEntries, int i Log.Debug( "PTables merge finished in {elapsed} ([{entryCount}] entries merged into {dumpedEntryCount}).", watch.Elapsed, string.Join(", ", tables.Select(x => x.Count)), dumpedEntryCount); - return new PTable(outputFile, Guid.NewGuid(), initialReaders, maxReaders, cacheDepth, skipIndexVerify, useBloomFilter, lruCacheSize); + return new PTable(outputFile, Guid.NewGuid(), cacheDepth, skipIndexVerify, useBloomFilter, lruCacheSize); } finally { foreach (var enumerator in enumerators) { enumerator.Dispose(); @@ -428,8 +415,6 @@ private static PTable MergeTo2(IList tables, long numIndexEntries, int i string outputFile, byte version, Func> shouldKeep, - int initialReaders, - int maxReaders, int cacheDepth = 16, bool skipIndexVerify = false, bool useBloomFilter = true, @@ -475,8 +460,7 @@ private static PTable MergeTo2(IList tables, long numIndexEntries, int i AppendRecordTo(bs, buffer, version, enumerator.Current, indexEntrySize); // WRITE BLOOM FILTER ENTRY if (bloomFilter != null && current.Stream != previousHash) { - var streamHash = current.Stream; - bloomFilter.Add(GetSpan(ref streamHash)); + bloomFilter.Add(Span.AsReadOnlyBytes(in current.Stream)); previousHash = current.Stream; } keptCount++; @@ -546,7 +530,7 @@ private static PTable MergeTo2(IList tables, long numIndexEntries, int i "PTable scavenge finished in {elapsed} ({droppedCount} entries removed, {keptCount} remaining).", watch.Elapsed, droppedCount, keptCount); - var scavengedTable = new PTable(outputFile, Guid.NewGuid(), initialReaders, maxReaders, cacheDepth, skipIndexVerify, useBloomFilter, lruCacheSize); + var scavengedTable = new PTable(outputFile, Guid.NewGuid(), cacheDepth, skipIndexVerify, useBloomFilter, lruCacheSize); return (scavengedTable, table._size - scavengedTable._size); } catch (Exception) { try { diff --git a/src/KurrentDB.Core/Index/PTableFooter.cs b/src/KurrentDB.Core/Index/PTableFooter.cs index 3e1bfdf7cfc..ec229700358 100644 --- a/src/KurrentDB.Core/Index/PTableFooter.cs +++ b/src/KurrentDB.Core/Index/PTableFooter.cs @@ -3,58 +3,67 @@ using System; using System.IO; +using System.Runtime.InteropServices; +using DotNext.Buffers; +using DotNext.Buffers.Binary; using KurrentDB.Core.Exceptions; +using Microsoft.Win32.SafeHandles; namespace KurrentDB.Core.Index; -public class PTableFooter { - private const int Size = 128; - public readonly FileType FileType; +[StructLayout(LayoutKind.Auto)] +public readonly struct PTableFooter : IBinaryFormattable { + public const int Size = 128; + private const byte FileType = (byte)Index.FileType.PTableFile; + public readonly byte Version; public readonly uint NumMidpointsCached; - public static int GetSize(byte version) { - if (version >= PTableVersions.IndexV4) - return Size; - return 0; - } - public PTableFooter(byte version, uint numMidpointsCached) { - FileType = FileType.PTableFile; Version = version; NumMidpointsCached = numMidpointsCached; } - public byte[] AsByteArray() { - var array = new byte[Size]; - array[0] = (byte)FileType.PTableFile; - array[1] = Version; - uint numMidpoints = NumMidpointsCached; - for (int i = 0; i < 4; i++) { - array[i + 2] = (byte)(numMidpoints & 0xFF); - numMidpoints >>= 8; - } - - return array; - } - - public static PTableFooter FromStream(Stream stream) { - var type = stream.ReadByte(); - if (type != (int)FileType.PTableFile) + private PTableFooter(ref SpanReader reader) { + if (reader.Read() is not FileType) throw new CorruptIndexException("Corrupted PTable.", new InvalidFileException("Wrong type of PTable.")); - var version = stream.ReadByte(); - if (version == -1) - throw new CorruptIndexException("Couldn't read version of PTable from footer.", - new InvalidFileException("Invalid PTable file.")); - if (!(version >= PTableVersions.IndexV4)) + + Version = reader.Read(); + if (Version < PTableVersions.IndexV4) throw new CorruptIndexException( "PTable footer with version < 4 found. PTable footers are supported as from version 4.", new InvalidFileException("Invalid PTable file.")); - byte[] buffer = new byte[4]; - stream.Read(buffer, 0, 4); - uint numMidpointsCached = BitConverter.ToUInt32(buffer, 0); + NumMidpointsCached = reader.ReadLittleEndian(); + } + + public static int GetSize(byte version) + => version >= PTableVersions.IndexV4 ? Size : 0; + + static int IBinaryFormattable.Size => Size; - return new PTableFooter((byte)version, numMidpointsCached); + public static PTableFooter Parse(ReadOnlySpan source) { + var reader = new SpanReader(source); + return new(ref reader); + } + + public static PTableFooter Parse(SafeFileHandle handle, long fileOffset) { + Span buffer = stackalloc byte[Size]; + return RandomAccess.Read(handle, buffer, fileOffset) == buffer.Length + ? Parse(buffer) + : throw new CorruptIndexException("Corrupted PTable footer.", new InvalidFileException("Wrong file size.")); + } + + public void Format(Span buffer) { + var writer = new SpanWriter(buffer); + writer.Add(FileType); + writer.Add(Version); + writer.WriteLittleEndian(NumMidpointsCached); + } + + public byte[] AsByteArray() { + var result = new byte[Size]; + Format(result); + return result; } } diff --git a/src/KurrentDB.Core/Index/PTableHeader.cs b/src/KurrentDB.Core/Index/PTableHeader.cs index 5680bb6455c..5babd490c5f 100644 --- a/src/KurrentDB.Core/Index/PTableHeader.cs +++ b/src/KurrentDB.Core/Index/PTableHeader.cs @@ -1,37 +1,57 @@ // Copyright (c) Kurrent, Inc and/or licensed to Kurrent, Inc under one or more agreements. // Kurrent, Inc licenses this file to you under the Kurrent License v1 (see LICENSE.md). +using System; using System.IO; +using System.Runtime.InteropServices; +using DotNext.Buffers; +using DotNext.Buffers.Binary; using KurrentDB.Core.Exceptions; +using Microsoft.Win32.SafeHandles; namespace KurrentDB.Core.Index; -public class PTableHeader { +[StructLayout(LayoutKind.Auto)] +public readonly struct PTableHeader : IBinaryFormattable { public const int Size = 128; + private const byte FileType = (byte)Index.FileType.PTableFile; - public readonly FileType FileType; public readonly byte Version; public PTableHeader(byte version) { - FileType = FileType.PTableFile; Version = version; } + private PTableHeader(ref SpanReader reader) { + if (reader.Read() is not FileType) + throw new CorruptIndexException("Corrupted PTable.", new InvalidFileException("Wrong type of PTable.")); + + Version = reader.Read(); + } + + public void Format(Span destination) { + var writer = new SpanWriter(destination); + writer.Add(FileType); + writer.Add(Version); + } + public byte[] AsByteArray() { - var array = new byte[Size]; - array[0] = (byte)FileType.PTableFile; - array[1] = Version; - return array; + var result = new byte[Size]; + Format(result); + return result; } - public static PTableHeader FromStream(Stream stream) { - var type = stream.ReadByte(); - if (type != (int)FileType.PTableFile) - throw new CorruptIndexException("Corrupted PTable.", new InvalidFileException("Wrong type of PTable.")); - var version = stream.ReadByte(); - if (version == -1) - throw new CorruptIndexException("Couldn't read version of PTable from header.", - new InvalidFileException("Invalid PTable file.")); - return new PTableHeader((byte)version); + public static PTableHeader Parse(ReadOnlySpan source) { + var reader = new SpanReader(source); + return new(ref reader); } + + public static PTableHeader Parse(SafeFileHandle handle, long fileOffset) { + Span buffer = stackalloc byte[Size]; + return RandomAccess.Read(handle, buffer, fileOffset) == buffer.Length + ? Parse(buffer) + : throw new CorruptIndexException("Corrupted PTable header.", new InvalidFileException("Wrong file size.")); + } + + static int IBinaryFormattable.Size => Size; } diff --git a/src/KurrentDB.Core/Index/TableIndex.cs b/src/KurrentDB.Core/Index/TableIndex.cs index 1ffca18303f..c678c11f25b 100644 --- a/src/KurrentDB.Core/Index/TableIndex.cs +++ b/src/KurrentDB.Core/Index/TableIndex.cs @@ -72,7 +72,6 @@ public long PrepareCheckpoint { private bool _initialized; private readonly int _maxAutoMergeIndexLevel; - private readonly int _pTableMaxReaderCount; public TableIndex(string directory, IHasher lowHasher, @@ -82,7 +81,6 @@ public TableIndex(string directory, ITransactionFileReader tfReader, byte ptableVersion, int maxAutoMergeIndexLevel, - int pTableMaxReaderCount, int maxSizeForMemory = 1000000, int maxTablesPerLevel = 4, bool additionalReclaim = false, @@ -100,7 +98,6 @@ public TableIndex(string directory, Ensure.NotNull(highHasher, "highHasher"); ArgumentNullException.ThrowIfNull(tfReader); Ensure.Positive(initializationThreads, "initializationThreads"); - Ensure.Positive(pTableMaxReaderCount, "pTableMaxReaderCount"); ArgumentOutOfRangeException.ThrowIfLessThanOrEqual(maxTablesPerLevel, 1); @@ -129,7 +126,6 @@ public TableIndex(string directory, _emptyStreamId = emptyStreamId; _maxAutoMergeIndexLevel = maxAutoMergeIndexLevel; - _pTableMaxReaderCount = pTableMaxReaderCount; } public void Initialize(long chaserCheckpoint) { @@ -141,7 +137,7 @@ public void Initialize(long chaserCheckpoint) { _initialized = true; if (_inMem) { - _indexMap = IndexMap.CreateEmpty(_maxTablesPerLevel, int.MaxValue, _pTableMaxReaderCount); + _indexMap = IndexMap.CreateEmpty(_maxTablesPerLevel, int.MaxValue); _prepareCheckpoint = _indexMap.PrepareCheckpoint; _commitCheckpoint = _indexMap.CommitCheckpoint; return; @@ -163,8 +159,7 @@ public void Initialize(long chaserCheckpoint) { useBloomFilter: _useBloomFilter, lruCacheSize: _lruCacheSize, threads: _initializationThreads, - maxAutoMergeLevel: _maxAutoMergeIndexLevel, - pTableMaxReaderCount: _pTableMaxReaderCount); + maxAutoMergeLevel: _maxAutoMergeIndexLevel); if (_indexMap.CommitCheckpoint >= chaserCheckpoint) { _indexMap.Dispose(TimeSpan.FromMilliseconds(5000)); throw new CorruptIndexException(String.Format( @@ -186,8 +181,7 @@ public void Initialize(long chaserCheckpoint) { useBloomFilter: _useBloomFilter, lruCacheSize: _lruCacheSize, threads: _initializationThreads, - maxAutoMergeLevel: _maxAutoMergeIndexLevel, - pTableMaxReaderCount: _pTableMaxReaderCount); + maxAutoMergeLevel: _maxAutoMergeIndexLevel); } _prepareCheckpoint = _indexMap.PrepareCheckpoint; @@ -343,8 +337,6 @@ private void ReadOffQueue() { if (memtable != null) { memtable.MarkForConversion(); ptable = PTable.FromMemtable(memtable, _fileNameProvider.GetFilenameNewTable(), - ESConsts.PTableInitialReaderCount, - _pTableMaxReaderCount, _indexCacheDepth, _skipIndexVerify, useBloomFilter: _useBloomFilter, @@ -511,8 +503,6 @@ private void ReclaimMemoryIfNeeded(List awaitingMemTables) { Log.Debug("Putting awaiting file as PTable instead of MemTable [{id}].", memtable.Id); var ptable = PTable.FromMemtable(memtable, _fileNameProvider.GetFilenameNewTable(), - ESConsts.PTableInitialReaderCount, - _pTableMaxReaderCount, _indexCacheDepth, _skipIndexVerify, useBloomFilter: _useBloomFilter,