Skip to content

Commit 0681ab2

Browse files
horghclaude
andcommitted
Replace ArrayBuffer with anonymous mmap for Memory mode and stream construction
ArrayBuffer was backed by byte[], which is limited to ~2.1 GiB in .NET. This adds a new MemoryBuffer class that uses anonymous (non-file-backed) MemoryMappedFile, allowing FileAccessMode.Memory and the Reader(Stream) constructor to handle databases larger than 2 GiB. ArrayBuffer is retained only for small internal byte[] uses in TypeActivatorCreator and tests. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
1 parent 35bda8b commit 0681ab2

File tree

8 files changed

+309
-78
lines changed

8 files changed

+309
-78
lines changed

CLAUDE.md

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -69,8 +69,9 @@ The library has three main architectural layers:
6969

7070
1. **Buffer Layer** - File access abstraction
7171
- `Buffer` (abstract base): Defines read interface for binary data
72-
- `MemoryMapBuffer`: Memory-mapped file implementation (default, best balance)
73-
- `ArrayBuffer`: In-memory byte array implementation (fastest lookups, highest RAM usage)
72+
- `MemoryMapBuffer`: Named memory-mapped file implementation (default, supports cross-process sharing)
73+
- `MemoryBuffer`: Anonymous memory-mapped file implementation (used by `FileAccessMode.Memory` and stream construction)
74+
- `ArrayBuffer`: In-memory byte array implementation (used internally for small buffers in `TypeActivatorCreator` and tests)
7475

7576
2. **Reader Layer** - Database navigation and IP lookup
7677
- `Reader`: Main entry point for IP address lookups

MaxMind.Db.Test/PointerTest.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ public void TestWithPointers()
1616
{
1717
var path = Path.Combine(TestUtils.TestDirectory, "TestData", "MaxMind-DB", "test-data", "maps-with-pointers.raw");
1818

19-
using var database = new ArrayBuffer(path);
19+
using var database = new ArrayBuffer(File.ReadAllBytes(path));
2020
var decoder = new Decoder(database, 0);
2121

2222
var node = decoder.Decode<Dictionary<string, object>>(0, out _);

MaxMind.Db/ArrayBuffer.cs

Lines changed: 0 additions & 69 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,7 @@
11
#region
22

33
using System;
4-
using System.IO;
54
using System.Text;
6-
using System.Threading.Tasks;
75

86
#endregion
97

@@ -19,73 +17,6 @@ public ArrayBuffer(byte[] array)
1917
_fileBytes = array;
2018
}
2119

22-
public ArrayBuffer(string file) : this(File.ReadAllBytes(file))
23-
{
24-
}
25-
26-
internal ArrayBuffer(Stream stream) : this(BytesFromStream(stream))
27-
{
28-
}
29-
30-
public static async Task<ArrayBuffer> CreateAsync(string file)
31-
{
32-
using var stream = new FileStream(file, FileMode.Open, FileAccess.Read, FileShare.Read, 4096, true);
33-
return await CreateAsync(stream).ConfigureAwait(false);
34-
}
35-
36-
internal static async Task<ArrayBuffer> CreateAsync(Stream stream)
37-
{
38-
return new ArrayBuffer(await BytesFromStreamAsync(stream).ConfigureAwait(false));
39-
}
40-
41-
private static byte[] BytesFromStream(Stream stream)
42-
{
43-
if (stream == null)
44-
{
45-
throw new ArgumentNullException(nameof(stream), "The database stream must not be null.");
46-
}
47-
48-
byte[] bytes;
49-
50-
using (var memoryStream = new MemoryStream())
51-
{
52-
stream.CopyTo(memoryStream);
53-
bytes = memoryStream.ToArray();
54-
}
55-
56-
if (bytes.Length == 0)
57-
{
58-
throw new InvalidDatabaseException(
59-
"There are zero bytes left in the stream. Perhaps you need to reset the stream's position.");
60-
}
61-
62-
return bytes;
63-
}
64-
65-
private static async Task<byte[]> BytesFromStreamAsync(Stream stream)
66-
{
67-
if (stream == null)
68-
{
69-
throw new ArgumentNullException(nameof(stream), "The database stream must not be null.");
70-
}
71-
72-
byte[] bytes;
73-
74-
using (var memoryStream = new MemoryStream())
75-
{
76-
await stream.CopyToAsync(memoryStream).ConfigureAwait(false);
77-
bytes = memoryStream.ToArray();
78-
}
79-
80-
if (bytes.Length == 0)
81-
{
82-
throw new InvalidDatabaseException(
83-
"There are zero bytes left in the stream. Perhaps you need to reset the stream's position.");
84-
}
85-
86-
return bytes;
87-
}
88-
8920
public override byte[] Read(long offset, int count)
9021
{
9122
var bytes = new byte[count];

MaxMind.Db/GlobalSuppressions.cs

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,6 @@
1010
[assembly: SuppressMessage("Design", "CA1034:Nested types should not be visible", Justification = "<Pending>", Scope = "type", Target = "~T:MaxMind.Db.Reader.ReaderIteratorNode`1")]
1111
[assembly: SuppressMessage("Performance", "CA1815:Override equals and operator equals on value types", Justification = "<Pending>", Scope = "type", Target = "~T:MaxMind.Db.Reader.ReaderIteratorNode`1")]
1212
[assembly: SuppressMessage("Interoperability", "CA1416:Validate platform compatibility", Justification = "<Pending>", Scope = "member", Target = "~M:MaxMind.Db.MemoryMapBuffer.#ctor(System.String,System.Boolean,System.IO.FileInfo)")]
13-
[assembly: SuppressMessage("Major Code Smell", "S4457:Parameter validation in \"async\"/\"await\" methods should be wrapped", Justification = "<Pending>", Scope = "member", Target = "~M:MaxMind.Db.ArrayBuffer.BytesFromStreamAsync(System.IO.Stream)~System.Threading.Tasks.Task{System.Byte[]}")]
1413
[assembly: SuppressMessage("Style", "IDE0056:Use index operator", Justification = "<Pending>", Scope = "member", Target = "~M:MaxMind.Db.Buffer.ReadBigInteger(System.Int64,System.Int32)~System.Numerics.BigInteger")]
1514
[assembly: SuppressMessage("Style", "IDE0056:Use index operator", Justification = "<Pending>", Scope = "member", Target = "~M:MaxMind.Db.Reader.FindAll``1(MaxMind.Db.InjectableValues,System.Int32)~System.Collections.Generic.IEnumerable{MaxMind.Db.Reader.ReaderIteratorNode{``0}}")]
1615

0 commit comments

Comments
 (0)