diff --git a/compiler/src/test/java/org/capnproto/test/EncodingTest.java b/compiler/src/test/java/org/capnproto/test/EncodingTest.java index 3b653a8e..60272a60 100644 --- a/compiler/src/test/java/org/capnproto/test/EncodingTest.java +++ b/compiler/src/test/java/org/capnproto/test/EncodingTest.java @@ -28,7 +28,7 @@ public void testAllTypes() { @Test public void testAllTypesMultiSegment() { - MessageBuilder message = new MessageBuilder(5, BuilderArena.AllocationStrategy.FIXED_SIZE); + MessageBuilder message = new MessageBuilder(5, Allocator.AllocationStrategy.FIXED_SIZE); org.capnproto.test.Test.TestAllTypes.Builder allTypes = message.initRoot(org.capnproto.test.Test.TestAllTypes.factory); TestUtil.initTestMessage(allTypes); diff --git a/runtime/pom.xml b/runtime/pom.xml index d49cd2f9..e529e245 100644 --- a/runtime/pom.xml +++ b/runtime/pom.xml @@ -31,8 +31,8 @@ UTF-8 - 1.8 - 1.8 + 9 + 9 @@ -59,9 +59,11 @@ org.apache.maven.plugins maven-compiler-plugin - 3.6.2 + 3.11.0 -Xlint:unchecked + 9 + 9 @@ -71,10 +73,10 @@ jdk9FF - (1.8,) + (9,) - 8 + 9 diff --git a/runtime/src/main/java/org/capnproto/Allocator.java b/runtime/src/main/java/org/capnproto/Allocator.java index a67bb7f3..29f9efc0 100644 --- a/runtime/src/main/java/org/capnproto/Allocator.java +++ b/runtime/src/main/java/org/capnproto/Allocator.java @@ -4,10 +4,19 @@ * An object that allocates memory for a Cap'n Proto message as it is being built. */ public interface Allocator { + public enum AllocationStrategy { + FIXED_SIZE, + GROW_HEURISTICALLY + } /** * Allocates a ByteBuffer to be used as a segment in a message. The returned * buffer must contain at least `minimumSize` bytes, all of which MUST be * set to zero. */ public java.nio.ByteBuffer allocateSegment(int minimumSize); + + /** + * set the size for the next buffer allocation in bytes + */ + public void setNextAllocationSizeBytes(int nextSize); } diff --git a/runtime/src/main/java/org/capnproto/BuilderArena.java b/runtime/src/main/java/org/capnproto/BuilderArena.java index 4f063cd7..e687f00b 100644 --- a/runtime/src/main/java/org/capnproto/BuilderArena.java +++ b/runtime/src/main/java/org/capnproto/BuilderArena.java @@ -24,27 +24,38 @@ import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.util.ArrayList; +import org.capnproto.Allocator.AllocationStrategy; public final class BuilderArena implements Arena { - public enum AllocationStrategy { - FIXED_SIZE, - GROW_HEURISTICALLY + // allocator to use, default BYTE_BUFFER is faster but is limited by + // available process memory limit + public enum AllocatorType { + BYTE_BUFFER, + MEMORY_MAPPED } public static final int SUGGESTED_FIRST_SEGMENT_WORDS = 1024; public static final AllocationStrategy SUGGESTED_ALLOCATION_STRATEGY = AllocationStrategy.GROW_HEURISTICALLY; + public static final AllocatorType SUGGESTED_ALLOCATOR_TYPE = + AllocatorType.BYTE_BUFFER; + public final ArrayList segments; private final Allocator allocator; - public BuilderArena(int firstSegmentSizeWords, AllocationStrategy allocationStrategy) { + // public BuilderArena(int firstSegmentSizeWords, AllocationStrategy allocationStrategy) { + // this.segments = new ArrayList(); + // this.allocator = new DefaultAllocator(allocationStrategy); + // this.allocator.setNextAllocationSizeBytes( + // firstSegmentSizeWords * Constants.BYTES_PER_WORD); + // } + + public BuilderArena(int firstSegmentSizeWords, AllocationStrategy allocationStrategy, AllocatorType allocatorType) { this.segments = new ArrayList(); - { - DefaultAllocator allocator = new DefaultAllocator(allocationStrategy); - allocator.setNextAllocationSizeBytes(firstSegmentSizeWords * Constants.BYTES_PER_WORD); - this.allocator = allocator; - } + this.allocator = createAllocator(allocatorType, allocationStrategy); + this.allocator.setNextAllocationSizeBytes( + firstSegmentSizeWords * Constants.BYTES_PER_WORD); } public BuilderArena(Allocator allocator) { @@ -77,15 +88,12 @@ public BuilderArena(Allocator allocator, ByteBuffer firstSegment) { segmentBuilder.id = ii; segmentBuilder.pos = segmentBuilder.capacity(); // buffer is pre-filled segments.add(segmentBuilder); - // Find the largest segment for the allocation strategy. largestSegment = Math.max(largestSegment, segment.buffer.capacity()); } - DefaultAllocator defaultAllocator = new DefaultAllocator(SUGGESTED_ALLOCATION_STRATEGY); - - // Use largest segment as next size. - defaultAllocator.setNextAllocationSizeBytes(largestSegment); - this.allocator = defaultAllocator; + AllocatorType allocatorType = suggestAllocator(largestSegment); + this.allocator = createAllocator(allocatorType, SUGGESTED_ALLOCATION_STRATEGY); + this.allocator.setNextAllocationSizeBytes(largestSegment); } @Override @@ -152,4 +160,31 @@ public final ByteBuffer[] getSegmentsForOutput() { } return result; } + + private static Allocator createAllocator(AllocatorType allocatorType, AllocationStrategy allocationStrategy) { + switch (allocatorType) { + case BYTE_BUFFER: + DefaultAllocator allocator1 = new DefaultAllocator(allocationStrategy); + return allocator1; + + case MEMORY_MAPPED: + MemoryMappedAllocator allocator2 = new MemoryMappedAllocator( + "capnp_mmf", allocationStrategy); + return allocator2; + default: + throw new AssertionError( + "Allocator must be BYTE_BUFFER or MEMORY_MAPPED"); + } + } + + public static AllocatorType suggestAllocator(int sizeToAllocate) { + Runtime runtime = Runtime.getRuntime(); + long freeMemory = runtime.freeMemory(); // Free memory in bytes + long totalMemory = runtime.totalMemory(); // Total memory in JVM + long maxMemory = runtime.maxMemory(); // Max memory the JVM will attempt to use + if (freeMemory - sizeToAllocate < 0) + return AllocatorType.MEMORY_MAPPED; + + return AllocatorType.BYTE_BUFFER; + }; } diff --git a/runtime/src/main/java/org/capnproto/DefaultAllocator.java b/runtime/src/main/java/org/capnproto/DefaultAllocator.java index 1d1d1bb6..3b33cc8d 100644 --- a/runtime/src/main/java/org/capnproto/DefaultAllocator.java +++ b/runtime/src/main/java/org/capnproto/DefaultAllocator.java @@ -2,7 +2,6 @@ import java.nio.ByteBuffer; -import org.capnproto.BuilderArena.AllocationStrategy; public class DefaultAllocator implements Allocator { @@ -45,6 +44,7 @@ public DefaultAllocator(AllocationStrategy allocationStrategy, this.allocationStyle = style; } + @Override public void setNextAllocationSizeBytes(int nextSize) { this.nextSize = nextSize; } diff --git a/runtime/src/main/java/org/capnproto/MemoryMappedAllocator.java b/runtime/src/main/java/org/capnproto/MemoryMappedAllocator.java new file mode 100644 index 00000000..d47967c6 --- /dev/null +++ b/runtime/src/main/java/org/capnproto/MemoryMappedAllocator.java @@ -0,0 +1,207 @@ +// Copyright (c) 2013-2014 Sandstorm Development Group, Inc. and contributors +// Licensed under the MIT License: +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package org.capnproto; + +import java.io.File; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.lang.ref.Cleaner; +import java.nio.MappedByteBuffer; +import java.nio.channels.FileChannel; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Random; + + +public class MemoryMappedAllocator implements Allocator { + // cleaner for file cleanup when this is GCed + private static final Cleaner cleaner = Cleaner.create(); + private final Cleaner.Cleanable cleanable; + + // the length of the random prefix part of the random filename string + private final int PREFIX_LENGTH = 5; + // the used charset for creating random filenames + private static final String CHARSET = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"; + + // (minimum) number of bytes in the next allocation + private int nextSize = BuilderArena.SUGGESTED_FIRST_SEGMENT_WORDS; + // the maximum allocateable size + public int maxSegmentBytes = Integer.MAX_VALUE - 2; + + // the memory mapped file buffer name prefix + private final String rPrefix; + + // the allocation strategy with which the allocation size grows + public AllocationStrategy allocationStrategy = + AllocationStrategy.GROW_HEURISTICALLY; + + // hashmaps used for keeping track of files + private final Map randomAccFiles = Collections.synchronizedMap(new HashMap<>()); + private final Map channelMap = Collections.synchronizedMap(new HashMap<>()); + + public MemoryMappedAllocator(String baseFileName) { + // create random file name with baseFileNamePrefix: + // ${baseFileName}_XXXXX_000001 + Random random = new Random(); + StringBuilder sb = new StringBuilder(PREFIX_LENGTH); + for (int i = 0; i < PREFIX_LENGTH; i++) { + int index = random.nextInt(CHARSET.length()); + sb.append(CHARSET.charAt(index)); + } + rPrefix = baseFileName + "_" + sb.toString(); + this.cleanable = cleaner.register(this, new State(this.randomAccFiles, rPrefix)); + } + + + public MemoryMappedAllocator(String baseFileName, AllocationStrategy allocationStrategy) { + // create random file name with baseFileNamePrefix: + // ${baseFileName}_XXXXX_000001 + Random random = new Random(); + StringBuilder sb = new StringBuilder(PREFIX_LENGTH); + for (int i = 0; i < PREFIX_LENGTH; i++) { + int index = random.nextInt(CHARSET.length()); + sb.append(CHARSET.charAt(index)); + } + rPrefix = baseFileName + "_" + sb.toString(); + this.cleanable = cleaner.register(this, new State(this.randomAccFiles, rPrefix)); + this.allocationStrategy = allocationStrategy; + } + + + private static String nameForInt(String prefix, int key) { + String ret = prefix + "_" + String.format("%05d", key); + return ret; + } + + + private Integer generateFile() throws IOException { + int fCount; + synchronized (randomAccFiles) { + fCount = randomAccFiles.size(); + String newFileName = nameForInt(rPrefix, fCount); + RandomAccessFile newFile = new RandomAccessFile(newFileName, "rw"); + randomAccFiles.put(fCount, newFile); + File test = new File(newFileName); + test.deleteOnExit(); + } + return fCount; + } + + + /** + * set the grow size of the memory mapped file + */ + @Override + public void setNextAllocationSizeBytes(int nextSize) { + this.nextSize = nextSize; + } + + + private FileChannel createSegment(int segmentSize) throws IOException { + int fileKey = generateFile(); + FileChannel channel = null; + synchronized (channelMap) { + if (!channelMap.containsKey(fileKey)) { + synchronized (randomAccFiles) { + RandomAccessFile file = randomAccFiles.get(fileKey); + file.setLength(segmentSize); + channel = file.getChannel(); + channelMap.put(fileKey, channel); + } + } + } + return channel; + } + + + @Override + public java.nio.ByteBuffer allocateSegment(int minimumSize) { + int size = Math.max(minimumSize, this.nextSize); + MappedByteBuffer result = null; + try { + FileChannel channel = createSegment(size); + result = channel.map(FileChannel.MapMode.READ_WRITE, 0, size); + } + catch (IOException e) + { + System.err.println("IOException: allocateSegment failed with:" + e); + } + + + switch (this.allocationStrategy) { + case GROW_HEURISTICALLY: + if (size < this.maxSegmentBytes - this.nextSize) { + this.nextSize += size; + } else { + this.nextSize = maxSegmentBytes; + } + break; + case FIXED_SIZE: + break; + } + + + + // if (size < this.maxSegmentBytes - this.nextSize) { + // this.nextSize += size; + // } else { + // this.nextSize = maxSegmentBytes; + // } + return result; + } + + + private static class State implements Runnable { + private final Map randomAccFiles; + private final String rPrefix; + + State(Map files, String rPrefix) { + this.randomAccFiles = files; + this.rPrefix = rPrefix; + } + + @Override + public void run() { + // Cleanup logic: delete all files + for (Map.Entry entry : randomAccFiles.entrySet()) { + try { + entry.getValue().close(); + } + catch (IOException e) + { + } + String name = nameForInt(rPrefix, entry.getKey()); + File file = new File(name); + file.delete(); + } + } + } + + /** + * Explicit cleanup: WARNING: this invalidates all alloceted buffers, + * use close() after using the ByteBuffers. + */ + public void close() { + cleanable.clean(); + } +} diff --git a/runtime/src/main/java/org/capnproto/MessageBuilder.java b/runtime/src/main/java/org/capnproto/MessageBuilder.java index 8d75b6d3..d27400e0 100644 --- a/runtime/src/main/java/org/capnproto/MessageBuilder.java +++ b/runtime/src/main/java/org/capnproto/MessageBuilder.java @@ -27,17 +27,28 @@ public final class MessageBuilder { public MessageBuilder() { this.arena = new BuilderArena(BuilderArena.SUGGESTED_FIRST_SEGMENT_WORDS, - BuilderArena.SUGGESTED_ALLOCATION_STRATEGY); + BuilderArena.SUGGESTED_ALLOCATION_STRATEGY, + BuilderArena.SUGGESTED_ALLOCATOR_TYPE); } public MessageBuilder(int firstSegmentWords) { this.arena = new BuilderArena(firstSegmentWords, - BuilderArena.SUGGESTED_ALLOCATION_STRATEGY); + BuilderArena.SUGGESTED_ALLOCATION_STRATEGY, + BuilderArena.SUGGESTED_ALLOCATOR_TYPE); } - public MessageBuilder(int firstSegmentWords, BuilderArena.AllocationStrategy allocationStrategy) { + public MessageBuilder(int firstSegmentWords, Allocator.AllocationStrategy allocationStrategy) { this.arena = new BuilderArena(firstSegmentWords, - allocationStrategy); + allocationStrategy, + BuilderArena.SUGGESTED_ALLOCATOR_TYPE); + } + + + public MessageBuilder(int firstSegmentWords, Allocator.AllocationStrategy allocationStrategy, + BuilderArena.AllocatorType allocatorType) { + this.arena = new BuilderArena(firstSegmentWords, + allocationStrategy, + allocatorType); } /** diff --git a/runtime/src/main/java/org/capnproto/SegmentBuilder.java b/runtime/src/main/java/org/capnproto/SegmentBuilder.java index 9727b586..f0c40276 100644 --- a/runtime/src/main/java/org/capnproto/SegmentBuilder.java +++ b/runtime/src/main/java/org/capnproto/SegmentBuilder.java @@ -74,9 +74,7 @@ public final void put(int index, long value) { } public final void clear() { - for (int ii = 0; ii < this.pos; ++ii) { - this.put(ii, 0); - } - this.pos = 0; + byte[] bytes = new byte[this.pos]; + this.buffer.put(bytes); } } diff --git a/runtime/src/test/java/org/capnproto/DefaultAllocatorTest.java b/runtime/src/test/java/org/capnproto/DefaultAllocatorTest.java index d139195a..9be40810 100644 --- a/runtime/src/test/java/org/capnproto/DefaultAllocatorTest.java +++ b/runtime/src/test/java/org/capnproto/DefaultAllocatorTest.java @@ -1,15 +1,13 @@ package org.capnproto; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - import static org.junit.jupiter.api.Assertions.assertEquals; +import org.junit.jupiter.api.Test; public class DefaultAllocatorTest { @Test public void maxSegmentBytes() { DefaultAllocator allocator = new DefaultAllocator(); - assertEquals(BuilderArena.AllocationStrategy.GROW_HEURISTICALLY, allocator.allocationStrategy); + assertEquals(Allocator.AllocationStrategy.GROW_HEURISTICALLY, allocator.allocationStrategy); allocator.maxSegmentBytes = (1 << 25) - 1; int allocationSize = 1 << 24; diff --git a/runtime/src/test/java/org/capnproto/LayoutTest.java b/runtime/src/test/java/org/capnproto/LayoutTest.java index acd0a81e..ebda023d 100644 --- a/runtime/src/test/java/org/capnproto/LayoutTest.java +++ b/runtime/src/test/java/org/capnproto/LayoutTest.java @@ -1,13 +1,11 @@ package org.capnproto; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - import java.nio.ByteBuffer; import java.nio.ByteOrder; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; +import org.junit.jupiter.api.Test; public class LayoutTest { @@ -144,7 +142,21 @@ public void testStructRoundTripOneSegment() { buffer.order(ByteOrder.LITTLE_ENDIAN); SegmentBuilder segment = new SegmentBuilder(buffer, new BuilderArena(BuilderArena.SUGGESTED_FIRST_SEGMENT_WORDS, - BuilderArena.SUGGESTED_ALLOCATION_STRATEGY)); + BuilderArena.SUGGESTED_ALLOCATION_STRATEGY, BuilderArena.SUGGESTED_ALLOCATOR_TYPE)); + BareStructBuilder factory = new BareStructBuilder(new StructSize((short) 2, (short) 4)); + StructBuilder builder = WireHelpers.initStructPointer(factory, 0, segment, factory.structSize()); + + setUpStruct(builder); + checkStruct(builder); + } + + @Test + public void testStructRoundTripOneSegmentMemoryMapped() { + ByteBuffer buffer = ByteBuffer.allocate(1024*8); + buffer.order(ByteOrder.LITTLE_ENDIAN); + + SegmentBuilder segment = new SegmentBuilder(buffer, new BuilderArena(BuilderArena.SUGGESTED_FIRST_SEGMENT_WORDS, + BuilderArena.SUGGESTED_ALLOCATION_STRATEGY, BuilderArena.AllocatorType.MEMORY_MAPPED)); BareStructBuilder factory = new BareStructBuilder(new StructSize((short) 2, (short) 4)); StructBuilder builder = WireHelpers.initStructPointer(factory, 0, segment, factory.structSize()); diff --git a/runtime/src/test/java/org/capnproto/MemoryMappedAllocatorTest.java b/runtime/src/test/java/org/capnproto/MemoryMappedAllocatorTest.java new file mode 100644 index 00000000..d15464f8 --- /dev/null +++ b/runtime/src/test/java/org/capnproto/MemoryMappedAllocatorTest.java @@ -0,0 +1,140 @@ +// Copyright (c) 2018 Sandstorm Development Group, Inc. and contributors +// Licensed under the MIT License: +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package org.capnproto; + +import java.io.IOException; +import java.nio.BufferOverflowException; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; +import java.util.Arrays; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import org.junit.jupiter.api.Test; + + +public class MemoryMappedAllocatorTest { + @Test + public void testInitializey() throws java.io.IOException { + MemoryMappedAllocator allocator = new MemoryMappedAllocator("test"); + } + + @Test + public void maxSegmentBytes() { + MemoryMappedAllocator allocator = new MemoryMappedAllocator("test"); + allocator.maxSegmentBytes = (1 << 25) - 1; + + int allocationSize = 1 << 24; + allocator.setNextAllocationSizeBytes(allocationSize); + + assertEquals(allocationSize, + allocator.allocateSegment(allocationSize).capacity()); + + assertEquals(allocator.maxSegmentBytes, + allocator.allocateSegment(allocationSize).capacity()); + + assertEquals(allocator.maxSegmentBytes, + allocator.allocateSegment(allocationSize).capacity()); + } + + @Test + public void writeToBuffer() { + MemoryMappedAllocator allocator = new MemoryMappedAllocator("test"); + allocator.maxSegmentBytes = (1 << 25) - 1; + + int allocationSize = 1 << 24; + allocator.setNextAllocationSizeBytes(allocationSize); + + ByteBuffer buffer = allocator.allocateSegment(allocationSize); + assertEquals(allocationSize, buffer.capacity()); + byte[] aBytes = new byte[Math.min(allocationSize, 8192)]; + Arrays.fill(aBytes, (byte) 'A'); + int remaining = allocationSize; + + while (remaining > 0) { + int chunkSize = Math.min(aBytes.length, remaining); + buffer.put(aBytes, 0, chunkSize); // updates the position on the buffer after the put + remaining -= chunkSize; + } + allocator.close(); + } + + private void fillBuffer(ByteBuffer buffer, int allocationSize, byte filling) { + byte[] aBytes = new byte[Math.min(allocationSize, 8192)]; + Arrays.fill(aBytes, filling); + int remaining = allocationSize; + + while (remaining > 0) { + int chunkSize = Math.min(aBytes.length, remaining); + buffer.put(aBytes, 0, chunkSize); // updates the position on the buffer after the put + remaining -= chunkSize; + } + } + + @Test + public void checkBufferOverflow() { + MemoryMappedAllocator allocator = new MemoryMappedAllocator("tmpFileBuffered"); + allocator.maxSegmentBytes = (1 << 25) - 1; + + int allocationSize = 1 << 24; + allocator.setNextAllocationSizeBytes(allocationSize); + + ByteBuffer buffer = allocator.allocateSegment(allocationSize); + assertEquals(allocationSize, buffer.capacity()); + fillBuffer(buffer, allocationSize, (byte) 'A'); + // Now test that writing 1 more byte over the allocated size, + // it causes BufferOverflowException + assertThrows(BufferOverflowException.class, () -> { + fillBuffer(buffer, 1, (byte) 'A'); + }); + } + + + + @Test + public void verifyBufferContent() { + MemoryMappedAllocator allocator = new MemoryMappedAllocator("tmpFileBuffered"); + allocator.maxSegmentBytes = (1 << 25) - 1; + + int allocationSize = 1 << 24; + allocator.setNextAllocationSizeBytes(allocationSize); + + ByteBuffer buffer = allocator.allocateSegment(allocationSize); + assertEquals(allocationSize, buffer.capacity()); + fillBuffer(buffer, allocationSize, (byte) 'C'); + + + Map channelMap = TestUtils.getChannelMap(allocator); + assertNotNull(channelMap); + + for (FileChannel channel : channelMap.values()) { + // Veryfi content + try { + TestUtils.assertAllBytes(channel, 'C'); + } catch (IOException e) { + assertEquals(false, true); + } + } + } +} diff --git a/runtime/src/test/java/org/capnproto/TestUtils.java b/runtime/src/test/java/org/capnproto/TestUtils.java new file mode 100644 index 00000000..73de833d --- /dev/null +++ b/runtime/src/test/java/org/capnproto/TestUtils.java @@ -0,0 +1,68 @@ +// Copyright (c) 2018 Sandstorm Development Group, Inc. and contributors +// Licensed under the MIT License: +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package org.capnproto; + +import java.io.IOException; +import java.lang.reflect.Field; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; +import java.util.Map; + +public class TestUtils { + @SuppressWarnings("unchecked") + public static Map getChannelMap(MemoryMappedAllocator allocator) { + try { + Field channelMapField = MemoryMappedAllocator.class.getDeclaredField("channelMap"); + channelMapField.setAccessible(true); + return (Map) channelMapField.get(allocator); + } catch (NoSuchFieldException | IllegalAccessException e) { + e.printStackTrace(); + return null; + } + } + + public static void assertAllBytes(FileChannel channel, char allocator) throws IOException { + long size = channel.size(); + long position = 0; + int bufferSize = 8192; + ByteBuffer buffer = ByteBuffer.allocate(bufferSize); + + while (position < size) { + buffer.clear(); + int bytesToRead = (int) Math.min(bufferSize, size - position); + buffer.limit(bytesToRead); + + int read = channel.read(buffer, position); + if (read == -1) break; // EOF + + buffer.flip(); + for (int i = 0; i < read; i++) { + byte b = buffer.get(); + if (b != allocator) { + throw new AssertionError("File content is not all 'A'. Found byte: " + b + " at position " + (position + i)); + } + } + position += read; + } + } + +}