Skip to content

Commit a22210b

Browse files
aaronriekenbergGitHub Enterprise
aaronriekenberg
authored and
GitHub Enterprise
committed
Merge pull request #25 from AaronRiekenberg/041322_examples
Add examples submodule.
2 parents f4cc3ee + 33a7941 commit a22210b

File tree

18 files changed

+666
-18
lines changed

18 files changed

+666
-18
lines changed

‎README.md‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,10 @@ This project provides the following components:
3333

3434
## Example Usage
3535

36+
See the `examples` directory for complete runnable example applications.
37+
38+
Basic steps to use NativeMemoryAllocator:
39+
3640
Define a map value object.
3741

3842
```
@@ -118,7 +122,7 @@ The [Dokka](https://github.com/Kotlin/dokka) plugin is present in `build.gradle.
118122
To generate HTML documentation use the following command:
119123

120124
```
121-
./gradlew dokkaHtml
125+
./gradlew -PdokkaEnabled=true dokkaHtml
122126
```
123127

124128
Open `build/dokka/html/index.html` to view generated documentation.

‎build.gradle.kts‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,16 @@ plugins {
2020
id("com.jfrog.artifactory") version "4.26.2"
2121
`maven-publish`
2222
jacoco
23-
id("org.jetbrains.dokka") version "1.6.10"
23+
id("org.jetbrains.dokka") version "1.6.10" apply false
24+
}
25+
26+
// Conditionally enable dokka only when dokkaEnabled=true property is set.
27+
// Latest version 1.6.10 depends on vulnerable versions of jackson/jsoup/etc.
28+
val dokkaEnabled = (project.properties["dokkaEnabled"]?.toString()?.toBoolean()) ?: false
29+
project.logger.lifecycle("dokkaEnabled = $dokkaEnabled")
30+
31+
if (dokkaEnabled) {
32+
apply(plugin = "org.jetbrains.dokka")
2433
}
2534

2635
apply(plugin = "nebula.release")

‎examples/map/README.md‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
# map examples
2+
3+
## running examples
4+
5+
All examples have the gradle `shadowJar` plugin configured to easily run as a standalone app:
6+
7+
```
8+
cd examples/map/offheap
9+
java -jar build/libs/*shadowjar*.jar
10+
```
11+
12+
## onheap example
13+
14+
* Put 20,000 CacheObject instances into a ConcurrentHashMap.
15+
* Each CacheObject instance contains a random string of length 500KB.
16+
* This is a total of 10GB of data stored on the normal Java heap.
17+
* This application does not make use of any features of the native_memory_allocator library. It is provided for
18+
comparison with NativeMemoryMap demos using off-heap storage.
19+
* As this application uses > 10GB of Java heap space, you may have to increase the Java heap size (e.g. `-Xmx20g`).
20+
21+
## offheap example
22+
23+
* Same as onheap but uses NativeMemoryMap with off-heap storage.
24+
* Uses the ConcurrentHashMap backend for NativeMemoryMap, which does not support eviction.
25+
* Total of 10GB of data in off-heap memory.
26+
27+
## offheap-eviction example
28+
29+
* Same as offheap but uses the Caffeine backend with maximumSize of 10,000 entries.
30+
* Expect 10,000 map entries at end of run with 10,000 total evictions.
31+
32+
## offheap-eviction-operationcounters example
33+
34+
* Same as offheap-eviction but enables operation counters and logs them at the end of the demo.

‎examples/map/offheap-eviction-operationcounters/build.gradle.kts‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar
2+
3+
plugins {
4+
kotlin("jvm")
5+
id("com.github.johnrengelman.shadow") version "7.1.2"
6+
}
7+
8+
repositories {
9+
mavenCentral()
10+
}
11+
12+
dependencies {
13+
implementation(project(":examples:map:utils"))
14+
}
15+
16+
tasks {
17+
named<ShadowJar>("shadowJar") {
18+
archiveBaseName.set("offheap-eviction-operationcounters-shadowjar")
19+
manifest {
20+
attributes(mapOf("Main-Class" to "com.target.nativememoryallocator.examples.map.offheap.eviction.operationcounters.OffHeapEvictionOperationCountersKt"))
21+
}
22+
}
23+
}
24+
25+
tasks {
26+
build {
27+
dependsOn(shadowJar)
28+
}
29+
}

‎examples/map/offheap-eviction-operationcounters/src/main/kotlin/com/target/nativememoryallocator/examples/map/offheap.eviction.operationcounters/OffHeapEvictionOperationCounters.kt‎

Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,95 @@
1+
package com.target.nativememoryallocator.examples.map.offheap.eviction.operationcounters
2+
3+
import com.target.nativememoryallocator.allocator.NativeMemoryAllocatorBuilder
4+
import com.target.nativememoryallocator.examples.map.utils.CacheObject
5+
import com.target.nativememoryallocator.examples.map.utils.CacheObjectSerializer
6+
import com.target.nativememoryallocator.examples.map.utils.buildRandomString
7+
import com.target.nativememoryallocator.map.NativeMemoryMapBackend
8+
import com.target.nativememoryallocator.map.NativeMemoryMapBuilder
9+
import kotlinx.coroutines.*
10+
import mu.KotlinLogging
11+
import kotlin.random.Random
12+
13+
private val logger = KotlinLogging.logger {}
14+
15+
16+
/**
17+
* Same as OffHeapEviction but enables operation counters
18+
*/
19+
private class OffHeapEvictionOperationCounters {
20+
21+
private val numEntries = 20_000
22+
23+
private val randomIndex = Random.nextInt(0, numEntries)
24+
25+
private val nativeMemoryAllocator = NativeMemoryAllocatorBuilder(
26+
pageSizeBytes = 4_096, // 4 KB
27+
nativeMemorySizeBytes = (20L * 1_024L * 1_024L * 1_024L), // 20 GB
28+
).build()
29+
30+
private val nativeMemoryMap = NativeMemoryMapBuilder<Int, CacheObject>(
31+
valueSerializer = CacheObjectSerializer(),
32+
nativeMemoryAllocator = nativeMemoryAllocator,
33+
backend = NativeMemoryMapBackend.CAFFEINE,
34+
caffeineConfigFunction = { caffeineConfigBuilder ->
35+
caffeineConfigBuilder
36+
.maximumSize(10_000)
37+
.recordStats()
38+
},
39+
enableOperationCounters = true
40+
).build()
41+
42+
private fun putValueIntoMap(i: Int) {
43+
if ((i % 100) == 0) {
44+
logger.info { "put i = $i" }
45+
}
46+
val value = buildRandomString(length = 500 * 1_024)
47+
if (i == randomIndex) {
48+
logger.info { "put randomIndex = $randomIndex value.length = ${value.length}" }
49+
logger.info { "value.substring(0,20) = ${value.substring(0, 20)}" }
50+
}
51+
nativeMemoryMap.put(
52+
key = i,
53+
value = CacheObject(
54+
s = value,
55+
),
56+
)
57+
}
58+
59+
suspend fun run() {
60+
logger.info { "begin run randomIndex = $randomIndex" }
61+
62+
coroutineScope {
63+
(0 until numEntries).forEach { i ->
64+
launch {
65+
putValueIntoMap(i = i)
66+
}
67+
}
68+
}
69+
70+
logger.info { "nativeMemoryMap.size = ${nativeMemoryMap.size}" }
71+
logger.info { "nativeMemoryAllocator.nativeMemoryAllocatorMetadata = ${nativeMemoryAllocator.nativeMemoryAllocatorMetadata}" }
72+
73+
val randomIndexValue = nativeMemoryMap.get(key = randomIndex)
74+
randomIndexValue?.let {
75+
logger.info { "get randomIndex = $randomIndex" }
76+
logger.info { "randomIndexValue.s.length = ${it.s.length}" }
77+
logger.info { "randomIndexValue.s.substring(0,20) = ${it.s.substring(0, 20)}" }
78+
}
79+
80+
logger.info { "caffeine eviction count = ${nativeMemoryMap.stats.caffeineStats?.evictionCount()}" }
81+
82+
logger.info { "nativeMemoryMap.operationCounters = ${nativeMemoryMap.operationCounters}" }
83+
84+
while (true) {
85+
delay(1_000)
86+
}
87+
}
88+
89+
}
90+
91+
suspend fun main() {
92+
withContext(Dispatchers.Default) {
93+
OffHeapEvictionOperationCounters().run()
94+
}
95+
}

‎examples/map/offheap-eviction/build.gradle.kts‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar
2+
3+
plugins {
4+
kotlin("jvm")
5+
id("com.github.johnrengelman.shadow") version "7.1.2"
6+
}
7+
8+
repositories {
9+
mavenCentral()
10+
}
11+
12+
dependencies {
13+
implementation(project(":examples:map:utils"))
14+
}
15+
16+
tasks {
17+
named<ShadowJar>("shadowJar") {
18+
archiveBaseName.set("offheap-eviction-shadowjar")
19+
manifest {
20+
attributes(mapOf("Main-Class" to "com.target.nativememoryallocator.examples.map.offheap.eviction.OffHeapEvictionKt"))
21+
}
22+
}
23+
}
24+
25+
tasks {
26+
build {
27+
dependsOn(shadowJar)
28+
}
29+
}

‎examples/map/offheap-eviction/src/main/kotlin/com/target/nativememoryallocator/examples/map/offheap.eviction/OffHeapEviction.kt‎

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
package com.target.nativememoryallocator.examples.map.offheap.eviction
2+
3+
import com.target.nativememoryallocator.allocator.NativeMemoryAllocatorBuilder
4+
import com.target.nativememoryallocator.examples.map.utils.CacheObject
5+
import com.target.nativememoryallocator.examples.map.utils.CacheObjectSerializer
6+
import com.target.nativememoryallocator.examples.map.utils.buildRandomString
7+
import com.target.nativememoryallocator.map.NativeMemoryMapBackend
8+
import com.target.nativememoryallocator.map.NativeMemoryMapBuilder
9+
import kotlinx.coroutines.*
10+
import mu.KotlinLogging
11+
import kotlin.random.Random
12+
13+
private val logger = KotlinLogging.logger {}
14+
15+
16+
/**
17+
* Same as OffHeap but uses the Caffeine backend with maximumSize of 10,000 entries.
18+
*/
19+
private class OffHeapEviction {
20+
21+
private val numEntries = 20_000
22+
23+
private val randomIndex = Random.nextInt(0, numEntries)
24+
25+
private val nativeMemoryAllocator = NativeMemoryAllocatorBuilder(
26+
pageSizeBytes = 4_096, // 4 KB
27+
nativeMemorySizeBytes = (20L * 1_024L * 1_024L * 1_024L), // 20 GB
28+
).build()
29+
30+
private val nativeMemoryMap = NativeMemoryMapBuilder<Int, CacheObject>(
31+
valueSerializer = CacheObjectSerializer(),
32+
nativeMemoryAllocator = nativeMemoryAllocator,
33+
backend = NativeMemoryMapBackend.CAFFEINE,
34+
caffeineConfigFunction = { caffeineConfigBuilder ->
35+
caffeineConfigBuilder
36+
.maximumSize(10_000)
37+
.recordStats()
38+
}
39+
).build()
40+
41+
private fun putValueIntoMap(i: Int) {
42+
if ((i % 100) == 0) {
43+
logger.info { "put i = $i" }
44+
}
45+
val value = buildRandomString(length = 500 * 1_024)
46+
if (i == randomIndex) {
47+
logger.info { "put randomIndex = $randomIndex value.length = ${value.length}" }
48+
logger.info { "value.substring(0,20) = ${value.substring(0, 20)}" }
49+
}
50+
nativeMemoryMap.put(
51+
key = i,
52+
value = CacheObject(
53+
s = value,
54+
),
55+
)
56+
}
57+
58+
suspend fun run() {
59+
logger.info { "begin run randomIndex = $randomIndex" }
60+
61+
coroutineScope {
62+
(0 until numEntries).forEach { i ->
63+
launch {
64+
putValueIntoMap(i = i)
65+
}
66+
}
67+
}
68+
69+
logger.info { "nativeMemoryMap.size = ${nativeMemoryMap.size}" }
70+
logger.info { "nativeMemoryAllocator.nativeMemoryAllocatorMetadata = ${nativeMemoryAllocator.nativeMemoryAllocatorMetadata}" }
71+
72+
val randomIndexValue = nativeMemoryMap.get(key = randomIndex)
73+
randomIndexValue?.let {
74+
logger.info { "get randomIndex = $randomIndex" }
75+
logger.info { "randomIndexValue.s.length = ${it.s.length}" }
76+
logger.info { "randomIndexValue.s.substring(0,20) = ${it.s.substring(0, 20)}" }
77+
}
78+
79+
logger.info { "caffeine eviction count = ${nativeMemoryMap.stats.caffeineStats?.evictionCount()}" }
80+
81+
while (true) {
82+
delay(1_000)
83+
}
84+
}
85+
86+
}
87+
88+
suspend fun main() {
89+
withContext(Dispatchers.Default) {
90+
OffHeapEviction().run()
91+
}
92+
}

‎examples/map/offheap/build.gradle.kts‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar
2+
3+
plugins {
4+
kotlin("jvm")
5+
id("com.github.johnrengelman.shadow") version "7.1.2"
6+
}
7+
8+
repositories {
9+
mavenCentral()
10+
}
11+
12+
dependencies {
13+
implementation(project(":examples:map:utils"))
14+
}
15+
16+
tasks {
17+
named<ShadowJar>("shadowJar") {
18+
archiveBaseName.set("offheap-shadowjar")
19+
manifest {
20+
attributes(mapOf("Main-Class" to "com.target.nativememoryallocator.examples.map.offheap.OffHeapKt"))
21+
}
22+
}
23+
}
24+
25+
tasks {
26+
build {
27+
dependsOn(shadowJar)
28+
}
29+
}

0 commit comments

Comments
 (0)