Skip to content

Commit 0b860b9

Browse files
manami-projectmanami-project
committed
Only download dataset files once per day.
1 parent 06d9739 commit 0b860b9

File tree

7 files changed

+162
-25
lines changed

7 files changed

+162
-25
lines changed

‎README.md‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,3 +32,11 @@ Manami creates an index file for the anime that you already watched and stored o
3232
* Requires JDK 21 and the possibility to run JavaFX
3333
* Download the *.jar file of the latest release
3434
* No installation or additional setup needed. Just Download the `*.jar` and start it by double click or via console `java -jar manami.jar`.
35+
36+
## Configuration
37+
38+
See ["Configuration Management"](https://github.com/manami-project/modb-core/tree/master#configuration-management)
39+
40+
| parameter | type | default | description |
41+
|------------------------------|-----------|---------|--------------------------------------------------------------------------------------------------------------|
42+
| `manami.cache.useLocalFiles` | `Boolean` | `true` | Downloads anime-offline-database files once and stores them next to the *.jar file. Redownload after 24 hrs. |

‎manami-app/src/main/kotlin/io/github/manamiproject/manami/app/Manami.kt‎

Lines changed: 11 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -21,10 +21,13 @@ import io.github.manamiproject.manami.app.search.SearchHandler
2121
import io.github.manamiproject.manami.app.versioning.DefaultLatestVersionChecker
2222
import io.github.manamiproject.modb.anidb.AnidbConfig
2323
import io.github.manamiproject.modb.anilist.AnilistConfig
24+
import io.github.manamiproject.modb.core.coroutines.ModbDispatchers
2425
import io.github.manamiproject.modb.core.logging.LoggerDelegate
2526
import io.github.manamiproject.modb.kitsu.KitsuConfig
2627
import io.github.manamiproject.modb.myanimelist.MyanimelistConfig
28+
import kotlinx.coroutines.launch
2729
import kotlinx.coroutines.runBlocking
30+
import kotlinx.coroutines.withContext
2831
import java.net.URI
2932
import java.util.concurrent.Executors
3033
import java.util.concurrent.atomic.AtomicReference
@@ -51,12 +54,14 @@ class Manami(
5154
log.info {"Starting manami" }
5255
SimpleEventBus.subscribe(this)
5356
runInBackground {
54-
DefaultLatestVersionChecker().checkLatestVersion()
55-
AnimeCachePopulator().populate(DefaultAnimeCache.instance)
56-
DeadEntriesCachePopulator(config = AnidbConfig, url = URI("$DEAD_ENTRIES_BASE_URL/anidb.zip").toURL()).populate(DefaultAnimeCache.instance)
57-
DeadEntriesCachePopulator(config = AnilistConfig, url = URI("$DEAD_ENTRIES_BASE_URL/anilist.zip").toURL()).populate(DefaultAnimeCache.instance)
58-
DeadEntriesCachePopulator(config = KitsuConfig, url = URI("$DEAD_ENTRIES_BASE_URL/kitsu.zip").toURL()).populate(DefaultAnimeCache.instance)
59-
DeadEntriesCachePopulator(config = MyanimelistConfig, url = URI("$DEAD_ENTRIES_BASE_URL/myanimelist.zip").toURL()).populate(DefaultAnimeCache.instance)
57+
withContext(ModbDispatchers.LIMITED_CPU) {
58+
launch { DefaultLatestVersionChecker().checkLatestVersion() }
59+
launch { AnimeCachePopulator().populate(DefaultAnimeCache.instance) }
60+
launch { DeadEntriesCachePopulator(config = AnidbConfig, url = URI("$DEAD_ENTRIES_BASE_URL/anidb.zip").toURL()).populate(DefaultAnimeCache.instance) }
61+
launch { DeadEntriesCachePopulator(config = AnilistConfig, url = URI("$DEAD_ENTRIES_BASE_URL/anilist.zip").toURL()).populate(DefaultAnimeCache.instance) }
62+
launch { DeadEntriesCachePopulator(config = KitsuConfig, url = URI("$DEAD_ENTRIES_BASE_URL/kitsu.zip").toURL()).populate(DefaultAnimeCache.instance) }
63+
launch { DeadEntriesCachePopulator(config = MyanimelistConfig, url = URI("$DEAD_ENTRIES_BASE_URL/myanimelist.zip").toURL()).populate(DefaultAnimeCache.instance) }
64+
}
6065
}
6166
}
6267

‎manami-app/src/main/kotlin/io/github/manamiproject/manami/app/cache/populator/AnimeCachePopulator.kt‎

Lines changed: 54 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5,24 +5,74 @@ import io.github.manamiproject.manami.app.cache.CacheEntry
55
import io.github.manamiproject.manami.app.cache.PresentValue
66
import io.github.manamiproject.manami.app.events.EventBus
77
import io.github.manamiproject.manami.app.events.SimpleEventBus
8+
import io.github.manamiproject.modb.core.config.BooleanPropertyDelegate
9+
import io.github.manamiproject.modb.core.config.ConfigRegistry
10+
import io.github.manamiproject.modb.core.config.DefaultConfigRegistry
11+
import io.github.manamiproject.modb.core.coroutines.ModbDispatchers.LIMITED_FS
12+
import io.github.manamiproject.modb.core.extensions.regularFileExists
13+
import io.github.manamiproject.modb.core.extensions.writeToFile
14+
import io.github.manamiproject.modb.core.httpclient.DefaultHttpClient
15+
import io.github.manamiproject.modb.core.httpclient.HttpClient
816
import io.github.manamiproject.modb.core.logging.LoggerDelegate
917
import io.github.manamiproject.modb.core.models.Anime
1018
import io.github.manamiproject.modb.serde.json.AnimeListJsonStringDeserializer
1119
import io.github.manamiproject.modb.serde.json.DefaultExternalResourceJsonDeserializer
1220
import io.github.manamiproject.modb.serde.json.ExternalResourceJsonDeserializer
1321
import io.github.manamiproject.modb.serde.json.models.Dataset
22+
import kotlinx.coroutines.withContext
1423
import java.net.URI
24+
import java.nio.file.Files
25+
import java.nio.file.attribute.BasicFileAttributes
26+
import java.time.LocalDate
27+
import java.time.ZoneId
28+
import java.time.temporal.ChronoUnit
29+
import kotlin.io.path.Path
1530

1631
internal class AnimeCachePopulator(
17-
private val uri: URI = URI("https://raw.githubusercontent.com/manami-project/anime-offline-database/master/anime-offline-database.zip"),
32+
private val fileName: String = "anime-offline-database.zip",
33+
private val uri: URI = URI("https://raw.githubusercontent.com/manami-project/anime-offline-database/master/$fileName"),
1834
private val parser: ExternalResourceJsonDeserializer<Dataset> = DefaultExternalResourceJsonDeserializer(deserializer = AnimeListJsonStringDeserializer.instance),
1935
private val eventBus: EventBus = SimpleEventBus,
36+
private val httpClient: HttpClient = DefaultHttpClient.instance,
37+
configRegistry: ConfigRegistry = DefaultConfigRegistry.instance,
2038
) : CachePopulator<URI, CacheEntry<Anime>> {
2139

40+
private val isUseLocalFiles by BooleanPropertyDelegate(
41+
namespace = "manami.cache.useLocalFiles",
42+
configRegistry = configRegistry,
43+
default = true
44+
)
45+
2246
override suspend fun populate(cache: Cache<URI, CacheEntry<Anime>>) {
23-
log.info {"Populating cache with anime from [$uri]." }
47+
val parsedAnime = if (isUseLocalFiles) {
48+
val file = Path(fileName)
49+
50+
val isDownloadFile = if (!file.regularFileExists()) {
51+
true
52+
} else {
53+
val attrs = withContext(LIMITED_FS) {
54+
Files.readAttributes(file, BasicFileAttributes::class.java)
55+
}
56+
val creationTime = attrs.creationTime()
57+
.toInstant()
58+
.atZone(ZoneId.systemDefault())
59+
.toLocalDate()
60+
61+
ChronoUnit.DAYS.between(creationTime, LocalDate.now()) >= 1L
62+
}
2463

25-
val parsedAnime = parser.deserialize(uri.toURL()).data
64+
if (isDownloadFile) {
65+
log.info {"Downloading dataset from [$uri], because a local file doesn't exist." }
66+
httpClient.get(uri.toURL()).body.writeToFile(file)
67+
}
68+
69+
log.info {"Populating cache with anime." }
70+
71+
parser.deserialize(file).data
72+
} else {
73+
log.info {"Populating cache with anime from [$uri]." }
74+
parser.deserialize(uri.toURL()).data
75+
}
2676

2777
parsedAnime.forEach { anime ->
2878
anime.sources.forEach { source ->
@@ -39,7 +89,7 @@ internal class AnimeCachePopulator(
3989
eventBus.post(NumberOfEntriesPerMetaDataProviderEvent(numberOfEntriesPerMetaDataProvider))
4090

4191
eventBus.post(CachePopulatorFinishedEvent)
42-
log.info { "Finished populating cache with anime from [$uri]." }
92+
log.info { "Finished populating cache with anime." }
4393
}
4494

4595
private companion object {

‎manami-app/src/main/kotlin/io/github/manamiproject/manami/app/cache/populator/DeadEntriesCachePopulator.kt‎

Lines changed: 52 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3,31 +3,78 @@ package io.github.manamiproject.manami.app.cache.populator
33
import io.github.manamiproject.manami.app.cache.Cache
44
import io.github.manamiproject.manami.app.cache.CacheEntry
55
import io.github.manamiproject.manami.app.cache.DeadEntry
6+
import io.github.manamiproject.modb.core.config.BooleanPropertyDelegate
7+
import io.github.manamiproject.modb.core.config.ConfigRegistry
8+
import io.github.manamiproject.modb.core.config.DefaultConfigRegistry
69
import io.github.manamiproject.modb.core.config.MetaDataProviderConfig
10+
import io.github.manamiproject.modb.core.coroutines.ModbDispatchers.LIMITED_FS
11+
import io.github.manamiproject.modb.core.extensions.regularFileExists
12+
import io.github.manamiproject.modb.core.extensions.writeToFile
13+
import io.github.manamiproject.modb.core.httpclient.DefaultHttpClient
14+
import io.github.manamiproject.modb.core.httpclient.HttpClient
715
import io.github.manamiproject.modb.core.logging.LoggerDelegate
816
import io.github.manamiproject.modb.core.models.Anime
917
import io.github.manamiproject.modb.serde.json.DeadEntriesJsonStringDeserializer
1018
import io.github.manamiproject.modb.serde.json.DefaultExternalResourceJsonDeserializer
1119
import io.github.manamiproject.modb.serde.json.ExternalResourceJsonDeserializer
1220
import io.github.manamiproject.modb.serde.json.models.DeadEntries
13-
import kotlinx.coroutines.runBlocking
21+
import kotlinx.coroutines.withContext
1422
import java.net.URI
1523
import java.net.URL
24+
import java.nio.file.Files
25+
import java.nio.file.attribute.BasicFileAttributes
26+
import java.time.LocalDate
27+
import java.time.ZoneId
28+
import java.time.temporal.ChronoUnit
29+
import kotlin.io.path.Path
1630

1731
internal class DeadEntriesCachePopulator(
1832
private val config: MetaDataProviderConfig,
1933
private val url: URL,
2034
private val parser: ExternalResourceJsonDeserializer<DeadEntries> = DefaultExternalResourceJsonDeserializer(deserializer = DeadEntriesJsonStringDeserializer.instance),
35+
private val httpClient: HttpClient = DefaultHttpClient.instance,
36+
configRegistry: ConfigRegistry = DefaultConfigRegistry.instance,
2137
) : CachePopulator<URI, CacheEntry<Anime>> {
2238

39+
private val isUseLocalFiles by BooleanPropertyDelegate(
40+
namespace = "manami.cache.useLocalFiles",
41+
configRegistry = configRegistry,
42+
default = true
43+
)
44+
2345
override suspend fun populate(cache: Cache<URI, CacheEntry<Anime>>) {
2446
log.info { "Populating cache with dead entries from [${config.hostname()}]" }
2547

26-
runBlocking {
27-
parser.deserialize(url).deadEntries.forEach { animeId ->
28-
val source = config.buildAnimeLink(animeId)
29-
cache.populate(source, DeadEntry())
48+
val parsedData = if (isUseLocalFiles) {
49+
val file = Path("${config.hostname()}.zip")
50+
51+
val isDownloadFile = if (!file.regularFileExists()) {
52+
true
53+
} else {
54+
val attrs = withContext(LIMITED_FS) {
55+
Files.readAttributes(file, BasicFileAttributes::class.java)
56+
}
57+
val creationTime = attrs.creationTime()
58+
.toInstant()
59+
.atZone(ZoneId.systemDefault())
60+
.toLocalDate()
61+
62+
ChronoUnit.DAYS.between(creationTime, LocalDate.now()) >= 1L
63+
}
64+
65+
if (isDownloadFile) {
66+
log.info {"Downloading dead entries file from [$url], because a local file doesn't exist." }
67+
httpClient.get(url).body.writeToFile(file)
3068
}
69+
70+
parser.deserialize(file)
71+
} else {
72+
parser.deserialize(url)
73+
}
74+
75+
parsedData.deadEntries.forEach { animeId ->
76+
val source = config.buildAnimeLink(animeId)
77+
cache.populate(source, DeadEntry())
3178
}
3279

3380
log.info { "Finished populating cache with dead entries from [${config.hostname()}]" }

‎manami-app/src/test/kotlin/io/github/manamiproject/manami/app/cache/TestingAssets.kt‎

Lines changed: 17 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,7 @@
11
package io.github.manamiproject.manami.app.cache
22

33
import io.github.manamiproject.manami.app.cache.loader.CacheLoader
4-
import io.github.manamiproject.modb.core.config.AnimeId
5-
import io.github.manamiproject.modb.core.config.FileSuffix
6-
import io.github.manamiproject.modb.core.config.Hostname
7-
import io.github.manamiproject.modb.core.config.MetaDataProviderConfig
4+
import io.github.manamiproject.modb.core.config.*
85
import io.github.manamiproject.modb.core.converter.AnimeConverter
96
import io.github.manamiproject.modb.core.downloader.Downloader
107
import io.github.manamiproject.modb.core.httpclient.HttpClient
@@ -15,6 +12,9 @@ import io.github.manamiproject.modb.core.models.Tag
1512
import io.github.manamiproject.modb.test.shouldNotBeInvoked
1613
import java.net.URI
1714
import java.net.URL
15+
import java.time.LocalDate
16+
import java.time.LocalDateTime
17+
import java.time.OffsetDateTime
1818

1919
internal object TestCacheLoader : CacheLoader {
2020
override fun hostname(): Hostname = shouldNotBeInvoked()
@@ -53,4 +53,17 @@ internal object TestAnimeCache: AnimeCache {
5353
override fun fetch(key: URI): CacheEntry<Anime> = shouldNotBeInvoked()
5454
override fun populate(key: URI, value: CacheEntry<Anime>) = shouldNotBeInvoked()
5555
override fun clear() = shouldNotBeInvoked()
56+
}
57+
58+
internal object TestConfigRegistry: ConfigRegistry {
59+
override fun boolean(key: String): Boolean = shouldNotBeInvoked()
60+
override fun double(key: String): Double = shouldNotBeInvoked()
61+
override fun int(key: String): Int = shouldNotBeInvoked()
62+
override fun <T : Any> list(key: String): List<T> = shouldNotBeInvoked()
63+
override fun localDate(key: String): LocalDate = shouldNotBeInvoked()
64+
override fun localDateTime(key: String): LocalDateTime = shouldNotBeInvoked()
65+
override fun long(key: String): Long = shouldNotBeInvoked()
66+
override fun <T : Any> map(key: String): Map<String, T> = shouldNotBeInvoked()
67+
override fun offsetDateTime(key: String): OffsetDateTime = shouldNotBeInvoked()
68+
override fun string(key: String): String? = shouldNotBeInvoked()
5669
}

‎manami-app/src/test/kotlin/io/github/manamiproject/manami/app/cache/populator/AnimeCachePopulatorTest.kt‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,9 +5,11 @@ import com.github.tomakehurst.wiremock.client.WireMock.*
55
import io.github.manamiproject.manami.app.cache.DefaultAnimeCache
66
import io.github.manamiproject.manami.app.cache.PresentValue
77
import io.github.manamiproject.manami.app.cache.TestCacheLoader
8+
import io.github.manamiproject.manami.app.cache.TestConfigRegistry
89
import io.github.manamiproject.manami.app.events.Event
910
import io.github.manamiproject.manami.app.events.EventBus
1011
import io.github.manamiproject.manami.app.events.TestEventBus
12+
import io.github.manamiproject.modb.core.config.ConfigRegistry
1113
import io.github.manamiproject.modb.core.models.Anime
1214
import io.github.manamiproject.modb.core.models.Anime.Status.FINISHED
1315
import io.github.manamiproject.modb.core.models.Anime.Type.TV
@@ -114,9 +116,14 @@ internal class AnimeCachePopulatorTest: MockServerTestCase<WireMockServer> by Wi
114116
}
115117
}
116118

119+
val testConfigRegistry = object: ConfigRegistry by TestConfigRegistry {
120+
override fun boolean(key: String): Boolean = false
121+
}
122+
117123
val animeCachePopulator = AnimeCachePopulator(
118-
uri = URI("http://localhost:$port/anime/1535"),
119-
eventBus = testEventBus,
124+
uri = URI("http://localhost:$port/anime/1535"),
125+
eventBus = testEventBus,
126+
configRegistry = testConfigRegistry,
120127
)
121128

122129
serverInstance.stubFor(

‎manami-app/src/test/kotlin/io/github/manamiproject/manami/app/cache/populator/DeadEntriesCachePopulatorTest.kt‎

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,13 @@ package io.github.manamiproject.manami.app.cache.populator
22

33
import com.github.tomakehurst.wiremock.WireMockServer
44
import com.github.tomakehurst.wiremock.client.WireMock.*
5-
import io.github.manamiproject.manami.app.cache.DeadEntry
5+
import io.github.manamiproject.manami.app.cache.*
66
import io.github.manamiproject.manami.app.cache.DefaultAnimeCache
77
import io.github.manamiproject.manami.app.cache.MetaDataProviderTestConfig
88
import io.github.manamiproject.manami.app.cache.TestCacheLoader
9+
import io.github.manamiproject.manami.app.cache.TestConfigRegistry
910
import io.github.manamiproject.modb.core.config.AnimeId
11+
import io.github.manamiproject.modb.core.config.ConfigRegistry
1012
import io.github.manamiproject.modb.core.config.Hostname
1113
import io.github.manamiproject.modb.core.config.MetaDataProviderConfig
1214
import io.github.manamiproject.modb.test.MockServerTestCase
@@ -28,9 +30,14 @@ internal class DeadEntriesCachePopulatorTest: MockServerTestCase<WireMockServer>
2830
override fun buildAnimeLink(id: AnimeId): URI = URI("https://${hostname()}/anime/$id")
2931
}
3032

31-
val animeCachePopulator = DeadEntriesCachePopulator(
33+
val testConfigRegistry = object: ConfigRegistry by TestConfigRegistry {
34+
override fun boolean(key: String): Boolean = false
35+
}
36+
37+
val cachePopulator = DeadEntriesCachePopulator(
3238
config = testConfig,
33-
url = URI("http://localhost:$port/dead-entires/all.json").toURL()
39+
url = URI("http://localhost:$port/dead-entires/all.json").toURL(),
40+
configRegistry = testConfigRegistry,
3441
)
3542

3643
serverInstance.stubFor(
@@ -57,7 +64,7 @@ internal class DeadEntriesCachePopulatorTest: MockServerTestCase<WireMockServer>
5764

5865
// when
5966
runBlocking {
60-
animeCachePopulator.populate(testCache)
67+
cachePopulator.populate(testCache)
6168
}
6269

6370
// then

0 commit comments

Comments
 (0)