diff --git a/Cargo.lock b/Cargo.lock index aa2a02b32d5..ec6377f8aa0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -12393,7 +12393,9 @@ name = "starknet_patricia_storage" version = "0.0.0" dependencies = [ "hex", + "libmdbx", "lru 0.12.5", + "page_size", "rstest", "serde", "serde_json", diff --git a/crates/starknet_patricia_storage/Cargo.toml b/crates/starknet_patricia_storage/Cargo.toml index 923d772a2db..1f522f900f7 100644 --- a/crates/starknet_patricia_storage/Cargo.toml +++ b/crates/starknet_patricia_storage/Cargo.toml @@ -14,7 +14,9 @@ workspace = true [dependencies] hex.workspace = true +libmdbx.workspace = true lru.workspace = true +page_size.workspace = true serde = { workspace = true, features = ["derive"] } serde_json.workspace = true starknet-types-core.workspace = true diff --git a/crates/starknet_patricia_storage/src/lib.rs b/crates/starknet_patricia_storage/src/lib.rs index 4d95a4a8685..4ef92af19cc 100644 --- a/crates/starknet_patricia_storage/src/lib.rs +++ b/crates/starknet_patricia_storage/src/lib.rs @@ -3,4 +3,5 @@ pub mod errors; pub mod map_storage; #[cfg(test)] pub mod map_storage_test; +pub mod mdbx_storage; pub mod storage_trait; diff --git a/crates/starknet_patricia_storage/src/mdbx_storage.rs b/crates/starknet_patricia_storage/src/mdbx_storage.rs new file mode 100644 index 00000000000..06cfc6cdb8f --- /dev/null +++ b/crates/starknet_patricia_storage/src/mdbx_storage.rs @@ -0,0 +1,111 @@ +use std::path::Path; + +use libmdbx::{ + Database as MdbxDb, + DatabaseFlags, + Geometry, + PageSize, + TableFlags, + WriteFlags, + WriteMap, +}; +use page_size; + +use crate::storage_trait::{DbHashMap, DbKey, DbValue, PatriciaStorageResult, Storage}; + +pub struct MdbxStorage { + db: MdbxDb, +} + +// Size in bytes. +const MDBX_MIN_PAGESIZE: usize = 1 << 8; // 256 bytes +const MDBX_MAX_PAGESIZE: usize = 1 << 16; // 64KB + +fn get_page_size(os_page_size: usize) -> PageSize { + let mut page_size = os_page_size.clamp(MDBX_MIN_PAGESIZE, MDBX_MAX_PAGESIZE); + + // Page size must be power of two. + if !page_size.is_power_of_two() { + page_size = page_size.next_power_of_two() / 2; + } + + PageSize::Set(page_size) +} + +impl MdbxStorage { + pub fn open(path: &Path) -> PatriciaStorageResult { + // TODO(tzahi): geometry and related definitions are taken from apollo_storage. Check if + // there are better configurations for the committer and consider moving boh crates mdbx + // code to a common location. + let db = MdbxDb::::new() + .set_geometry(Geometry { + size: Some(1 << 20..1 << 40), + growth_step: Some(1 << 32), + page_size: Some(get_page_size(page_size::get())), + ..Default::default() + }) + .set_flags(DatabaseFlags { + // As DbKeys are hashed, there is no locality of pages in the database almost at + // all, so readahead will fill the RAM with garbage. + // See https://libmdbx.dqdkfa.ru/group__c__opening.html#gga9138119a904355d245777c4119534061a16a07f878f8053cc79990063ca9510e7 + no_rdahead: true, + // LIFO policy for recycling a Garbage Collection items should be faster when using + // disks with write-back cache. + liforeclaim: true, + ..Default::default() + }) + .open(path)?; + let txn = db.begin_rw_txn()?; + txn.create_table(None, TableFlags::empty())?; + txn.commit()?; + Ok(Self { db }) + } +} + +impl Storage for MdbxStorage { + fn get(&mut self, key: &DbKey) -> PatriciaStorageResult> { + let txn = self.db.begin_ro_txn()?; + let table = txn.open_table(None)?; + Ok(txn.get(&table, &key.0)?.map(DbValue)) + } + + fn set(&mut self, key: DbKey, value: DbValue) -> PatriciaStorageResult> { + let txn = self.db.begin_rw_txn()?; + let table = txn.open_table(None)?; + let prev_val = txn.get(&table, &key.0)?.map(DbValue); + txn.put(&table, key.0, value.0, WriteFlags::UPSERT)?; + txn.commit()?; + Ok(prev_val) + } + + fn mget(&mut self, keys: &[&DbKey]) -> PatriciaStorageResult>> { + let txn = self.db.begin_ro_txn()?; + let table = txn.open_table(None)?; + let mut res = Vec::with_capacity(keys.len()); + for key in keys { + res.push(txn.get(&table, &key.0)?.map(DbValue)); + } + Ok(res) + } + + fn mset(&mut self, key_to_value: DbHashMap) -> PatriciaStorageResult<()> { + let txn = self.db.begin_rw_txn()?; + let table = txn.open_table(None)?; + for (key, value) in key_to_value { + txn.put(&table, key.0, value.0, WriteFlags::UPSERT)?; + } + txn.commit()?; + Ok(()) + } + + fn delete(&mut self, key: &DbKey) -> PatriciaStorageResult> { + let txn = self.db.begin_rw_txn()?; + let table = txn.open_table(None)?; + let prev_val = txn.get(&table, &key.0)?.map(DbValue); + if prev_val.is_some() { + txn.del(&table, &key.0, None)?; + txn.commit()?; + } + Ok(prev_val) + } +} diff --git a/crates/starknet_patricia_storage/src/storage_trait.rs b/crates/starknet_patricia_storage/src/storage_trait.rs index 0ccf161ce37..7a2ea8f01ad 100644 --- a/crates/starknet_patricia_storage/src/storage_trait.rs +++ b/crates/starknet_patricia_storage/src/storage_trait.rs @@ -13,7 +13,11 @@ pub type DbHashMap = HashMap; /// An error that can occur when interacting with the database. #[derive(thiserror::Error, Debug)] -pub enum PatriciaStorageError {} +pub enum PatriciaStorageError { + /// An error that occurred in the database library. + #[error(transparent)] + Mdbx(#[from] libmdbx::Error), +} pub type PatriciaStorageResult = Result;