diff --git a/Cargo.toml b/Cargo.toml index ad03b3b2..113e9a03 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -34,6 +34,8 @@ base64 = "0.13" hex = "0.4" regex = "1.5" lru = "0.10" +urlencoding = "2.1.2" +dyn-clone = "1.0.11" [dev-dependencies] tempfile = "3.1" diff --git a/benches/bench.rs b/benches/bench.rs index e5522071..f86c2ceb 100644 --- a/benches/bench.rs +++ b/benches/bench.rs @@ -4,12 +4,13 @@ extern crate test; use tempfile::tempdir; use terminus_store; use terminus_store::layer::ValueTriple; +use terminus_store::storage::directory::NoFilenameEncoding; use test::Bencher; #[bench] fn bench_add_string_triple(b: &mut Bencher) { let dir = tempdir().unwrap(); - let sync_store = terminus_store::open_sync_directory_store(dir.path()); + let sync_store = terminus_store::open_sync_directory_store(dir.path(), NoFilenameEncoding); let layer_builder = sync_store.create_base_layer().unwrap(); let mut count = 1; b.iter(|| { diff --git a/benches/builder/main.rs b/benches/builder/main.rs index 9557c93b..63a48a41 100644 --- a/benches/builder/main.rs +++ b/benches/builder/main.rs @@ -5,6 +5,7 @@ mod data; use rand::prelude::*; use tempfile::tempdir; use terminus_store; +use terminus_store::storage::directory::NoFilenameEncoding; use test::Bencher; use data::*; @@ -12,7 +13,7 @@ use data::*; #[bench] fn build_empty_base_layer(b: &mut Bencher) { let dir = tempdir().unwrap(); - let store = terminus_store::open_sync_directory_store(dir.path()); + let store = terminus_store::open_sync_directory_store(dir.path(), NoFilenameEncoding); b.iter(|| { let builder = store.create_base_layer().unwrap(); @@ -23,7 +24,7 @@ fn build_empty_base_layer(b: &mut Bencher) { #[bench] fn build_base_layer_1000(b: &mut Bencher) { let dir = tempdir().unwrap(); - let store = terminus_store::open_sync_directory_store(dir.path()); + let store = terminus_store::open_sync_directory_store(dir.path(), NoFilenameEncoding); let seed = b"the quick brown fox jumped over "; let rand = StdRng::from_seed(*seed); @@ -48,7 +49,7 @@ fn build_base_layer_1000(b: &mut Bencher) { #[bench] fn build_empty_child_layer_on_empty_base_layer(b: &mut Bencher) { let dir = tempdir().unwrap(); - let store = terminus_store::open_sync_directory_store(dir.path()); + let store = terminus_store::open_sync_directory_store(dir.path(), NoFilenameEncoding); let builder = store.create_base_layer().unwrap(); let base_layer = builder.commit().unwrap(); @@ -61,7 +62,7 @@ fn build_empty_child_layer_on_empty_base_layer(b: &mut Bencher) { #[bench] fn build_nonempty_child_layer_on_empty_base_layer(b: &mut Bencher) { let dir = tempdir().unwrap(); - let store = terminus_store::open_sync_directory_store(dir.path()); + let store = terminus_store::open_sync_directory_store(dir.path(), NoFilenameEncoding); let builder = store.create_base_layer().unwrap(); let base_layer = builder.commit().unwrap(); @@ -88,7 +89,7 @@ fn build_nonempty_child_layer_on_empty_base_layer(b: &mut Bencher) { #[bench] fn build_nonempty_child_layer_on_nonempty_base_layer(b: &mut Bencher) { let dir = tempdir().unwrap(); - let store = terminus_store::open_sync_directory_store(dir.path()); + let store = terminus_store::open_sync_directory_store(dir.path(), NoFilenameEncoding); let seed = b"the quick brown fox jumped over "; let rand = StdRng::from_seed(*seed); diff --git a/examples/create_graph.rs b/examples/create_graph.rs index fba7644f..7bbacedc 100644 --- a/examples/create_graph.rs +++ b/examples/create_graph.rs @@ -1,5 +1,6 @@ use std::env; +use terminus_store::storage::directory::NoFilenameEncoding; use terminus_store::*; use tokio; @@ -10,7 +11,7 @@ async fn main() { println!("usage: {} ", args[0]); } else { // open a store at the given path. the directory has to exist. - let store = open_directory_store(&args[1]); + let store = open_directory_store(&args[1], NoFilenameEncoding); // then create a graph. if the graph already exists, this will error. store.create(&args[2]).await.unwrap(); diff --git a/examples/print_graph.rs b/examples/print_graph.rs index 85513e91..ae07b1a3 100644 --- a/examples/print_graph.rs +++ b/examples/print_graph.rs @@ -1,12 +1,13 @@ use std::env; use std::io; +use terminus_store::storage::directory::NoFilenameEncoding; use terminus_store::structure::TdbDataType; use terminus_store::*; use tokio; async fn print_graph(store_path: &str, graph: &str) -> io::Result<()> { - let store = open_directory_store(store_path); + let store = open_directory_store(store_path, NoFilenameEncoding); let graph = store .open(graph) .await? diff --git a/examples/write_to_graph.rs b/examples/write_to_graph.rs index 5a67ceb6..cb68507d 100644 --- a/examples/write_to_graph.rs +++ b/examples/write_to_graph.rs @@ -2,6 +2,7 @@ use std::env; use lazy_static::lazy_static; use regex::Regex; +use terminus_store::storage::directory::NoFilenameEncoding; use terminus_store::*; use tokio; use tokio::io::{self, AsyncBufReadExt}; @@ -52,7 +53,7 @@ async fn parse_command(s: &str) -> io::Result { } async fn process_commands(store_path: &str, graph: &str) -> io::Result<()> { - let store = open_directory_store(store_path); + let store = open_directory_store(store_path, NoFilenameEncoding); let graph = store .open(graph) .await? diff --git a/src/storage/directory.rs b/src/storage/directory.rs index 54702adc..bb5e0c4f 100644 --- a/src/storage/directory.rs +++ b/src/storage/directory.rs @@ -1,6 +1,7 @@ //! Directory-based implementation of storage traits. use bytes::{Bytes, BytesMut}; +use dyn_clone::{clone_trait_object, DynClone}; use locking::*; use std::collections::HashMap; use std::io::{self, SeekFrom}; @@ -9,6 +10,7 @@ use std::sync::Arc; use tokio::fs::{self, *}; use tokio::io::{AsyncReadExt, AsyncSeekExt, AsyncWriteExt, BufWriter}; use tokio::sync::RwLock; +use urlencoding::{decode, encode}; use async_trait::async_trait; @@ -181,14 +183,55 @@ impl PersistentLayerStore for DirectoryLayerStore { } } +/// Encoding of the file names +pub trait FilenameEncoding: DynClone + Send + Sync { + fn encode(&self, str: String) -> String; + fn decode(&self, str: String) -> String; +} + +clone_trait_object!(FilenameEncoding); + +/// Bypassing encoding of the file name +#[derive(Clone)] +pub struct NoFilenameEncoding; + +impl FilenameEncoding for NoFilenameEncoding { + fn encode(&self, str: String) -> String { + str + } + fn decode(&self, str: String) -> String { + str + } +} + +/// URL encoding of the file name +#[derive(Clone)] +pub struct URLFilenameEncoding; + +impl FilenameEncoding for URLFilenameEncoding { + fn encode(&self, str: String) -> String { + return encode(str.as_str()).to_string(); + } + fn decode(&self, str: String) -> String { + return decode(str.as_str()).unwrap().to_string(); + } +} + #[derive(Clone)] pub struct DirectoryLabelStore { path: PathBuf, + filename_encoding: Box, } impl DirectoryLabelStore { - pub fn new>(path: P) -> DirectoryLabelStore { - DirectoryLabelStore { path: path.into() } + pub fn new>( + path: P, + filename_encoding: impl FilenameEncoding + 'static, + ) -> DirectoryLabelStore { + DirectoryLabelStore { + path: path.into(), + filename_encoding: Box::new(filename_encoding), + } } } @@ -235,28 +278,27 @@ fn get_label_from_data(name: String, data: &[u8]) -> io::Result