Add full-text search bench

.github/workflows/ci.yml

@@ -23,7 +23,7 @@ defaults:
 jobs:
   format:
     name: Check format
-    runs-on: [runner-amd64-large]
+    runs-on: [ runner-amd64-large ]
     continue-on-error: true
     steps:
       - name: Install stable toolchain
@@ -45,7 +45,7 @@ jobs:
 
   clippy:
     name: Check clippy
-    runs-on: [runner-amd64-large]
+    runs-on: [ runner-amd64-large ]
     continue-on-error: true
     steps:
       - name: Install stable toolchain
@@ -67,7 +67,7 @@ jobs:
 
   test:
     name: Check tests
-    runs-on: [runner-amd64-large]
+    runs-on: [ runner-amd64-large ]
     continue-on-error: true
     steps:
       - name: Install stable toolchain
@@ -122,7 +122,7 @@ jobs:
 
   docker-build:
     name: Build Docker image
-    runs-on: [runner-amd64-large-private]
+    runs-on: [ runner-amd64-large-private ]
     needs: build
     continue-on-error: true
     steps:
@@ -193,7 +193,7 @@ jobs:
   benchmark:
     name: Benchmark ${{ matrix.description }}
     needs: build
-    runs-on: [runner-amd64-large-private]
+    runs-on: [ runner-amd64-large-private ]
     continue-on-error: true
     strategy:
       fail-fast: false
@@ -204,142 +204,169 @@ jobs:
             database: arangodb
             enabled: true
             description: ArangoDB
+            fulltext: false
           # Cassandra
           - name: cassandra
             database: cassandra
             enabled: false
             description: Cassandra
+            fulltext: false
             skipped: Cassandra benchmark not yet implemented
           # Dragonfly
           - name: dragonfly
             database: dragonfly
             enabled: true
             description: Dragonfly
+            fulltext: false
           # Dry
           - name: dry
             database: dry
             enabled: true
             description: Dry
+            fulltext: false
           # EchoDB
           - name: echodb
             database: echodb
             enabled: true
             description: EchoDB
+            fulltext: false
           # Fjall
           - name: Fjall
             database: fjall
             enabled: true
             description: Fjall
+            fulltext: false
           # KeyDB
           - name: keydb
             database: keydb
             enabled: true
             description: KeyDB
+            fulltext: false
           # LMDB
           - name: lmdb
             database: lmdb
             enabled: true
             description: LMDB
+            fulltext: false
           # Map
           - name: map
             database: map
             enabled: true
             description: Map
+            fulltext: false
           # MemoDB
           - name: memodb
             database: memodb
             enabled: true
             description: MemoDB
+            fulltext: false
           # MongoDB
           - name: mongodb
             database: mongodb
             enabled: true
             description: MongoDB
+            fulltext: false
           # MySQL
           - name: mysql
             database: mysql
             enabled: true
             description: MySQL
+            fulltext: false
           # Neo4j
           - name: neo4j
             database: neo4j
             enabled: true
             description: Neo4j
+            fulltext: false
           # Postgres
           - name: postgres
             database: postgres
             enabled: true
             description: Postgres
+            fulltext: false
           # Redb
           - name: redb
             database: redb
             enabled: false
             description: ReDB
+            fulltext: false
             skipped: ReDB benchmark skipped due to excessive benchmark time
           # Redis
           - name: redis
             database: redis
             enabled: true
             description: Redis
+            fulltext: false
           # RocksDB
           - name: rocksdb
             database: rocksdb
             enabled: true
             description: RocksDB
+            fulltext: false
           # Scylladb
           - name: scylladb
             database: scylladb
             enabled: false
             description: ScyllaDB
+            fulltext: false
             skipped: ScyllaDB benchmark not yet implemented
           # SQLite
           - name: sqlite
             database: sqlite
             enabled: true
             description: SQLite
+            fulltext: false
           # SurrealDB + Memory
           - name: surrealdb-memory
             database: surrealdb-memory
             enabled: true
             description: SurrealDB with in-memory storage
+            fulltext: true
           # SurrealDB + RocksDB
           - name: surrealdb-rocksdb
             database: surrealdb-rocksdb
             enabled: true
             description: SurrealDB with RocksDB storage
+            fulltext: true
           # SurrealDB + SurrealKV
           - name: surrealdb-surrealkv
             database: surrealdb-surrealkv
             enabled: true
             description: SurrealDB with SurrealKV storage
+            fulltext: true
           # SurrealDB Memory Engine
           - name: surrealdb-embedded-memory
             database: surrealdb
             enabled: true
             endpoint: -e memory
             description: SurrealDB embedded with in-memory storage
+            fulltext: true
           # SurrealDB RocksDB Engine
           - name: surrealdb-embedded-rocksdb
             database: surrealdb
             enabled: true
             endpoint: -e rocksdb:~/crud-bench
             description: SurrealDB embedded with RocksDB storage
+            fulltext: true
           # SurrealDB SurrealKV Engine
           - name: surrealdb-embedded-surrealkv
             database: surrealdb
             enabled: true
             endpoint: -e surrealkv:~/crud-bench
             description: SurrealDB embedded with SurrealKV storage
+            fulltext: true
           # SurrealKV
           - name: surrealkv
             database: surrealkv
             enabled: true
             description: SurrealKV
+            fulltext: false
           # SurrealKV Memory
           - name: surrealkv-memory
             database: surrealkv-memory
             enabled: true
             description: SurrealKV with in-memory storage
+            fulltext: false
     steps:
       - name: Download artifacts
         uses: actions/download-artifact@v4
@@ -448,6 +475,17 @@ jobs:
         env:
           CRUD_BENCH_VALUE: '{ "text": "text:50", "integer": "int", "nested": { "text": "text:1000", "array": [ "string:50", "string:50", "string:50", "string:50", "string:50" ] } }'
 
+      - name: Run benchmarks (100,000 samples / 128 clients / 48 threads / key string26 / random / words)
+        timeout-minutes: 30
+        if: ${{ matrix.enabled && matrix.fulltext && (success() || failure()) }}
+        run: |
+          ${{ github.workspace }}/artifacts/crud-bench -d ${{ matrix.database }} ${{ matrix.endpoint || '' }} -s 1000 -c 10 -t 10 -k string26 -r -n string26-random-1k
+          docker container kill crud-bench &>/dev/null || docker container prune --force &>/dev/null || docker volume prune --all --force &>/dev/null || true
+        env:
+          CRUD_BENCH_PREPARE: 'DEFINE ANALYZER IF NOT EXISTS simple TOKENIZERS class FILTERS lowercase,ascii; DEFINE INDEX ft ON record COLUMNS words FULLTEXT ANALYZER simple BM25;'
+          CRUD_BENCH_VALUE: '{"words": "words:50;how,are,you,a,an,foo,bar,hello,world"}'
+          CRUD_BENCH_SCANS: '[{ "name": "fulltext", "samples": 1000, "condition": "words @@ \"foo\"", "limit": 1000 }]'
+
       - name: Upload result artifacts
         uses: actions/upload-artifact@v4
         if: ${{ matrix.enabled && (success() || failure()) }}

src/arangodb.rs

@@ -85,13 +85,16 @@ async fn create_arango_client(
 }
 
 impl BenchmarkClient for ArangoDBClient {
-	async fn startup(&self) -> Result<()> {
+	async fn startup(&self, prepare: Option<&String>) -> Result<()> {
 		// Ensure we drop the database first.
 		// We can drop the database initially
 		// because the other clients will be
 		// created subsequently, and will then
 		// create the database as necessary.
 		self.connection.drop_database("crud-bench").await?;
+		if prepare.is_some() {
+			bail!(NOT_SUPPORTED_ERROR);
+		}
 		// Everything ok
 		Ok(())
 	}

src/benchmark.rs

@@ -42,6 +42,8 @@ pub(crate) struct Benchmark {
 	pub(crate) sync: bool,
 	/// Whether to enable disk persistence (specific to surrealkv for now)
 	pub(crate) disk_persistence: bool,
+	/// Statement(s) to run before the benchmark
+	pub(crate) prepare: Option<String>,
 }
 impl Benchmark {
 	pub(crate) fn new(args: &Args) -> Self {
@@ -56,6 +58,7 @@ impl Benchmark {
 			sync: args.sync,
 			pid: args.pid,
 			disk_persistence: true,
+			prepare: args.prepare.to_owned(),
 		}
 	}
 	/// Run the benchmark for the desired benchmark engine
@@ -74,7 +77,7 @@ impl Benchmark {
 		// Generate a value sample for the report
 		let sample = vp.generate_value::<D>();
 		// Setup the datastore
-		self.wait_for_client(&engine).await?.startup().await?;
+		self.wait_for_client(&engine).await?.startup(self.prepare.as_ref()).await?;
 		// Setup the clients
 		let clients = self.setup_clients(&engine).await?;
 		// Run the "creates" benchmark

src/engine.rs

@@ -28,7 +28,10 @@ where
 /// running benchmark tests for a client or connection.
 pub(crate) trait BenchmarkClient: Sync + Send + 'static {
 	/// Initialise the store at startup
-	async fn startup(&self) -> Result<()> {
+	async fn startup(&self, prepare: Option<&String>) -> Result<()> {
+		if prepare.is_some() {
+			bail!(NOT_SUPPORTED_ERROR);
+		}
 		Ok(())
 	}
 

src/main.rs

@@ -136,6 +136,10 @@ pub(crate) struct Args {
 		]"#
 	)]
 	pub(crate) scans: String,
+
+	/// Statement(s) to execute before the benchmark
+	#[arg(long, env = "CRUD_BENCH_PREPARE")]
+	pub(crate) prepare: Option<String>,
 }
 
 #[derive(Debug, ValueEnum, Clone, Copy)]
@@ -340,6 +344,7 @@ mod test {
 			scans: r#"[{"name": "limit", "start": 50, "limit": 100, "expect": 100}]"#.to_string(),
 			show_sample: false,
 			pid: None,
+			prepare: None,
 		})
 	}
 

src/mysql.rs

@@ -56,7 +56,7 @@ pub(crate) struct MysqlClient {
 }
 
 impl BenchmarkClient for MysqlClient {
-	async fn startup(&self) -> Result<()> {
+	async fn startup(&self, prepare: Option<&String>) -> Result<()> {
 		let id_type = match self.kt {
 			KeyType::Integer => "SERIAL",
 			KeyType::String26 => "VARCHAR(26)",
@@ -85,9 +85,13 @@ impl BenchmarkClient for MysqlClient {
 			})
 			.collect::<Vec<String>>()
 			.join(", ");
-		let stm =
+		let mut stm =
 			format!("DROP TABLE IF EXISTS record; CREATE TABLE record ( id {id_type} PRIMARY KEY, {fields}) ENGINE=InnoDB;");
+		if let Some(prepare) = prepare {
+			stm.push_str(prepare);
+		}
 		self.conn.lock().await.query_drop(&stm).await?;
+
 		Ok(())
 	}
 

src/neo4j.rs

@@ -68,9 +68,12 @@ pub(crate) struct Neo4jClient {
 }
 
 impl BenchmarkClient for Neo4jClient {
-	async fn startup(&self) -> Result<()> {
-		let stm = "CREATE INDEX FOR (r:Record) ON (r.id);";
-		self.graph.execute(query(stm)).await?.next().await.ok();
+	async fn startup(&self, prepare: Option<&String>) -> Result<()> {
+		let mut stm = "CREATE INDEX FOR (r:Record) ON (r.id);".to_string();
+		if let Some(prepare) = prepare {
+			stm.push_str(prepare);
+		}
+		self.graph.execute(query(&stm)).await?.next().await.ok();
 		Ok(())
 	}
 

src/postgres.rs

@@ -61,7 +61,7 @@ pub(crate) struct PostgresClient {
 }
 
 impl BenchmarkClient for PostgresClient {
-	async fn startup(&self) -> Result<()> {
+	async fn startup(&self, prepare: Option<&String>) -> Result<()> {
 		let id_type = match self.kt {
 			KeyType::Integer => "SERIAL",
 			KeyType::String26 => "VARCHAR(26)",
@@ -90,7 +90,10 @@ impl BenchmarkClient for PostgresClient {
 			})
 			.collect::<Vec<String>>()
 			.join(", ");
-		let stm = format!("DROP TABLE IF EXISTS record; CREATE TABLE record ( id {id_type} PRIMARY KEY, {fields});");
+		let mut stm = format!("DROP TABLE IF EXISTS record; CREATE TABLE record ( id {id_type} PRIMARY KEY, {fields});");
+		if let Some(prepare) = prepare {
+			stm.push_str(prepare);
+		}
 		self.client.batch_execute(&stm).await?;
 		Ok(())
 	}

src/scylladb.rs

@@ -55,7 +55,7 @@ pub(crate) struct ScylladbClient {
 }
 
 impl BenchmarkClient for ScylladbClient {
-	async fn startup(&self) -> Result<()> {
+	async fn startup(&self, prepare: Option<&String>) -> Result<()> {
 		self.session
 			.query_unpaged(
 				"
@@ -96,6 +96,9 @@ impl BenchmarkClient for ScylladbClient {
 				(),
 			)
 			.await?;
+		if let Some(prepare) = prepare {
+			self.session.query_unpaged(prepare.to_owned(), ()).await?;
+		}
 		Ok(())
 	}
 	async fn create_u32(&self, key: u32, val: Value) -> Result<()> {