Metastore on sqlite

.cargo/config.toml

@@ -2,7 +2,7 @@
 WEB_ASSETS_SOURCE_PATH = { value = "ui/dist", relative = true }
 WEB_ASSETS_TARBALL_PATH = { value = "ui/dist.tar", relative = true }
 LIBSQLITE3_FLAGS = """-DSQLITE_ENABLE_COLUMN_METADATA=1 \
-    -DSQLITE_THREADSAFE=2 \
+    -DSQLITE_THREADSAFE=1 \
     -DSQLITE_ENABLE_LOAD_EXTENSION=1 \
     -DSQLITE_ENABLE_FTS5=1 \
     -DSQLITE_ENABLE_DBSTAT_VTAB=1 \

.github/workflows/rustdoc.yaml

@@ -0,0 +1,26 @@
+name: rustdoc comments
+
+on: 
+  workflow_dispatch:
+
+  pull_request:
+    branches:
+      - yaro/sqlite-metastore
+
+env:
+  CARGO_TERM_COLOR: always
+
+jobs:
+  run_patchdog:
+
+    runs-on: ubuntu-latest
+    permissions: 
+      contents: write
+      pull-requests: write
+
+    steps:
+      - name: Patchdog
+        uses: YuraLitvinov/[email protected]
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          api_key_gemini: ${{ secrets.API_KEY_GEMINI }}

Cargo.toml

@@ -105,6 +105,8 @@ insta = { version = "1.42.0", features = ["json", "filters", "redactions"] }
 cfg-if = { version = "1.0.3" }
 rusqlite = { version = "0.37.0", features = ["blob", "trace", "bundled"] }
 deadpool-sqlite = { version = "0.12.1", features = ["tracing"] }
+deadpool = { version = "0.12.3" }
+deadpool-diesel = {  version = "0.6.1", features = ["sqlite", "tracing"] }
 
 [patch.crates-io]
 datafusion = { git = "https://github.com/Embucket/datafusion.git", rev = "832c278922863064571c0a7c5716a3ff87ce5201" }

crates/api-iceberg-rest/src/error.rs

@@ -96,6 +96,11 @@ impl IntoResponse for Error {
             | core_metastore::Error::Serde { .. }
             | core_metastore::Error::TableMetadataBuilder { .. }
             | core_metastore::Error::TableObjectStoreNotFound { .. }
+            | core_metastore::Error::CreateDir { .. }
+            | core_metastore::Error::CoreSqlite { .. }
+            | core_metastore::Error::CreateTables { .. }
+            | core_metastore::Error::Deadpool { .. }
+            | core_metastore::Error::Diesel { .. }
             | core_metastore::Error::UrlParse { .. } => http::StatusCode::INTERNAL_SERVER_ERROR,
         };
 

crates/api-iceberg-rest/src/state.rs

@@ -1,4 +1,4 @@
-use core_metastore::metastore::Metastore;
+use core_metastore::Metastore;
 use std::sync::Arc;
 
 use serde::{Deserialize, Serialize};

crates/api-internal-rest/src/error.rs

@@ -143,6 +143,11 @@ impl IntoResponse for Error {
                 | core_metastore::Error::Serde { .. }
                 | core_metastore::Error::TableMetadataBuilder { .. }
                 | core_metastore::Error::TableObjectStoreNotFound { .. }
+                | core_metastore::Error::CreateDir { .. }
+                | core_metastore::Error::CoreSqlite { .. }
+                | core_metastore::Error::CreateTables { .. }
+                | core_metastore::Error::Deadpool { .. }
+                | core_metastore::Error::Diesel { .. }
                 | core_metastore::Error::UrlParse { .. } => http::StatusCode::INTERNAL_SERVER_ERROR,
             },
         };

crates/api-internal-rest/src/handlers.rs

@@ -72,7 +72,7 @@ pub async fn create_volume(
         .context(error::CreateVolumeSnafu)?;
     state
         .metastore
-        .create_volume(&volume.ident.clone(), volume)
+        .create_volume(volume)
         .await
         .context(error::CreateVolumeSnafu)
         .map(|v| Json(hide_sensitive(v)))
@@ -168,7 +168,7 @@ pub async fn create_database(
         .context(error::CreateDatabaseSnafu)?;
     state
         .metastore
-        .create_database(&database.ident.clone(), database)
+        .create_database(database)
         .await
         .context(error::CreateDatabaseSnafu)
         .map(Json)

crates/api-internal-rest/src/state.rs

@@ -1,5 +1,5 @@
 use core_history::HistoryStore;
-use core_metastore::metastore::Metastore;
+use core_metastore::Metastore;
 use std::sync::Arc;
 
 // Define a State struct that contains shared services or repositories

crates/api-snowflake-rest/src/server/handlers.rs

@@ -144,8 +144,9 @@ pub async fn abort(
         request_id,
     }): Json<AbortRequestBody>,
 ) -> Result<Json<serde_json::value::Value>> {
-    state
+    let _query_status = state
         .execution_svc
-        .abort_query(RunningQueryId::ByRequestId(request_id, sql_text))?;
+        .abort_query(RunningQueryId::ByRequestId(request_id, sql_text))
+        .await?;
     Ok(Json(serde_json::value::Value::Null))
 }

crates/api-snowflake-rest/src/server/test_server.rs

@@ -11,8 +11,9 @@ pub async fn run_test_rest_api_server(data_format: &str) -> SocketAddr {
     let app_cfg = Config::new(data_format)
         .expect("Failed to create server config")
         .with_demo_credentials("embucket".to_string(), "embucket".to_string());
-
-    run_test_rest_api_server_with_config(app_cfg, UtilsConfig::default()).await
+    let exec_cfg = UtilsConfig::default()
+        .with_max_concurrency_level(2);
+    run_test_rest_api_server_with_config(app_cfg, exec_cfg).await
 }
 
 #[allow(clippy::unwrap_used, clippy::expect_used)]
@@ -39,13 +40,15 @@ pub async fn run_test_rest_api_server_with_config(
         .with_line_number(true)
         .with_span_events(FmtSpan::NONE)
         .with_level(true)
-        .with_max_level(tracing_subscriber::filter::LevelFilter::DEBUG)
+        .with_max_level(tracing_subscriber::filter::LevelFilter::TRACE)
         .finish();
 
     // ignoring error: as with parralel tests execution, just first thread is able to set it successfully
     // since all tests run in a single process
     let _ = tracing::subscriber::set_global_default(subscriber);
 
+    tracing::info!("Starting server at {}", addr);
+
     let metastore = SlateDBMetastore::new_in_memory().await;
     let history = SlateDBHistoryStore::new_in_memory().await;
 

crates/api-snowflake-rest/src/tests/client.rs

@@ -31,6 +31,7 @@ pub async fn http_req_with_headers<T: serde::de::DeserializeOwned>(
     url: &String,
     payload: String,
 ) -> Result<(HeaderMap, T), TestHttpError> {
+    tracing::trace!("Request: {method} {url}");
     let res = client
         .request(method.clone(), url)
         .headers(headers)

crates/api-snowflake-rest/src/tests/test_requests_abort.rs

@@ -106,6 +106,7 @@ mod tests {
         let mut results = Vec::new();
         // start retry_count from 1, to ensure it works with any retry_count as well
         for retry_count in 1_u16..20_u16 {
+            tokio::time::sleep(Duration::from_millis(100)).await;
             let result = query::<JsonResponse>(
                 &query_client,
                 &addr,
@@ -116,7 +117,7 @@ mod tests {
                 false,
             )
             .await;
-            eprintln!("Retry count: {}, Result: {}", retry_count, result.is_ok());
+            eprintln!("Retry count: {}, Result: {:?}", retry_count, result);
             if result.is_ok() {
                 results.push(result);
                 break;

crates/api-ui/src/tests/queries.rs

@@ -427,7 +427,7 @@ async fn test_ui_async_query_infer_default_exec_mode() {
     .await
     .expect_err("Get query error");
 
-    std::thread::sleep(std::time::Duration::from_millis(1000));
+    tokio::time::sleep(std::time::Duration::from_millis(1000)).await;
 
     let QueryGetResponse(query_record) = http_req::<QueryGetResponse>(
         &client,