Add dual store support

ndr-ds · ndr-ds · commit 64724027edbb · 2025-05-07T11:39:49.000-03:00
diff --git a/docker/Dockerfile b/docker/Dockerfile
@@ -11,6 +11,7 @@
 #   `aarch64-unknown-linux-gnu`.
 # - `build_flag` is either "--release" or an empty string.
 # - `build_folder` is either "release" or "debug".
+# - `build_features` is a comma-separated list of features to build the binaries with.
 
 # Stage 1 - Generate recipe file for dependencies
 
@@ -24,12 +25,14 @@ ARG binaries=
 ARG copy=${binaries:+_copy}
 ARG build_flag=--release
 ARG build_folder=release
+ARG build_features=scylladb,metrics
 
 FROM rust:1.74-slim-bookworm AS builder
 ARG git_commit
 ARG target
 ARG build_flag
 ARG build_folder
+ARG build_features
 
 RUN apt-get update && apt-get install -y \
     pkg-config \
@@ -69,7 +72,7 @@ RUN cargo build ${build_flag:+"$build_flag"} \
     --bin linera \
     --bin linera-proxy \
     --bin linera-server \
-    --features scylladb,metrics
+    --features $build_features
 
 RUN mv \
     target/"$target"/"$build_folder"/linera \
diff --git a/docker/compose-server-entrypoint.sh b/docker/compose-server-entrypoint.sh
@@ -1,7 +1,9 @@
 #!/bin/sh
 
+storage=$1
+
 exec ./linera-server run \
-  --storage scylladb:tcp:scylla:9042 \
+  --storage $storage \
   --server /config/server.json \
   --shard 0 \
   --genesis /config/genesis.json
diff --git a/docker/compose-server-init.sh b/docker/compose-server-init.sh
@@ -1,7 +1,9 @@
 #!/bin/sh
 
+storage=$1
+
 while true; do
-  ./linera storage check-existence --storage "scylladb:tcp:scylla:9042"
+  ./linera storage check-existence --storage $storage
   status=$?
 
   if [ $status -eq 0 ]; then
@@ -10,7 +12,7 @@ while true; do
   elif [ $status -eq 1 ]; then
     echo "Database does not exist, attempting to initialize..."
     if ./linera-server initialize \
-      --storage scylladb:tcp:scylla:9042 \
+      --storage $storage \
       --genesis /config/genesis.json; then
       echo "Initialization successful."
       exit 0
diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml
@@ -30,7 +30,7 @@ services:
     image: "${LINERA_IMAGE:-linera}"
     deploy:
       replicas: 4
-    command: [ "./compose-server-entrypoint.sh" ]
+    command: [ "./compose-server-entrypoint.sh", "scylladb:tcp:scylla:9042" ]
     volumes:
       - .:/config
     labels:
@@ -42,7 +42,7 @@ services:
   shard-init:
     image: "${LINERA_IMAGE:-linera}"
     container_name: shard-init
-    command: [ "./compose-server-init.sh" ]
+    command: [ "./compose-server-init.sh", "scylladb:tcp:scylla:9042" ]
     volumes:
       - .:/config
     depends_on:
diff --git a/docker/server-entrypoint.sh b/docker/server-entrypoint.sh
@@ -2,9 +2,10 @@
 
 # Extract the ordinal number from the pod hostname
 ORDINAL="${HOSTNAME##*-}"
+storage=$1
 
 exec ./linera-server run \
-  --storage scylladb:tcp:scylla-client.scylla.svc.cluster.local:9042 \
+  --storage $storage \
   --server /config/server.json \
   --shard $ORDINAL \
   --genesis /config/genesis.json
diff --git a/docker/server-init.sh b/docker/server-init.sh
@@ -1,7 +1,9 @@
 #!/bin/sh
 
+storage=$1
+
 while true; do
-  ./linera storage check-existence --storage "scylladb:tcp:scylla-client.scylla.svc.cluster.local:9042"
+  ./linera storage check-existence --storage $storage
   status=$?
 
   if [ $status -eq 0 ]; then
@@ -10,7 +12,7 @@ while true; do
   elif [ $status -eq 1 ]; then
     echo "Database does not exist, attempting to initialize..."
     if ./linera-server initialize \
-      --storage scylladb:tcp:scylla-client.scylla.svc.cluster.local:9042 \
+      --storage $storage \
       --genesis /config/genesis.json; then
       echo "Initialization successful."
       exit 0
diff --git a/kubernetes/linera-validator/scylla-manager.values.yaml b/kubernetes/linera-validator/scylla-manager.values.yaml
@@ -14,7 +14,7 @@ scylla:
     tag: 5.4.3
   agentImage:
     tag: 3.2.8
-  datacenter: manager-dc
+  datacenter: validator
   racks:
   - name: manager-rack
     members: 1
diff --git a/kubernetes/linera-validator/templates/shards.yaml b/kubernetes/linera-validator/templates/shards.yaml
@@ -35,7 +35,7 @@ spec:
         - name: linera-server-initializer
           image: {{ .Values.lineraImage }}
           imagePullPolicy: {{ .Values.lineraImagePullPolicy }}
-          command: ["./server-init.sh"]
+          command: ["./server-init.sh", {{ .Values.storage | quote }}]
           env:
             - name: RUST_LOG
               value: {{ .Values.logLevel }}
@@ -45,18 +45,26 @@ spec:
             - name: config
               mountPath: "/config"
               readOnly: true
+            {{- if .Values.dualStore }}
+            - name: linera-db
+              mountPath: "/linera.db"
+            {{- end }}
       containers:
         - name: linera-server
           image: {{ .Values.lineraImage }}
           imagePullPolicy: {{ .Values.lineraImagePullPolicy }}
-          command: ["./server-entrypoint.sh"]
+          command: ["./server-entrypoint.sh", {{ .Values.storage | quote }}]
           env:
             - name: RUST_LOG
               value: {{ .Values.logLevel }}
           volumeMounts:
             - name: config
               mountPath: "/config"
               readOnly: true
+            {{- if .Values.dualStore }}
+            - name: linera-db
+              mountPath: "/linera.db"
+            {{- end }}
       volumes:
         - name: config
           configMap:
@@ -66,3 +74,13 @@ spec:
                 path: server.json
               - key: genesisConfig
                 path: genesis.json
+  {{- if .Values.dualStore }}
+  volumeClaimTemplates:
+    - metadata:
+        name: linera-db
+      spec:
+        accessModes: ["ReadWriteOnce"]
+        resources:
+          requests:
+            storage: {{ .Values.rocksdbStorageSize }}
+  {{- end }}
diff --git a/kubernetes/linera-validator/values-local.yaml.gotmpl b/kubernetes/linera-validator/values-local.yaml.gotmpl
@@ -4,9 +4,13 @@
 lineraImage: {{ env "LINERA_HELMFILE_LINERA_IMAGE" | default "linera:latest" }}
 lineraImagePullPolicy: Never
 logLevel: "debug"
+# Size of the RocksDB storage per shard, IF using `DualStore`. Otherwise will be ignored.
+rocksdbStorageSize: {{ env "LINERA_HELMFILE_SET_ROCKSDB_STORAGE_SIZE" | default "2Gi" }}
 proxyPort: 19100
 metricsPort: 21100
 numShards: {{ env "LINERA_HELMFILE_SET_NUM_SHARDS" | default 10 }}
+storage: {{ env "LINERA_HELMFILE_SET_STORAGE" | default "scylladb:tcp:scylla-client.scylla.svc.cluster.local:9042" }}
+dualStore: {{ env "LINERA_HELMFILE_SET_DUAL_STORE" | default "false" }}
 
 # Loki
 loki-stack:
diff --git a/linera-service/src/cli_wrappers/docker.rs b/linera-service/src/cli_wrappers/docker.rs
@@ -24,6 +24,7 @@ impl DockerImage {
         binaries: &BuildArg,
         github_root: &PathBuf,
         build_mode: &BuildMode,
+        dual_store: bool,
     ) -> Result<Self> {
         let build_arg = match binaries {
             BuildArg::Directory(bin_path) => {
@@ -73,6 +74,10 @@ impl DockerImage {
             }
         }
 
+        if dual_store {
+            command.args(["--build-arg", "build_features=rocksdb,scylladb,metrics"]);
+        }
+
         #[cfg(not(with_testing))]
         command
             .args([
diff --git a/linera-service/src/cli_wrappers/helmfile.rs b/linera-service/src/cli_wrappers/helmfile.rs
@@ -17,14 +17,25 @@ impl HelmFile {
         num_shards: usize,
         cluster_id: u32,
         docker_image_name: String,
+        dual_store: bool,
     ) -> Result<()> {
         let chart_dir = format!("{}/kubernetes/linera-validator", github_root.display());
 
         let temp_dir = tempfile::tempdir()?;
         fs_extra::copy_items(&[&chart_dir], temp_dir.path(), &CopyOptions::new())?;
 
-        Command::new("helmfile")
-            .current_dir(temp_dir.path().join("linera-validator"))
+        let mut command = Command::new("helmfile");
+        command.current_dir(temp_dir.path().join("linera-validator"));
+
+        if dual_store {
+            command.env(
+                "LINERA_HELMFILE_SET_STORAGE",
+                "dualrocksdbscylladb:/linera.db:spawn_blocking:tcp:scylla-client.scylla.svc.cluster.local:9042",
+            );
+            command.env("LINERA_HELMFILE_SET_DUAL_STORE", "true");
+        }
+
+        command
             .env(
                 "LINERA_HELMFILE_SET_SERVER_CONFIG",
                 format!("working/server_{server_config_id}.json"),
diff --git a/linera-service/src/cli_wrappers/local_kubernetes_net.rs b/linera-service/src/cli_wrappers/local_kubernetes_net.rs
@@ -71,6 +71,7 @@ pub struct LocalKubernetesNetConfig {
     pub docker_image_name: String,
     pub build_mode: BuildMode,
     pub policy_config: ResourceControlPolicyConfig,
+    pub dual_store: bool,
 }
 
 /// A wrapper of [`LocalKubernetesNetConfig`] to create a shared local Kubernetes network
@@ -93,6 +94,7 @@ pub struct LocalKubernetesNet {
     kind_clusters: Vec<KindCluster>,
     num_initial_validators: usize,
     num_shards: usize,
+    dual_store: bool,
 }
 
 #[cfg(with_testing)]
@@ -127,6 +129,7 @@ impl SharedLocalKubernetesNetTestingConfig {
             docker_image_name: String::from("linera:latest"),
             build_mode: BuildMode::Release,
             policy_config: ResourceControlPolicyConfig::Testnet,
+            dual_store: false,
         })
     }
 }
@@ -159,6 +162,7 @@ impl LineraNetConfig for LocalKubernetesNetConfig {
             clusters,
             self.num_initial_validators,
             self.num_shards,
+            self.dual_store,
         )?;
 
         let client = net.make_client().await;
@@ -338,6 +342,7 @@ impl LocalKubernetesNet {
         kind_clusters: Vec<KindCluster>,
         num_initial_validators: usize,
         num_shards: usize,
+        dual_store: bool,
     ) -> Result<Self> {
         Ok(Self {
             network,
@@ -352,6 +357,7 @@ impl LocalKubernetesNet {
             kind_clusters,
             num_initial_validators,
             num_shards,
+            dual_store,
         })
     }
 
@@ -433,6 +439,7 @@ impl LocalKubernetesNet {
                 &self.binaries,
                 &github_root,
                 &self.build_mode,
+                self.dual_store,
             )
             .await?;
             self.docker_image_name.clone()
@@ -460,6 +467,7 @@ impl LocalKubernetesNet {
             let tmp_dir_path = tmp_dir_path_clone.clone();
 
             let docker_image_name = docker_image_name.clone();
+            let dual_store = self.dual_store;
             let future = async move {
                 let cluster_id = kind_cluster.id();
                 kind_cluster.load_docker_image(&docker_image_name).await?;
@@ -470,7 +478,15 @@ impl LocalKubernetesNet {
                     base_dir.join(&server_config_filename),
                 )?;
 
-                HelmFile::sync(i, &github_root, num_shards, cluster_id, docker_image_name).await?;
+                HelmFile::sync(
+                    i,
+                    &github_root,
+                    num_shards,
+                    cluster_id,
+                    docker_image_name,
+                    dual_store,
+                )
+                .await?;
 
                 let mut kubectl_instance = kubectl_instance.lock().await;
                 let output = kubectl_instance.get_pods(cluster_id).await?;
diff --git a/linera-service/src/linera/command.rs b/linera-service/src/linera/command.rs
@@ -991,6 +991,12 @@ pub enum NetCommand {
         /// The number of block exporters per validator in the local test network. Default is 0.
         #[arg(long, default_value = "0")]
         block_exporters: u32,
+
+        /// Use dual store (rocksdb and scylladb) instead of just scylladb. This is exclusive for
+        /// kubernetes deployments.
+        #[cfg(feature = "kubernetes")]
+        #[arg(long, default_value = "false")]
+        dual_store: bool,
     },
 
     /// Print a bash helper script to make `linera net up` easier to use. The script is
diff --git a/linera-service/src/linera/main.rs b/linera-service/src/linera/main.rs
@@ -2000,6 +2000,7 @@ async fn run(options: &ClientOptions) -> Result<i32, Error> {
                 faucet_chain,
                 faucet_port,
                 faucet_amount,
+                dual_store,
                 ..
             } => {
                 net_up_utils::handle_net_up_kubernetes(
@@ -2017,6 +2018,7 @@ async fn run(options: &ClientOptions) -> Result<i32, Error> {
                     *faucet_chain,
                     *faucet_port,
                     *faucet_amount,
+                    *dual_store,
                 )
                 .boxed()
                 .await?;
diff --git a/linera-service/src/linera/net_up_utils.rs b/linera-service/src/linera/net_up_utils.rs
@@ -119,6 +119,7 @@ pub async fn handle_net_up_kubernetes(
     faucet_chain: Option<u32>,
     faucet_port: NonZeroU16,
     faucet_amount: Amount,
+    dual_store: bool,
 ) -> anyhow::Result<()> {
     if num_initial_validators < 1 {
         panic!("The local test network must have at least one validator.");
@@ -148,6 +149,7 @@ pub async fn handle_net_up_kubernetes(
         docker_image_name,
         build_mode,
         policy_config,
+        dual_store,
     };
     let (mut net, client) = config.instantiate().await?;
     let faucet_service = print_messages_and_create_faucet(
