Properly deal with tempo and memory profiling and stop using scripts for entrypoint/init

.github/workflows/docker_image.yml

@@ -9,7 +9,7 @@ permissions:
   contents: read
 
 jobs:
-  build-and-push:
+  build-and-push-base:
     runs-on: ubuntu-latest-16-cores
     timeout-minutes: 40
 
@@ -28,11 +28,63 @@ jobs:
           credentials_json: ${{ secrets.GCP_SA_ACTIONS_RUNNER_KEY }}
       - name: Set up Google Cloud SDK
         uses: google-github-actions/setup-gcloud@v1
-
+
+      - name: Authenticate Docker with GCP
+        run: |
+          gcloud auth configure-docker us-docker.pkg.dev
+
+      - name: Set env variables
+        run: |
+          echo "GIT_COMMIT_SHORT=${GITHUB_SHA:0:7}" >> $GITHUB_ENV
+          echo "GIT_COMMIT_LONG=${GITHUB_SHA}" >> $GITHUB_ENV
+          if [ -n "${{ github.head_ref }}" ]; then
+            echo "BRANCH_NAME=${{ github.head_ref }}" >> $GITHUB_ENV
+          else
+            echo "BRANCH_NAME=${{ github.ref_name }}" >> $GITHUB_ENV
+          fi
+
+      - name: Set docker image env variable
+        run: |
+          echo "DOCKER_IMAGE=us-docker.pkg.dev/linera-io-dev/linera-public-registry/linera" >> $GITHUB_ENV
+
+      - name: Build base Docker image
+        run: |
+          docker build --build-arg git_commit=${{ env.GIT_COMMIT_LONG }} \
+            -f docker/Dockerfile . \
+            -t ${{ env.DOCKER_IMAGE }}:${{ env.BRANCH_NAME }}_base \
+            -t ${{ env.DOCKER_IMAGE }}:${{ env.GIT_COMMIT_SHORT }}_base \
+            -t ${{ env.DOCKER_IMAGE }}:${{ env.GIT_COMMIT_LONG }}_base
+
+      - name: Push base Docker image to Google Artifact Registry
+        run: |
+          docker push ${{ env.DOCKER_IMAGE }}:${{ env.BRANCH_NAME }}_base
+          docker push ${{ env.DOCKER_IMAGE }}:${{ env.GIT_COMMIT_SHORT }}_base
+          docker push ${{ env.DOCKER_IMAGE }}:${{ env.GIT_COMMIT_LONG }}_base
+
+  build-and-push-full:
+    runs-on: ubuntu-latest-16-cores
+    timeout-minutes: 40
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - uses: actions-rust-lang/setup-rust-toolchain@v1
+      - name: Install Protoc
+        uses: arduino/setup-protoc@v3
+        with:
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+      - name: Auth service account
+        uses: google-github-actions/auth@v1
+        with:
+          credentials_json: ${{ secrets.GCP_SA_ACTIONS_RUNNER_KEY }}
+      - name: Set up Google Cloud SDK
+        uses: google-github-actions/setup-gcloud@v1
+
       - name: Authenticate Docker with GCP
         run: |
           gcloud auth configure-docker us-docker.pkg.dev
-  
+
       - name: Set env variables
         run: |
           echo "GIT_COMMIT_SHORT=${GITHUB_SHA:0:7}" >> $GITHUB_ENV
@@ -46,17 +98,18 @@ jobs:
       - name: Set docker image env variable
         run: |
           echo "DOCKER_IMAGE=us-docker.pkg.dev/linera-io-dev/linera-public-registry/linera" >> $GITHUB_ENV
-  
-      - name: Build Docker image
+
+      - name: Build full Docker image
         run: |
           docker build --build-arg git_commit=${{ env.GIT_COMMIT_LONG }} \
+            --build-arg additional_features=tempo,memory-profiling \
             -f docker/Dockerfile . \
-            -t ${{ env.DOCKER_IMAGE }}:${{ env.BRANCH_NAME }} \
-            -t ${{ env.DOCKER_IMAGE }}:${{ env.GIT_COMMIT_SHORT }} \
-            -t ${{ env.DOCKER_IMAGE }}:${{ env.GIT_COMMIT_LONG }}
-  
-      - name: Push Docker image to Google Artifact Registry
-        run: |
-          docker push ${{ env.DOCKER_IMAGE }}:${{ env.BRANCH_NAME }}
-          docker push ${{ env.DOCKER_IMAGE }}:${{ env.GIT_COMMIT_SHORT }}
-          docker push ${{ env.DOCKER_IMAGE }}:${{ env.GIT_COMMIT_LONG }}
+            -t ${{ env.DOCKER_IMAGE }}:${{ env.BRANCH_NAME }}_full \
+            -t ${{ env.DOCKER_IMAGE }}:${{ env.GIT_COMMIT_SHORT }}_full \
+            -t ${{ env.DOCKER_IMAGE }}:${{ env.GIT_COMMIT_LONG }}_full
+
+      - name: Push full Docker image to Google Artifact Registry
+        run: |
+          docker push ${{ env.DOCKER_IMAGE }}:${{ env.BRANCH_NAME }}_full
+          docker push ${{ env.DOCKER_IMAGE }}:${{ env.GIT_COMMIT_SHORT }}_full
+          docker push ${{ env.DOCKER_IMAGE }}:${{ env.GIT_COMMIT_LONG }}_full

CLI.md

@@ -144,6 +144,9 @@ Client implementation and command-line tool for the Linera blockchain
 * `--max-retries <MAX_RETRIES>` — Number of times to retry connecting to a validator
 
   Default value: `10`
+* `--chrome-trace-exporter` — Enable OpenTelemetry Chrome JSON exporter for trace data analysis
+* `--otel-trace-file <OTEL_TRACE_FILE>` — Output file path for Chrome trace JSON format. Can be visualized in chrome://tracing or Perfetto UI
+* `--otel-exporter-otlp-endpoint <OTEL_EXPORTER_OTLP_ENDPOINT>` — OpenTelemetry OTLP exporter endpoint (requires tempo feature)
 * `--wait-for-outgoing-messages` — Whether to wait until a quorum of validators has confirmed that all sent cross-chain messages have been delivered
 * `--long-lived-services` — (EXPERIMENTAL) Whether application services can persist in some cases between queries
 * `--blanket-message-policy <BLANKET_MESSAGE_POLICY>` — The policy for handling incoming messages

Cargo.toml

@@ -259,6 +259,7 @@ tonic-web-wasm-client = "0.6.0"
 tower = "0.4.13"
 tower-http = "0.6.6"
 tracing = { version = "0.1.40", features = ["release_max_level_debug"] }
+tracing-chrome = "0.7.2"
 tracing-opentelemetry = "0.31.0"
 tracing-subscriber = { version = "0.3.18", default-features = false, features = [
     "env-filter",

docker/Dockerfile

@@ -25,7 +25,8 @@ ARG binaries=
 ARG copy=${binaries:+_copy}
 ARG build_flag=--release
 ARG build_folder=release
-ARG build_features=scylladb,metrics,memory-profiling,tempo
+ARG build_features=scylladb,metrics
+ARG additional_features=
 ARG rustflags="-C force-frame-pointers=yes"
 
 FROM rust:1.74-slim-bookworm AS builder
@@ -34,6 +35,7 @@ ARG target
 ARG build_flag
 ARG build_folder
 ARG build_features
+ARG additional_features
 ARG rustflags
 
 RUN apt-get update && apt-get install -y \
@@ -74,12 +76,17 @@ COPY rust-toolchain* Cargo.* ./
 ENV GIT_COMMIT=${git_commit}
 ENV RUSTFLAGS=${rustflags}
 
-RUN cargo build ${build_flag:+"$build_flag"} \
+RUN if [ -n "$additional_features" ]; then \
+        features="$build_features,$additional_features"; \
+    else \
+        features="$build_features"; \
+    fi && \
+    cargo build ${build_flag:+"$build_flag"} \
     --target "$target" \
     --bin linera \
     --bin linera-proxy \
     --bin linera-server \
-    --features $build_features
+    --features "$features"
 
 RUN mv \
     target/"$target"/"$build_folder"/linera \
@@ -119,13 +126,3 @@ COPY --from=binaries \
     linera-server \
     linera-proxy \
     ./
-
-COPY --chmod=755 \
-    docker/server-entrypoint.sh \
-    docker/server-init.sh \
-    docker/proxy-entrypoint.sh \
-    docker/proxy-init.sh \
-    docker/compose-server-entrypoint.sh \
-    docker/compose-proxy-entrypoint.sh \
-    docker/compose-server-init.sh \
-    ./

docker/docker-compose.indexer-test.yml

@@ -6,7 +6,7 @@ services:
     ports:
       - "${LINERA_STORAGE_SERVICE_PORT:-1235}:${LINERA_STORAGE_SERVICE_PORT:-1235}"
     command: >
-      sh -c "./linera-storage-server memory --endpoint 0.0.0.0:${LINERA_STORAGE_SERVICE_PORT:-1235}"
+      ./linera-storage-server memory --endpoint 0.0.0.0:${LINERA_STORAGE_SERVICE_PORT:-1235}
     environment:
       - LINERA_STORAGE_SERVICE_PORT=${LINERA_STORAGE_SERVICE_PORT:-1235}
     healthcheck:
@@ -51,10 +51,10 @@ services:
       - "${BLOCK_EXPORTER_PORT:-8882}:${BLOCK_EXPORTER_PORT:-8882}"
       - "${METRICS_PORT:-9091}:${METRICS_PORT:-9091}"
     command: >
-      sh -c "./linera-exporter 
-      --storage service:tcp:linera-storage-service:${LINERA_STORAGE_SERVICE_PORT:-1235}:linera_storage_service_server_0_db 
-      --config-path /exporter-config.toml 
-      --metrics-port ${METRICS_PORT:-9091}"
+      ./linera-exporter
+      --storage service:tcp:linera-storage-service:${LINERA_STORAGE_SERVICE_PORT:-1235}:linera_storage_service_server_0_db
+      --config-path /exporter-config.toml
+      --metrics-port ${METRICS_PORT:-9091}
     environment:
       - LINERA_STORAGE_SERVICE_PORT=${LINERA_STORAGE_SERVICE_PORT:-1235}
       - BLOCK_EXPORTER_PORT=${BLOCK_EXPORTER_PORT:-8882}
@@ -82,14 +82,14 @@ services:
       - "${FAUCET_PORT:-8080}:${FAUCET_PORT:-8080}"
       - "13001:13001"
     command: >
-      sh -c "./linera net 
+      ./linera net
       --storage service:tcp:linera-storage-service:${LINERA_STORAGE_SERVICE_PORT:-1235}:linera_storage_service
       up
       --with-faucet
       --faucet-port ${FAUCET_PORT:-8080}
       --with-block-exporter
       --exporter-address linera-block-exporter
-      --exporter-port ${BLOCK_EXPORTER_PORT:-8882}"
+      --exporter-port ${BLOCK_EXPORTER_PORT:-8882}
     environment:
       - RUST_LOG=linera=info
       - LINERA_STORAGE_SERVICE_PORT=${LINERA_STORAGE_SERVICE_PORT:-1235}

docker/docker-compose.yml

@@ -47,7 +47,14 @@ services:
     container_name: proxy
     ports:
       - "19100:19100"
-    command: ["./compose-proxy-entrypoint.sh", "1"]
+    command:
+      - "sh"
+      - "-c"
+      - |
+        exec ./linera-proxy \
+          --storage scylladb:tcp:scylla:9042 \
+          --storage-replication-factor 1 \
+          /config/server.json
     volumes:
       - .:/config
     labels:
@@ -60,7 +67,15 @@ services:
     image: "${LINERA_IMAGE:-us-docker.pkg.dev/linera-io-dev/linera-public-registry/linera:testnet_conway_release}"
     deploy:
       replicas: 4
-    command: ["./compose-server-entrypoint.sh", "scylladb:tcp:scylla:9042", "1"]
+    command:
+      - "sh"
+      - "-c"
+      - |
+        exec ./linera-server run \
+          --storage scylladb:tcp:scylla:9042 \
+          --server /config/server.json \
+          --shard 0 \
+          --storage-replication-factor 1
     volumes:
       - .:/config
     labels:
@@ -72,7 +87,31 @@ services:
   shard-init:
     image: "${LINERA_IMAGE:-us-docker.pkg.dev/linera-io-dev/linera-public-registry/linera:testnet_conway_release}"
     container_name: shard-init
-    command: ["./compose-server-init.sh", "scylladb:tcp:scylla:9042", "1"]
+    command:
+      - "sh"
+      - "-c"
+      - |
+        while true; do
+          ./linera storage check-existence --storage scylladb:tcp:scylla:9042
+          status=$?
+
+          if [ $status -eq 0 ]; then
+            echo "Database already exists, no need to initialize."
+            exit 0
+          elif [ $status -eq 1 ]; then
+            echo "Database does not exist, attempting to initialize..."
+            if ./linera storage initialize --storage scylladb:tcp:scylla:9042 --genesis /config/genesis.json; then
+              echo "Initialization successful."
+              exit 0
+            else
+              echo "Initialization failed, retrying in 5 seconds..."
+              sleep 5
+            fi
+          else
+            echo "An unexpected error occurred (status: $status), retrying in 5 seconds..."
+            sleep 5
+          fi
+        done
     volumes:
       - .:/config
     depends_on: