From 07c92d5618ff2ab564b085cb50667eba3fbb518a Mon Sep 17 00:00:00 2001
From: Reinaldy Rafli <github@reinaldyrafli.com>
Date: Thu, 31 Jul 2025 20:57:32 +0700
Subject: [PATCH 01/11] feat: seaweedfs as s3 nodestore backend

---
 .env                              |  4 ---
 docker-compose.yml                | 46 ++++++++++++++++++++++---------
 garage.toml                       | 19 -------------
 install.sh                        |  2 +-
 install/bootstrap-garage.sh       | 37 -------------------------
 install/bootstrap-s3-nodestore.sh | 24 ++++++++++++++++
 install/create-docker-volumes.sh  |  1 +
 sentry/sentry.conf.example.py     | 26 +++++++++++------
 8 files changed, 77 insertions(+), 82 deletions(-)
 delete mode 100644 garage.toml
 delete mode 100644 install/bootstrap-garage.sh
 create mode 100644 install/bootstrap-s3-nodestore.sh

diff --git a/.env b/.env
index 916e535334..ee11f6bd67 100644
--- a/.env
+++ b/.env
@@ -24,7 +24,3 @@ HEALTHCHECK_RETRIES=10
 POSTGRES_MAX_CONNECTIONS=100
 # Set SETUP_JS_SDK_ASSETS to 1 to enable the setup of JS SDK assets
 # SETUP_JS_SDK_ASSETS=1
-# Storage size for node store in Garage
-# If you need to change this after first install, you'd need to change the Garage layout
-# See https://garagehq.deuxfleurs.fr/documentation/quick-start/#creating-a-cluster-layout
-GARAGE_STORAGE_SIZE=100G
diff --git a/docker-compose.yml b/docker-compose.yml
index 984f4ea0f3..89504974ce 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -36,7 +36,7 @@ x-sentry-defaults: &sentry_defaults
       <<: *depends_on-default
     smtp:
       <<: *depends_on-default
-    garage:
+    seaweedfs:
       <<: *depends_on-default
     snuba-api:
       <<: *depends_on-default
@@ -212,19 +212,38 @@ services:
       interval: 10s
       timeout: 10s
       retries: 30
-  garage:
-    image: dxflrs/garage:v1.0.1
+  seaweedfs:
+    image: "chrislusf/seaweedfs:3.94_large_disk"
+    entrypoint: "weed"
+    command: >-
+        -filer=true
+        -filer.port=8888
+        -filer.defaultReplicaPlacement=000
+        -master.port=9333
+        -metricsPort=9091
+        -s3=true
+        -s3.port=8333
+        -volume=true
+        -volume.dir.idx=/data/idx
+        -volume.index=leveldbLarge
+        -volume.max=30
+        -volume.minFreeSpace=50GiB
+        -volume.preStopSeconds=8
+        -volume.readMode=redirect
+        -volume.port=8080
+        -master=true
+        -webdav=false
+    environment:
+      AWS_ACCESS_KEY_ID: sentry
+      AWS_SECRET_ACCESS_KEY: sentry
     volumes:
-      - type: bind
-        read_only: true
-        source: ./garage.toml
-        target: /etc/garage.toml
-      - "sentry-garage:/var/lib/garage/"
+      - "sentry-seaweedfs:/data"
     healthcheck:
-      test: ["CMD", "/garage", "status"]
-      interval: 10s
-      timeout: 2s
-      retries: 2
+      test: ["CMD", "wget", "-q", "-O-", "http://seaweedfs:8080/healthz"]
+      interval: 30s
+      timeout: 20s
+      retries: 5
+      start_period: 60s
   snuba-api:
     <<: *snuba_defaults
   # Kafka consumer responsible for feeding events into Clickhouse
@@ -621,6 +640,8 @@ volumes:
     external: true
   sentry-symbolicator:
     external: true
+  sentry-seaweedfs:
+    external: true
   # This volume stores JS SDK assets and the data inside this volume should
   # be cleaned periodically on upgrades.
   sentry-nginx-www:
@@ -640,4 +661,3 @@ volumes:
   sentry-kafka-log:
   sentry-smtp-log:
   sentry-clickhouse-log:
-  sentry-garage:
diff --git a/garage.toml b/garage.toml
deleted file mode 100644
index d86df63e76..0000000000
--- a/garage.toml
+++ /dev/null
@@ -1,19 +0,0 @@
-metadata_dir = "/var/lib/garage/meta"
-data_dir = "/var/lib/garage/data"
-db_engine = "sqlite"
-
-replication_factor = 1
-# idk why rcp_ is needed without replication but ok
-rpc_bind_addr = "[::]:3901"
-rpc_public_addr = "0.0.0.0:3901"
-# This secret will not ever be used as we are in a single node setup
-# so this static value is OK. Don't forget to change it if you ever
-# decide to use garage in a multi-node setup.
-rpc_secret = "28f2fc651d4f280cea95e166f9113fe589039f9711d6c4f461872c1c5aaf7dec"
-
-# Setting 0 chooses the default which is 3 for now
-compression_level = 0
-
-[s3_api]
-s3_region = "garage"
-api_bind_addr = "[::]:3900"
diff --git a/install.sh b/install.sh
index 04305ab136..c6b3a62a17 100755
--- a/install.sh
+++ b/install.sh
@@ -36,7 +36,7 @@ source install/ensure-relay-credentials.sh
 source install/generate-secret-key.sh
 source install/update-docker-images.sh
 source install/build-docker-images.sh
-source install/bootstrap-garage.sh
+source install/bootstrap-s3-nodestore.sh
 source install/bootstrap-snuba.sh
 source install/upgrade-postgres.sh
 source install/ensure-correct-permissions-profiles-dir.sh
diff --git a/install/bootstrap-garage.sh b/install/bootstrap-garage.sh
deleted file mode 100644
index 568675ed01..0000000000
--- a/install/bootstrap-garage.sh
+++ /dev/null
@@ -1,37 +0,0 @@
-echo "${_group}Bootstrapping garage (node store)..."
-
-$dc up --wait garage postgres
-garage="$dc exec garage /garage"
-
-if [[ $($garage bucket list | tail -1 | awk '{print $1}') != 'nodestore' ]]; then
-  node_id=$($garage status | tail -1 | awk '{print $1}')
-  $garage layout assign -z local -c $GARAGE_STORAGE_SIZE "$node_id"
-  $garage layout apply --version 1
-
-  # Only touch if no existing nodestore config is found
-  if ! grep -q "SENTRY_NODESTORE" $SENTRY_CONFIG_PY; then
-    nodestore_config=$(sed -n '/SENTRY_NODESTORE/,/[}]/{p}' sentry/sentry.conf.example.py)
-    if [[ $($dc exec postgres psql -qAt -U postgres -c "select exists (select * from nodestore_node limit 1)") = "f" ]]; then
-      nodestore_config=$(echo -e "$nodestore_config" | sed '$s/\}/    "read_through": True,\n    "delete_through": True,\n\}/')
-    fi
-    echo "$nodestore_config" >>$SENTRY_CONFIG_PY
-  fi
-
-  $garage bucket create nodestore
-  key_info=$($garage key create nodestore-key | head -3 | tail -2)
-  echo "$key_info"
-  key_id=$(echo "$key_info" | head -1 | awk '{print $3}')
-  key_secret=$(echo "$key_info" | tail -1 | awk '{print $3}')
-
-  $garage bucket allow --read --write --owner nodestore --key nodestore-key
-
-  if grep -q "<GARAGE_KEY_ID>" $SENTRY_CONFIG_PY; then
-    sed -i -e "s/<GARAGE_KEY_ID>/$key_id/" $SENTRY_CONFIG_PY
-    sed -i -e "s/<GARAGE_SECRET_KEY>/$key_secret/" $SENTRY_CONFIG_PY
-    echo "Set Garage keys for SENTRY_NODESTORE_OPTIONS in $SENTRY_CONFIG_PY"
-  fi
-else
-  echo "Node store already exists, skipping..."
-fi
-
-echo "${_endgroup}"
diff --git a/install/bootstrap-s3-nodestore.sh b/install/bootstrap-s3-nodestore.sh
new file mode 100644
index 0000000000..a2d6cd30d9
--- /dev/null
+++ b/install/bootstrap-s3-nodestore.sh
@@ -0,0 +1,24 @@
+echo "${_group}Bootstrapping seaweedfs (node store)..."
+
+$dc up --wait seaweedfs postgres
+$dc exec seaweedfs apk add --no-cache s3cmd
+s3cmd="$dc exec seaweedfs s3cmd"
+
+bucket_list=$($s3cmd --access_key=sentry --secret_key=sentry --no-ssl --region=us-east-1 --host=localhost:8333 --host-bucket='localhost:8333/%(bucket)' ls)
+
+if [[ $($bucket_list | tail -1 | awk '{print $3}') != 's3://nodestore' ]]; then
+  # Only touch if no existing nodestore config is found
+  if ! grep -q "SENTRY_NODESTORE" $SENTRY_CONFIG_PY; then
+    nodestore_config=$(sed -n '/SENTRY_NODESTORE/,/[}]/{p}' sentry/sentry.conf.example.py)
+    if [[ $($dc exec postgres psql -qAt -U postgres -c "select exists (select * from nodestore_node limit 1)") = "f" ]]; then
+      nodestore_config=$(echo -e "$nodestore_config" | sed '$s/\}/    "read_through": True,\n    "delete_through": True,\n\}/')
+    fi
+    echo "$nodestore_config" >>$SENTRY_CONFIG_PY
+  fi
+
+  $s3cmd --access_key=foo --secret_key=bar --no-ssl --region=us-east-1 --host=localhost:8333 --host-bucket='localhost:8333/%(bucket)' mb s3://nodestore
+else
+  echo "Node store already exists, skipping..."
+fi
+
+echo "${_endgroup}"
diff --git a/install/create-docker-volumes.sh b/install/create-docker-volumes.sh
index fdbecc2288..c0437c5563 100644
--- a/install/create-docker-volumes.sh
+++ b/install/create-docker-volumes.sh
@@ -17,5 +17,6 @@ echo "Created $(create_volume sentry-kafka)."
 echo "Created $(create_volume sentry-postgres)."
 echo "Created $(create_volume sentry-redis)."
 echo "Created $(create_volume sentry-symbolicator)."
+echo "Created $(create_volume sentry-seaweedfs)."
 
 echo "${_endgroup}"
diff --git a/sentry/sentry.conf.example.py b/sentry/sentry.conf.example.py
index f4aa9901eb..12ee85db0f 100644
--- a/sentry/sentry.conf.example.py
+++ b/sentry/sentry.conf.example.py
@@ -95,19 +95,29 @@ def get_internal_network():
 # See https://develop.sentry.dev/self-hosted/experimental/errors-only/
 SENTRY_SELF_HOSTED_ERRORS_ONLY = env("COMPOSE_PROFILES") != "feature-complete"
 
-##############
-# Node Store #
-##############
+################
+# Node Storage #
+################
+
+# Sentry uses an abstraction layer called "node storage" to store raw events.
+# Previously, it used PostgreSQL as the backend, but this didn't scale for
+# high-throughput environments. Read more about this in the documentation:
+# https://develop.sentry.dev/backend/application-domains/nodestore/
+#
+# Through this setting, you can use the provided blob storage or
+# your own S3-compatible API from your infrastructure.
+# Other backend implementations for node storage developed by the community
+# are available in public GitHub repositories.
 
 SENTRY_NODESTORE = "sentry_nodestore_s3.S3PassthroughDjangoNodeStorage"
 SENTRY_NODESTORE_OPTIONS = {
-    "compression": False,  # we have compression enabled in Garage itself
-    "endpoint_url": "http://garage:3900",
+    "compression": True,
+    "endpoint_url": "http://seaweedfs:8333",
     "bucket_path": "nodestore",
     "bucket_name": "nodestore",
-    "region_name": "garage",
-    "aws_access_key_id": "<GARAGE_KEY_ID>",
-    "aws_secret_access_key": "<GARAGE_SECRET_KEY>",
+    "region_name": "us-east-1",
+    "aws_access_key_id": "sentry",
+    "aws_secret_access_key": "sentry",
 }
 
 #########

From 160cca7116e5c484b3769542e825caf11b0bd0c2 Mon Sep 17 00:00:00 2001
From: Reinaldy Rafli <github@reinaldyrafli.com>
Date: Thu, 31 Jul 2025 21:21:36 +0700
Subject: [PATCH 02/11] fix: 'server' was missing for seaweed

---
 docker-compose.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docker-compose.yml b/docker-compose.yml
index 89504974ce..8da0ed3132 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -216,6 +216,7 @@ services:
     image: "chrislusf/seaweedfs:3.94_large_disk"
     entrypoint: "weed"
     command: >-
+        server
         -filer=true
         -filer.port=8888
         -filer.defaultReplicaPlacement=000

From fb303bd4ae03031c1fa4346e912382d298e60320 Mon Sep 17 00:00:00 2001
From: Reinaldy Rafli <github@reinaldyrafli.com>
Date: Mon, 4 Aug 2025 08:11:38 +0700
Subject: [PATCH 03/11] feat: remove minimum volume free space

---
 docker-compose.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/docker-compose.yml b/docker-compose.yml
index 8da0ed3132..76c55be749 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -220,19 +220,19 @@ services:
         -filer=true
         -filer.port=8888
         -filer.defaultReplicaPlacement=000
+        -master=true
         -master.port=9333
+        -master.volumeSizeLimitMB=1024
         -metricsPort=9091
         -s3=true
         -s3.port=8333
         -volume=true
         -volume.dir.idx=/data/idx
         -volume.index=leveldbLarge
-        -volume.max=30
-        -volume.minFreeSpace=50GiB
+        -volume.max=0
         -volume.preStopSeconds=8
         -volume.readMode=redirect
         -volume.port=8080
-        -master=true
         -webdav=false
     environment:
       AWS_ACCESS_KEY_ID: sentry

From e1e1a23580d5ff0d0ae89932b03838b0ed706f11 Mon Sep 17 00:00:00 2001
From: Reinaldy Rafli <github@reinaldyrafli.com>
Date: Mon, 4 Aug 2025 08:26:46 +0700
Subject: [PATCH 04/11] feat: specify hostname on ip

---
 docker-compose.yml | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/docker-compose.yml b/docker-compose.yml
index 76c55be749..996dc2af4b 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -213,16 +213,17 @@ services:
       timeout: 10s
       retries: 30
   seaweedfs:
-    image: "chrislusf/seaweedfs:3.94_large_disk"
+    image: "chrislusf/seaweedfs:3.96_large_disk"
     entrypoint: "weed"
     command: >-
         server
         -filer=true
         -filer.port=8888
+        -filer.grpcPort=18888
         -filer.defaultReplicaPlacement=000
         -master=true
         -master.port=9333
-        -master.volumeSizeLimitMB=1024
+        -master.grpcPort=19333
         -metricsPort=9091
         -s3=true
         -s3.port=8333
@@ -233,6 +234,8 @@ services:
         -volume.preStopSeconds=8
         -volume.readMode=redirect
         -volume.port=8080
+        -volume.grpcPort=18080
+        -ip=seaweedfs
         -webdav=false
     environment:
       AWS_ACCESS_KEY_ID: sentry

From b09f4bbcea5b573900d55a6b192caca9ff4c1b45 Mon Sep 17 00:00:00 2001
From: Reinaldy Rafli <github@reinaldyrafli.com>
Date: Tue, 5 Aug 2025 08:01:56 +0700
Subject: [PATCH 05/11] fix: grpc port on seaweed should be
 `-{service}.port.grpc` instead of `-{service}.grpcPort`

---
 docker-compose.yml | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/docker-compose.yml b/docker-compose.yml
index 996dc2af4b..6cf33a0324 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -219,14 +219,15 @@ services:
         server
         -filer=true
         -filer.port=8888
-        -filer.grpcPort=18888
+        -filer.port.grpc=18888
         -filer.defaultReplicaPlacement=000
         -master=true
         -master.port=9333
-        -master.grpcPort=19333
+        -master.port.grpc=19333
         -metricsPort=9091
         -s3=true
         -s3.port=8333
+        -s3.port.grpc=18333
         -volume=true
         -volume.dir.idx=/data/idx
         -volume.index=leveldbLarge
@@ -234,8 +235,9 @@ services:
         -volume.preStopSeconds=8
         -volume.readMode=redirect
         -volume.port=8080
-        -volume.grpcPort=18080
+        -volume.port.grpc=18080
         -ip=seaweedfs
+        -ip.bind=0.0.0.0
         -webdav=false
     environment:
       AWS_ACCESS_KEY_ID: sentry

From 6fea950c755e99d86f8bbf2c206e11b6b10eedb2 Mon Sep 17 00:00:00 2001
From: Reinaldy Rafli <github@reinaldyrafli.com>
Date: Tue, 5 Aug 2025 11:15:48 +0700
Subject: [PATCH 06/11] fix: wrong access key & secret key; use localhost for
 internal comms

---
 docker-compose.yml                | 2 +-
 install/bootstrap-s3-nodestore.sh | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/docker-compose.yml b/docker-compose.yml
index 6cf33a0324..c9247f6126 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -236,7 +236,7 @@ services:
         -volume.readMode=redirect
         -volume.port=8080
         -volume.port.grpc=18080
-        -ip=seaweedfs
+        -ip=127.0.0.1
         -ip.bind=0.0.0.0
         -webdav=false
     environment:
diff --git a/install/bootstrap-s3-nodestore.sh b/install/bootstrap-s3-nodestore.sh
index a2d6cd30d9..b89f028397 100644
--- a/install/bootstrap-s3-nodestore.sh
+++ b/install/bootstrap-s3-nodestore.sh
@@ -16,7 +16,7 @@ if [[ $($bucket_list | tail -1 | awk '{print $3}') != 's3://nodestore' ]]; then
     echo "$nodestore_config" >>$SENTRY_CONFIG_PY
   fi
 
-  $s3cmd --access_key=foo --secret_key=bar --no-ssl --region=us-east-1 --host=localhost:8333 --host-bucket='localhost:8333/%(bucket)' mb s3://nodestore
+  $s3cmd --access_key=sentry --secret_key=sentry --no-ssl --region=us-east-1 --host=localhost:8333 --host-bucket='localhost:8333/%(bucket)' mb s3://nodestore
 else
   echo "Node store already exists, skipping..."
 fi

From 38d9c12330dd06c669ea7078b9121654d178f0a9 Mon Sep 17 00:00:00 2001
From: Reinaldy Rafli <github@reinaldyrafli.com>
Date: Tue, 5 Aug 2025 14:32:32 +0700
Subject: [PATCH 07/11] fix: create index directory

---
 install/bootstrap-s3-nodestore.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/install/bootstrap-s3-nodestore.sh b/install/bootstrap-s3-nodestore.sh
index b89f028397..c418fef9c5 100644
--- a/install/bootstrap-s3-nodestore.sh
+++ b/install/bootstrap-s3-nodestore.sh
@@ -15,7 +15,7 @@ if [[ $($bucket_list | tail -1 | awk '{print $3}') != 's3://nodestore' ]]; then
     fi
     echo "$nodestore_config" >>$SENTRY_CONFIG_PY
   fi
-
+  $dc exec seaweedfs mkdir -p /data/idx/
   $s3cmd --access_key=sentry --secret_key=sentry --no-ssl --region=us-east-1 --host=localhost:8333 --host-bucket='localhost:8333/%(bucket)' mb s3://nodestore
 else
   echo "Node store already exists, skipping..."

From 30a11e738427b328d4844a12dbcfda431b47d275 Mon Sep 17 00:00:00 2001
From: Reinaldy Rafli <github@reinaldyrafli.com>
Date: Tue, 5 Aug 2025 15:16:48 +0700
Subject: [PATCH 08/11] test: add sentry-seaweedfs volume into expected volumes

---
 _unit-test/create-docker-volumes-test.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/_unit-test/create-docker-volumes-test.sh b/_unit-test/create-docker-volumes-test.sh
index 2cb9b962a8..ce82b99ed3 100755
--- a/_unit-test/create-docker-volumes-test.sh
+++ b/_unit-test/create-docker-volumes-test.sh
@@ -14,7 +14,8 @@ sentry-data
 sentry-kafka
 sentry-postgres
 sentry-redis
-sentry-symbolicator"
+sentry-symbolicator
+sentry-seaweedfs"
 
 before=$(get_volumes)
 

From 3a8c05036b196d4b3638feef49c1a07629918a82 Mon Sep 17 00:00:00 2001
From: Reinaldy Rafli <github@reinaldyrafli.com>
Date: Tue, 5 Aug 2025 15:25:42 +0700
Subject: [PATCH 09/11] debug:
 aaaaaaaaaaaaaaaaaaaaaaarrrrggggggghhhhhhhhhhhhhhh

---
 _unit-test/create-docker-volumes-test.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/_unit-test/create-docker-volumes-test.sh b/_unit-test/create-docker-volumes-test.sh
index ce82b99ed3..5169cf9e06 100755
--- a/_unit-test/create-docker-volumes-test.sh
+++ b/_unit-test/create-docker-volumes-test.sh
@@ -24,6 +24,7 @@ test "$before" == "" || test "$before" == "$expected_volumes"
 source install/create-docker-volumes.sh
 
 after=$(get_volumes)
+echo "$after"
 test "$after" == "$expected_volumes"
 
 report_success

From 0bdf72b8cb8e27527cf5ff93fb33b16f2412b7e6 Mon Sep 17 00:00:00 2001
From: Reinaldy Rafli <github@reinaldyrafli.com>
Date: Tue, 5 Aug 2025 15:29:40 +0700
Subject: [PATCH 10/11] test: correct ordering for expected volumes

---
 _unit-test/create-docker-volumes-test.sh | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/_unit-test/create-docker-volumes-test.sh b/_unit-test/create-docker-volumes-test.sh
index 5169cf9e06..83591d3aa8 100755
--- a/_unit-test/create-docker-volumes-test.sh
+++ b/_unit-test/create-docker-volumes-test.sh
@@ -14,8 +14,8 @@ sentry-data
 sentry-kafka
 sentry-postgres
 sentry-redis
-sentry-symbolicator
-sentry-seaweedfs"
+sentry-seaweedfs
+sentry-symbolicator"
 
 before=$(get_volumes)
 
@@ -24,7 +24,6 @@ test "$before" == "" || test "$before" == "$expected_volumes"
 source install/create-docker-volumes.sh
 
 after=$(get_volumes)
-echo "$after"
 test "$after" == "$expected_volumes"
 
 report_success

From d25faa3e796f605b63b202a6a0df2f68f389379a Mon Sep 17 00:00:00 2001
From: Reinaldy Rafli <github@reinaldyrafli.com>
Date: Tue, 5 Aug 2025 15:55:54 +0700
Subject: [PATCH 11/11] chore: seaweedfs healthcheck to multiple urls

See https://stackoverflow.com/a/14578575/3153224
---
 docker-compose.yml                | 2 +-
 install/bootstrap-s3-nodestore.sh | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/docker-compose.yml b/docker-compose.yml
index c9247f6126..3d139e4570 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -245,7 +245,7 @@ services:
     volumes:
       - "sentry-seaweedfs:/data"
     healthcheck:
-      test: ["CMD", "wget", "-q", "-O-", "http://seaweedfs:8080/healthz"]
+      test: ["CMD", "wget", "-q", "-O-", "http://seaweedfs:8080/healthz", "http://seaweedfs:9333/cluster/healthz", "http://seaweedfs:8333/healthz"]
       interval: 30s
       timeout: 20s
       retries: 5
diff --git a/install/bootstrap-s3-nodestore.sh b/install/bootstrap-s3-nodestore.sh
index c418fef9c5..4ff465c146 100644
--- a/install/bootstrap-s3-nodestore.sh
+++ b/install/bootstrap-s3-nodestore.sh
@@ -2,6 +2,7 @@ echo "${_group}Bootstrapping seaweedfs (node store)..."
 
 $dc up --wait seaweedfs postgres
 $dc exec seaweedfs apk add --no-cache s3cmd
+$dc exec seaweedfs mkdir -p /data/idx/
 s3cmd="$dc exec seaweedfs s3cmd"
 
 bucket_list=$($s3cmd --access_key=sentry --secret_key=sentry --no-ssl --region=us-east-1 --host=localhost:8333 --host-bucket='localhost:8333/%(bucket)' ls)