feat: use seaweedfs as nodestore backend (#3842)

* feat: seaweedfs as s3 nodestore backend

* fix: 'server' was missing for seaweed

* feat: remove minimum volume free space

* feat: specify hostname on ip

* fix: grpc port on seaweed should be `-{service}.port.grpc` instead of `-{service}.grpcPort`

* fix: wrong access key & secret key; use localhost for internal comms

* fix: create index directory

* test: add sentry-seaweedfs volume into expected volumes

* debug: aaaaaaaaaaaaaaaaaaaaaaarrrrggggggghhhhhhhhhhhhhhh

* test: correct ordering for expected volumes

* chore: seaweedfs healthcheck to multiple urls

See https://stackoverflow.com/a/14578575/3153224

Author: aldy505

.env

@@ -24,7 +24,3 @@ HEALTHCHECK_RETRIES=10
 POSTGRES_MAX_CONNECTIONS=100
 # Set SETUP_JS_SDK_ASSETS to 1 to enable the setup of JS SDK assets
 # SETUP_JS_SDK_ASSETS=1
-# Storage size for node store in Garage
-# If you need to change this after first install, you'd need to change the Garage layout
-# See https://garagehq.deuxfleurs.fr/documentation/quick-start/#creating-a-cluster-layout
-GARAGE_STORAGE_SIZE=100G

_unit-test/create-docker-volumes-test.sh

@@ -14,6 +14,7 @@ sentry-data
 sentry-kafka
 sentry-postgres
 sentry-redis
+sentry-seaweedfs
 sentry-symbolicator"
 
 before=$(get_volumes)

docker-compose.yml

@@ -36,7 +36,7 @@ x-sentry-defaults: &sentry_defaults
       <<: *depends_on-default
     smtp:
       <<: *depends_on-default
-    garage:
+    seaweedfs:
       <<: *depends_on-default
     snuba-api:
       <<: *depends_on-default
@@ -212,19 +212,44 @@ services:
       interval: 10s
       timeout: 10s
       retries: 30
-  garage:
-    image: dxflrs/garage:v1.0.1
+  seaweedfs:
+    image: "chrislusf/seaweedfs:3.96_large_disk"
+    entrypoint: "weed"
+    command: >-
+        server
+        -filer=true
+        -filer.port=8888
+        -filer.port.grpc=18888
+        -filer.defaultReplicaPlacement=000
+        -master=true
+        -master.port=9333
+        -master.port.grpc=19333
+        -metricsPort=9091
+        -s3=true
+        -s3.port=8333
+        -s3.port.grpc=18333
+        -volume=true
+        -volume.dir.idx=/data/idx
+        -volume.index=leveldbLarge
+        -volume.max=0
+        -volume.preStopSeconds=8
+        -volume.readMode=redirect
+        -volume.port=8080
+        -volume.port.grpc=18080
+        -ip=127.0.0.1
+        -ip.bind=0.0.0.0
+        -webdav=false
+    environment:
+      AWS_ACCESS_KEY_ID: sentry
+      AWS_SECRET_ACCESS_KEY: sentry
     volumes:
-      - type: bind
-        read_only: true
-        source: ./garage.toml
-        target: /etc/garage.toml
-      - "sentry-garage:/var/lib/garage/"
+      - "sentry-seaweedfs:/data"
     healthcheck:
-      test: ["CMD", "/garage", "status"]
-      interval: 10s
-      timeout: 2s
-      retries: 2
+      test: ["CMD", "wget", "-q", "-O-", "http://seaweedfs:8080/healthz", "http://seaweedfs:9333/cluster/healthz", "http://seaweedfs:8333/healthz"]
+      interval: 30s
+      timeout: 20s
+      retries: 5
+      start_period: 60s
   snuba-api:
     <<: *snuba_defaults
   # Kafka consumer responsible for feeding events into Clickhouse
@@ -621,6 +646,8 @@ volumes:
     external: true
   sentry-symbolicator:
     external: true
+  sentry-seaweedfs:
+    external: true
   # This volume stores JS SDK assets and the data inside this volume should
   # be cleaned periodically on upgrades.
   sentry-nginx-www:
@@ -640,4 +667,3 @@ volumes:
   sentry-kafka-log:
   sentry-smtp-log:
   sentry-clickhouse-log:
-  sentry-garage:

garage.toml

@@ -36,7 +36,7 @@ source install/ensure-relay-credentials.sh
 source install/generate-secret-key.sh
 source install/update-docker-images.sh
 source install/build-docker-images.sh
-source install/bootstrap-garage.sh
+source install/bootstrap-s3-nodestore.sh
 source install/bootstrap-snuba.sh
 source install/upgrade-postgres.sh
 source install/ensure-correct-permissions-profiles-dir.sh

install.sh

@@ -0,0 +1,25 @@
+echo "${_group}Bootstrapping seaweedfs (node store)..."
+
+$dc up --wait seaweedfs postgres
+$dc exec seaweedfs apk add --no-cache s3cmd
+$dc exec seaweedfs mkdir -p /data/idx/
+s3cmd="$dc exec seaweedfs s3cmd"
+
+bucket_list=$($s3cmd --access_key=sentry --secret_key=sentry --no-ssl --region=us-east-1 --host=localhost:8333 --host-bucket='localhost:8333/%(bucket)' ls)
+
+if [[ $($bucket_list | tail -1 | awk '{print $3}') != 's3://nodestore' ]]; then
+  # Only touch if no existing nodestore config is found
+  if ! grep -q "SENTRY_NODESTORE" $SENTRY_CONFIG_PY; then
+    nodestore_config=$(sed -n '/SENTRY_NODESTORE/,/[}]/{p}' sentry/sentry.conf.example.py)
+    if [[ $($dc exec postgres psql -qAt -U postgres -c "select exists (select * from nodestore_node limit 1)") = "f" ]]; then
+      nodestore_config=$(echo -e "$nodestore_config" | sed '$s/\}/    "read_through": True,\n    "delete_through": True,\n\}/')
+    fi
+    echo "$nodestore_config" >>$SENTRY_CONFIG_PY
+  fi
+  $dc exec seaweedfs mkdir -p /data/idx/
+  $s3cmd --access_key=sentry --secret_key=sentry --no-ssl --region=us-east-1 --host=localhost:8333 --host-bucket='localhost:8333/%(bucket)' mb s3://nodestore
+else
+  echo "Node store already exists, skipping..."
+fi
+
+echo "${_endgroup}"

install/bootstrap-garage.sh

@@ -17,5 +17,6 @@ echo "Created $(create_volume sentry-kafka)."
 echo "Created $(create_volume sentry-postgres)."
 echo "Created $(create_volume sentry-redis)."
 echo "Created $(create_volume sentry-symbolicator)."
+echo "Created $(create_volume sentry-seaweedfs)."
 
 echo "${_endgroup}"

install/bootstrap-s3-nodestore.sh

@@ -95,19 +95,29 @@ def get_internal_network():
 # See https://develop.sentry.dev/self-hosted/experimental/errors-only/
 SENTRY_SELF_HOSTED_ERRORS_ONLY = env("COMPOSE_PROFILES") != "feature-complete"
 
-##############
-# Node Store #
-##############
+################
+# Node Storage #
+################
+
+# Sentry uses an abstraction layer called "node storage" to store raw events.
+# Previously, it used PostgreSQL as the backend, but this didn't scale for
+# high-throughput environments. Read more about this in the documentation:
+# https://develop.sentry.dev/backend/application-domains/nodestore/
+#
+# Through this setting, you can use the provided blob storage or
+# your own S3-compatible API from your infrastructure.
+# Other backend implementations for node storage developed by the community
+# are available in public GitHub repositories.
 
 SENTRY_NODESTORE = "sentry_nodestore_s3.S3PassthroughDjangoNodeStorage"
 SENTRY_NODESTORE_OPTIONS = {
-    "compression": False,  # we have compression enabled in Garage itself
-    "endpoint_url": "http://garage:3900",
+    "compression": True,
+    "endpoint_url": "http://seaweedfs:8333",
     "bucket_path": "nodestore",
     "bucket_name": "nodestore",
-    "region_name": "garage",
-    "aws_access_key_id": "<GARAGE_KEY_ID>",
-    "aws_secret_access_key": "<GARAGE_SECRET_KEY>",
+    "region_name": "us-east-1",
+    "aws_access_key_id": "sentry",
+    "aws_secret_access_key": "sentry",
 }
 
 #########