diff --git a/.env b/.env index 916e535334..ee11f6bd67 100644 --- a/.env +++ b/.env @@ -24,7 +24,3 @@ HEALTHCHECK_RETRIES=10 POSTGRES_MAX_CONNECTIONS=100 # Set SETUP_JS_SDK_ASSETS to 1 to enable the setup of JS SDK assets # SETUP_JS_SDK_ASSETS=1 -# Storage size for node store in Garage -# If you need to change this after first install, you'd need to change the Garage layout -# See https://garagehq.deuxfleurs.fr/documentation/quick-start/#creating-a-cluster-layout -GARAGE_STORAGE_SIZE=100G diff --git a/_unit-test/create-docker-volumes-test.sh b/_unit-test/create-docker-volumes-test.sh index 2cb9b962a8..83591d3aa8 100755 --- a/_unit-test/create-docker-volumes-test.sh +++ b/_unit-test/create-docker-volumes-test.sh @@ -14,6 +14,7 @@ sentry-data sentry-kafka sentry-postgres sentry-redis +sentry-seaweedfs sentry-symbolicator" before=$(get_volumes) diff --git a/docker-compose.yml b/docker-compose.yml index 984f4ea0f3..3d139e4570 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -36,7 +36,7 @@ x-sentry-defaults: &sentry_defaults <<: *depends_on-default smtp: <<: *depends_on-default - garage: + seaweedfs: <<: *depends_on-default snuba-api: <<: *depends_on-default @@ -212,19 +212,44 @@ services: interval: 10s timeout: 10s retries: 30 - garage: - image: dxflrs/garage:v1.0.1 + seaweedfs: + image: "chrislusf/seaweedfs:3.96_large_disk" + entrypoint: "weed" + command: >- + server + -filer=true + -filer.port=8888 + -filer.port.grpc=18888 + -filer.defaultReplicaPlacement=000 + -master=true + -master.port=9333 + -master.port.grpc=19333 + -metricsPort=9091 + -s3=true + -s3.port=8333 + -s3.port.grpc=18333 + -volume=true + -volume.dir.idx=/data/idx + -volume.index=leveldbLarge + -volume.max=0 + -volume.preStopSeconds=8 + -volume.readMode=redirect + -volume.port=8080 + -volume.port.grpc=18080 + -ip=127.0.0.1 + -ip.bind=0.0.0.0 + -webdav=false + environment: + AWS_ACCESS_KEY_ID: sentry + AWS_SECRET_ACCESS_KEY: sentry volumes: - - type: bind - read_only: true - source: ./garage.toml - target: /etc/garage.toml - - "sentry-garage:/var/lib/garage/" + - "sentry-seaweedfs:/data" healthcheck: - test: ["CMD", "/garage", "status"] - interval: 10s - timeout: 2s - retries: 2 + test: ["CMD", "wget", "-q", "-O-", "http://seaweedfs:8080/healthz", "http://seaweedfs:9333/cluster/healthz", "http://seaweedfs:8333/healthz"] + interval: 30s + timeout: 20s + retries: 5 + start_period: 60s snuba-api: <<: *snuba_defaults # Kafka consumer responsible for feeding events into Clickhouse @@ -621,6 +646,8 @@ volumes: external: true sentry-symbolicator: external: true + sentry-seaweedfs: + external: true # This volume stores JS SDK assets and the data inside this volume should # be cleaned periodically on upgrades. sentry-nginx-www: @@ -640,4 +667,3 @@ volumes: sentry-kafka-log: sentry-smtp-log: sentry-clickhouse-log: - sentry-garage: diff --git a/garage.toml b/garage.toml deleted file mode 100644 index d86df63e76..0000000000 --- a/garage.toml +++ /dev/null @@ -1,19 +0,0 @@ -metadata_dir = "/var/lib/garage/meta" -data_dir = "/var/lib/garage/data" -db_engine = "sqlite" - -replication_factor = 1 -# idk why rcp_ is needed without replication but ok -rpc_bind_addr = "[::]:3901" -rpc_public_addr = "0.0.0.0:3901" -# This secret will not ever be used as we are in a single node setup -# so this static value is OK. Don't forget to change it if you ever -# decide to use garage in a multi-node setup. -rpc_secret = "28f2fc651d4f280cea95e166f9113fe589039f9711d6c4f461872c1c5aaf7dec" - -# Setting 0 chooses the default which is 3 for now -compression_level = 0 - -[s3_api] -s3_region = "garage" -api_bind_addr = "[::]:3900" diff --git a/install.sh b/install.sh index 04305ab136..c6b3a62a17 100755 --- a/install.sh +++ b/install.sh @@ -36,7 +36,7 @@ source install/ensure-relay-credentials.sh source install/generate-secret-key.sh source install/update-docker-images.sh source install/build-docker-images.sh -source install/bootstrap-garage.sh +source install/bootstrap-s3-nodestore.sh source install/bootstrap-snuba.sh source install/upgrade-postgres.sh source install/ensure-correct-permissions-profiles-dir.sh diff --git a/install/bootstrap-garage.sh b/install/bootstrap-garage.sh deleted file mode 100644 index 568675ed01..0000000000 --- a/install/bootstrap-garage.sh +++ /dev/null @@ -1,37 +0,0 @@ -echo "${_group}Bootstrapping garage (node store)..." - -$dc up --wait garage postgres -garage="$dc exec garage /garage" - -if [[ $($garage bucket list | tail -1 | awk '{print $1}') != 'nodestore' ]]; then - node_id=$($garage status | tail -1 | awk '{print $1}') - $garage layout assign -z local -c $GARAGE_STORAGE_SIZE "$node_id" - $garage layout apply --version 1 - - # Only touch if no existing nodestore config is found - if ! grep -q "SENTRY_NODESTORE" $SENTRY_CONFIG_PY; then - nodestore_config=$(sed -n '/SENTRY_NODESTORE/,/[}]/{p}' sentry/sentry.conf.example.py) - if [[ $($dc exec postgres psql -qAt -U postgres -c "select exists (select * from nodestore_node limit 1)") = "f" ]]; then - nodestore_config=$(echo -e "$nodestore_config" | sed '$s/\}/ "read_through": True,\n "delete_through": True,\n\}/') - fi - echo "$nodestore_config" >>$SENTRY_CONFIG_PY - fi - - $garage bucket create nodestore - key_info=$($garage key create nodestore-key | head -3 | tail -2) - echo "$key_info" - key_id=$(echo "$key_info" | head -1 | awk '{print $3}') - key_secret=$(echo "$key_info" | tail -1 | awk '{print $3}') - - $garage bucket allow --read --write --owner nodestore --key nodestore-key - - if grep -q "" $SENTRY_CONFIG_PY; then - sed -i -e "s//$key_id/" $SENTRY_CONFIG_PY - sed -i -e "s//$key_secret/" $SENTRY_CONFIG_PY - echo "Set Garage keys for SENTRY_NODESTORE_OPTIONS in $SENTRY_CONFIG_PY" - fi -else - echo "Node store already exists, skipping..." -fi - -echo "${_endgroup}" diff --git a/install/bootstrap-s3-nodestore.sh b/install/bootstrap-s3-nodestore.sh new file mode 100644 index 0000000000..4ff465c146 --- /dev/null +++ b/install/bootstrap-s3-nodestore.sh @@ -0,0 +1,25 @@ +echo "${_group}Bootstrapping seaweedfs (node store)..." + +$dc up --wait seaweedfs postgres +$dc exec seaweedfs apk add --no-cache s3cmd +$dc exec seaweedfs mkdir -p /data/idx/ +s3cmd="$dc exec seaweedfs s3cmd" + +bucket_list=$($s3cmd --access_key=sentry --secret_key=sentry --no-ssl --region=us-east-1 --host=localhost:8333 --host-bucket='localhost:8333/%(bucket)' ls) + +if [[ $($bucket_list | tail -1 | awk '{print $3}') != 's3://nodestore' ]]; then + # Only touch if no existing nodestore config is found + if ! grep -q "SENTRY_NODESTORE" $SENTRY_CONFIG_PY; then + nodestore_config=$(sed -n '/SENTRY_NODESTORE/,/[}]/{p}' sentry/sentry.conf.example.py) + if [[ $($dc exec postgres psql -qAt -U postgres -c "select exists (select * from nodestore_node limit 1)") = "f" ]]; then + nodestore_config=$(echo -e "$nodestore_config" | sed '$s/\}/ "read_through": True,\n "delete_through": True,\n\}/') + fi + echo "$nodestore_config" >>$SENTRY_CONFIG_PY + fi + $dc exec seaweedfs mkdir -p /data/idx/ + $s3cmd --access_key=sentry --secret_key=sentry --no-ssl --region=us-east-1 --host=localhost:8333 --host-bucket='localhost:8333/%(bucket)' mb s3://nodestore +else + echo "Node store already exists, skipping..." +fi + +echo "${_endgroup}" diff --git a/install/create-docker-volumes.sh b/install/create-docker-volumes.sh index fdbecc2288..c0437c5563 100644 --- a/install/create-docker-volumes.sh +++ b/install/create-docker-volumes.sh @@ -17,5 +17,6 @@ echo "Created $(create_volume sentry-kafka)." echo "Created $(create_volume sentry-postgres)." echo "Created $(create_volume sentry-redis)." echo "Created $(create_volume sentry-symbolicator)." +echo "Created $(create_volume sentry-seaweedfs)." echo "${_endgroup}" diff --git a/sentry/sentry.conf.example.py b/sentry/sentry.conf.example.py index f4aa9901eb..12ee85db0f 100644 --- a/sentry/sentry.conf.example.py +++ b/sentry/sentry.conf.example.py @@ -95,19 +95,29 @@ def get_internal_network(): # See https://develop.sentry.dev/self-hosted/experimental/errors-only/ SENTRY_SELF_HOSTED_ERRORS_ONLY = env("COMPOSE_PROFILES") != "feature-complete" -############## -# Node Store # -############## +################ +# Node Storage # +################ + +# Sentry uses an abstraction layer called "node storage" to store raw events. +# Previously, it used PostgreSQL as the backend, but this didn't scale for +# high-throughput environments. Read more about this in the documentation: +# https://develop.sentry.dev/backend/application-domains/nodestore/ +# +# Through this setting, you can use the provided blob storage or +# your own S3-compatible API from your infrastructure. +# Other backend implementations for node storage developed by the community +# are available in public GitHub repositories. SENTRY_NODESTORE = "sentry_nodestore_s3.S3PassthroughDjangoNodeStorage" SENTRY_NODESTORE_OPTIONS = { - "compression": False, # we have compression enabled in Garage itself - "endpoint_url": "http://garage:3900", + "compression": True, + "endpoint_url": "http://seaweedfs:8333", "bucket_path": "nodestore", "bucket_name": "nodestore", - "region_name": "garage", - "aws_access_key_id": "", - "aws_secret_access_key": "", + "region_name": "us-east-1", + "aws_access_key_id": "sentry", + "aws_secret_access_key": "sentry", } #########