diff --git a/cmd/elastic.sh b/cmd/elastic.sh index 701c19c9..f627d279 100644 --- a/cmd/elastic.sh +++ b/cmd/elastic.sh @@ -5,7 +5,7 @@ function elastic_schema_drop(){ compose_run 'schema' node scripts/drop_index "$@ function elastic_schema_create(){ compose_run 'schema' ./bin/create_index; } function elastic_start(){ mkdir -p $DATA_DIR/elasticsearch - # attemp to set proper permissions if running as root + # Attempt to set proper permissions if running as root chown $DOCKER_USER $DATA_DIR/elasticsearch 2>/dev/null || true compose_exec up -d elasticsearch } @@ -24,11 +24,11 @@ function elastic_status(){ --output /dev/null \ --silent \ --write-out "%{http_code}" \ - "http://${ELASTIC_HOST:-localhost:9200}/_cluster/health?wait_for_status=yellow&timeout=1s" \ + "http://${ELASTIC_HOST:-localhost}:${ELASTIC_PORT:-9200}/_cluster/health?wait_for_status=yellow&timeout=1s" \ || true; } -# the same function but with a trailing newline +# Function to get HTTP status with a trailing newline, using optional port function elastic_status_newline(){ echo $(elastic_status); } register 'elastic' 'status' 'HTTP status code of the elasticsearch service' elastic_status_newline @@ -39,10 +39,10 @@ function elastic_wait(){ i=1 while [[ "$i" -le "$retry_count" ]]; do if [[ $(elastic_status) -eq 200 ]]; then - echo "Elasticsearch up!" + echo "Elasticsearch up on port ${ELASTIC_PORT:-9200}!" exit 0 elif [[ $(elastic_status) -eq 408 ]]; then - # 408 indicates the server is up but not yet yellow status + # 408 indicates the server is up but has not reached yellow status yet printf ":" else printf "." @@ -58,11 +58,13 @@ function elastic_wait(){ register 'elastic' 'wait' 'wait for elasticsearch to start up' elastic_wait -function elastic_info(){ curl -s "http://${ELASTIC_HOST:-localhost:9200}/"; } +# Function to get Elasticsearch version and build info with an optional port argument +function elastic_info(){ curl -s "http://${ELASTIC_HOST:-localhost}:${ELASTIC_PORT:-9200}/"; } register 'elastic' 'info' 'display elasticsearch version and build info' elastic_info +# Function to display a summary of document counts per source/layer with optional port function elastic_stats(){ - curl -s "http://${ELASTIC_HOST:-localhost:9200}/pelias/_search?request_cache=true&timeout=10s&pretty=true" \ + curl -s "http://${ELASTIC_HOST:-localhost}:${ELASTIC_PORT:-9200}/pelias/_search?request_cache=true&timeout=10s&pretty=true" \ -H 'Content-Type: application/json' \ -d '{ "aggs": { diff --git a/projects/germany/.env b/projects/germany/.env index c27c8edb..0b44f0ba 100644 --- a/projects/germany/.env +++ b/projects/germany/.env @@ -1,2 +1,2 @@ -COMPOSE_PROJECT_NAME=pelias +COMPOSE_PROJECT_NAME=pelias_germany DATA_DIR=./data diff --git a/projects/germany/README.md b/projects/germany/README.md index 68e78bdb..6ab11727 100644 --- a/projects/germany/README.md +++ b/projects/germany/README.md @@ -14,17 +14,10 @@ Please ensure that's all working fine before continuing. # Run a Build -To run a complete build, execute the following commands: - -```bash -pelias compose pull -pelias elastic start -pelias elastic wait -pelias elastic create -pelias download all -pelias prepare all -pelias import all -pelias compose up +To run a complete build, execute +``` +chmod +x generate.sh +./generate.sh ``` # Make an Example Query diff --git a/projects/germany/docker-compose.yml b/projects/germany/docker-compose.yml index 5399075d..ef3b2d43 100644 --- a/projects/germany/docker-compose.yml +++ b/projects/germany/docker-compose.yml @@ -1,7 +1,8 @@ -version: '3' networks: default: driver: bridge + shared-net: + external: true # Dies gibt an, dass es sich um ein externes Netzwerk handelt services: libpostal: image: pelias/libpostal-service @@ -9,12 +10,18 @@ services: user: "${DOCKER_USER}" restart: always ports: [ "127.0.0.1:4400:4400" ] + networks: + - default + - shared-net schema: image: pelias/schema:master container_name: pelias_schema user: "${DOCKER_USER}" volumes: - "./pelias.json:/code/pelias.json" + networks: + - default + - shared-net api: image: pelias/api:master container_name: pelias_api @@ -24,6 +31,9 @@ services: ports: [ "0.0.0.0:4000:4000" ] volumes: - "./pelias.json:/code/pelias.json" + networks: + - default + - shared-net placeholder: image: pelias/placeholder:master container_name: pelias_placeholder @@ -35,6 +45,9 @@ services: - "./pelias.json:/code/pelias.json" - "${DATA_DIR}:/data" - "./blacklist/:/data/blacklist" + networks: + - default + - shared-net whosonfirst: image: pelias/whosonfirst:master container_name: pelias_whosonfirst @@ -42,6 +55,9 @@ services: volumes: - "./pelias.json:/code/pelias.json" - "${DATA_DIR}:/data" + networks: + - default + - shared-net openstreetmap: image: pelias/openstreetmap:master container_name: pelias_openstreetmap @@ -49,6 +65,9 @@ services: volumes: - "./pelias.json:/code/pelias.json" - "${DATA_DIR}:/data" + networks: + - default + - shared-net openaddresses: image: pelias/openaddresses:master container_name: pelias_openaddresses @@ -56,6 +75,9 @@ services: volumes: - "./pelias.json:/code/pelias.json" - "${DATA_DIR}:/data" + networks: + - default + - shared-net csv-importer: image: pelias/csv-importer:master container_name: pelias_csv_importer @@ -64,6 +86,9 @@ services: - "./pelias.json:/code/pelias.json" - "${DATA_DIR}:/data" - "./blacklist/:/data/blacklist" + networks: + - default + - shared-net polylines: image: pelias/polylines:master container_name: pelias_polylines @@ -71,6 +96,9 @@ services: volumes: - "./pelias.json:/code/pelias.json" - "${DATA_DIR}:/data" + networks: + - default + - shared-net interpolation: image: pelias/interpolation:master container_name: pelias_interpolation @@ -81,6 +109,9 @@ services: volumes: - "./pelias.json:/code/pelias.json" - "${DATA_DIR}:/data" + networks: + - default + - shared-net pip: image: pelias/pip-service:master container_name: pelias_pip-service @@ -91,6 +122,9 @@ services: volumes: - "./pelias.json:/code/pelias.json" - "${DATA_DIR}:/data" + networks: + - default + - shared-net elasticsearch: image: pelias/elasticsearch:7.16.1 container_name: pelias_elasticsearch @@ -110,3 +144,6 @@ services: cap_add: [ "IPC_LOCK" ] security_opt: - seccomp=unconfined + networks: + - default + - shared-net diff --git a/projects/germany/generate.sh b/projects/germany/generate.sh new file mode 100644 index 00000000..55e8cf48 --- /dev/null +++ b/projects/germany/generate.sh @@ -0,0 +1,161 @@ +#!/bin/bash + +# Logdatei setzen +LOGFILE="logfile.txt" + +# Terminal-Sitzung in die Logdatei schreiben +exec > >(tee -a $LOGFILE) 2>&1 + +# Maximale Anzahl der Versuche +MAX_RETRIES=3 + +# Standardmäßig werden alle Schritte ausgeführt +SKIP_DOWNLOAD=false +SKIP_PREPARE=false +SKIP_IMPORT=false + +# Fehlerbehandlungsfunktion +handle_error() { + local error_message="Fehler: $1" + echo "$error_message" + + # E-Mail senden über msmtp + { + echo "From: no-reply@druckplanet24.de" + echo "To: admin@druckplanet24.de" + echo "Subject: Fehler im Pelias Setup" + echo + echo "$error_message" + echo + echo "Logfile-Inhalt:" + tail -n 50 "$LOGFILE" + } | msmtp admin@druckplanet24.de + + exit 1 +} + +# Sicheres Ausführen von Befehlen mit Wiederholungslogik +safe_execute() { + echo "Ausführen: $1" + local command=$1 + local retries=0 + + until [ $retries -ge $MAX_RETRIES ] + do + eval $command + if [ $? -eq 0 ]; then + echo "$command abgeschlossen" + return 0 + fi + + output=$(eval $command 2>&1) + if echo "$output" | grep -q "no such service"; then + echo "Dienst existiert nicht, überspringe: $command" + return 0 + fi + + retries=$((retries+1)) + echo "Fehler beim Ausführen von: $command. Versuch $retries von $MAX_RETRIES." + done + + handle_error "$command nach $MAX_RETRIES Versuchen fehlgeschlagen" +} + +# Syntaxprüfung +echo "Starte Syntaxprüfung..." + +# Überprüfen der .env-Datei +if [ -f ".env" ]; then + if grep -Eq '^[^#]*=.*$' .env; then + echo ".env Syntax ist korrekt." + else + handle_error ".env hat einen Syntaxfehler." + fi +else + handle_error ".env Datei existiert nicht." +fi + +# Überprüfen der docker-compose.yml-Datei +if [ -f "docker-compose.yml" ]; then + docker-compose config -q || handle_error "docker-compose.yml hat einen Syntaxfehler." + echo "docker-compose.yml Syntax ist korrekt." +else + handle_error "docker-compose.yml Datei existiert nicht." +fi + +# Überprüfen der pelias.json-Datei +if [ -f "pelias.json" ]; then + jq empty pelias.json || handle_error "pelias.json hat einen Syntaxfehler." + echo "pelias.json Syntax ist korrekt." +else + handle_error "pelias.json Datei existiert nicht." +fi + +# Überprüfen, ob Pelias Docker läuft +if docker ps | grep -q "pelias"; then + echo "Pelias Docker läuft, beende ihn..." + pelias compose down || handle_error "pelias compose down" +else + echo "Pelias Docker läuft nicht." +fi + +# Pelias-Befehle ausführen +safe_execute "pelias compose pull" +safe_execute "pelias elastic start" +safe_execute "pelias elastic wait" + +# Reset-Abfrage vor pelias elastic create +read -t 10 -p "Möchten Sie Elasticsearch zurücksetzen? (Standard: Nein) [ja/Nein]: " reset_choice +reset_choice=${reset_choice:-nein} +if [[ "$reset_choice" =~ ^[Jj]a$ ]]; then + echo "Zurücksetzen von Elasticsearch..." + pelias elastic drop || handle_error "pelias elastic drop" +else + echo "Kein Zurücksetzen von Elasticsearch durchgeführt." +fi + +# Elasticsearch Index erstellen, wenn er nicht bereits existiert +create_output=$(pelias elastic create 2>&1) +if echo "$create_output" | grep -q "resource_already_exists_exception"; then + echo "Index existiert bereits, weiter mit dem nächsten Schritt." +elif echo "$create_output" | grep -q "acknowledged"; then + echo "Index erfolgreich erstellt, weiter mit dem nächsten Schritt." +else + handle_error "pelias elastic create: $create_output" +fi + +# Download-Schritt (optional überspringen) +if [ "$SKIP_DOWNLOAD" = "false" ]; then + safe_execute "pelias download all" +else + echo "Pelias Download-Schritt wird übersprungen." +fi + +# Vorbereitungsschritte und Valhalla-Tiles erstellen (optional überspringen) +if [ "$SKIP_PREPARE" = "false" ]; then + safe_execute "mkdir -p data/valhalla" + safe_execute "docker run --rm -it \ + --network shared-net \ + --user 1000:1000 \ + -v '/nfs-data/docker/appdata/pelias/germany/data/valhalla:/data/valhalla' \ + -v '/nfs-data/docker/appdata/pelias/germany/data/openstreetmap:/data/openstreetmap' \ + -v '/nfs-data/docker/appdata/pelias/germany/data/polylines:/data/polylines' \ + pelias/valhalla_baseimage \ + /bin/bash -c \"./scripts/build_tiles.sh && \ + find /data/valhalla/valhalla_tiles | sort -n | tar cf /data/valhalla/valhalla_tiles.tar --no-recursion -T - && \ + ./scripts/export_edges.sh\"" + safe_execute "pelias prepare interpolation" + safe_execute "pelias prepare placeholder" +else + echo "Vorbereitungsschritte und Valhalla-Tiles-Erstellung werden übersprungen." +fi + +# Daten importieren (optional überspringen) +if [ "$SKIP_IMPORT" = "false" ]; then + safe_execute "pelias import all" +else + echo "Datenimport wird übersprungen." +fi + +# Pelias-Dienste starten +safe_execute "pelias compose up" diff --git a/projects/germany/pelias.json b/projects/germany/pelias.json index b7b83957..f3ff723b 100644 --- a/projects/germany/pelias.json +++ b/projects/germany/pelias.json @@ -4,7 +4,7 @@ "timestamp": false }, "esclient": { - "apiVersion": "7.5", + "apiVersion": "7.x", "hosts": [ { "host": "elasticsearch" } ] @@ -16,8 +16,45 @@ "number_of_replicas": "0", "number_of_shards": "1" } + }, + "accessLog": "common", + "host": "http://maps.druckplanet24.de/", + "indexName": "pelias", + "version": "1.0", + "targets": { + "auto_discover": true, + "canonical_sources": ["whosonfirst", "openstreetmap", "openaddresses", "geonames"], + "layers_by_source": { + "openstreetmap": [ "address", "venue", "street" ], + "openaddresses": [ "address" ], + "geonames": [ + "country", "macroregion", "region", "county", "localadmin", "locality", "borough", + "neighbourhood", "venue" + ], + "whosonfirst": [ + "continent", "empire", "country", "dependency", "macroregion", "region", "locality", + "localadmin", "macrocounty", "county", "macrohood", "borough", "neighbourhood", + "microhood", "disputed", "venue", "postalcode", "ocean", "marinearea" + ] + }, + "source_aliases": { + "osm": [ "openstreetmap" ], + "oa": [ "openaddresses" ], + "gn": [ "geonames" ], + "wof": [ "whosonfirst" ] + }, + "layer_aliases": { + "coarse": [ + "continent", "empire", "country", "dependency", "macroregion", "region", "locality", + "localadmin", "macrocounty", "county", "macrohood", "borough", "neighbourhood", + "microhood", "disputed", "postalcode", "ocean", "marinearea" + ] + } } }, + "schema": { + "indexName": "pelias" + }, "api": { "services": { "pip": { "url": "http://pip:4200" }, @@ -52,7 +89,7 @@ "whosonfirst": { "datapath": "/data/whosonfirst", "countryCode": "DE", - "importPostalcodes": false, + "importPostalcodes": true, "importPlace": [ "85633111" ] } }