From 1219f568c8da0b0f54aa8d9d3368a47605ff5468 Mon Sep 17 00:00:00 2001 From: Ricardo Garcia Silva Date: Sat, 27 Apr 2024 16:14:51 +0100 Subject: [PATCH 01/43] Adding compose for running CITE tests --- .gitignore | 5 +- tests/cite/bootstrap-teamengine.sh | 27 +++++++++ tests/cite/cite.config.yml | 76 ++------------------------ tests/cite/compose.test-cite.yaml | 88 ++++++++++++++++++++++++++++++ tests/load_es_data.py | 6 +- 5 files changed, 130 insertions(+), 72 deletions(-) create mode 100644 tests/cite/bootstrap-teamengine.sh create mode 100644 tests/cite/compose.test-cite.yaml diff --git a/.gitignore b/.gitignore index 3a434a1dd..f1fe3b252 100644 --- a/.gitignore +++ b/.gitignore @@ -105,7 +105,10 @@ ENV/ # development setup examples example-config.yml -example-openapi.yml +example-openapi.yml + +# CITE openapi document (must be (re)generated before runing CITE tests) +tests/cite/cite.openapi.yml # misc *.swp diff --git a/tests/cite/bootstrap-teamengine.sh b/tests/cite/bootstrap-teamengine.sh new file mode 100644 index 000000000..13ac57e54 --- /dev/null +++ b/tests/cite/bootstrap-teamengine.sh @@ -0,0 +1,27 @@ +# bootstrap the teamengine service +# 1. ensure service is up +# 2. register a test user for running tests + +user_registration_curl_command=$( + printf '%s' \ + "curl -v " \ + "--silent " \ + "--output /dev/null " \ + "--write-out '%{http_code}' " \ + "--data firstName=teamengine " \ + "--data lastName=user " \ + "--data username=teamengine " \ + "--data password=tester " \ + "--data repeat_password=tester " \ + "--data email=noone@noplace.com " \ + "--data organization=none " \ + "--data acceptPrivacy=on " \ + "--data disclaimer=on " \ + "http://localhost:8080/teamengine/registrationHandler" +) +response_status=$(docker exec pygeoapi-cite-teamengine-1 \ + bash -c 'curl --silent --output /dev/null --write-out "%{http_code}" localhost:8080/teamengine/') && \ +echo "Response status: ${response_status}" && \ +user_registration_response_status=$(docker exec pygeoapi-cite-teamengine-1 \ + bash -c "${user_registration_curl_command}") && \ +echo "User registration status: ${user_registration_response_status}" diff --git a/tests/cite/cite.config.yml b/tests/cite/cite.config.yml index 104fb09b5..de7cd4a6c 100644 --- a/tests/cite/cite.config.yml +++ b/tests/cite/cite.config.yml @@ -1,16 +1,14 @@ server: bind: host: 0.0.0.0 - port: 5001 - url: http://localhost:5001 + port: ${PYGEOAPI_PORT:-5001} + url: ${PYGEOAPI_PUBLIC_URL:-http://localhost:5001} mimetype: application/json; charset=UTF-8 encoding: utf-8 language: en-US cors: true pretty_print: true - limits: - default_items: 10 - max_items: 10 + limit: 100 # templates: /path/to/templates map: url: https://tile.openstreetmap.org/{z}/{x}/{y}.png @@ -58,35 +56,6 @@ metadata: role: pointOfContact resources: - icoads-sst: - type: collection - title: International Comprehensive Ocean-Atmosphere Data Set (ICOADS) - description: International Comprehensive Ocean-Atmosphere Data Set (ICOADS) - keywords: - - icoads - - sst - - air temperature - extents: - spatial: - bbox: [-180,-90,180,90] - crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 - temporal: - begin: 2000-01-16T06:00:00Z - end: 2000-12-16T01:20:06Z - links: - - type: text/html - rel: canonical - title: information - href: https://psl.noaa.gov/data/gridded/data.coads.1deg.html - hreflang: en-US - providers: - - type: edr - name: xarray-edr - data: ../data/coads_sst.nc - format: - name: NetCDF - mimetype: application/x-netcdf - canada-hydat-daily-mean-02hc003: type: collection title: Daily Mean of Water Level or Flow @@ -127,47 +96,14 @@ resources: temporal: begin: 1850-01-01T00:00:00Z end: null # or empty + providers: - type: feature - name: TinyDB - data: ../data/canada-hydat-daily-mean-02hc003.tinydb + name: Elasticsearch + data: ${PYGEOAPI_ELASTICSEARCH_SERVER_URL:-http://localhost:9200}/canada-hydat-daily-mean-02hc003 id_field: IDENTIFIER time_field: DATE - lakes: - type: collection - title: Large Lakes - description: lakes of the world, public domain - keywords: - - lakes - crs: - - CRS84 - links: - - type: text/html - rel: canonical - title: information - href: http://www.naturalearthdata.com/ - hreflang: en-US - extents: - spatial: - bbox: [-180,-90,180,90] - crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 - temporal: - begin: 2011-11-11T00:00:00Z - end: null # or empty - providers: - - type: tile - name: MVT-tippecanoe - data: ../data/tiles/ne_110m_lakes - options: - bounds: [[-124.953634,-16.536406],[109.929807,66.969298]] - zoom: - min: 0 - max: 5 - format: - name: pbf - mimetype: application/vnd.mapbox-vector-tile - hello-world: type: process processor: diff --git a/tests/cite/compose.test-cite.yaml b/tests/cite/compose.test-cite.yaml new file mode 100644 index 000000000..141ac2187 --- /dev/null +++ b/tests/cite/compose.test-cite.yaml @@ -0,0 +1,88 @@ +# compose file suitable for running CITE tests in a local dev environment +# This is meant to emulate the conditions found on pygeoapi's CI + +# initialization: +# +# 1 - Stand up this stack: +# docker compose -f docker/compose.test-cite.yaml up --detach +# +# 2 - install pygeoapi dev and provider requirements: +# docker exec -ti pygeoapi-integration-pygeoapi-1 pip3 install -r requirements-dev.txt -r requirements-provider.txt + +# 3 - Upgrade the elasticsearch-dsl package for version 8.x of elasticsearch +# docker exec -ti pygeoapi-integration-pygeoapi-1 pip3 install 'elasticsearch-dsl>=8.0.0,<9.0.0' +# +# 4 - load elasticsearch data: +# docker exec -ti pygeoapi-integration-pygeoapi-1 python3 tests/load_es_data.py tests/data/ne_110m_populated_places_simple.geojson geonameid +# docker exec -ti pygeoapi-integration-pygeoapi-1 python3 tests/load_es_data.py tests/cite/canada-hydat-daily-mean-02HC003.geojson IDENTIFIER +# +# 5 - install GDAL in the pygeoapi container: +# docker exec -ti pygeoapi-integration-pygeoapi-1 bash docker/install-gdal-python-bindings.sh + +# +# +# running tests: +# +# POSTGRESQL_PASSWORD=pass pytest -v tests/test_postgresql_provider.py +# +# docker exec -ti pygeoapi-integration-pygeoapi-1 bash -c 'POSTGRESQL_PASSWORD=pass pytest -v tests/test_postgresql_provider.py' + +name: pygeoapi-cite + +services: + + pygeoapi: + image: 'ghcr.io/geopython/pygeoapi:${CURRENT_GIT_BRANCH:-latest}' + ports: + - target: 5000 + published: 5001 + environment: + PYGEOAPI_PORT: 5000 + PYGEOAPI_PUBLIC_URL: http://localhost:5001 + PYGEOAPI_CONFIG: /pygeoapi/tests/cite/cite.config.yml + PYGEOAPI_OPENAPI: /pygeoapi/example-openapi.yml + PYGEOAPI_ELASTICSEARCH_SERVER_URL: http://elasticsearch-server:9200 + volumes: + - type: bind + source: $PWD + target: /pygeoapi + depends_on: + elasticsearch-server: + condition: service_healthy + entrypoint: ['/bin/bash', '-c'] + command: + - | + pip3 install 'elasticsearch-dsl>=8.0.0,<9.0.0' + python3 tests/load_es_data.py tests/data/ne_110m_populated_places_simple.geojson geonameid + python3 tests/load_es_data.py tests/cite/canada-hydat-daily-mean-02HC003.geojson IDENTIFIER + gunicorn --bind 0.0.0.0:5000 pygeoapi.flask_app:APP + + elasticsearch-server: + image: elasticsearch:8.12.2 + ports: + - target: 9200 + published: 9200 + - target: 9300 + published: 9300 + environment: + ES_JAVA_OPTS: -Xms2g -Xmx2g + discovery.type: single-node + xpack.security.enabled: false + xpack.security.enrollment.enabled: false + healthcheck: + interval: 10s + timeout: 3s + start_period: 1m + retries: 10 + test: | + response=$$(curl --silent --write-out "%{http_code}" --output /dev/null localhost:9200) + if [ $${response} = '200' ]; + then exit 0; + else echo "++++++++++ elasticsearch is not ready ++++++++++"; exit 1; + fi + + teamengine: + image: ogccite/teamengine-production:1.0-SNAPSHOT + extra_hosts: + - "host.docker.internal:host-gateway" + network_mode: bridge diff --git a/tests/load_es_data.py b/tests/load_es_data.py index 3324a69ae..353a01ce9 100644 --- a/tests/load_es_data.py +++ b/tests/load_es_data.py @@ -29,10 +29,14 @@ import json from pathlib import Path +import os import sys from elasticsearch import Elasticsearch, helpers -es = Elasticsearch('http://localhost:9200') + +elastic_url = os.getenv( + 'PYGEOAPI_ELASTICSEARCH_SERVER_URL', 'http://localhost:9200') +es = Elasticsearch(elastic_url) if len(sys.argv) < 3: print(f'Usage: {sys.argv[0]} ') From 63815dd43db1cd61e22f1dd28b6c5dc7f6832f60 Mon Sep 17 00:00:00 2001 From: Ricardo Garcia Silva Date: Tue, 9 Jul 2024 23:44:36 +0100 Subject: [PATCH 02/43] Adding compose for running CITE tests --- tests/cite/compose.test-cite.yaml | 40 ++++++++++++++----------------- 1 file changed, 18 insertions(+), 22 deletions(-) diff --git a/tests/cite/compose.test-cite.yaml b/tests/cite/compose.test-cite.yaml index 141ac2187..18c3f8c46 100644 --- a/tests/cite/compose.test-cite.yaml +++ b/tests/cite/compose.test-cite.yaml @@ -4,28 +4,19 @@ # initialization: # # 1 - Stand up this stack: -# docker compose -f docker/compose.test-cite.yaml up --detach +# docker compose -f tests/cite/compose.test-cite.yaml up --detach # -# 2 - install pygeoapi dev and provider requirements: -# docker exec -ti pygeoapi-integration-pygeoapi-1 pip3 install -r requirements-dev.txt -r requirements-provider.txt - -# 3 - Upgrade the elasticsearch-dsl package for version 8.x of elasticsearch -# docker exec -ti pygeoapi-integration-pygeoapi-1 pip3 install 'elasticsearch-dsl>=8.0.0,<9.0.0' +# 2 - Bootstrap the teamengine service, basically create a user for running tests) +# ./tests/cite/bootstrap-teamengine.sh # -# 4 - load elasticsearch data: -# docker exec -ti pygeoapi-integration-pygeoapi-1 python3 tests/load_es_data.py tests/data/ne_110m_populated_places_simple.geojson geonameid -# docker exec -ti pygeoapi-integration-pygeoapi-1 python3 tests/load_es_data.py tests/cite/canada-hydat-daily-mean-02HC003.geojson IDENTIFIER +# 3 - Access the teamengine GUI at http://localhost:8080/teamengine and login +# with: +# - user: teamengine +# - password: tester # -# 5 - install GDAL in the pygeoapi container: -# docker exec -ti pygeoapi-integration-pygeoapi-1 bash docker/install-gdal-python-bindings.sh +# 4 - Use the teamengine GUI to run tests suite(s) + -# -# -# running tests: -# -# POSTGRESQL_PASSWORD=pass pytest -v tests/test_postgresql_provider.py -# -# docker exec -ti pygeoapi-integration-pygeoapi-1 bash -c 'POSTGRESQL_PASSWORD=pass pytest -v tests/test_postgresql_provider.py' name: pygeoapi-cite @@ -40,7 +31,7 @@ services: PYGEOAPI_PORT: 5000 PYGEOAPI_PUBLIC_URL: http://localhost:5001 PYGEOAPI_CONFIG: /pygeoapi/tests/cite/cite.config.yml - PYGEOAPI_OPENAPI: /pygeoapi/example-openapi.yml + PYGEOAPI_OPENAPI: /pygeoapi/tests/cite/cite.openapi.yml PYGEOAPI_ELASTICSEARCH_SERVER_URL: http://elasticsearch-server:9200 volumes: - type: bind @@ -55,6 +46,7 @@ services: pip3 install 'elasticsearch-dsl>=8.0.0,<9.0.0' python3 tests/load_es_data.py tests/data/ne_110m_populated_places_simple.geojson geonameid python3 tests/load_es_data.py tests/cite/canada-hydat-daily-mean-02HC003.geojson IDENTIFIER + pygeoapi openapi generate /pygeoapi/tests/cite/cite.config.yml --output-file /pygeoapi/tests/cite/cite.openapi.yml gunicorn --bind 0.0.0.0:5000 pygeoapi.flask_app:APP elasticsearch-server: @@ -83,6 +75,10 @@ services: teamengine: image: ogccite/teamengine-production:1.0-SNAPSHOT - extra_hosts: - - "host.docker.internal:host-gateway" - network_mode: bridge + ports: + - target: 8080 + published: 8888 + # use network_mode: host in order to allow teamengine to access pygeoapi + # at localhost:5001 - this is necessary in order to have teamengine be + # able to follow pygeoapi URLs + network_mode: host From 1199423b2cc83b446c5387ad2701f2b9443b0a6f Mon Sep 17 00:00:00 2001 From: Ricardo Garcia Silva Date: Fri, 12 Jul 2024 11:54:14 +0100 Subject: [PATCH 03/43] Adding compose for running CITE tests --- .github/workflows/cite.yml | 24 ++++++++++++++++++++++++ tests/cite/compose.test-cite.yaml | 5 ++--- 2 files changed, 26 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/cite.yml diff --git a/.github/workflows/cite.yml b/.github/workflows/cite.yml new file mode 100644 index 000000000..d3e02a69b --- /dev/null +++ b/.github/workflows/cite.yml @@ -0,0 +1,24 @@ +name: 'Perform CITE tests against a local teamengine instance' + +on: + push: + pull_request: + + +jobs: + perform-cite-testing: + runs-on: ubuntu:22.04 + steps: + + - name: grab code + uses: actions/checkout@v4 + + - name: start pygeoapi with suitable CITE data and config + run: > + docker compose -f tests/cite/cite.config.yml up --detach + + - name: test ogcapi-features compliancy + uses: OSGEO/ogc-cite-action + with: + test-suite-identifier: ogcapi-features-1.0 + test-session-arguments: iut=http://localhost:5001 noofcollections=-1 \ No newline at end of file diff --git a/tests/cite/compose.test-cite.yaml b/tests/cite/compose.test-cite.yaml index 18c3f8c46..840e7bca4 100644 --- a/tests/cite/compose.test-cite.yaml +++ b/tests/cite/compose.test-cite.yaml @@ -3,6 +3,8 @@ # initialization: # +# 0 - Navigate to the root of the pygeoapi repo +# # 1 - Stand up this stack: # docker compose -f tests/cite/compose.test-cite.yaml up --detach # @@ -75,9 +77,6 @@ services: teamengine: image: ogccite/teamengine-production:1.0-SNAPSHOT - ports: - - target: 8080 - published: 8888 # use network_mode: host in order to allow teamengine to access pygeoapi # at localhost:5001 - this is necessary in order to have teamengine be # able to follow pygeoapi URLs From b1d5212d431b2abdb1f6aeb51cee3465c138eaed Mon Sep 17 00:00:00 2001 From: Ricardo Garcia Silva Date: Fri, 12 Jul 2024 11:56:35 +0100 Subject: [PATCH 04/43] Adding compose for running CITE tests --- .github/workflows/cite.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cite.yml b/.github/workflows/cite.yml index d3e02a69b..f59173d96 100644 --- a/.github/workflows/cite.yml +++ b/.github/workflows/cite.yml @@ -18,7 +18,7 @@ jobs: docker compose -f tests/cite/cite.config.yml up --detach - name: test ogcapi-features compliancy - uses: OSGEO/ogc-cite-action + uses: OSGEO/ogc-cite-action@main with: test-suite-identifier: ogcapi-features-1.0 test-session-arguments: iut=http://localhost:5001 noofcollections=-1 \ No newline at end of file From bf986a09dc820b9c3b7eb82914d7166e39773b66 Mon Sep 17 00:00:00 2001 From: Ricardo Garcia Silva Date: Fri, 12 Jul 2024 20:31:26 +0100 Subject: [PATCH 05/43] Fix bug in workflow --- .github/workflows/cite.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cite.yml b/.github/workflows/cite.yml index f59173d96..d80b8c812 100644 --- a/.github/workflows/cite.yml +++ b/.github/workflows/cite.yml @@ -7,7 +7,7 @@ on: jobs: perform-cite-testing: - runs-on: ubuntu:22.04 + runs-on: ubuntu_22.04 steps: - name: grab code From f301f88afb4397f974ccb1f85f27b5aa7428b9d1 Mon Sep 17 00:00:00 2001 From: Ricardo Garcia Silva Date: Fri, 12 Jul 2024 20:44:59 +0100 Subject: [PATCH 06/43] Fix bug in workflow --- .github/workflows/cite.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cite.yml b/.github/workflows/cite.yml index d80b8c812..c0a7ff8d9 100644 --- a/.github/workflows/cite.yml +++ b/.github/workflows/cite.yml @@ -7,7 +7,7 @@ on: jobs: perform-cite-testing: - runs-on: ubuntu_22.04 + runs-on: ubuntu-22.04 steps: - name: grab code From 99b83d1a1ea518dd8914979d673a5ec4d066220e Mon Sep 17 00:00:00 2001 From: Ricardo Garcia Silva Date: Fri, 12 Jul 2024 20:47:38 +0100 Subject: [PATCH 07/43] Fix bug in workflow --- .github/workflows/cite.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cite.yml b/.github/workflows/cite.yml index c0a7ff8d9..0a2276744 100644 --- a/.github/workflows/cite.yml +++ b/.github/workflows/cite.yml @@ -15,7 +15,7 @@ jobs: - name: start pygeoapi with suitable CITE data and config run: > - docker compose -f tests/cite/cite.config.yml up --detach + docker compose -f tests/cite/compose.test-cite.yaml up --detach - name: test ogcapi-features compliancy uses: OSGEO/ogc-cite-action@main From 5639e6c39314045304065d69fd96534140ef43b5 Mon Sep 17 00:00:00 2001 From: Ricardo Garcia Silva Date: Sat, 13 Jul 2024 00:11:55 +0100 Subject: [PATCH 08/43] Trying out the github action --- tests/cite/compose.test-cite.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/cite/compose.test-cite.yaml b/tests/cite/compose.test-cite.yaml index 840e7bca4..f66fecc99 100644 --- a/tests/cite/compose.test-cite.yaml +++ b/tests/cite/compose.test-cite.yaml @@ -75,9 +75,9 @@ services: else echo "++++++++++ elasticsearch is not ready ++++++++++"; exit 1; fi - teamengine: - image: ogccite/teamengine-production:1.0-SNAPSHOT - # use network_mode: host in order to allow teamengine to access pygeoapi - # at localhost:5001 - this is necessary in order to have teamengine be - # able to follow pygeoapi URLs - network_mode: host +# teamengine: +# image: ogccite/teamengine-production:1.0-SNAPSHOT +# # use network_mode: host in order to allow teamengine to access pygeoapi +# # at localhost:5001 - this is necessary in order to have teamengine be +# # able to follow pygeoapi URLs +# network_mode: host \ No newline at end of file From 61988020efe36001c6157947f8cd1cb69a586c19 Mon Sep 17 00:00:00 2001 From: Ricardo Garcia Silva Date: Sat, 13 Jul 2024 00:20:31 +0100 Subject: [PATCH 09/43] Trying out the github action --- .github/workflows/cite.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/cite.yml b/.github/workflows/cite.yml index 0a2276744..9192c6a9c 100644 --- a/.github/workflows/cite.yml +++ b/.github/workflows/cite.yml @@ -21,4 +21,6 @@ jobs: uses: OSGEO/ogc-cite-action@main with: test-suite-identifier: ogcapi-features-1.0 - test-session-arguments: iut=http://localhost:5001 noofcollections=-1 \ No newline at end of file + test-session-arguments: > + --test-suite-input iut http://localhost:5001 + --test-suite-input noofcollections -1 \ No newline at end of file From 33bf014a9fa75da8a0017890c2e58c6112855c65 Mon Sep 17 00:00:00 2001 From: Ricardo Garcia Silva Date: Sat, 13 Jul 2024 01:08:34 +0100 Subject: [PATCH 10/43] Added healthcheck for pygeoapi service --- .github/workflows/cite.yml | 6 ++++++ tests/cite/compose.test-cite.yaml | 11 +++++++++++ 2 files changed, 17 insertions(+) diff --git a/.github/workflows/cite.yml b/.github/workflows/cite.yml index 9192c6a9c..9b620df2a 100644 --- a/.github/workflows/cite.yml +++ b/.github/workflows/cite.yml @@ -17,6 +17,12 @@ jobs: run: > docker compose -f tests/cite/compose.test-cite.yaml up --detach + - name: wait for pygeoapi to be usable + uses: raschmitt/wait-for-healthy-container@v1.0.1 + with: + container-name: pygeoapi-cite-pygeoapi-1 + timeout: 120 + - name: test ogcapi-features compliancy uses: OSGEO/ogc-cite-action@main with: diff --git a/tests/cite/compose.test-cite.yaml b/tests/cite/compose.test-cite.yaml index f66fecc99..62922840d 100644 --- a/tests/cite/compose.test-cite.yaml +++ b/tests/cite/compose.test-cite.yaml @@ -50,6 +50,17 @@ services: python3 tests/load_es_data.py tests/cite/canada-hydat-daily-mean-02HC003.geojson IDENTIFIER pygeoapi openapi generate /pygeoapi/tests/cite/cite.config.yml --output-file /pygeoapi/tests/cite/cite.openapi.yml gunicorn --bind 0.0.0.0:5000 pygeoapi.flask_app:APP + healthcheck: + interval: 10s + timeout: 3s + start_period: 1m + retries: 10 + test: | + response=$$(python3 -c 'import urllib.request; print(urllib.request.urlopen("http://localhost:5000").status)') + if [ $${response} = '200' ]; + then exit 0; + else echo "++++++++++ pygeoapi is not ready ++++++++++"; exit 1; + fi elasticsearch-server: image: elasticsearch:8.12.2 From 1bafb481e57b13519626be6d91028b3638fc8507 Mon Sep 17 00:00:00 2001 From: Ricardo Garcia Silva Date: Sat, 13 Jul 2024 01:13:42 +0100 Subject: [PATCH 11/43] Increased timeout for checking if pygeoapi service is healthy --- .github/workflows/cite.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cite.yml b/.github/workflows/cite.yml index 9b620df2a..91c769b98 100644 --- a/.github/workflows/cite.yml +++ b/.github/workflows/cite.yml @@ -21,7 +21,7 @@ jobs: uses: raschmitt/wait-for-healthy-container@v1.0.1 with: container-name: pygeoapi-cite-pygeoapi-1 - timeout: 120 + timeout: 240 - name: test ogcapi-features compliancy uses: OSGEO/ogc-cite-action@main From 5dee390d252b88cabe9e1d0740d86878cbf04c48 Mon Sep 17 00:00:00 2001 From: Ricardo Garcia Silva Date: Sat, 13 Jul 2024 01:27:11 +0100 Subject: [PATCH 12/43] Increased timeout for checking if pygeoapi service is healthy --- .github/workflows/cite.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cite.yml b/.github/workflows/cite.yml index 91c769b98..1b2246cf1 100644 --- a/.github/workflows/cite.yml +++ b/.github/workflows/cite.yml @@ -21,7 +21,7 @@ jobs: uses: raschmitt/wait-for-healthy-container@v1.0.1 with: container-name: pygeoapi-cite-pygeoapi-1 - timeout: 240 + timeout: 480 - name: test ogcapi-features compliancy uses: OSGEO/ogc-cite-action@main From 54e102d6ea7b3fde94c0a9c4be5740824bbe8ace Mon Sep 17 00:00:00 2001 From: Ricardo Garcia Silva Date: Sat, 13 Jul 2024 09:49:37 +0100 Subject: [PATCH 13/43] Adding way to inspect logs on github workflow --- .github/workflows/cite.yml | 9 ++++++++- tests/cite/compose.test-cite.yaml | 1 + 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/cite.yml b/.github/workflows/cite.yml index 1b2246cf1..0985cf9d9 100644 --- a/.github/workflows/cite.yml +++ b/.github/workflows/cite.yml @@ -21,7 +21,14 @@ jobs: uses: raschmitt/wait-for-healthy-container@v1.0.1 with: container-name: pygeoapi-cite-pygeoapi-1 - timeout: 480 + timeout: 120 + + - name: collect docker logs on failure + if: failure() + uses: jwalton/gh-docker-logs@v2.2.2 + with: + images: pygeoapi + tail: 500 - name: test ogcapi-features compliancy uses: OSGEO/ogc-cite-action@main diff --git a/tests/cite/compose.test-cite.yaml b/tests/cite/compose.test-cite.yaml index 62922840d..48a91079c 100644 --- a/tests/cite/compose.test-cite.yaml +++ b/tests/cite/compose.test-cite.yaml @@ -45,6 +45,7 @@ services: entrypoint: ['/bin/bash', '-c'] command: - | + pip3 install -r requirements.txt pip3 install 'elasticsearch-dsl>=8.0.0,<9.0.0' python3 tests/load_es_data.py tests/data/ne_110m_populated_places_simple.geojson geonameid python3 tests/load_es_data.py tests/cite/canada-hydat-daily-mean-02HC003.geojson IDENTIFIER From ccde58bc704eb720c697d69e709e0a4f33348b02 Mon Sep 17 00:00:00 2001 From: Ricardo Garcia Silva Date: Sat, 13 Jul 2024 10:02:47 +0100 Subject: [PATCH 14/43] Fixed unrecognized image name on step that collects logs --- .github/workflows/cite.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cite.yml b/.github/workflows/cite.yml index 0985cf9d9..a7284c4c6 100644 --- a/.github/workflows/cite.yml +++ b/.github/workflows/cite.yml @@ -27,7 +27,7 @@ jobs: if: failure() uses: jwalton/gh-docker-logs@v2.2.2 with: - images: pygeoapi + images: ghcr.io/geopython/pygeoapi tail: 500 - name: test ogcapi-features compliancy From 62fa84a95faca5d88b090ec739aa9c13d5c60901 Mon Sep 17 00:00:00 2001 From: Ricardo Garcia Silva Date: Sat, 13 Jul 2024 10:20:27 +0100 Subject: [PATCH 15/43] Fixing compose so that it runs on CI --- tests/cite/compose.test-cite.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/cite/compose.test-cite.yaml b/tests/cite/compose.test-cite.yaml index 48a91079c..194e0d8bb 100644 --- a/tests/cite/compose.test-cite.yaml +++ b/tests/cite/compose.test-cite.yaml @@ -49,6 +49,7 @@ services: pip3 install 'elasticsearch-dsl>=8.0.0,<9.0.0' python3 tests/load_es_data.py tests/data/ne_110m_populated_places_simple.geojson geonameid python3 tests/load_es_data.py tests/cite/canada-hydat-daily-mean-02HC003.geojson IDENTIFIER + pip3 install -e . pygeoapi openapi generate /pygeoapi/tests/cite/cite.config.yml --output-file /pygeoapi/tests/cite/cite.openapi.yml gunicorn --bind 0.0.0.0:5000 pygeoapi.flask_app:APP healthcheck: From b42d5db7310ef5ad0c69bad4b20a18e36fd1ce0b Mon Sep 17 00:00:00 2001 From: Ricardo Garcia Silva Date: Sun, 14 Jul 2024 18:14:42 +0100 Subject: [PATCH 16/43] Trying out workflow with matrix to run multiple test suites --- .github/workflows/cite.yml | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/.github/workflows/cite.yml b/.github/workflows/cite.yml index a7284c4c6..ecb9d5bec 100644 --- a/.github/workflows/cite.yml +++ b/.github/workflows/cite.yml @@ -6,7 +6,20 @@ on: jobs: + perform-cite-testing: + strategy: + matrix: + test-suite: + - suite-id: ogcapi-features-1.0 + arguments: > + --test-suite-input iut http://localhost:5001 + --test-suite-input noofcollections -1 + - suite-id: ogcapi-processes-1.0 + arguments: > + --test-suite-input iut http://localhost:5001 + --test-suite-input noofcollections -1 + runs-on: ubuntu-22.04 steps: @@ -33,7 +46,5 @@ jobs: - name: test ogcapi-features compliancy uses: OSGEO/ogc-cite-action@main with: - test-suite-identifier: ogcapi-features-1.0 - test-session-arguments: > - --test-suite-input iut http://localhost:5001 - --test-suite-input noofcollections -1 \ No newline at end of file + test-suite-identifier: ${{ matrix.test-suite.suite-id }} + test-session-arguments: ${{ matrix.test-suite.arguments }} \ No newline at end of file From a9a2e6de4743d1649a18aab76466754faed7a0da Mon Sep 17 00:00:00 2001 From: Ricardo Garcia Silva Date: Mon, 23 Dec 2024 18:02:55 +0000 Subject: [PATCH 17/43] Test updated cite action --- .github/workflows/cite.yml | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/.github/workflows/cite.yml b/.github/workflows/cite.yml index ecb9d5bec..4f0bb98d4 100644 --- a/.github/workflows/cite.yml +++ b/.github/workflows/cite.yml @@ -13,12 +13,12 @@ jobs: test-suite: - suite-id: ogcapi-features-1.0 arguments: > - --test-suite-input iut http://localhost:5001 - --test-suite-input noofcollections -1 + iut=http://localhost:5001 + noofcollections=-1 - suite-id: ogcapi-processes-1.0 arguments: > - --test-suite-input iut http://localhost:5001 - --test-suite-input noofcollections -1 + iut=http://localhost:5001 + noofcollections=-1 runs-on: ubuntu-22.04 steps: @@ -47,4 +47,7 @@ jobs: uses: OSGEO/ogc-cite-action@main with: test-suite-identifier: ${{ matrix.test-suite.suite-id }} - test-session-arguments: ${{ matrix.test-suite.arguments }} \ No newline at end of file + test-session-arguments: ${{ matrix.test-suite.arguments }} + env: + teamengine_username: ${{ secrets.TEAMENGINE_USERNAME }} + teamengine_password: ${{ secrets.TEAMENGINE_PASSWORD }} From 043549f73287dd422112863ff7a63db11df0b737 Mon Sep 17 00:00:00 2001 From: Ricardo Garcia Silva Date: Mon, 30 Dec 2024 15:42:49 +0000 Subject: [PATCH 18/43] Test updated cite action --- .github/workflows/cite.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cite.yml b/.github/workflows/cite.yml index 4f0bb98d4..873d39692 100644 --- a/.github/workflows/cite.yml +++ b/.github/workflows/cite.yml @@ -46,8 +46,8 @@ jobs: - name: test ogcapi-features compliancy uses: OSGEO/ogc-cite-action@main with: - test-suite-identifier: ${{ matrix.test-suite.suite-id }} - test-session-arguments: ${{ matrix.test-suite.arguments }} + test_suite_identifier: ${{ matrix.test-suite.suite-id }} + test_session_arguments: ${{ matrix.test-suite.arguments }} env: teamengine_username: ${{ secrets.TEAMENGINE_USERNAME }} teamengine_password: ${{ secrets.TEAMENGINE_PASSWORD }} From 7353c678d030fbf08247c02848ae23131c81f515 Mon Sep 17 00:00:00 2001 From: Ricardo Garcia Silva Date: Thu, 2 Jan 2025 14:01:55 +0000 Subject: [PATCH 19/43] Test updated cite action --- .github/workflows/cite.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/cite.yml b/.github/workflows/cite.yml index 873d39692..6bef462c9 100644 --- a/.github/workflows/cite.yml +++ b/.github/workflows/cite.yml @@ -48,6 +48,10 @@ jobs: with: test_suite_identifier: ${{ matrix.test-suite.suite-id }} test_session_arguments: ${{ matrix.test-suite.arguments }} + treat_skipped_tests_as_failures: "true" + exit_with_error_on_suite_failed_result: "true" env: teamengine_username: ${{ secrets.TEAMENGINE_USERNAME }} teamengine_password: ${{ secrets.TEAMENGINE_PASSWORD }} + treat_skipped_tests_as_failures: "true" + exit_with_error_on_suite_failed_result: "true" From ce575e346fee48a88187e74fd23ed62a278f83e8 Mon Sep 17 00:00:00 2001 From: Ricardo Garcia Silva Date: Thu, 2 Jan 2025 18:04:13 +0000 Subject: [PATCH 20/43] Added COLUMNS env variable --- .github/workflows/cite.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/cite.yml b/.github/workflows/cite.yml index 6bef462c9..027bfc303 100644 --- a/.github/workflows/cite.yml +++ b/.github/workflows/cite.yml @@ -4,6 +4,9 @@ on: push: pull_request: +env: + COLUMNS: 120 + jobs: @@ -50,8 +53,3 @@ jobs: test_session_arguments: ${{ matrix.test-suite.arguments }} treat_skipped_tests_as_failures: "true" exit_with_error_on_suite_failed_result: "true" - env: - teamengine_username: ${{ secrets.TEAMENGINE_USERNAME }} - teamengine_password: ${{ secrets.TEAMENGINE_PASSWORD }} - treat_skipped_tests_as_failures: "true" - exit_with_error_on_suite_failed_result: "true" From a44e87b48b6e6e39f7ba84b63ee577fa3b10665d Mon Sep 17 00:00:00 2001 From: Ricardo Garcia Silva Date: Thu, 20 Mar 2025 15:54:43 +0000 Subject: [PATCH 21/43] Added ogcapi-edr CITE runner --- .github/workflows/cite.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cite.yml b/.github/workflows/cite.yml index 027bfc303..79015d9f2 100644 --- a/.github/workflows/cite.yml +++ b/.github/workflows/cite.yml @@ -15,13 +15,17 @@ jobs: matrix: test-suite: - suite-id: ogcapi-features-1.0 - arguments: > + arguments: >- iut=http://localhost:5001 noofcollections=-1 - suite-id: ogcapi-processes-1.0 - arguments: > + arguments: >- iut=http://localhost:5001 noofcollections=-1 + - suite-id: ogcapi-edr10 + arguments: >- + iut=http://localhost:5001 + apiDefinition=http://localhost:5001/openapi runs-on: ubuntu-22.04 steps: From 218d4a50fb45aa00ee9d2a72bc5fd93f55460e59 Mon Sep 17 00:00:00 2001 From: Ricardo Garcia Silva Date: Thu, 27 Mar 2025 09:59:01 +0000 Subject: [PATCH 22/43] Adapted cite-runner arguments --- .github/workflows/cite.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/cite.yml b/.github/workflows/cite.yml index 79015d9f2..980d2778b 100644 --- a/.github/workflows/cite.yml +++ b/.github/workflows/cite.yml @@ -51,9 +51,7 @@ jobs: tail: 500 - name: test ogcapi-features compliancy - uses: OSGEO/ogc-cite-action@main + uses: OSGEO/cite-runner@main with: test_suite_identifier: ${{ matrix.test-suite.suite-id }} test_session_arguments: ${{ matrix.test-suite.arguments }} - treat_skipped_tests_as_failures: "true" - exit_with_error_on_suite_failed_result: "true" From 652ae6d64714a8e0d3f81a1bede9e2adc497301e Mon Sep 17 00:00:00 2001 From: Ricardo Garcia Silva Date: Thu, 27 Mar 2025 11:36:23 +0000 Subject: [PATCH 23/43] Renaming cite testing jobs to make them easier to distinguish --- .github/workflows/cite.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/cite.yml b/.github/workflows/cite.yml index 980d2778b..d68dcf0ec 100644 --- a/.github/workflows/cite.yml +++ b/.github/workflows/cite.yml @@ -11,6 +11,7 @@ env: jobs: perform-cite-testing: + name: ${{ matrix.test-suite.suite-id }} CITE testing strategy: matrix: test-suite: From 03e8940de045c8caef12e749607dade122ad576b Mon Sep 17 00:00:00 2001 From: Ricardo Garcia Silva Date: Thu, 10 Apr 2025 15:09:46 +0100 Subject: [PATCH 24/43] Added ogcapi-tiles to the CITE CI workflow --- .github/workflows/cite.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.github/workflows/cite.yml b/.github/workflows/cite.yml index d68dcf0ec..83dea9310 100644 --- a/.github/workflows/cite.yml +++ b/.github/workflows/cite.yml @@ -12,6 +12,7 @@ jobs: perform-cite-testing: name: ${{ matrix.test-suite.suite-id }} CITE testing + continue-on-error: true strategy: matrix: test-suite: @@ -27,6 +28,16 @@ jobs: arguments: >- iut=http://localhost:5001 apiDefinition=http://localhost:5001/openapi + - suite-id: ogcapi-tiles-1.0 + arguments: >- + iut=http://localhost:5001 + tilematrixsetdefinitionuri=http://www.opengis.net/def/tilematrixset/OGC/1.0/WebMercatorQuad + urltemplatefortiles=http://localhost:5001/collections/lakes/tiles/WebMercatorQuad/{tileMatrix}/{tileRow}/{tileCol}?f=mvt + tilematrix=0 + mintilerow=0 + maxtilerow=1 + mintilecol=0 + maxtilecol=1 runs-on: ubuntu-22.04 steps: From d589c1bce493bb2d65c4bb29d366e3b5b8f02b47 Mon Sep 17 00:00:00 2001 From: Ricardo Garcia Silva Date: Thu, 10 Apr 2025 16:35:02 +0100 Subject: [PATCH 25/43] Adding job exit with error on suite failed result --- .github/workflows/cite.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/cite.yml b/.github/workflows/cite.yml index 83dea9310..4236b963d 100644 --- a/.github/workflows/cite.yml +++ b/.github/workflows/cite.yml @@ -22,11 +22,11 @@ jobs: noofcollections=-1 - suite-id: ogcapi-processes-1.0 arguments: >- - iut=http://localhost:5001 + iut=http://localhost:5001 noofcollections=-1 - suite-id: ogcapi-edr10 arguments: >- - iut=http://localhost:5001 + iut=http://localhost:5001 apiDefinition=http://localhost:5001/openapi - suite-id: ogcapi-tiles-1.0 arguments: >- @@ -62,8 +62,9 @@ jobs: images: ghcr.io/geopython/pygeoapi tail: 500 - - name: test ogcapi-features compliancy + - name: test {{ matrix.test-suite.suite-id }} compliancy uses: OSGEO/cite-runner@main with: test_suite_identifier: ${{ matrix.test-suite.suite-id }} test_session_arguments: ${{ matrix.test-suite.arguments }} + exit_with_error_on_suite_failed_result: "true" From 04f238d703f4f4dc481127b651721c0beccf550b Mon Sep 17 00:00:00 2001 From: Ricardo Garcia Silva Date: Mon, 14 Apr 2025 18:14:55 +0100 Subject: [PATCH 26/43] Adapt to changed parameter names --- .github/workflows/cite.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/cite.yml b/.github/workflows/cite.yml index 4236b963d..00070f453 100644 --- a/.github/workflows/cite.yml +++ b/.github/workflows/cite.yml @@ -62,9 +62,8 @@ jobs: images: ghcr.io/geopython/pygeoapi tail: 500 - - name: test {{ matrix.test-suite.suite-id }} compliancy + - name: test {{ matrix.test-suite.suite-id }} compliance uses: OSGEO/cite-runner@main with: test_suite_identifier: ${{ matrix.test-suite.suite-id }} test_session_arguments: ${{ matrix.test-suite.arguments }} - exit_with_error_on_suite_failed_result: "true" From 96f1da00471f22c52a9dd87021ced87c3c2bed8e Mon Sep 17 00:00:00 2001 From: Ricardo Garcia Silva Date: Thu, 24 Apr 2025 12:03:22 +0100 Subject: [PATCH 27/43] Update to use a tagged version of cite-runner --- .github/workflows/cite.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cite.yml b/.github/workflows/cite.yml index 00070f453..a9aa88eff 100644 --- a/.github/workflows/cite.yml +++ b/.github/workflows/cite.yml @@ -63,7 +63,7 @@ jobs: tail: 500 - name: test {{ matrix.test-suite.suite-id }} compliance - uses: OSGEO/cite-runner@main + uses: OSGeo/cite-runner@v0.1.4 with: test_suite_identifier: ${{ matrix.test-suite.suite-id }} test_session_arguments: ${{ matrix.test-suite.arguments }} From fa9b12c480b7b53e1622d8ccd15abf89866ad278 Mon Sep 17 00:00:00 2001 From: Ricardo Garcia Silva Date: Thu, 24 Apr 2025 12:10:52 +0100 Subject: [PATCH 28/43] Update to use a tagged version of cite-runner --- .github/workflows/cite.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cite.yml b/.github/workflows/cite.yml index a9aa88eff..4798d8602 100644 --- a/.github/workflows/cite.yml +++ b/.github/workflows/cite.yml @@ -63,7 +63,7 @@ jobs: tail: 500 - name: test {{ matrix.test-suite.suite-id }} compliance - uses: OSGeo/cite-runner@v0.1.4 + uses: OSGeo/cite-runner@v0.1.0 with: test_suite_identifier: ${{ matrix.test-suite.suite-id }} test_session_arguments: ${{ matrix.test-suite.arguments }} From 87490532a2383e98c137d5eaf494b0db8b97eca3 Mon Sep 17 00:00:00 2001 From: Ricardo Garcia Silva Date: Thu, 24 Apr 2025 14:53:24 +0100 Subject: [PATCH 29/43] Revert back to cite-runner@main --- .github/workflows/cite.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cite.yml b/.github/workflows/cite.yml index 4798d8602..b4abf10f4 100644 --- a/.github/workflows/cite.yml +++ b/.github/workflows/cite.yml @@ -63,7 +63,7 @@ jobs: tail: 500 - name: test {{ matrix.test-suite.suite-id }} compliance - uses: OSGeo/cite-runner@v0.1.0 + uses: OSGeo/cite-runner@main with: test_suite_identifier: ${{ matrix.test-suite.suite-id }} test_session_arguments: ${{ matrix.test-suite.arguments }} From 2c916bdaa5a3fd64bb0bc2ee5b1fc60a578867d7 Mon Sep 17 00:00:00 2001 From: Ricardo Garcia Silva Date: Thu, 8 May 2025 14:51:34 +0100 Subject: [PATCH 30/43] Update arguments for cite-runner --- .github/workflows/cite.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/cite.yml b/.github/workflows/cite.yml index b4abf10f4..eb836bda6 100644 --- a/.github/workflows/cite.yml +++ b/.github/workflows/cite.yml @@ -18,15 +18,15 @@ jobs: test-suite: - suite-id: ogcapi-features-1.0 arguments: >- - iut=http://localhost:5001 + iut=http://host.docker.internal:5001 noofcollections=-1 - suite-id: ogcapi-processes-1.0 arguments: >- - iut=http://localhost:5001 + iut=http://host.docker.internal:5001 noofcollections=-1 - suite-id: ogcapi-edr10 arguments: >- - iut=http://localhost:5001 + iut=http://host.docker.internal:5001 apiDefinition=http://localhost:5001/openapi - suite-id: ogcapi-tiles-1.0 arguments: >- From 2a36fc4956a7d1e54afa271dc5dc705c93b39464 Mon Sep 17 00:00:00 2001 From: Ricardo Garcia Silva Date: Thu, 8 May 2025 14:55:45 +0100 Subject: [PATCH 31/43] Update arguments for cite-runner --- .github/workflows/cite.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cite.yml b/.github/workflows/cite.yml index eb836bda6..d83f870a9 100644 --- a/.github/workflows/cite.yml +++ b/.github/workflows/cite.yml @@ -30,7 +30,7 @@ jobs: apiDefinition=http://localhost:5001/openapi - suite-id: ogcapi-tiles-1.0 arguments: >- - iut=http://localhost:5001 + iut=http://host.docker.internal:5001 tilematrixsetdefinitionuri=http://www.opengis.net/def/tilematrixset/OGC/1.0/WebMercatorQuad urltemplatefortiles=http://localhost:5001/collections/lakes/tiles/WebMercatorQuad/{tileMatrix}/{tileRow}/{tileCol}?f=mvt tilematrix=0 From 95153523fec0e7ae66de49c61247e58a49a2556c Mon Sep 17 00:00:00 2001 From: Ricardo Garcia Silva Date: Thu, 8 May 2025 15:09:06 +0100 Subject: [PATCH 32/43] Adding final workflow job to set overall status based on matrix results --- .github/workflows/cite.yml | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/.github/workflows/cite.yml b/.github/workflows/cite.yml index d83f870a9..d37790064 100644 --- a/.github/workflows/cite.yml +++ b/.github/workflows/cite.yml @@ -54,16 +54,30 @@ jobs: with: container-name: pygeoapi-cite-pygeoapi-1 timeout: 120 - - - name: collect docker logs on failure - if: failure() - uses: jwalton/gh-docker-logs@v2.2.2 - with: - images: ghcr.io/geopython/pygeoapi - tail: 500 +# +# - name: collect docker logs on failure +# if: failure() +# uses: jwalton/gh-docker-logs@v2.2.2 +# with: +# images: ghcr.io/geopython/pygeoapi +# tail: 500 - name: test {{ matrix.test-suite.suite-id }} compliance uses: OSGeo/cite-runner@main with: test_suite_identifier: ${{ matrix.test-suite.suite-id }} test_session_arguments: ${{ matrix.test-suite.arguments }} + + set-final-status: + needs: perform-cite-testing + runs-on: ubuntu-22.04 + if: always() + steps: + - name: check status of workflow + run: | + if [[ "${{ contains(needs.*.result, 'failure') }}" == "true" ]]; then + echo "One or more workflow jobs have failed" + exit 1 + else + echo "All workflow jobs have passed" + fi \ No newline at end of file From 2f356ccc8712feb7ebb1bfc8f553317faf8a49a4 Mon Sep 17 00:00:00 2001 From: Ricardo Garcia Silva Date: Thu, 8 May 2025 15:27:45 +0100 Subject: [PATCH 33/43] Adding final workflow job to set overall status based on matrix results --- .github/workflows/cite.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cite.yml b/.github/workflows/cite.yml index d37790064..0fc9b4d0e 100644 --- a/.github/workflows/cite.yml +++ b/.github/workflows/cite.yml @@ -75,7 +75,7 @@ jobs: steps: - name: check status of workflow run: | - if [[ "${{ contains(needs.*.result, 'failure') }}" == "true" ]]; then + if [ "${{ contains(needs.*.result, 'failure') }}" = "true" ]; then echo "One or more workflow jobs have failed" exit 1 else From fb5582fe942e5a9585d6601c72fae4e8430c44b1 Mon Sep 17 00:00:00 2001 From: Ricardo Garcia Silva Date: Thu, 8 May 2025 15:43:22 +0100 Subject: [PATCH 34/43] Adding final workflow job to set overall status based on matrix results --- .github/workflows/cite.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/cite.yml b/.github/workflows/cite.yml index 0fc9b4d0e..68e800287 100644 --- a/.github/workflows/cite.yml +++ b/.github/workflows/cite.yml @@ -75,6 +75,8 @@ jobs: steps: - name: check status of workflow run: | + echo "${{ needs }}" + echo "${{ needs.*.result }}" if [ "${{ contains(needs.*.result, 'failure') }}" = "true" ]; then echo "One or more workflow jobs have failed" exit 1 From bd0e7622e9241f8f426632971cbbf97d35f1dd01 Mon Sep 17 00:00:00 2001 From: Ricardo Garcia Silva Date: Thu, 8 May 2025 15:48:02 +0100 Subject: [PATCH 35/43] Adding final workflow job to set overall status based on matrix results --- .github/workflows/cite.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cite.yml b/.github/workflows/cite.yml index 68e800287..890acae64 100644 --- a/.github/workflows/cite.yml +++ b/.github/workflows/cite.yml @@ -75,8 +75,8 @@ jobs: steps: - name: check status of workflow run: | - echo "${{ needs }}" - echo "${{ needs.*.result }}" + echo "${{ toJSON(needs) }}" + echo "${{ toJSON(needs.*.result) }}" if [ "${{ contains(needs.*.result, 'failure') }}" = "true" ]; then echo "One or more workflow jobs have failed" exit 1 From 88b34bda10da07f6471d4e514fd1ccce29e23c1c Mon Sep 17 00:00:00 2001 From: Ricardo Garcia Silva Date: Thu, 8 May 2025 16:15:17 +0100 Subject: [PATCH 36/43] Adjusting workflow status when a job fails --- .github/workflows/cite.yml | 22 ++++------------------ 1 file changed, 4 insertions(+), 18 deletions(-) diff --git a/.github/workflows/cite.yml b/.github/workflows/cite.yml index 890acae64..af43b11b9 100644 --- a/.github/workflows/cite.yml +++ b/.github/workflows/cite.yml @@ -12,8 +12,10 @@ jobs: perform-cite-testing: name: ${{ matrix.test-suite.suite-id }} CITE testing - continue-on-error: true +# continue-on-error: true + continue-on-error: false strategy: + fail-fast: false matrix: test-suite: - suite-id: ogcapi-features-1.0 @@ -66,20 +68,4 @@ jobs: uses: OSGeo/cite-runner@main with: test_suite_identifier: ${{ matrix.test-suite.suite-id }} - test_session_arguments: ${{ matrix.test-suite.arguments }} - - set-final-status: - needs: perform-cite-testing - runs-on: ubuntu-22.04 - if: always() - steps: - - name: check status of workflow - run: | - echo "${{ toJSON(needs) }}" - echo "${{ toJSON(needs.*.result) }}" - if [ "${{ contains(needs.*.result, 'failure') }}" = "true" ]; then - echo "One or more workflow jobs have failed" - exit 1 - else - echo "All workflow jobs have passed" - fi \ No newline at end of file + test_session_arguments: ${{ matrix.test-suite.arguments }} \ No newline at end of file From 8c2ecee4b2e73e2f6eba6e2f4d97dfdc4036049c Mon Sep 17 00:00:00 2001 From: Ricardo Garcia Silva Date: Thu, 8 May 2025 16:22:28 +0100 Subject: [PATCH 37/43] Adapt edr suite arguments --- .github/workflows/cite.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cite.yml b/.github/workflows/cite.yml index af43b11b9..67f44855c 100644 --- a/.github/workflows/cite.yml +++ b/.github/workflows/cite.yml @@ -29,7 +29,7 @@ jobs: - suite-id: ogcapi-edr10 arguments: >- iut=http://host.docker.internal:5001 - apiDefinition=http://localhost:5001/openapi + apiDefinition=http://host.docker.internal:5001/openapi - suite-id: ogcapi-tiles-1.0 arguments: >- iut=http://host.docker.internal:5001 From 3ffd9b50f5a9a73d7b5438fe35c0ba0bdd7e2d5c Mon Sep 17 00:00:00 2001 From: Ricardo Garcia Silva Date: Mon, 12 May 2025 15:19:18 +0100 Subject: [PATCH 38/43] Updated cite-runner action version --- .github/workflows/cite.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cite.yml b/.github/workflows/cite.yml index 67f44855c..df303ccc7 100644 --- a/.github/workflows/cite.yml +++ b/.github/workflows/cite.yml @@ -65,7 +65,7 @@ jobs: # tail: 500 - name: test {{ matrix.test-suite.suite-id }} compliance - uses: OSGeo/cite-runner@main + uses: OSGeo/cite-runner@v0.2.0 with: test_suite_identifier: ${{ matrix.test-suite.suite-id }} test_session_arguments: ${{ matrix.test-suite.arguments }} \ No newline at end of file From 0cf747816c5b95aaee39c14c809703a41f7cbeb5 Mon Sep 17 00:00:00 2001 From: Ricardo Garcia Silva Date: Mon, 12 May 2025 15:49:05 +0100 Subject: [PATCH 39/43] Added more test datasets --- tests/cite/cite.config.yml | 266 ++++++++++++++++++------------ tests/cite/compose.test-cite.yaml | 8 +- 2 files changed, 166 insertions(+), 108 deletions(-) diff --git a/tests/cite/cite.config.yml b/tests/cite/cite.config.yml index de7cd4a6c..785a2526a 100644 --- a/tests/cite/cite.config.yml +++ b/tests/cite/cite.config.yml @@ -1,110 +1,174 @@ server: - bind: - host: 0.0.0.0 - port: ${PYGEOAPI_PORT:-5001} - url: ${PYGEOAPI_PUBLIC_URL:-http://localhost:5001} - mimetype: application/json; charset=UTF-8 - encoding: utf-8 - language: en-US - cors: true - pretty_print: true - limit: 100 - # templates: /path/to/templates - map: - url: https://tile.openstreetmap.org/{z}/{x}/{y}.png - attribution: '© OpenStreetMap contributors' - manager: - name: TinyDB - connection: /tmp/pygeoapi-process-manager.db - output_dir: /tmp/ + bind: + host: 0.0.0.0 + port: ${PYGEOAPI_PORT:-5001} + url: ${PYGEOAPI_PUBLIC_URL:-http://localhost:5001} + mimetype: application/json; charset=UTF-8 + encoding: utf-8 + language: en-US + cors: true + pretty_print: true + limit: 100 + # templates: /path/to/templates + map: + url: https://tile.openstreetmap.org/{z}/{x}/{y}.png + attribution: '© OpenStreetMap contributors' + manager: + name: TinyDB + connection: /tmp/pygeoapi-process-manager.db + output_dir: /tmp/ logging: - level: ERROR - #logfile: /tmp/pygeoapi.log + level: ERROR + #logfile: /tmp/pygeoapi.log metadata: - identification: - title: pygeoapi CITE instance - description: pygeoapi instance in support of OGC CITE compliance - keywords: - - geospatial - - data - - api - keywords_type: theme - terms_of_service: https://creativecommons.org/licenses/by/4.0/ - url: https://pygeoapi.io - license: - name: CC-BY 4.0 license - url: https://creativecommons.org/licenses/by/4.0/ - provider: - name: pygeoapi - url: https://pygeoapi.io - contact: - name: Tom Kralidis - position: Senior Systems Scientist - address: Mailing Address - city: City - stateorprovince: Administrative Area - postalcode: Zip or Postal Code - country: Country - phone: +xx-xxx-xxx-xxxx - fax: +xx-xxx-xxx-xxxx - email: pygeoapi@lists.osgeo.org - url: https://pygeoapi.io/community/ - hours: Mo-Fr 08:00-17:00 - instructions: During hours of service. Off on weekends. - role: pointOfContact + identification: + title: pygeoapi CITE instance + description: pygeoapi instance in support of OGC CITE compliance + keywords: + - geospatial + - data + - api + keywords_type: theme + terms_of_service: https://creativecommons.org/licenses/by/4.0/ + url: https://pygeoapi.io + license: + name: CC-BY 4.0 license + url: https://creativecommons.org/licenses/by/4.0/ + provider: + name: pygeoapi + url: https://pygeoapi.io + contact: + name: Tom Kralidis + position: Senior Systems Scientist + address: Mailing Address + city: City + stateorprovince: Administrative Area + postalcode: Zip or Postal Code + country: Country + phone: +xx-xxx-xxx-xxxx + fax: +xx-xxx-xxx-xxxx + email: pygeoapi@lists.osgeo.org + url: https://pygeoapi.io/community/ + hours: Mo-Fr 08:00-17:00 + instructions: During hours of service. Off on weekends. + role: pointOfContact resources: - canada-hydat-daily-mean-02hc003: - type: collection - title: Daily Mean of Water Level or Flow - description: The daily mean is the average of all unit values for a given day. - keywords: [Daily, Daily Mean, Water Level, Flow, Discharge] - crs: - - CRS84 - links: - - type: text/html - rel: canonical - title: Water Level and Flow - Environment Canada - href: https://wateroffice.ec.gc.ca - hreflang: en-CA - - type: text/html - rel: canonical - title: Niveau d'eau et débit - Environnement Canada - href: https://wateroffice.ec.gc.ca/index_f.html - hreflang: fr-CA - - type: text/html - rel: download - title: "National water data archive: HYDAT - Canada.ca" - href: https://www.canada.ca/en/environment-climate-change/services/water-overview/quantity/monitoring/survey/data-products-services/national-archive-hydat.html - hreflang: en-CA - - type: text/html - rel: download - title: "Archives nationales des données hydrologiques : HYDAT - Canada.ca" - href: https://www.canada.ca/fr/environnement-changement-climatique/services/eau-apercu/volume/surveillance/releves/produits-donnees-services/archives-nationales-hydat.html - hreflang: fr-CA - - type: application/zip - rel: download - title: download data - href: https://collaboration.cmc.ec.gc.ca/cmc/hydrometrics/www - hreflang: en-CA - extents: - spatial: - bbox: [-142, 52, -52, 84] - crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 - temporal: - begin: 1850-01-01T00:00:00Z - end: null # or empty + icoads-sst: + type: collection + title: International Comprehensive Ocean-Atmosphere Data Set (ICOADS) + description: International Comprehensive Ocean-Atmosphere Data Set (ICOADS) + keywords: + - icoads + - sst + - air temperature + extents: + spatial: + bbox: [ -180,-90,180,90 ] + crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 + temporal: + begin: 2000-01-16T06:00:00Z + end: 2000-12-16T01:20:06Z + links: + - type: text/html + rel: canonical + title: information + href: https://psl.noaa.gov/data/gridded/data.coads.1deg.html + hreflang: en-US + providers: + - type: edr + name: xarray-edr + data: ${PYGEOAPI_DATA_ROOT}/coads_sst.nc + format: + name: NetCDF + mimetype: application/x-netcdf - providers: - - type: feature - name: Elasticsearch - data: ${PYGEOAPI_ELASTICSEARCH_SERVER_URL:-http://localhost:9200}/canada-hydat-daily-mean-02hc003 - id_field: IDENTIFIER - time_field: DATE - hello-world: - type: process - processor: - name: HelloWorld + canada-hydat-daily-mean-02hc003: + type: collection + title: Daily Mean of Water Level or Flow + description: The daily mean is the average of all unit values for a given day. + keywords: [ Daily, Daily Mean, Water Level, Flow, Discharge ] + crs: + - CRS84 + links: + - type: text/html + rel: canonical + title: Water Level and Flow - Environment Canada + href: https://wateroffice.ec.gc.ca + hreflang: en-CA + - type: text/html + rel: canonical + title: Niveau d'eau et débit - Environnement Canada + href: https://wateroffice.ec.gc.ca/index_f.html + hreflang: fr-CA + - type: text/html + rel: download + title: "National water data archive: HYDAT - Canada.ca" + href: https://www.canada.ca/en/environment-climate-change/services/water-overview/quantity/monitoring/survey/data-products-services/national-archive-hydat.html + hreflang: en-CA + - type: text/html + rel: download + title: "Archives nationales des données hydrologiques : HYDAT - Canada.ca" + href: https://www.canada.ca/fr/environnement-changement-climatique/services/eau-apercu/volume/surveillance/releves/produits-donnees-services/archives-nationales-hydat.html + hreflang: fr-CA + - type: application/zip + rel: download + title: download data + href: https://collaboration.cmc.ec.gc.ca/cmc/hydrometrics/www + hreflang: en-CA + extents: + spatial: + bbox: [ -142, 52, -52, 84 ] + crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 + temporal: + begin: 1850-01-01T00:00:00Z + end: null # or empty + + providers: + - type: feature + name: Elasticsearch + data: ${PYGEOAPI_ELASTICSEARCH_SERVER_URL:-http://localhost:9200}/canada-hydat-daily-mean-02hc003 + id_field: IDENTIFIER + time_field: DATE + + lakes: + type: collection + title: Large Lakes + description: lakes of the world, public domain + keywords: + - lakes + crs: + - CRS84 + links: + - type: text/html + rel: canonical + title: information + href: http://www.naturalearthdata.com/ + hreflang: en-US + extents: + spatial: + bbox: [ -180,-90,180,90 ] + crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 + temporal: + begin: 2011-11-11T00:00:00Z + end: null # or empty + providers: + - type: tile + name: MVT-tippecanoe + data: ${PYGEOAPI_DATA_ROOT}/tiles/ne_110m_lakes + options: + bounds: [ [ -124.953634,-16.536406 ],[ 109.929807,66.969298 ] ] + zoom: + min: 0 + max: 5 + format: + name: pbf + mimetype: application/vnd.mapbox-vector-tile + + hello-world: + type: process + processor: + name: HelloWorld diff --git a/tests/cite/compose.test-cite.yaml b/tests/cite/compose.test-cite.yaml index 194e0d8bb..c1187138f 100644 --- a/tests/cite/compose.test-cite.yaml +++ b/tests/cite/compose.test-cite.yaml @@ -35,6 +35,7 @@ services: PYGEOAPI_CONFIG: /pygeoapi/tests/cite/cite.config.yml PYGEOAPI_OPENAPI: /pygeoapi/tests/cite/cite.openapi.yml PYGEOAPI_ELASTICSEARCH_SERVER_URL: http://elasticsearch-server:9200 + PYGEOAPI_DATA_ROOT: /pygeoapi/tests/data volumes: - type: bind source: $PWD @@ -87,10 +88,3 @@ services: then exit 0; else echo "++++++++++ elasticsearch is not ready ++++++++++"; exit 1; fi - -# teamengine: -# image: ogccite/teamengine-production:1.0-SNAPSHOT -# # use network_mode: host in order to allow teamengine to access pygeoapi -# # at localhost:5001 - this is necessary in order to have teamengine be -# # able to follow pygeoapi URLs -# network_mode: host \ No newline at end of file From e352c6f9722e110dbe88b323986bed413280389b Mon Sep 17 00:00:00 2001 From: Ricardo Garcia Silva Date: Mon, 12 May 2025 17:31:11 +0100 Subject: [PATCH 40/43] Rebase on top of current pygeoapi master --- tests/cite/cite.config.docker.yml | 174 ++++++++++++++++++++++++++++++ tests/cite/cite.config.yml | 28 ++--- tests/cite/compose.test-cite.yaml | 74 +++++-------- 3 files changed, 214 insertions(+), 62 deletions(-) create mode 100644 tests/cite/cite.config.docker.yml diff --git a/tests/cite/cite.config.docker.yml b/tests/cite/cite.config.docker.yml new file mode 100644 index 000000000..064c1bb11 --- /dev/null +++ b/tests/cite/cite.config.docker.yml @@ -0,0 +1,174 @@ +server: + bind: + host: 0.0.0.0 + port: ${PYGEOAPI_PORT:-5001} + url: ${PYGEOAPI_PUBLIC_URL:-http://localhost:5001} + mimetype: application/json; charset=UTF-8 + encoding: utf-8 + language: en-US + cors: true + pretty_print: true + limit: 100 + # templates: /path/to/templates + map: + url: https://tile.openstreetmap.org/{z}/{x}/{y}.png + attribution: '© OpenStreetMap contributors' + manager: + name: TinyDB + connection: /tmp/pygeoapi-process-manager.db + output_dir: /tmp/ + +logging: + level: ERROR + #logfile: /tmp/pygeoapi.log + +metadata: + identification: + title: pygeoapi CITE instance + description: pygeoapi instance in support of OGC CITE compliance + keywords: + - geospatial + - data + - api + keywords_type: theme + terms_of_service: https://creativecommons.org/licenses/by/4.0/ + url: https://pygeoapi.io + license: + name: CC-BY 4.0 license + url: https://creativecommons.org/licenses/by/4.0/ + provider: + name: pygeoapi + url: https://pygeoapi.io + contact: + name: Tom Kralidis + position: Senior Systems Scientist + address: Mailing Address + city: City + stateorprovince: Administrative Area + postalcode: Zip or Postal Code + country: Country + phone: +xx-xxx-xxx-xxxx + fax: +xx-xxx-xxx-xxxx + email: pygeoapi@lists.osgeo.org + url: https://pygeoapi.io/community/ + hours: Mo-Fr 08:00-17:00 + instructions: During hours of service. Off on weekends. + role: pointOfContact + +resources: + icoads-sst: + type: collection + title: International Comprehensive Ocean-Atmosphere Data Set (ICOADS) + description: International Comprehensive Ocean-Atmosphere Data Set (ICOADS) + keywords: + - icoads + - sst + - air temperature + extents: + spatial: + bbox: [ -180,-90,180,90 ] + crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 + temporal: + begin: 2000-01-16T06:00:00Z + end: 2000-12-16T01:20:06Z + links: + - type: text/html + rel: canonical + title: information + href: https://psl.noaa.gov/data/gridded/data.coads.1deg.html + hreflang: en-US + providers: + - type: edr + name: xarray-edr + data: ${PYGEOAPI_DATA_ROOT}/coads_sst.nc + format: + name: NetCDF + mimetype: application/x-netcdf + + + canada-hydat-daily-mean-02hc003: + type: collection + title: Daily Mean of Water Level or Flow + description: The daily mean is the average of all unit values for a given day. + keywords: [ Daily, Daily Mean, Water Level, Flow, Discharge ] + crs: + - CRS84 + links: + - type: text/html + rel: canonical + title: Water Level and Flow - Environment Canada + href: https://wateroffice.ec.gc.ca + hreflang: en-CA + - type: text/html + rel: canonical + title: Niveau d'eau et débit - Environnement Canada + href: https://wateroffice.ec.gc.ca/index_f.html + hreflang: fr-CA + - type: text/html + rel: download + title: "National water data archive: HYDAT - Canada.ca" + href: https://www.canada.ca/en/environment-climate-change/services/water-overview/quantity/monitoring/survey/data-products-services/national-archive-hydat.html + hreflang: en-CA + - type: text/html + rel: download + title: "Archives nationales des données hydrologiques : HYDAT - Canada.ca" + href: https://www.canada.ca/fr/environnement-changement-climatique/services/eau-apercu/volume/surveillance/releves/produits-donnees-services/archives-nationales-hydat.html + hreflang: fr-CA + - type: application/zip + rel: download + title: download data + href: https://collaboration.cmc.ec.gc.ca/cmc/hydrometrics/www + hreflang: en-CA + extents: + spatial: + bbox: [ -142, 52, -52, 84 ] + crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 + temporal: + begin: 1850-01-01T00:00:00Z + end: null # or empty + + providers: + - type: feature + name: TinyDB + data: ${PYGEOAPI_DATA_ROOT}/canada-hydat-daily-mean-02HC003.tinydb + id_field: IDENTIFIER + time_field: DATE + + lakes: + type: collection + title: Large Lakes + description: lakes of the world, public domain + keywords: + - lakes + crs: + - CRS84 + links: + - type: text/html + rel: canonical + title: information + href: http://www.naturalearthdata.com/ + hreflang: en-US + extents: + spatial: + bbox: [ -180,-90,180,90 ] + crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 + temporal: + begin: 2011-11-11T00:00:00Z + end: null # or empty + providers: + - type: tile + name: MVT-tippecanoe + data: ${PYGEOAPI_DATA_ROOT}/tiles/ne_110m_lakes + options: + bounds: [ [ -124.953634,-16.536406 ],[ 109.929807,66.969298 ] ] + zoom: + min: 0 + max: 5 + format: + name: pbf + mimetype: application/vnd.mapbox-vector-tile + + hello-world: + type: process + processor: + name: HelloWorld diff --git a/tests/cite/cite.config.yml b/tests/cite/cite.config.yml index 785a2526a..f39d2bdb0 100644 --- a/tests/cite/cite.config.yml +++ b/tests/cite/cite.config.yml @@ -1,14 +1,16 @@ server: bind: host: 0.0.0.0 - port: ${PYGEOAPI_PORT:-5001} - url: ${PYGEOAPI_PUBLIC_URL:-http://localhost:5001} + port: 5001 + url: http://localhost:5001 mimetype: application/json; charset=UTF-8 encoding: utf-8 language: en-US cors: true pretty_print: true - limit: 100 + limits: + default_items: 10 + max_items: 10 # templates: /path/to/templates map: url: https://tile.openstreetmap.org/{z}/{x}/{y}.png @@ -66,7 +68,7 @@ resources: - air temperature extents: spatial: - bbox: [ -180,-90,180,90 ] + bbox: [-180,-90,180,90] crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 temporal: begin: 2000-01-16T06:00:00Z @@ -80,17 +82,16 @@ resources: providers: - type: edr name: xarray-edr - data: ${PYGEOAPI_DATA_ROOT}/coads_sst.nc + data: ../data/coads_sst.nc format: name: NetCDF mimetype: application/x-netcdf - canada-hydat-daily-mean-02hc003: type: collection title: Daily Mean of Water Level or Flow description: The daily mean is the average of all unit values for a given day. - keywords: [ Daily, Daily Mean, Water Level, Flow, Discharge ] + keywords: [Daily, Daily Mean, Water Level, Flow, Discharge] crs: - CRS84 links: @@ -121,16 +122,15 @@ resources: hreflang: en-CA extents: spatial: - bbox: [ -142, 52, -52, 84 ] + bbox: [-142, 52, -52, 84] crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 temporal: begin: 1850-01-01T00:00:00Z end: null # or empty - providers: - type: feature - name: Elasticsearch - data: ${PYGEOAPI_ELASTICSEARCH_SERVER_URL:-http://localhost:9200}/canada-hydat-daily-mean-02hc003 + name: TinyDB + data: ../data/canada-hydat-daily-mean-02hc003.tinydb id_field: IDENTIFIER time_field: DATE @@ -150,7 +150,7 @@ resources: hreflang: en-US extents: spatial: - bbox: [ -180,-90,180,90 ] + bbox: [-180,-90,180,90] crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 temporal: begin: 2011-11-11T00:00:00Z @@ -158,9 +158,9 @@ resources: providers: - type: tile name: MVT-tippecanoe - data: ${PYGEOAPI_DATA_ROOT}/tiles/ne_110m_lakes + data: ../data/tiles/ne_110m_lakes options: - bounds: [ [ -124.953634,-16.536406 ],[ 109.929807,66.969298 ] ] + bounds: [[-124.953634,-16.536406],[109.929807,66.969298]] zoom: min: 0 max: 5 diff --git a/tests/cite/compose.test-cite.yaml b/tests/cite/compose.test-cite.yaml index c1187138f..72847991f 100644 --- a/tests/cite/compose.test-cite.yaml +++ b/tests/cite/compose.test-cite.yaml @@ -1,22 +1,28 @@ -# compose file suitable for running CITE tests in a local dev environment -# This is meant to emulate the conditions found on pygeoapi's CI - -# initialization: +# docker compose file suitable for running CITE tests in both pygeoapi CI and in a local +# dev environment # -# 0 - Navigate to the root of the pygeoapi repo +# How to use this to run CITE tests locally: # -# 1 - Stand up this stack: -# docker compose -f tests/cite/compose.test-cite.yaml up --detach +# 0. install cite-runner +# pipx install cite-runner # -# 2 - Bootstrap the teamengine service, basically create a user for running tests) -# ./tests/cite/bootstrap-teamengine.sh +# 1. Navigate to the root of the pygeoapi repo +# +# 2. Stand up this stack: +# docker compose -f tests/cite/compose.test-cite.yaml up --detach # -# 3 - Access the teamengine GUI at http://localhost:8080/teamengine and login -# with: -# - user: teamengine -# - password: tester +# 3. Stand up a local teamengine instance +# docker run --rm --name teamengine \ +# --add-host=host.docker.internal:host-gateway \ +# --publish 9081:8080 \ +# ogccite/teamengine-production:1.0-SNAPSHOT # -# 4 - Use the teamengine GUI to run tests suite(s) +# 4. Use cite-runner to run CITE tests +# cite-runner \ +# execute-test-suite \ +# http://localhost:9081/teamengine \ +# ogcapi-features-1.0 \ +# --suite-input iut http://host.docker.internal:5001 @@ -31,8 +37,8 @@ services: published: 5001 environment: PYGEOAPI_PORT: 5000 - PYGEOAPI_PUBLIC_URL: http://localhost:5001 - PYGEOAPI_CONFIG: /pygeoapi/tests/cite/cite.config.yml + PYGEOAPI_PUBLIC_URL: http://host.docker.internal:5001 + PYGEOAPI_CONFIG: /pygeoapi/tests/cite/cite.config.docker.yml PYGEOAPI_OPENAPI: /pygeoapi/tests/cite/cite.openapi.yml PYGEOAPI_ELASTICSEARCH_SERVER_URL: http://elasticsearch-server:9200 PYGEOAPI_DATA_ROOT: /pygeoapi/tests/data @@ -40,19 +46,15 @@ services: - type: bind source: $PWD target: /pygeoapi - depends_on: - elasticsearch-server: - condition: service_healthy entrypoint: ['/bin/bash', '-c'] command: - | pip3 install -r requirements.txt - pip3 install 'elasticsearch-dsl>=8.0.0,<9.0.0' - python3 tests/load_es_data.py tests/data/ne_110m_populated_places_simple.geojson geonameid - python3 tests/load_es_data.py tests/cite/canada-hydat-daily-mean-02HC003.geojson IDENTIFIER + # pip3 install 'elasticsearch-dsl>=8.0.0,<9.0.0' + # python3 tests/load_es_data.py /pygeoapi/tests/data/ne_110m_populated_places_simple.geojson geonameid pip3 install -e . - pygeoapi openapi generate /pygeoapi/tests/cite/cite.config.yml --output-file /pygeoapi/tests/cite/cite.openapi.yml - gunicorn --bind 0.0.0.0:5000 pygeoapi.flask_app:APP + pygeoapi openapi generate /pygeoapi/tests/cite/cite.config.docker.yml --output-file /pygeoapi/tests/cite/cite.openapi.yml + gunicorn --access-logfile - --bind 0.0.0.0:5000 pygeoapi.flask_app:APP healthcheck: interval: 10s timeout: 3s @@ -64,27 +66,3 @@ services: then exit 0; else echo "++++++++++ pygeoapi is not ready ++++++++++"; exit 1; fi - - elasticsearch-server: - image: elasticsearch:8.12.2 - ports: - - target: 9200 - published: 9200 - - target: 9300 - published: 9300 - environment: - ES_JAVA_OPTS: -Xms2g -Xmx2g - discovery.type: single-node - xpack.security.enabled: false - xpack.security.enrollment.enabled: false - healthcheck: - interval: 10s - timeout: 3s - start_period: 1m - retries: 10 - test: | - response=$$(curl --silent --write-out "%{http_code}" --output /dev/null localhost:9200) - if [ $${response} = '200' ]; - then exit 0; - else echo "++++++++++ elasticsearch is not ready ++++++++++"; exit 1; - fi From 0669f32ac4381c3ee2e82ef936897bf51d1269e7 Mon Sep 17 00:00:00 2001 From: Ricardo Garcia Silva Date: Mon, 12 May 2025 17:35:26 +0100 Subject: [PATCH 41/43] Remove commented out workflow step --- .github/workflows/cite.yml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/.github/workflows/cite.yml b/.github/workflows/cite.yml index df303ccc7..3a058d6a1 100644 --- a/.github/workflows/cite.yml +++ b/.github/workflows/cite.yml @@ -56,13 +56,6 @@ jobs: with: container-name: pygeoapi-cite-pygeoapi-1 timeout: 120 -# -# - name: collect docker logs on failure -# if: failure() -# uses: jwalton/gh-docker-logs@v2.2.2 -# with: -# images: ghcr.io/geopython/pygeoapi -# tail: 500 - name: test {{ matrix.test-suite.suite-id }} compliance uses: OSGeo/cite-runner@v0.2.0 From 2ddd3ec05c08b73cf32ba2460d5121640850be54 Mon Sep 17 00:00:00 2001 From: Ricardo Garcia Silva Date: Mon, 12 May 2025 17:36:58 +0100 Subject: [PATCH 42/43] Remove unused teamengine bootstrap file --- tests/cite/bootstrap-teamengine.sh | 27 --------------------------- 1 file changed, 27 deletions(-) delete mode 100644 tests/cite/bootstrap-teamengine.sh diff --git a/tests/cite/bootstrap-teamengine.sh b/tests/cite/bootstrap-teamengine.sh deleted file mode 100644 index 13ac57e54..000000000 --- a/tests/cite/bootstrap-teamengine.sh +++ /dev/null @@ -1,27 +0,0 @@ -# bootstrap the teamengine service -# 1. ensure service is up -# 2. register a test user for running tests - -user_registration_curl_command=$( - printf '%s' \ - "curl -v " \ - "--silent " \ - "--output /dev/null " \ - "--write-out '%{http_code}' " \ - "--data firstName=teamengine " \ - "--data lastName=user " \ - "--data username=teamengine " \ - "--data password=tester " \ - "--data repeat_password=tester " \ - "--data email=noone@noplace.com " \ - "--data organization=none " \ - "--data acceptPrivacy=on " \ - "--data disclaimer=on " \ - "http://localhost:8080/teamengine/registrationHandler" -) -response_status=$(docker exec pygeoapi-cite-teamengine-1 \ - bash -c 'curl --silent --output /dev/null --write-out "%{http_code}" localhost:8080/teamengine/') && \ -echo "Response status: ${response_status}" && \ -user_registration_response_status=$(docker exec pygeoapi-cite-teamengine-1 \ - bash -c "${user_registration_curl_command}") && \ -echo "User registration status: ${user_registration_response_status}" From 468971ec77b6ecbd024552e794f3e21f63563352 Mon Sep 17 00:00:00 2001 From: Ricardo Garcia Silva Date: Wed, 18 Jun 2025 15:28:59 +0100 Subject: [PATCH 43/43] Update cite action --- .github/workflows/cite.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/cite.yml b/.github/workflows/cite.yml index 3a058d6a1..7f9b525d8 100644 --- a/.github/workflows/cite.yml +++ b/.github/workflows/cite.yml @@ -4,6 +4,9 @@ on: push: pull_request: + schedule: + - cron: '43 4 * * *' # run once per day at 04h43 + env: COLUMNS: 120 @@ -58,7 +61,7 @@ jobs: timeout: 120 - name: test {{ matrix.test-suite.suite-id }} compliance - uses: OSGeo/cite-runner@v0.2.0 + uses: OSGeo/ogc-cite-runner@v0.3.0 with: test_suite_identifier: ${{ matrix.test-suite.suite-id }} test_session_arguments: ${{ matrix.test-suite.arguments }} \ No newline at end of file