diff --git a/.github/workflows/cite.yml b/.github/workflows/cite.yml new file mode 100644 index 000000000..7f9b525d8 --- /dev/null +++ b/.github/workflows/cite.yml @@ -0,0 +1,67 @@ +name: 'Perform CITE tests against a local teamengine instance' + +on: + push: + pull_request: + + schedule: + - cron: '43 4 * * *' # run once per day at 04h43 + +env: + COLUMNS: 120 + + +jobs: + + perform-cite-testing: + name: ${{ matrix.test-suite.suite-id }} CITE testing +# continue-on-error: true + continue-on-error: false + strategy: + fail-fast: false + matrix: + test-suite: + - suite-id: ogcapi-features-1.0 + arguments: >- + iut=http://host.docker.internal:5001 + noofcollections=-1 + - suite-id: ogcapi-processes-1.0 + arguments: >- + iut=http://host.docker.internal:5001 + noofcollections=-1 + - suite-id: ogcapi-edr10 + arguments: >- + iut=http://host.docker.internal:5001 + apiDefinition=http://host.docker.internal:5001/openapi + - suite-id: ogcapi-tiles-1.0 + arguments: >- + iut=http://host.docker.internal:5001 + tilematrixsetdefinitionuri=http://www.opengis.net/def/tilematrixset/OGC/1.0/WebMercatorQuad + urltemplatefortiles=http://localhost:5001/collections/lakes/tiles/WebMercatorQuad/{tileMatrix}/{tileRow}/{tileCol}?f=mvt + tilematrix=0 + mintilerow=0 + maxtilerow=1 + mintilecol=0 + maxtilecol=1 + + runs-on: ubuntu-22.04 + steps: + + - name: grab code + uses: actions/checkout@v4 + + - name: start pygeoapi with suitable CITE data and config + run: > + docker compose -f tests/cite/compose.test-cite.yaml up --detach + + - name: wait for pygeoapi to be usable + uses: raschmitt/wait-for-healthy-container@v1.0.1 + with: + container-name: pygeoapi-cite-pygeoapi-1 + timeout: 120 + + - name: test {{ matrix.test-suite.suite-id }} compliance + uses: OSGeo/ogc-cite-runner@v0.3.0 + with: + test_suite_identifier: ${{ matrix.test-suite.suite-id }} + test_session_arguments: ${{ matrix.test-suite.arguments }} \ No newline at end of file diff --git a/.gitignore b/.gitignore index 3a434a1dd..f1fe3b252 100644 --- a/.gitignore +++ b/.gitignore @@ -105,7 +105,10 @@ ENV/ # development setup examples example-config.yml -example-openapi.yml +example-openapi.yml + +# CITE openapi document (must be (re)generated before runing CITE tests) +tests/cite/cite.openapi.yml # misc *.swp diff --git a/tests/cite/cite.config.docker.yml b/tests/cite/cite.config.docker.yml new file mode 100644 index 000000000..064c1bb11 --- /dev/null +++ b/tests/cite/cite.config.docker.yml @@ -0,0 +1,174 @@ +server: + bind: + host: 0.0.0.0 + port: ${PYGEOAPI_PORT:-5001} + url: ${PYGEOAPI_PUBLIC_URL:-http://localhost:5001} + mimetype: application/json; charset=UTF-8 + encoding: utf-8 + language: en-US + cors: true + pretty_print: true + limit: 100 + # templates: /path/to/templates + map: + url: https://tile.openstreetmap.org/{z}/{x}/{y}.png + attribution: '© OpenStreetMap contributors' + manager: + name: TinyDB + connection: /tmp/pygeoapi-process-manager.db + output_dir: /tmp/ + +logging: + level: ERROR + #logfile: /tmp/pygeoapi.log + +metadata: + identification: + title: pygeoapi CITE instance + description: pygeoapi instance in support of OGC CITE compliance + keywords: + - geospatial + - data + - api + keywords_type: theme + terms_of_service: https://creativecommons.org/licenses/by/4.0/ + url: https://pygeoapi.io + license: + name: CC-BY 4.0 license + url: https://creativecommons.org/licenses/by/4.0/ + provider: + name: pygeoapi + url: https://pygeoapi.io + contact: + name: Tom Kralidis + position: Senior Systems Scientist + address: Mailing Address + city: City + stateorprovince: Administrative Area + postalcode: Zip or Postal Code + country: Country + phone: +xx-xxx-xxx-xxxx + fax: +xx-xxx-xxx-xxxx + email: pygeoapi@lists.osgeo.org + url: https://pygeoapi.io/community/ + hours: Mo-Fr 08:00-17:00 + instructions: During hours of service. Off on weekends. + role: pointOfContact + +resources: + icoads-sst: + type: collection + title: International Comprehensive Ocean-Atmosphere Data Set (ICOADS) + description: International Comprehensive Ocean-Atmosphere Data Set (ICOADS) + keywords: + - icoads + - sst + - air temperature + extents: + spatial: + bbox: [ -180,-90,180,90 ] + crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 + temporal: + begin: 2000-01-16T06:00:00Z + end: 2000-12-16T01:20:06Z + links: + - type: text/html + rel: canonical + title: information + href: https://psl.noaa.gov/data/gridded/data.coads.1deg.html + hreflang: en-US + providers: + - type: edr + name: xarray-edr + data: ${PYGEOAPI_DATA_ROOT}/coads_sst.nc + format: + name: NetCDF + mimetype: application/x-netcdf + + + canada-hydat-daily-mean-02hc003: + type: collection + title: Daily Mean of Water Level or Flow + description: The daily mean is the average of all unit values for a given day. + keywords: [ Daily, Daily Mean, Water Level, Flow, Discharge ] + crs: + - CRS84 + links: + - type: text/html + rel: canonical + title: Water Level and Flow - Environment Canada + href: https://wateroffice.ec.gc.ca + hreflang: en-CA + - type: text/html + rel: canonical + title: Niveau d'eau et débit - Environnement Canada + href: https://wateroffice.ec.gc.ca/index_f.html + hreflang: fr-CA + - type: text/html + rel: download + title: "National water data archive: HYDAT - Canada.ca" + href: https://www.canada.ca/en/environment-climate-change/services/water-overview/quantity/monitoring/survey/data-products-services/national-archive-hydat.html + hreflang: en-CA + - type: text/html + rel: download + title: "Archives nationales des données hydrologiques : HYDAT - Canada.ca" + href: https://www.canada.ca/fr/environnement-changement-climatique/services/eau-apercu/volume/surveillance/releves/produits-donnees-services/archives-nationales-hydat.html + hreflang: fr-CA + - type: application/zip + rel: download + title: download data + href: https://collaboration.cmc.ec.gc.ca/cmc/hydrometrics/www + hreflang: en-CA + extents: + spatial: + bbox: [ -142, 52, -52, 84 ] + crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 + temporal: + begin: 1850-01-01T00:00:00Z + end: null # or empty + + providers: + - type: feature + name: TinyDB + data: ${PYGEOAPI_DATA_ROOT}/canada-hydat-daily-mean-02HC003.tinydb + id_field: IDENTIFIER + time_field: DATE + + lakes: + type: collection + title: Large Lakes + description: lakes of the world, public domain + keywords: + - lakes + crs: + - CRS84 + links: + - type: text/html + rel: canonical + title: information + href: http://www.naturalearthdata.com/ + hreflang: en-US + extents: + spatial: + bbox: [ -180,-90,180,90 ] + crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 + temporal: + begin: 2011-11-11T00:00:00Z + end: null # or empty + providers: + - type: tile + name: MVT-tippecanoe + data: ${PYGEOAPI_DATA_ROOT}/tiles/ne_110m_lakes + options: + bounds: [ [ -124.953634,-16.536406 ],[ 109.929807,66.969298 ] ] + zoom: + min: 0 + max: 5 + format: + name: pbf + mimetype: application/vnd.mapbox-vector-tile + + hello-world: + type: process + processor: + name: HelloWorld diff --git a/tests/cite/cite.config.yml b/tests/cite/cite.config.yml index 104fb09b5..f39d2bdb0 100644 --- a/tests/cite/cite.config.yml +++ b/tests/cite/cite.config.yml @@ -1,174 +1,174 @@ server: - bind: - host: 0.0.0.0 - port: 5001 - url: http://localhost:5001 - mimetype: application/json; charset=UTF-8 - encoding: utf-8 - language: en-US - cors: true - pretty_print: true - limits: - default_items: 10 - max_items: 10 - # templates: /path/to/templates - map: - url: https://tile.openstreetmap.org/{z}/{x}/{y}.png - attribution: '© OpenStreetMap contributors' - manager: - name: TinyDB - connection: /tmp/pygeoapi-process-manager.db - output_dir: /tmp/ + bind: + host: 0.0.0.0 + port: 5001 + url: http://localhost:5001 + mimetype: application/json; charset=UTF-8 + encoding: utf-8 + language: en-US + cors: true + pretty_print: true + limits: + default_items: 10 + max_items: 10 + # templates: /path/to/templates + map: + url: https://tile.openstreetmap.org/{z}/{x}/{y}.png + attribution: '© OpenStreetMap contributors' + manager: + name: TinyDB + connection: /tmp/pygeoapi-process-manager.db + output_dir: /tmp/ logging: - level: ERROR - #logfile: /tmp/pygeoapi.log + level: ERROR + #logfile: /tmp/pygeoapi.log metadata: - identification: - title: pygeoapi CITE instance - description: pygeoapi instance in support of OGC CITE compliance - keywords: - - geospatial - - data - - api - keywords_type: theme - terms_of_service: https://creativecommons.org/licenses/by/4.0/ - url: https://pygeoapi.io - license: - name: CC-BY 4.0 license - url: https://creativecommons.org/licenses/by/4.0/ - provider: - name: pygeoapi - url: https://pygeoapi.io - contact: - name: Tom Kralidis - position: Senior Systems Scientist - address: Mailing Address - city: City - stateorprovince: Administrative Area - postalcode: Zip or Postal Code - country: Country - phone: +xx-xxx-xxx-xxxx - fax: +xx-xxx-xxx-xxxx - email: pygeoapi@lists.osgeo.org - url: https://pygeoapi.io/community/ - hours: Mo-Fr 08:00-17:00 - instructions: During hours of service. Off on weekends. - role: pointOfContact + identification: + title: pygeoapi CITE instance + description: pygeoapi instance in support of OGC CITE compliance + keywords: + - geospatial + - data + - api + keywords_type: theme + terms_of_service: https://creativecommons.org/licenses/by/4.0/ + url: https://pygeoapi.io + license: + name: CC-BY 4.0 license + url: https://creativecommons.org/licenses/by/4.0/ + provider: + name: pygeoapi + url: https://pygeoapi.io + contact: + name: Tom Kralidis + position: Senior Systems Scientist + address: Mailing Address + city: City + stateorprovince: Administrative Area + postalcode: Zip or Postal Code + country: Country + phone: +xx-xxx-xxx-xxxx + fax: +xx-xxx-xxx-xxxx + email: pygeoapi@lists.osgeo.org + url: https://pygeoapi.io/community/ + hours: Mo-Fr 08:00-17:00 + instructions: During hours of service. Off on weekends. + role: pointOfContact resources: - icoads-sst: - type: collection - title: International Comprehensive Ocean-Atmosphere Data Set (ICOADS) - description: International Comprehensive Ocean-Atmosphere Data Set (ICOADS) - keywords: - - icoads - - sst - - air temperature - extents: - spatial: - bbox: [-180,-90,180,90] - crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 - temporal: - begin: 2000-01-16T06:00:00Z - end: 2000-12-16T01:20:06Z - links: - - type: text/html - rel: canonical - title: information - href: https://psl.noaa.gov/data/gridded/data.coads.1deg.html - hreflang: en-US - providers: - - type: edr - name: xarray-edr - data: ../data/coads_sst.nc - format: - name: NetCDF - mimetype: application/x-netcdf + icoads-sst: + type: collection + title: International Comprehensive Ocean-Atmosphere Data Set (ICOADS) + description: International Comprehensive Ocean-Atmosphere Data Set (ICOADS) + keywords: + - icoads + - sst + - air temperature + extents: + spatial: + bbox: [-180,-90,180,90] + crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 + temporal: + begin: 2000-01-16T06:00:00Z + end: 2000-12-16T01:20:06Z + links: + - type: text/html + rel: canonical + title: information + href: https://psl.noaa.gov/data/gridded/data.coads.1deg.html + hreflang: en-US + providers: + - type: edr + name: xarray-edr + data: ../data/coads_sst.nc + format: + name: NetCDF + mimetype: application/x-netcdf - canada-hydat-daily-mean-02hc003: - type: collection - title: Daily Mean of Water Level or Flow - description: The daily mean is the average of all unit values for a given day. - keywords: [Daily, Daily Mean, Water Level, Flow, Discharge] - crs: - - CRS84 - links: - - type: text/html - rel: canonical - title: Water Level and Flow - Environment Canada - href: https://wateroffice.ec.gc.ca - hreflang: en-CA - - type: text/html - rel: canonical - title: Niveau d'eau et débit - Environnement Canada - href: https://wateroffice.ec.gc.ca/index_f.html - hreflang: fr-CA - - type: text/html - rel: download - title: "National water data archive: HYDAT - Canada.ca" - href: https://www.canada.ca/en/environment-climate-change/services/water-overview/quantity/monitoring/survey/data-products-services/national-archive-hydat.html - hreflang: en-CA - - type: text/html - rel: download - title: "Archives nationales des données hydrologiques : HYDAT - Canada.ca" - href: https://www.canada.ca/fr/environnement-changement-climatique/services/eau-apercu/volume/surveillance/releves/produits-donnees-services/archives-nationales-hydat.html - hreflang: fr-CA - - type: application/zip - rel: download - title: download data - href: https://collaboration.cmc.ec.gc.ca/cmc/hydrometrics/www - hreflang: en-CA - extents: - spatial: - bbox: [-142, 52, -52, 84] - crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 - temporal: - begin: 1850-01-01T00:00:00Z - end: null # or empty - providers: - - type: feature - name: TinyDB - data: ../data/canada-hydat-daily-mean-02hc003.tinydb - id_field: IDENTIFIER - time_field: DATE + canada-hydat-daily-mean-02hc003: + type: collection + title: Daily Mean of Water Level or Flow + description: The daily mean is the average of all unit values for a given day. + keywords: [Daily, Daily Mean, Water Level, Flow, Discharge] + crs: + - CRS84 + links: + - type: text/html + rel: canonical + title: Water Level and Flow - Environment Canada + href: https://wateroffice.ec.gc.ca + hreflang: en-CA + - type: text/html + rel: canonical + title: Niveau d'eau et débit - Environnement Canada + href: https://wateroffice.ec.gc.ca/index_f.html + hreflang: fr-CA + - type: text/html + rel: download + title: "National water data archive: HYDAT - Canada.ca" + href: https://www.canada.ca/en/environment-climate-change/services/water-overview/quantity/monitoring/survey/data-products-services/national-archive-hydat.html + hreflang: en-CA + - type: text/html + rel: download + title: "Archives nationales des données hydrologiques : HYDAT - Canada.ca" + href: https://www.canada.ca/fr/environnement-changement-climatique/services/eau-apercu/volume/surveillance/releves/produits-donnees-services/archives-nationales-hydat.html + hreflang: fr-CA + - type: application/zip + rel: download + title: download data + href: https://collaboration.cmc.ec.gc.ca/cmc/hydrometrics/www + hreflang: en-CA + extents: + spatial: + bbox: [-142, 52, -52, 84] + crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 + temporal: + begin: 1850-01-01T00:00:00Z + end: null # or empty + providers: + - type: feature + name: TinyDB + data: ../data/canada-hydat-daily-mean-02hc003.tinydb + id_field: IDENTIFIER + time_field: DATE - lakes: - type: collection - title: Large Lakes - description: lakes of the world, public domain - keywords: - - lakes - crs: - - CRS84 - links: - - type: text/html - rel: canonical - title: information - href: http://www.naturalearthdata.com/ - hreflang: en-US - extents: - spatial: - bbox: [-180,-90,180,90] - crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 - temporal: - begin: 2011-11-11T00:00:00Z - end: null # or empty - providers: - - type: tile - name: MVT-tippecanoe - data: ../data/tiles/ne_110m_lakes - options: - bounds: [[-124.953634,-16.536406],[109.929807,66.969298]] - zoom: - min: 0 - max: 5 - format: - name: pbf - mimetype: application/vnd.mapbox-vector-tile + lakes: + type: collection + title: Large Lakes + description: lakes of the world, public domain + keywords: + - lakes + crs: + - CRS84 + links: + - type: text/html + rel: canonical + title: information + href: http://www.naturalearthdata.com/ + hreflang: en-US + extents: + spatial: + bbox: [-180,-90,180,90] + crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 + temporal: + begin: 2011-11-11T00:00:00Z + end: null # or empty + providers: + - type: tile + name: MVT-tippecanoe + data: ../data/tiles/ne_110m_lakes + options: + bounds: [[-124.953634,-16.536406],[109.929807,66.969298]] + zoom: + min: 0 + max: 5 + format: + name: pbf + mimetype: application/vnd.mapbox-vector-tile - hello-world: - type: process - processor: - name: HelloWorld + hello-world: + type: process + processor: + name: HelloWorld diff --git a/tests/cite/compose.test-cite.yaml b/tests/cite/compose.test-cite.yaml new file mode 100644 index 000000000..72847991f --- /dev/null +++ b/tests/cite/compose.test-cite.yaml @@ -0,0 +1,68 @@ +# docker compose file suitable for running CITE tests in both pygeoapi CI and in a local +# dev environment +# +# How to use this to run CITE tests locally: +# +# 0. install cite-runner +# pipx install cite-runner +# +# 1. Navigate to the root of the pygeoapi repo +# +# 2. Stand up this stack: +# docker compose -f tests/cite/compose.test-cite.yaml up --detach +# +# 3. Stand up a local teamengine instance +# docker run --rm --name teamengine \ +# --add-host=host.docker.internal:host-gateway \ +# --publish 9081:8080 \ +# ogccite/teamengine-production:1.0-SNAPSHOT +# +# 4. Use cite-runner to run CITE tests +# cite-runner \ +# execute-test-suite \ +# http://localhost:9081/teamengine \ +# ogcapi-features-1.0 \ +# --suite-input iut http://host.docker.internal:5001 + + + +name: pygeoapi-cite + +services: + + pygeoapi: + image: 'ghcr.io/geopython/pygeoapi:${CURRENT_GIT_BRANCH:-latest}' + ports: + - target: 5000 + published: 5001 + environment: + PYGEOAPI_PORT: 5000 + PYGEOAPI_PUBLIC_URL: http://host.docker.internal:5001 + PYGEOAPI_CONFIG: /pygeoapi/tests/cite/cite.config.docker.yml + PYGEOAPI_OPENAPI: /pygeoapi/tests/cite/cite.openapi.yml + PYGEOAPI_ELASTICSEARCH_SERVER_URL: http://elasticsearch-server:9200 + PYGEOAPI_DATA_ROOT: /pygeoapi/tests/data + volumes: + - type: bind + source: $PWD + target: /pygeoapi + entrypoint: ['/bin/bash', '-c'] + command: + - | + pip3 install -r requirements.txt + # pip3 install 'elasticsearch-dsl>=8.0.0,<9.0.0' + # python3 tests/load_es_data.py /pygeoapi/tests/data/ne_110m_populated_places_simple.geojson geonameid + pip3 install -e . + pygeoapi openapi generate /pygeoapi/tests/cite/cite.config.docker.yml --output-file /pygeoapi/tests/cite/cite.openapi.yml + gunicorn --access-logfile - --bind 0.0.0.0:5000 pygeoapi.flask_app:APP + healthcheck: + interval: 10s + timeout: 3s + start_period: 1m + retries: 10 + test: | + response=$$(python3 -c 'import urllib.request; print(urllib.request.urlopen("http://localhost:5000").status)') + if [ $${response} = '200' ]; + then exit 0; + else echo "++++++++++ pygeoapi is not ready ++++++++++"; exit 1; + fi diff --git a/tests/load_es_data.py b/tests/load_es_data.py index 3324a69ae..353a01ce9 100644 --- a/tests/load_es_data.py +++ b/tests/load_es_data.py @@ -29,10 +29,14 @@ import json from pathlib import Path +import os import sys from elasticsearch import Elasticsearch, helpers -es = Elasticsearch('http://localhost:9200') + +elastic_url = os.getenv( + 'PYGEOAPI_ELASTICSEARCH_SERVER_URL', 'http://localhost:9200') +es = Elasticsearch(elastic_url) if len(sys.argv) < 3: print(f'Usage: {sys.argv[0]} ')