Skip to content

Commit 4d1d74c

Browse files
author
Phil Varner
authored
queryables for query (#403)
* add landing page queryables link rel and /queryables endpoint * add /collections/{cid}/queryables endpoint and link rels
1 parent 859622b commit 4d1d74c

14 files changed

+1195
-911
lines changed

.github/workflows/push.yaml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,8 @@ jobs:
1919
ports:
2020
- 4566:4566
2121
steps:
22-
- uses: actions/checkout@v2
23-
- uses: actions/setup-node@v2
22+
- uses: actions/checkout@v3
23+
- uses: actions/setup-node@v3
2424
with:
2525
node-version-file: ".nvmrc"
2626
cache: npm

CHANGELOG.md

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,13 @@ All notable changes to this project will be documented in this file.
55
The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
66
and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
77

8+
## [Unreleased] - TBD
9+
10+
### Added
11+
12+
- Added support for root and collection-level queryables to be used for Query Extension
13+
filtering.
14+
815
## [0.7.0] - 2023-02-09
916

1017
### Changed

README.md

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
- [Proxying Stac-server through CloudFront](#proxying-stac-server-through-cloudfront)
2828
- [Locking down transaction endpoints](#locking-down-transaction-endpoints)
2929
- [AWS WAF Rule Conflicts](#aws-waf-rule-conflicts)
30+
- [Queryables](#queryables)
3031
- [Ingesting Data](#ingesting-data)
3132
- [Ingesting large items](#ingesting-large-items)
3233
- [Subscribing to SNS Topics](#subscribing-to-sns-topics)
@@ -821,6 +822,19 @@ This is also triggered when using pystac_client with no filtering parameters.
821822
The fix is to disable the WAF SQL injection rule, which is unnecessary because
822823
stac-server does not use SQL.
823824

825+
## Queryables
826+
827+
STAC API supports the Query Extension. Unlike the Filter Extension (which is not supported),
828+
the Query Extension does not (yet) define a mechanism to advertise which terms may be
829+
used in expressions. However, an optional defintion may be added to it soon that defines
830+
queryables endpoints the same as used with Filter Extension. To define these for a Collection,
831+
add a field `queryables` with the value as the JSON Schema definition of the queryables
832+
for that collection. This will be used for a collection's queryables resource, and removed
833+
from the Collection entity whenever that is returned.
834+
835+
A non-configurable root-level queryables definition is defined with no named terms but
836+
`additionalProperties` set to `true`.
837+
824838
## Ingesting Data
825839

826840
STAC Collections and Items are ingested by the `ingest` Lambda function, however this Lambda is not invoked directly by a user, it consumes records from the `stac-server-<stage>-queue` SQS. To add STAC Items or Collections to the queue, publish them to the SNS Topic `stac-server-<stage>-ingest`.

package-lock.json

Lines changed: 889 additions & 887 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

package.json

Lines changed: 13 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,7 @@
6161
},
6262
"dependencies": {
6363
"@acuris/aws-es-connection": "^1.1.0",
64-
"@aws-sdk/client-secrets-manager": "^3.266.0",
64+
"@aws-sdk/client-secrets-manager": "^3.272.0",
6565
"@elastic/elasticsearch": "^7.9.0",
6666
"@mapbox/extent": "^0.4.0",
6767
"@opensearch-project/opensearch": "^2.2.0",
@@ -81,39 +81,39 @@
8181
"through2": "^4.0.2",
8282
"ts-loader": "^9.4.2",
8383
"winston": "^3.8.2",
84-
"zod": "^3.20.2"
84+
"zod": "^3.20.6"
8585
},
8686
"devDependencies": {
8787
"@ava/typescript": "^3.0.1",
8888
"@stoplight/spectral-cli": "^6.6.0",
8989
"@tsconfig/node16": "^1.0.3",
9090
"@types/aws-lambda": "^8.10.110",
9191
"@types/cors": "^2.8.13",
92-
"@types/express": "^4.17.16",
92+
"@types/express": "^4.17.17",
9393
"@types/http-errors": "^2.0.1",
9494
"@types/luxon": "^3.2.0",
9595
"@types/memorystream": "^0.3.0",
9696
"@types/morgan": "^1.9.4",
9797
"@types/node": "^16.18.9",
9898
"@types/sinon": "^10.0.13",
99-
"@typescript-eslint/eslint-plugin": "^5.49.0",
100-
"@typescript-eslint/parser": "^5.49.0",
101-
"ava": "^5.1.1",
99+
"@typescript-eslint/eslint-plugin": "^5.53.0",
100+
"@typescript-eslint/parser": "^5.53.0",
101+
"ava": "^5.2.0",
102102
"aws-event-mocks": "^0.0.0",
103-
"aws-sdk": "^2.1302.0",
103+
"aws-sdk": "^2.1319.0",
104104
"aws-sdk-client-mock": "^2.0.1",
105-
"c8": "^7.12.0",
105+
"c8": "^7.13.0",
106106
"copy-webpack-plugin": "^11.0.0",
107107
"crypto-random-string": "^5.0.0",
108-
"eslint": "^8.33.0",
108+
"eslint": "^8.34.0",
109109
"eslint-config-airbnb": "^19.0.4",
110110
"eslint-plugin-import": "^2.27.5",
111-
"eslint-plugin-jsdoc": "^39.8.0",
111+
"eslint-plugin-jsdoc": "^40.0.0",
112112
"luxon": "^3.2.1",
113113
"nock": "^13.3.0",
114114
"nodemon": "^2.0.20",
115115
"pre-commit": "^1.2.2",
116-
"prettier": "^2.8.3",
116+
"prettier": "^2.8.4",
117117
"prettier-eslint": "^15.0.1",
118118
"prettier-eslint-cli": "^7.1.0",
119119
"proxyquire": "^2.1.3",
@@ -122,8 +122,8 @@
122122
"shins": "^2.6.0",
123123
"sinon": "^15.0.1",
124124
"ts-node": "^10.9.1",
125-
"tslib": "^2.4.1",
126-
"typescript": "^4.9.4",
125+
"tslib": "^2.5.0",
126+
"typescript": "^4.9.5",
127127
"webpack": "^5.75.0",
128128
"webpack-cli": "^5.0.1",
129129
"widdershins": "^4.0.1",

src/lambdas/api/app.js

Lines changed: 39 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,15 @@ app.get('/conformance', async (_req, res, next) => {
7474
}
7575
})
7676

77+
app.get('/queryables', async (req, res, next) => {
78+
try {
79+
res.type('application/schema+json')
80+
res.json(await api.getQueryables(req.endpoint))
81+
} catch (error) {
82+
next(error)
83+
}
84+
})
85+
7786
app.get('/search', async (req, res, next) => {
7887
try {
7988
res.type('application/geo+json')
@@ -157,7 +166,7 @@ app.get('/collections/:collectionId', async (req, res, next) => {
157166
const { collectionId } = req.params
158167
try {
159168
const response = await api.getCollection(collectionId, database, req.endpoint)
160-
169+
delete response.queryables
161170
if (response instanceof Error) next(createError(404))
162171
else res.json(response)
163172
} catch (error) {
@@ -191,6 +200,35 @@ app.get('/collections/:collectionId/items', async (req, res, next) => {
191200
}
192201
})
193202

203+
const DEFAULT_QUERYABLES = {
204+
$schema: 'https://json-schema.org/draft/2020-12/schema',
205+
type: 'object',
206+
properties: {},
207+
additionalProperties: true
208+
}
209+
210+
app.get('/collections/:collectionId/queryables', async (req, res, next) => {
211+
const { collectionId } = req.params
212+
try {
213+
const collection = await api.getCollection(collectionId, database, req.endpoint)
214+
215+
if (collection instanceof Error) next(createError(404))
216+
else {
217+
const queryables = collection.queryables || { ...DEFAULT_QUERYABLES }
218+
queryables.$id = `${req.endpoint}/collections/${collectionId}/queryables`
219+
queryables.title = `Queryables for Collection ${collectionId}`
220+
res.type('application/schema+json')
221+
res.json(queryables)
222+
}
223+
} catch (error) {
224+
if (error instanceof ValidationError) {
225+
next(createError(400, error.message))
226+
} else {
227+
next(error)
228+
}
229+
}
230+
})
231+
194232
app.post('/collections/:collectionId/items', async (req, res, next) => {
195233
if (txnEnabled) {
196234
const { collectionId } = req.params

src/lib/api.js

Lines changed: 29 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -341,6 +341,12 @@ const addCollectionLinks = function (results, endpoint) {
341341
type: 'application/geo+json',
342342
href: `${endpoint}/collections/${id}/items`
343343
})
344+
// queryables
345+
links.push({
346+
rel: 'http://www.opengis.net/def/rel/ogc/1.0/queryables',
347+
type: 'application/schema+json',
348+
href: `${endpoint}/collections/${id}/queryables`
349+
})
344350
})
345351
return results
346352
}
@@ -707,6 +713,15 @@ const getConformance = async function (txnEnabled) {
707713
return { conformsTo }
708714
}
709715

716+
const getQueryables = async (endpoint = '') => ({
717+
$schema: 'https://json-schema.org/draft/2020-12/schema',
718+
$id: `${endpoint}/queryables`,
719+
type: 'object',
720+
title: `Queryables for ${process.env['STAC_TITLE'] || 'STAC API'}`,
721+
properties: {},
722+
additionalProperties: true
723+
})
724+
710725
const getCatalog = async function (txnEnabled, backend, endpoint = '') {
711726
const links = [
712727
{
@@ -755,7 +770,12 @@ const getCatalog = async function (txnEnabled, backend, endpoint = '') {
755770
rel: 'service-doc',
756771
type: 'text/html',
757772
href: `${endpoint}/api.html`
758-
}
773+
},
774+
{
775+
rel: 'http://www.opengis.net/def/rel/ogc/1.0/queryables',
776+
type: 'application/schema+json',
777+
href: `${endpoint}/queryables`
778+
},
759779
]
760780

761781
const docsUrl = process.env['STAC_DOCS_URL']
@@ -778,10 +798,14 @@ const getCatalog = async function (txnEnabled, backend, endpoint = '') {
778798
const getCollections = async function (backend, endpoint = '') {
779799
// TODO: implement proper pagination, as this will only return up to
780800
// COLLECTION_LIMIT collections
781-
const results = await backend.getCollections(1, COLLECTION_LIMIT)
782-
const linkedCollections = addCollectionLinks(results, endpoint)
801+
const collections = await backend.getCollections(1, COLLECTION_LIMIT)
802+
for (const collection of collections) {
803+
delete collection.queryables
804+
}
805+
806+
const linkedCollections = addCollectionLinks(collections, endpoint)
783807
const resp = {
784-
collections: results,
808+
collections,
785809
links: [
786810
{
787811
rel: 'self',
@@ -942,4 +966,5 @@ export default {
942966
aggregate,
943967
getItemThumbnail,
944968
healthCheck,
969+
getQueryables,
945970
}
Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
{
2+
"id": "landsat-8-l1-no-queryables",
3+
"type": "Collection",
4+
"stac_version": "1.0.0",
5+
"description": "Landat-8 L1 Collection-1 imagery radiometrically calibrated and orthorectified using gound points and Digital Elevation Model (DEM) data to correct relief displacement.",
6+
"links": [ ],
7+
"stac_extensions": [],
8+
"title": "Landsat-8 L1 Collection-1",
9+
"extent": {
10+
"spatial": {
11+
"bbox": [
12+
[
13+
-180,
14+
-90,
15+
180,
16+
90
17+
]
18+
]
19+
},
20+
"temporal": {
21+
"interval": [
22+
[
23+
"2013-06-01T00:00:00Z",
24+
null
25+
]
26+
]
27+
}
28+
}
29+
}

tests/fixtures/landsat-8-l1-collection.json

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -269,5 +269,17 @@
269269
]
270270
}
271271
},
272-
"license": "PDDL-1.0"
272+
"license": "PDDL-1.0",
273+
"queryables": {
274+
"$schema": "https://json-schema.org/draft/2020-12/schema",
275+
"$id": "",
276+
"type": "object",
277+
"title": "",
278+
"properties": {
279+
"eo:cloud_cover": {
280+
"$ref": "https://stac-extensions.github.io/eo/v1.0.0/schema.json#/definitions/fields/properties/eo:cloud_cover"
281+
}
282+
},
283+
"additionalProperties": true
284+
}
273285
}
Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
// @ts-nocheck
2+
3+
import test from 'ava'
4+
import { deleteAllIndices } from '../helpers/database.js'
5+
import { ingestItem } from '../helpers/ingest.js'
6+
import { randomId, loadFixture } from '../helpers/utils.js'
7+
import { setup } from '../helpers/system-tests.js'
8+
9+
test.before(async (t) => {
10+
await deleteAllIndices()
11+
const standUpResult = await setup()
12+
13+
t.context = standUpResult
14+
15+
t.context.collectionId = randomId('collection')
16+
})
17+
18+
test.after.always(async (t) => {
19+
if (t.context.api) await t.context.api.close()
20+
})
21+
22+
test('GET /collections/:collectionId/queryables returns queryables', async (t) => {
23+
const collection = await loadFixture(
24+
'landsat-8-l1-collection.json',
25+
{ id: t.context.collectionId }
26+
)
27+
28+
await ingestItem({
29+
ingestQueueUrl: t.context.ingestQueueUrl,
30+
ingestTopicArn: t.context.ingestTopicArn,
31+
item: collection
32+
})
33+
34+
const { collectionId } = t.context
35+
36+
const response = await t.context.api.client.get(`collections/${collectionId}/queryables`,
37+
{ resolveBodyOnly: false })
38+
39+
t.is(response.statusCode, 200)
40+
t.is(response.headers['content-type'], 'application/schema+json; charset=utf-8')
41+
// @ts-expect-error We need to validate these responses
42+
t.true(response.body.$id.endsWith(`/collections/${collectionId}/queryables`))
43+
t.is(response.body.title, `Queryables for Collection ${collectionId}`)
44+
t.is(response.body.$schema, 'https://json-schema.org/draft/2020-12/schema')
45+
t.is(response.body.type, 'object')
46+
t.deepEqual(response.body.properties, {
47+
'eo:cloud_cover': {
48+
$ref: 'https://stac-extensions.github.io/eo/v1.0.0/schema.json#/definitions/fields/properties/eo:cloud_cover',
49+
},
50+
})
51+
t.is(response.body.additionalProperties, true)
52+
})
53+
54+
test('GET /collections/:collectionId/queryables returns queryables even if not defined in Collection', async (t) => {
55+
const collection = await loadFixture(
56+
'collection-without-queryables.json',
57+
{ id: t.context.collectionId }
58+
)
59+
60+
await ingestItem({
61+
ingestQueueUrl: t.context.ingestQueueUrl,
62+
ingestTopicArn: t.context.ingestTopicArn,
63+
item: collection
64+
})
65+
66+
const { collectionId } = t.context
67+
68+
const response = await t.context.api.client.get(`collections/${collectionId}/queryables`,
69+
{ resolveBodyOnly: false })
70+
71+
t.is(response.statusCode, 200)
72+
t.is(response.headers['content-type'], 'application/schema+json; charset=utf-8')
73+
// @ts-expect-error We need to validate these responses
74+
t.true(response.body.$id.endsWith(`/collections/${collectionId}/queryables`))
75+
t.is(response.body.title, `Queryables for Collection ${collectionId}`)
76+
t.is(response.body.$schema, 'https://json-schema.org/draft/2020-12/schema')
77+
t.is(response.body.type, 'object')
78+
t.deepEqual(response.body.properties, {})
79+
t.is(response.body.additionalProperties, true)
80+
})
81+
82+
test('GET /collection/:collectionId/queryables for non-existent collection returns Not Found', async (t) => {
83+
const response = await t.context.api.client.get(
84+
'collections/DOES_NOT_EXIST/queryables',
85+
{ resolveBodyOnly: false, throwHttpErrors: false }
86+
)
87+
88+
t.is(response.statusCode, 404)
89+
})

0 commit comments

Comments
 (0)