Skip to content

Commit 0a1a655

Browse files
Allow Optimized searches for tiled requets. (#39)
* xyz tile search * add functions for bulk create_items and upsert_items * update tests, create migrations * add exitwhenfull flag Co-authored-by: vincentsarago <[email protected]>
1 parent 1c0694e commit 0a1a655

16 files changed

+2289
-10
lines changed

CHANGELOG.md

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,20 @@
11
# Changelog
22

3+
## [v0.3.4]
4+
5+
### Added
6+
7+
- add `geometrysearch`, `geojsonsearch` and `xyzsearch` for optimized searches for tiled requets ([#39](https://github.com/stac-utils/pgstac/pull/39))
8+
9+
## [v0.3.3]
10+
11+
## Fixed
12+
13+
- Fixed CQL term to be "id", not "ids" ([#46](https://github.com/stac-utils/pgstac/pull/46))
14+
- Make sure featureCollection response has empty features `[]` not `null` ([#46](https://github.com/stac-utils/pgstac/pull/46))
15+
- Fixed bugs for `sortby` and `pagination` ([#46](https://github.com/stac-utils/pgstac/pull/46))
16+
- Make sure pgtap errors get caught in CI ([#46](https://github.com/stac-utils/pgstac/pull/46))
17+
318
## [v0.3.2]
419

520
## Fixed

docker-compose.yml

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ services:
55
build:
66
context: .
77
dockerfile: Dockerfile.dev
8+
platform: linux/amd64
89
depends_on:
910
- database
1011
volumes:
@@ -20,6 +21,7 @@ services:
2021
build:
2122
context: .
2223
dockerfile: Dockerfile
24+
platform: linux/amd64
2325
environment:
2426
- POSTGRES_USER=username
2527
- POSTGRES_PASSWORD=password
@@ -29,7 +31,7 @@ services:
2931
- PGHOST=localhost
3032
- PGDATABASE=postgis
3133
ports:
32-
- "5432:5432"
34+
- "5439:5432"
3335
volumes:
3436
- pgstac-pgdata:/var/lib/postgresql/data
3537
- ./:/opt/src

pgstac.sql

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,5 +6,7 @@ BEGIN;
66
\i sql/002_collections.sql
77
\i sql/003_items.sql
88
\i sql/004_search.sql
9+
\i sql/005_tileutils.sql
10+
\i sql/006_tilesearch.sql
911
\i sql/999_version.sql
1012
COMMIT;

pypgstac/pypgstac/__init__.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,2 @@
11
"""PyPGStac Version."""
2-
__version__ = "0.3.3"
2+
__version__ = "0.3.4"
Lines changed: 200 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,200 @@
1+
SET SEARCH_PATH to pgstac, public;
2+
set check_function_bodies = off;
3+
4+
CREATE OR REPLACE FUNCTION pgstac.create_items(data jsonb)
5+
RETURNS void
6+
LANGUAGE sql
7+
SET search_path TO 'pgstac', 'public'
8+
AS $function$
9+
INSERT INTO items_staging (content)
10+
SELECT * FROM jsonb_array_elements(data);
11+
$function$
12+
;
13+
14+
CREATE OR REPLACE FUNCTION pgstac.ftime()
15+
RETURNS interval
16+
LANGUAGE sql
17+
AS $function$
18+
SELECT age(clock_timestamp(), transaction_timestamp());
19+
$function$
20+
;
21+
22+
CREATE OR REPLACE FUNCTION pgstac.geojsonsearch(geojson jsonb, queryhash text, fields jsonb DEFAULT NULL::jsonb, _scanlimit integer DEFAULT 10000, _limit integer DEFAULT 100, _timelimit interval DEFAULT '00:00:05'::interval, exitwhenfull boolean DEFAULT true, skipcovered boolean DEFAULT true)
23+
RETURNS jsonb
24+
LANGUAGE sql
25+
AS $function$
26+
SELECT * FROM geometrysearch(
27+
st_geomfromgeojson(geojson),
28+
queryhash,
29+
fields,
30+
_scanlimit,
31+
_limit,
32+
_timelimit,
33+
exitwhenfull,
34+
skipcovered
35+
);
36+
$function$
37+
;
38+
39+
CREATE OR REPLACE FUNCTION pgstac.geometrysearch(geom geometry, queryhash text, fields jsonb DEFAULT NULL::jsonb, _scanlimit integer DEFAULT 10000, _limit integer DEFAULT 100, _timelimit interval DEFAULT '00:00:05'::interval, exitwhenfull boolean DEFAULT true, skipcovered boolean DEFAULT true)
40+
RETURNS jsonb
41+
LANGUAGE plpgsql
42+
AS $function$
43+
DECLARE
44+
search searches%ROWTYPE;
45+
curs refcursor;
46+
_where text;
47+
query text;
48+
iter_record items%ROWTYPE;
49+
out_records jsonb[] := '{}'::jsonb[];
50+
exit_flag boolean := FALSE;
51+
counter int := 1;
52+
scancounter int := 1;
53+
remaining_limit int := _scanlimit;
54+
tilearea float;
55+
unionedgeom geometry;
56+
clippedgeom geometry;
57+
unionedgeom_area float := 0;
58+
prev_area float := 0;
59+
excludes text[];
60+
includes text[];
61+
62+
BEGIN
63+
-- If skipcovered is true then you will always want to exit when the passed in geometry is full
64+
IF skipcovered THEN
65+
exitwhenfull := TRUE;
66+
END IF;
67+
68+
SELECT * INTO search FROM searches WHERE hash=queryhash;
69+
70+
IF NOT FOUND THEN
71+
RAISE EXCEPTION 'Search with Query Hash % Not Found', queryhash;
72+
END IF;
73+
74+
tilearea := st_area(geom);
75+
_where := format('%s AND st_intersects(geometry, %L::geometry)', search._where, geom);
76+
77+
IF fields IS NOT NULL THEN
78+
IF fields ? 'fields' THEN
79+
fields := fields->'fields';
80+
END IF;
81+
IF fields ? 'exclude' THEN
82+
excludes=textarr(fields->'exclude');
83+
END IF;
84+
IF fields ? 'include' THEN
85+
includes=textarr(fields->'include');
86+
IF array_length(includes, 1)>0 AND NOT 'id' = ANY (includes) THEN
87+
includes = includes || '{id}';
88+
END IF;
89+
END IF;
90+
END IF;
91+
RAISE NOTICE 'fields: %, includes: %, excludes: %', fields, includes, excludes;
92+
93+
FOR query IN SELECT * FROM partition_queries(_where, search.orderby) LOOP
94+
query := format('%s LIMIT %L', query, remaining_limit);
95+
RAISE NOTICE '%', query;
96+
curs = create_cursor(query);
97+
LOOP
98+
FETCH curs INTO iter_record;
99+
EXIT WHEN NOT FOUND;
100+
IF exitwhenfull OR skipcovered THEN -- If we are not using exitwhenfull or skipcovered, we do not need to do expensive geometry operations
101+
clippedgeom := st_intersection(geom, iter_record.geometry);
102+
103+
IF unionedgeom IS NULL THEN
104+
unionedgeom := clippedgeom;
105+
ELSE
106+
unionedgeom := st_union(unionedgeom, clippedgeom);
107+
END IF;
108+
109+
unionedgeom_area := st_area(unionedgeom);
110+
111+
IF skipcovered AND prev_area = unionedgeom_area THEN
112+
scancounter := scancounter + 1;
113+
CONTINUE;
114+
END IF;
115+
116+
prev_area := unionedgeom_area;
117+
118+
RAISE NOTICE '% % % %', unionedgeom_area/tilearea, counter, scancounter, ftime();
119+
END IF;
120+
121+
IF fields IS NOT NULL THEN
122+
out_records := out_records || filter_jsonb(iter_record.content, includes, excludes);
123+
ELSE
124+
out_records := out_records || iter_record.content;
125+
END IF;
126+
IF counter >= _limit
127+
OR scancounter > _scanlimit
128+
OR ftime() > _timelimit
129+
OR (exitwhenfull AND unionedgeom_area >= tilearea)
130+
THEN
131+
exit_flag := TRUE;
132+
EXIT;
133+
END IF;
134+
counter := counter + 1;
135+
scancounter := scancounter + 1;
136+
137+
END LOOP;
138+
EXIT WHEN exit_flag;
139+
remaining_limit := _scanlimit - scancounter;
140+
END LOOP;
141+
142+
RETURN jsonb_build_object(
143+
'type', 'FeatureCollection',
144+
'features', array_to_json(out_records)::jsonb
145+
);
146+
END;
147+
$function$
148+
;
149+
150+
CREATE OR REPLACE FUNCTION pgstac.tileenvelope(zoom integer, x integer, y integer)
151+
RETURNS geometry
152+
LANGUAGE sql
153+
IMMUTABLE PARALLEL SAFE
154+
AS $function$
155+
WITH t AS (
156+
SELECT
157+
20037508.3427892 as merc_max,
158+
-20037508.3427892 as merc_min,
159+
(2 * 20037508.3427892) / (2 ^ zoom) as tile_size
160+
)
161+
SELECT st_makeenvelope(
162+
merc_min + (tile_size * x),
163+
merc_max - (tile_size * (y + 1)),
164+
merc_min + (tile_size * (x + 1)),
165+
merc_max - (tile_size * y),
166+
3857
167+
) FROM t;
168+
$function$
169+
;
170+
171+
CREATE OR REPLACE FUNCTION pgstac.upsert_items(data jsonb)
172+
RETURNS void
173+
LANGUAGE sql
174+
SET search_path TO 'pgstac', 'public'
175+
AS $function$
176+
INSERT INTO items_staging_upsert (content)
177+
SELECT * FROM jsonb_array_elements(data);
178+
$function$
179+
;
180+
181+
CREATE OR REPLACE FUNCTION pgstac.xyzsearch(_x integer, _y integer, _z integer, queryhash text, fields jsonb DEFAULT NULL::jsonb, _scanlimit integer DEFAULT 10000, _limit integer DEFAULT 100, _timelimit interval DEFAULT '00:00:05'::interval, exitwhenfull boolean DEFAULT true, skipcovered boolean DEFAULT true)
182+
RETURNS jsonb
183+
LANGUAGE sql
184+
AS $function$
185+
SELECT * FROM geometrysearch(
186+
st_transform(tileenvelope(_z, _x, _y), 4326),
187+
queryhash,
188+
fields,
189+
_scanlimit,
190+
_limit,
191+
_timelimit,
192+
exitwhenfull,
193+
skipcovered
194+
);
195+
$function$
196+
;
197+
198+
199+
200+
INSERT INTO migrations (version) VALUES ('0.3.4');

0 commit comments

Comments
 (0)