Skip to content

Commit 0b323db

Browse files
bkeepersbkeepers
committed
Atomic imports to support parallelism
1 parent f990eb6 commit 0b323db

10 files changed

Lines changed: 196 additions & 460 deletions

File tree

‎Makefile‎

Lines changed: 0 additions & 44 deletions
This file was deleted.

‎bin/download‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
#!/usr/bin/env bash
2+
# Downloads NOAA ENC data
3+
# Usage: bin/download
4+
5+
set -e
6+
7+
mkdir -p data
8+
echo "Downloading NOAA ENC data..."
9+
curl -L -o data/ALL_ENCs.zip https://charts.noaa.gov/ENCs/All_ENCs.zip
10+
echo "Extracting ENC data..."
11+
unzip -o data/ALL_ENCs.zip -d data

‎bin/s57-import‎

Lines changed: 88 additions & 60 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,14 @@
77
# tags features with chart metadata (DSNM, INTU), and pre-computes attributes
88
# for S-52 Conditional Symbology Procedures that require spatial queries.
99
#
10+
# The entire import runs in a single transaction: old data is deleted, new data
11+
# imported, and all post-processing runs atomically.
12+
#
13+
# Parallel imports are safe after the first chart has been imported (which creates
14+
# tables and columns). PostgreSQL MVCC ensures each transaction only sees its own
15+
# uncommitted rows, so concurrent `UPDATE WHERE dsnm IS NULL` tagging won't
16+
# cross-contaminate between parallel imports.
17+
#
1018
# Requires: PostGIS running (docker compose up postgis)
1119

1220
set -e
@@ -16,6 +24,9 @@ VERBOSE=${VERBOSE:-0}
1624

1725
if [ $# -lt 1 ]; then
1826
echo "Usage: $0 input.000 [input2.000 ...]"
27+
echo ""
28+
echo "Import all charts in parallel:"
29+
echo " find data/ENC_ROOT -name '*.000' | xargs -P \$(nproc) -I {} $0 {}"
1930
exit 1
2031
fi
2132

@@ -27,7 +38,7 @@ fi
2738
export PGHOST PGPORT PGDATABASE PGUSER PGPASSWORD
2839

2940
PG="PG:host=$PGHOST port=$PGPORT dbname=$PGDATABASE user=$PGUSER password=$PGPASSWORD"
30-
export OGR_S57_OPTIONS="SPLIT_MULTIPOINT=ON,ADD_SOUNDG_DEPTH=ON"
41+
export OGR_S57_OPTIONS="SPLIT_MULTIPOINT=ON,ADD_SOUNDG_DEPTH=ON,LIST_AS_STRING=YES"
3142

3243
for input in "$@"; do
3344
dsid=$(ogrinfo -q "$input" DSID 2>/dev/null)
@@ -41,77 +52,94 @@ for input in "$@"; do
4152

4253
echo "Importing $dsnm (INTU=$intu)..."
4354

44-
# Delete any existing data from this chart (for re-imports)
45-
tables=$(psql -tAq -c "
46-
SELECT table_name FROM information_schema.columns
55+
# Build PRELUDE: delete any existing data from this chart (for re-imports).
56+
# Wrapped in a DO block so missing tables don't cause errors.
57+
prelude="BEGIN;
58+
DO \$\$ DECLARE t text; BEGIN
59+
FOR t IN SELECT table_name FROM information_schema.columns
4760
WHERE table_schema = 'public' AND column_name = 'dsnm'
4861
GROUP BY table_name
49-
" 2>/dev/null || true)
50-
51-
if [ -n "$tables" ]; then
52-
for table in $tables; do
53-
psql -q -c "DELETE FROM \"$table\" WHERE dsnm = '$dsnm'" 2>/dev/null || true
54-
done
55-
fi
62+
LOOP
63+
EXECUTE format('DELETE FROM %I WHERE dsnm = \$1', t) USING '${dsnm}';
64+
END LOOP;
65+
END \$\$"
66+
67+
# Build CLOSING: tag features, preprocess, set SCAMIN defaults, dedup.
68+
closing="
69+
DO \$\$ DECLARE t text; BEGIN
70+
-- Add DSNM/INTU columns and tag new features
71+
FOR t IN SELECT table_name FROM information_schema.tables
72+
WHERE table_schema = 'public' AND table_type = 'BASE TABLE'
73+
AND table_name NOT IN ('spatial_ref_sys', 'geometry_columns', 'geography_columns')
74+
LOOP
75+
EXECUTE format('ALTER TABLE %I ADD COLUMN IF NOT EXISTS dsnm TEXT', t);
76+
EXECUTE format('ALTER TABLE %I ADD COLUMN IF NOT EXISTS intu INTEGER', t);
77+
EXECUTE format('UPDATE %I SET dsnm = \$1, intu = \$2 WHERE dsnm IS NULL', t) USING '${dsnm}', ${intu};
78+
79+
-- Set default SCAMIN where missing, based on INTU
80+
BEGIN
81+
EXECUTE format('UPDATE %I SET \"SCAMIN\" = CASE intu
82+
WHEN 1 THEN 19999999 WHEN 2 THEN 2999999 WHEN 3 THEN 499999
83+
WHEN 4 THEN 259999 WHEN 5 THEN 44999 WHEN 6 THEN 21999
84+
END WHERE \"SCAMIN\" IS NULL AND dsnm = \$1', t) USING '${dsnm}';
85+
EXCEPTION WHEN undefined_column THEN NULL;
86+
END;
87+
88+
-- Deduplicate by LNAM: keep highest-INTU version
89+
BEGIN
90+
EXECUTE format('DELETE FROM %I a USING %I b
91+
WHERE a.\"LNAM\" = b.\"LNAM\" AND a.intu < b.intu', t, t);
92+
EXCEPTION WHEN undefined_column THEN NULL;
93+
END;
94+
END LOOP;
95+
96+
-- UDWHAZ05: store containing DEPARE depth on hazard features
97+
BEGIN
98+
ALTER TABLE \"WRECKS\" ADD COLUMN IF NOT EXISTS _depare_drval1 REAL;
99+
UPDATE \"WRECKS\" w SET _depare_drval1 = d.\"DRVAL1\"
100+
FROM \"DEPARE\" d WHERE ST_Contains(d.geom, w.geom)
101+
AND w.dsnm = '${dsnm}' AND w._depare_drval1 IS NULL;
102+
EXCEPTION WHEN undefined_table THEN NULL;
103+
END;
104+
105+
BEGIN
106+
ALTER TABLE \"OBSTRN\" ADD COLUMN IF NOT EXISTS _depare_drval1 REAL;
107+
UPDATE \"OBSTRN\" w SET _depare_drval1 = d.\"DRVAL1\"
108+
FROM \"DEPARE\" d WHERE ST_Contains(d.geom, w.geom)
109+
AND w.dsnm = '${dsnm}' AND w._depare_drval1 IS NULL;
110+
EXCEPTION WHEN undefined_table THEN NULL;
111+
END;
112+
113+
-- TOPMAR01: detect floating platform
114+
BEGIN
115+
ALTER TABLE \"TOPMAR\" ADD COLUMN IF NOT EXISTS _floating INTEGER DEFAULT 0;
116+
FOR t IN SELECT unnest(ARRAY['BOYLAT','BOYSAW','BOYSPP','BOYCAR','BOYINB','BOYISD','LITFLT','LITVES'])
117+
LOOP
118+
BEGIN
119+
EXECUTE format('UPDATE \"TOPMAR\" SET _floating = 1
120+
WHERE dsnm = \$1 AND _floating = 0
121+
AND EXISTS (SELECT 1 FROM %I f WHERE ST_DWithin(\"TOPMAR\".geom, f.geom, 0.0001))', t)
122+
USING '${dsnm}';
123+
EXCEPTION WHEN undefined_table THEN NULL;
124+
END;
125+
END LOOP;
126+
EXCEPTION WHEN undefined_table THEN NULL;
127+
END;
128+
END \$\$;
129+
COMMIT"
56130

57-
# Import all layers into PostGIS
58131
ogr2ogr -f PostgreSQL "$PG" "$input" \
59132
-append \
60133
-skipfailures \
61134
-nlt PROMOTE_TO_MULTI \
62135
-lco GEOMETRY_NAME=geom \
63136
-lco SPATIAL_INDEX=GIST \
64137
-lco LAUNDER=NO \
138+
-ds_transaction \
65139
--config PG_USE_COPY YES \
140+
-doo "PRELUDE_STATEMENTS=$prelude" \
141+
-doo "CLOSING_STATEMENTS=$closing" \
66142
2>/dev/null
67143

68-
# Get the list of tables that were created/updated
69-
all_tables=$(psql -tAq -c "
70-
SELECT table_name FROM information_schema.tables
71-
WHERE table_schema = 'public' AND table_type = 'BASE TABLE'
72-
AND table_name NOT IN ('spatial_ref_sys', 'geometry_columns', 'geography_columns')
73-
")
74-
75-
# Add DSNM and INTU columns if they don't exist, then tag new features
76-
for table in $all_tables; do
77-
psql -q <<SQL 2>/dev/null || true
78-
ALTER TABLE "$table" ADD COLUMN IF NOT EXISTS dsnm TEXT;
79-
ALTER TABLE "$table" ADD COLUMN IF NOT EXISTS intu INTEGER;
80-
UPDATE "$table" SET dsnm = '$dsnm', intu = $intu WHERE dsnm IS NULL;
81-
SQL
82-
done
83-
84-
# --- Spatial preprocessing for S-52 CSPs ---
85-
86-
# UDWHAZ05: Store the DRVAL1 of the containing DEPARE on WRECKS and OBSTRN.
87-
# Style compares _DEPARE_DRVAL1 against the user's safety contour at render time.
88-
for hazard_table in WRECKS OBSTRN; do
89-
psql -q <<SQL 2>/dev/null || true
90-
ALTER TABLE "$hazard_table" ADD COLUMN IF NOT EXISTS _depare_drval1 REAL;
91-
UPDATE "$hazard_table" w SET _depare_drval1 = d."DRVAL1"
92-
FROM "DEPARE" d
93-
WHERE ST_Contains(d.geom, w.geom)
94-
AND w.dsnm = '$dsnm'
95-
AND w._depare_drval1 IS NULL;
96-
SQL
97-
done
98-
99-
# TOPMAR01: Determine if topmark is on a floating platform (buoy, light float,
100-
# light vessel) or rigid (beacon, daymark). Different symbol tables apply.
101-
psql -q <<SQL 2>/dev/null || true
102-
ALTER TABLE "TOPMAR" ADD COLUMN IF NOT EXISTS _floating INTEGER DEFAULT 0;
103-
SQL
104-
for float_table in BOYLAT BOYSAW BOYSPP BOYCAR BOYINB BOYISD LITFLT LITVES; do
105-
psql -q <<SQL 2>/dev/null || true
106-
UPDATE "TOPMAR" SET _floating = 1
107-
WHERE dsnm = '$dsnm'
108-
AND _floating = 0
109-
AND EXISTS (
110-
SELECT 1 FROM "$float_table" f
111-
WHERE ST_DWithin("TOPMAR".geom, f.geom, 0.0001)
112-
);
113-
SQL
114-
done
115-
116144
echo " Done: $dsnm"
117145
done

‎bin/s57-to-tiles‎

Lines changed: 0 additions & 143 deletions
This file was deleted.

‎bin/setup‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,3 +5,4 @@ set -e
55
git submodule update --init
66
npm install
77
npm run build
8+
docker compose up -d

0 commit comments

Comments
 (0)