Merge pull request #86 from minus34/202408

202408

Author: minus34

README.md

@@ -51,8 +51,8 @@ The behaviour of gnaf-loader can be controlled by specifying various command lin
 
 #### Optional Arguments
 * `--srid` Sets the coordinate system of the input data. Valid values are `4283` (the default: GDA94 lat/long) and `7844` (GDA2020 lat/long).
-* `--geoscape-version` Geoscape version number in YYYYMM format. Defaults to current year and last release month. e.g. `202405`.
-* `--previous-geoscape-version` Previous Geoscape release version number as YYYYMM; used for QA comparison. e.g. `202402`.
+* `--geoscape-version` Geoscape version number in YYYYMM format. Defaults to current year and last release month. e.g. `202408`.
+* `--previous-geoscape-version` Previous Geoscape release version number as YYYYMM; used for QA comparison. e.g. `202405`.
 * `--raw-gnaf-schema` schema name to store raw GNAF tables in. Defaults to `raw_gnaf_<geoscape_version>`.
 * `--raw-admin-schema` schema name to store raw admin boundary tables in. Defaults to `raw_admin_bdys_<geoscape_version>`.
 * `--gnaf-schema` destination schema name to store final GNAF tables in. Defaults to `gnaf_<geoscape_version>`.
@@ -69,7 +69,7 @@ The behaviour of gnaf-loader can be controlled by specifying various command lin
 * `--no-boundary-tag` DO NOT tag all addresses with some of the key admin boundary IDs for creating aggregates and choropleth maps.
 
 ### Example Command Line Arguments
-* Local Postgres server: `python load-gnaf.py --gnaf-tables-path="C:\temp\geoscape_202405\G-NAF" --admin-bdys-path="C:\temp\geoscape_202405\Administrative Boundaries"` Loads the GNAF tables to a Postgres server running locally. GNAF archives have been extracted to the folder `C:\temp\geoscape_202405\G-NAF`, and admin boundaries have been extracted to the `C:\temp\geoscape_202405\Administrative Boundaries` folder.
+* Local Postgres server: `python load-gnaf.py --gnaf-tables-path="C:\temp\geoscape_202408\G-NAF" --admin-bdys-path="C:\temp\geoscape_202408\Administrative Boundaries"` Loads the GNAF tables to a Postgres server running locally. GNAF archives have been extracted to the folder `C:\temp\geoscape_202408\G-NAF`, and admin boundaries have been extracted to the `C:\temp\geoscape_202408\Administrative Boundaries` folder.
 * Remote Postgres server: `python load-gnaf.py --gnaf-tables-path="\\svr\shared\gnaf" --local-server-dir="f:\shared\gnaf" --admin-bdys-path="c:\temp\unzipped\AdminBounds_ESRI"` Loads the GNAF tables which have been extracted to the shared folder `\\svr\shared\gnaf`. This shared folder corresponds to the local `f:\shared\gnaf` folder on the Postgres server. Admin boundaries have been extracted to the `c:\temp\unzipped\AdminBounds_ESRI` folder.
 * Loading only selected states: `python load-gnaf.py --states VIC TAS NT ...` Loads only the data for Victoria, Tasmania and Northern Territory
 
@@ -117,8 +117,8 @@ Should take 15-60 minutes.
 - A knowledge of [Postgres pg_restore parameters](https://www.postgresql.org/docs/14/app-pgrestore.html)
 
 ### Process
-1. Download the [GNAF dump file](https://minus34.com/opendata/geoscape-202405/gnaf-202405.dmp) or [GNAF GDA2020 dump file](https://minus34.com/opendata/geoscape-202405-gda2020/gnaf-202405.dmp) (~2.0Gb)
-2. Download the [Admin Bdys dump file](https://minus34.com/opendata/geoscape-202405/admin-bdys-202405.dmp) or [Admin Bdys GDA2020 dump file](https://minus34.com/opendata/geoscape-202405-gda2020/admin-bdys-202405.dmp) (~2.8Gb)
+1. Download the [GNAF dump file](https://minus34.com/opendata/geoscape-202408/gnaf-202408.dmp) or [GNAF GDA2020 dump file](https://minus34.com/opendata/geoscape-202408-gda2020/gnaf-202408.dmp) (~2.0Gb)
+2. Download the [Admin Bdys dump file](https://minus34.com/opendata/geoscape-202408/admin-bdys-202408.dmp) or [Admin Bdys GDA2020 dump file](https://minus34.com/opendata/geoscape-202408-gda2020/admin-bdys-202408.dmp) (~2.8Gb)
 3. Edit the _restore-gnaf-admin-bdys.bat_ or _.sh_ script in the supporting-files folder for your dump file names, database parameters and for the location of pg_restore
 5. Run the script, come back in 15-60 minutes and enjoy!
 
@@ -127,11 +127,11 @@ Geoparquet versions of the spatial tables, as well as parquet versions of the no
 
 Geometries have WGS84 lat/long coordinates (SRID/EPSG:4326). A sample query for analysing the data using [Apache Sedona](https://sedona.apache.org/), the spatial extension to [Apache Spark](https://spark.apache.org/) is in the `spark` folder.
 
-The files are here: `s3://minus34.com/opendata/geoscape-202405/geoparquet/`
+The files are here: `s3://minus34.com/opendata/geoscape-202408/geoparquet/`
 
 ### AWS CLI Examples:
-- List all datasets: `aws s3 ls s3://minus34.com/opendata/geoscape-202405/geoparquet/`
-- Copy all datasets: `aws s3 sync s3://minus34.com/opendata/geoscape-202405/geoparquet/ <my-local-folder>`
+- List all datasets: `aws s3 ls s3://minus34.com/opendata/geoscape-202408/geoparquet/`
+- Copy all datasets: `aws s3 sync s3://minus34.com/opendata/geoscape-202408/geoparquet/ <my-local-folder>`
 
 ## DATA LICENSES
 

docker/Dockerfile

@@ -1,7 +1,7 @@
 FROM debian:bookworm-slim
 
 # replaced the downloading of the Potgres dump files to use local files instead (for performance)
-# ARG BASE_URL="https://minus34.com/opendata/geoscape-202405"
+# ARG BASE_URL="https://minus34.com/opendata/geoscape-202408"
 # ENV BASE_URL ${BASE_URL}
 
 # Postgres user password - WARNING: change this to something a lot more secure
@@ -33,23 +33,23 @@ RUN echo "listen_addresses='*'" >> /etc/postgresql/15/main/postgresql.conf
 RUN mkdir -p /data
 WORKDIR /data
 
-ADD gnaf-202405.dmp .
-ADD admin-bdys-202405.dmp .
+ADD gnaf-202408.dmp .
+ADD admin-bdys-202408.dmp .
 
 # replace the add statements above if wanting to download Postgres dump files
 # RUN /data \
-#     && wget --quiet ${BASE_URL}/gnaf-202405.dmp \
-#     && wget --quiet ${BASE_URL}/admin-bdys-202405.dmp
+#     && wget --quiet ${BASE_URL}/gnaf-202408.dmp \
+#     && wget --quiet ${BASE_URL}/admin-bdys-202408.dmp
 
 RUN /etc/init.d/postgresql start \
-    && pg_restore -Fc -d postgres -h localhost -p 5432 -U postgres /data/gnaf-202405.dmp \
+    && pg_restore -Fc -d postgres -h localhost -p 5432 -U postgres /data/gnaf-202408.dmp \
     && /etc/init.d/postgresql stop \
-    && rm /data/gnaf-202405.dmp
+    && rm /data/gnaf-202408.dmp
 
 RUN /etc/init.d/postgresql start \
-    && pg_restore -Fc -d postgres -h localhost -p 5432 -U postgres /data/admin-bdys-202405.dmp \
+    && pg_restore -Fc -d postgres -h localhost -p 5432 -U postgres /data/admin-bdys-202408.dmp \
     && /etc/init.d/postgresql stop \
-    && rm /data/admin-bdys-202405.dmp
+    && rm /data/admin-bdys-202408.dmp
 
 EXPOSE 5432
 

docker/xx_code_snippets.sh

@@ -2,7 +2,7 @@
 cd /Users/$(whoami)/git/minus34/gnaf-loader/docker
 
 # build gnaf loader image
-docker build --squash --tag minus34/gnafloader:latest --tag minus34/gnafloader:202405 .
+docker build --squash --tag minus34/gnafloader:latest --tag minus34/gnafloader:202408 .
 
 # run gnaf loader container
 docker run --name=gnafloader --publish=5433:5432 minus34/gnafloader:latest

load-gnaf.py

@@ -214,7 +214,7 @@ def populate_raw_gnaf(pg_cur):
         # load all PSV files using multiprocessing
         geoscape.multiprocess_list("sql", sql_list, logger)
 
-        # fix missing geocodes (added due to missing data in 202405 release)
+        # fix missing geocodes (added due to missing data in 202408 release)
         sql = geoscape.open_sql_file("01-04-raw-gnaf-fix-missing-geocodes.sql")
         pg_cur.execute(sql)
 

postgres-scripts/01-04-raw-gnaf-fix-missing-geocodes.sql

@@ -1,4 +1,4 @@
--- workaround for missing default coordinates - 202405 release issue
+-- workaround for missing default coordinates - 202408 release issue
 with missing as (
     select address_detail_pid
     from raw_gnaf.address_default_geocode

postgres-scripts/02-02a-prep-admin-bdys-tables.sql

@@ -203,10 +203,10 @@ UPDATE admin_bdys.locality_bdys
 ;
 
 
--- -- add old locality_pids to unedited localities -- need to rollover old locality pids from GNAF 202405 release - not supplied in 202405 release
+-- -- add old locality_pids to unedited localities -- need to rollover old locality pids from GNAF 202408 release - not supplied in 202408 release
 -- UPDATE admin_bdys.locality_bdys as new
 --     SET old_locality_pid = old.old_locality_pid
--- FROM admin_bdys_202405.locality_bdys AS old
+-- FROM admin_bdys_202408.locality_bdys AS old
 -- WHERE new.locality_pid = old.locality_pid;
 
 

postgres-scripts/xx-04-02-manual-bdy-tags.sql

@@ -4,7 +4,7 @@
 
 
 -- fix 35 boatsheds
-update gnaf_202405.address_principal_admin_boundaries
+update gnaf_202408.address_principal_admin_boundaries
     set lga_pid = 'lgacbffb11990f2',
         lga_name = 'Hobart City'
 where locality_pid = 'loc0f7a581b85b7'

postgres-scripts/xx-add-elevation-to-gnaf.sql

@@ -43,7 +43,7 @@ DROP TABLE IF EXISTS temp_gnaf_100m_points;
 --
 -- SELECT ST_Value(dem.rast, gnaf.geom) as elevation,
 --        *
--- FROM gnaf_202405.address_principals as gnaf
--- INNER JOIN gnaf_202405.srtm_3s_dem as dem on ST_Intersects(gnaf.geom, dem.rast) limit 100;
+-- FROM gnaf_202408.address_principals as gnaf
+-- INNER JOIN gnaf_202408.srtm_3s_dem as dem on ST_Intersects(gnaf.geom, dem.rast) limit 100;
 
 

postgres-scripts/xx-alias-principals-with-different-coordinates.sql

@@ -6,9 +6,9 @@ SELECT als.gnaf_pid, als.street_locality_pid, als.locality_pid, als.alias_princi
 					ST_MakePoint(als.longitude, als.latitude)::geography,
 					ST_MakePoint(gnaf.longitude, gnaf.latitude)::geography
 				) as distance
-  FROM gnaf_202405.address_aliases as als
-  INNER JOIN gnaf_202405.address_alias_lookup as lkp on als.gnaf_pid = lkp.alias_pid
-  INNER JOIN gnaf_202405.address_principals as gnaf on lkp.principal_pid = gnaf.gnaf_pid
+  FROM gnaf_202408.address_aliases as als
+  INNER JOIN gnaf_202408.address_alias_lookup as lkp on als.gnaf_pid = lkp.alias_pid
+  INNER JOIN gnaf_202408.address_principals as gnaf on lkp.principal_pid = gnaf.gnaf_pid
   WHERE als.latitude <> gnaf.latitude
   OR als.longitude <> als.longitude
   order by ST_Distance(

postgres-scripts/xx-export-address-principals-to-csv.sql

@@ -6,5 +6,5 @@ COPY (
 				 address, locality_name, postcode, state, locality_postcode, confidence, 
 				 legal_parcel_id, mb_2016_code, mb_2021_code, latitude, longitude,
 				 geocode_type, reliability
-		FROM gnaf_202405.address_principals
+		FROM gnaf_202408.address_principals
 ) TO '/Users/hugh.saalmans/tmp/address_principals.psv' HEADER CSV;