Freeze CIA world factbook source

README.md

@@ -10,10 +10,20 @@ This Python package has the ability to scrap available information on the Intern
 Using public data from the the following websites:
 - Wikipedia
 - ITU-T
-- The CIA World Factbook
+- The CIA World Factbook (frozen — no longer refreshable)
 - txtNation
 - Ewen Gallic blog
 
+The CIA World Factbook has been sunset (https://www.cia.gov/stories/story/spotlighting-the-world-factbook-as-we-bid-a-fond-farewell/).
+This project keeps the last bundled World Factbook extract in `src/mcc_mnc_genlib/raw/world_fb.*`,
+but no longer supports refreshing it from the network.
+The frozen WFB data is still used by `gen_dataset.py` to enrich the country dictionaries
+(`p1_cc2` and `p1_cntr`) with the following fields:
+- `geo`: airports, ports, capital, coordinates, coastline, boundaries, region, WFB URL
+- `tel`: mobile/fixed/broadband subscriber counts, internet users, telecom notes
+- `codes`: GENC and STAN country codes (US government / NATO variants of ISO 3166)
+- `infos.nameset`: additional country name variants
+
 All raw content extracted is available in `src/mcc_mnc_genlib/raw/` as JSON and Python dictionnaries.
 This project aggregates and generates re-engineered dictionnaries from all those sources.
 All re-engineered content is available in `src/mcc_mnc_lut/`, as JSON and Python dictionnaries too.
@@ -158,10 +168,10 @@ The script put all resulting JSON and Python files into the `src/mcc_mnc_genlib/
 
 After checking several sources, it seems Wikipedia has the most complete, up-to-date and accurate information.
 Therefore, the tool primarily uses it to build the JSON / Python dictionnaries.
-Information related to MCC-MNC is completed with the csv listing from the txtNation website 
+Information related to MCC-MNC is completed with the csv listing from the txtNation website
 and the ITU-T operational bulletins 1162 and all following incremental updates.
 The list of Signaling Point Codes is extracted from ITU-T bulletin 1199.
-Geographical information are taken from the CIA World Factbook to gather information related to each country,
+Geographical information are taken from the CIA World Factbook (frozen source) to gather information related to each country,
 including borders and telephony-related.
 This is completed with the data provided on the _egallic_ blogpost for getting countries' proximity in addition to neighbours one.
 
@@ -195,7 +205,7 @@ Generally, installation is not required and every scripts can be run as-is.
 
 ### Source dataset update
 
-The Wikipedia, World Factbook and ITU-T bulletins source datasets can be updated with the
+The Wikipedia and ITU-T bulletins source datasets can be updated with the
 following scripts:
 
 ```console
@@ -211,15 +221,6 @@ optional arguments:
   -p          produce Python files (with suffix .py)
 ```
 
-```console
-$ mcc-mnc-parse-worldfactbook-infos --help
-usage: mcc-mnc-parse-worldfactbook-infos [-h] [-j] [-p]
-
-dump country-related informations from the CIA World Factbook into JSON or
-Python file
-[...]
-```
-
 ```console
 $ mcc-mnc-parse-itut-bulletins --help
 usage: mcc-mnc-parse-itut-bulletins [-h] [-d] [-b B] [-j] [-p]
@@ -301,7 +302,7 @@ $ mcc-mnc-gen-dataset
 
 The following one-liner can be used to update the whole final dataset (without downloading new ITU-T bulletins):
 ```console
-$ mcc-mnc-parse-wikipedia-tables -j -p && mcc-mnc-parse-worldfactbook-infos -j -p && mcc-mnc-parse-various-csv -j -p && mcc-mnc-parse-itut-bulletins -j -p && mcc-mnc-gen-dataset
+$ mcc-mnc-parse-wikipedia-tables -j -p && mcc-mnc-parse-various-csv -j -p && mcc-mnc-parse-itut-bulletins -j -p && mcc-mnc-gen-dataset
 ```
 
 ### Usage

pyproject.toml

@@ -51,7 +51,6 @@ mcc-mnc-gen-dataset = "mcc_mnc_genlib.scripts.gen_dataset:main"
 mcc-mnc-parse-itut-bulletins = "mcc_mnc_genlib.scripts.parse_itut_bulletins:main"
 mcc-mnc-parse-various-csv = "mcc_mnc_genlib.scripts.parse_various_csv:main"
 mcc-mnc-parse-wikipedia-tables = "mcc_mnc_genlib.scripts.parse_wikipedia_tables:main"
-mcc-mnc-parse-worldfactbook-infos = "mcc_mnc_genlib.scripts.parse_worldfactbook_infos:main"
 
 [tool.ruff]
 line-length = 79

src/mcc_mnc_genlib/core/patch_dataset.py

@@ -63,29 +63,20 @@
 ]
 
 
-import os
 import re
-import csv
 
 from mcc_mnc_genlib.core.patch_country_dep import COUNTRY_SPEC
-
 from mcc_mnc_genlib.scripts.parse_wikipedia_tables import (
     REC_ISO3166,
-    REC_MCC,
-    REC_MNC,
-    REC_BORDERS,
-)
-from mcc_mnc_genlib.scripts.parse_worldfactbook_infos import (
-    REC_COUNTRY,
 )
 
 try:
     from mcc_mnc_genlib.raw.wikip_borders import WIKIP_BORDERS
+    from mcc_mnc_genlib.raw.wikip_country import WIKIP_COUNTRY
     from mcc_mnc_genlib.raw.wikip_iso3166 import WIKIP_ISO3166
     from mcc_mnc_genlib.raw.wikip_mcc import WIKIP_MCC
     from mcc_mnc_genlib.raw.wikip_mnc import WIKIP_MNC
     from mcc_mnc_genlib.raw.wikip_msisdn import WIKIP_MSISDN
-    from mcc_mnc_genlib.raw.wikip_country import WIKIP_COUNTRY
     from mcc_mnc_genlib.raw.wikip_territory import WIKIP_TERRITORY
 except ImportError:
     raise (Exception('error: please run first mcc-mnc-parse-wikipedia-tables'))
@@ -94,7 +85,10 @@
     from mcc_mnc_genlib.raw.world_fb import WORLD_FB
 except ImportError:
     raise (
-        Exception('error: please run first mcc-mnc-parse-worldfactbook-infos')
+        Exception(
+            'error: missing bundled world_fb raw dataset '
+            '(mcc_mnc_genlib/raw/world_fb.py)'
+        )
     )
 
 try:
@@ -107,8 +101,8 @@
     from mcc_mnc_genlib.raw.itut_mnc_1111 import ITUT_MNC_1111
     from mcc_mnc_genlib.raw.itut_mnc_1162 import ITUT_MNC_1162
     from mcc_mnc_genlib.raw.itut_mnc_incr import ITUT_MNC_INCR
-    from mcc_mnc_genlib.raw.itut_spc_1199 import ITUT_SPC_1199
     from mcc_mnc_genlib.raw.itut_sanc_1125 import ITUT_SANC_1125
+    from mcc_mnc_genlib.raw.itut_spc_1199 import ITUT_SPC_1199
 except ImportError:
     raise (Exception('error: please run first mcc-mnc-parse-itut-bulletins'))
 
@@ -1013,8 +1007,8 @@ def patch_egal_min_dist():
         for dst in dst_dist:
             if dst not in CSV_EGAL_MIN_DIST:
                 print('>>> dst country %s in %s, not in src' % (dst, src))
-        if not src in isonameset:
-            if not src in SUBTERR_TO_COUNTRY:
+        if src not in isonameset:
+            if src not in SUBTERR_TO_COUNTRY:
                 print('>>> country %s, not matching any territory name' % src)
             else:
                 print('> country %s, matching only a sub-territory name' % src)

src/mcc_mnc_genlib/scripts/gen_dataset.py

@@ -820,7 +820,7 @@ def generate_init(path_pre):
 
 def main():
 
-    URL_SRC = 'data aggregated from Wikipedia, The World Factbook, ITU-T, Egallic blog and txtNation'
+    URL_SRC = 'data aggregated from Wikipedia, The World Factbook (frozen), ITU-T and Egallic blog'
     URL_LIC = 'produced by P1 Security, based on openly available data'
 
     generate_json(MNC, PATH_PRE + 'p1_mnc.json', [URL_SRC], URL_LIC)