feat: replace orphapacket by orphadata API access (#84) (#85)

Author: holtgrewe

.gitignore

@@ -1,3 +1,6 @@
+# Ignore all pickled data
+*.pickle*
+
 # Ignore the workflow directories.
 /work/
 /output/

Snakefile

@@ -86,8 +86,9 @@ rule all:
         #
         # genes
         f"work/download/genes/rcnv/2022/Collins_rCNV_2022.dosage_sensitivity_scores.tsv.gz",
-        f"work/download/genes/orphapacket/{DV.orphapacket}/orphapacket.tar.gz",
         "work/download/genes/alphamissense/1/AlphaMissense_gene_hg38.tsv.gz",
+        f"work/download/genes/ctd/{DV.today}/CTD_diseases.tsv.gz",
+        f"work/download/do/{DV.today}/omim-unmapped.csv",
         f"work/genes/dbnsfp/{DV.dbnsfp}/genes.tsv.gz",
         "work/genes/decipher/v3/decipher_hi_prediction.tsv.gz",
         f"work/genes/ensembl/{DV.ensembl}/ensembl_xlink.tsv",
@@ -96,7 +97,8 @@ rule all:
         f"work/genes/gnomad/{DV.gnomad_constraints}/gnomad_constraints.tsv",
         f"work/genes/hgnc/{DV.today}/hgnc_info.jsonl",
         f"work/genes/omim/{DV.hpo}+{DV.today}/omim_diseases.tsv",
-        f"work/genes/orphapacket/{DV.orphapacket}+{DV.today}/orpha_diseases.tsv",
+        f"work/genes/orphadata/{DV.orphadata}/orphadata.jsonl",
+        f"work/genes/mondo/{DV.today}/mondo.obo",
         "work/genes/rcnv/2022/rcnv_collins_2022.tsv",
         "work/genes/shet/2019/shet_weghorn_2019.tsv",
         f"work/genes/clingen/{DV.today}/ClinGen_gene_curation_list_GRCh37.tsv",
@@ -177,7 +179,7 @@ rule all:
         f"output/full/annonars/cons-grch37-{DV.ucsc_cons_37}+{PV.annonars}/rocksdb/IDENTITY",
         f"output/full/annonars/cons-grch38-{DV.ucsc_cons_38}+{PV.annonars}/rocksdb/IDENTITY",
         # ----- genes
-        f"output/full/annonars/genes-{DV.acmg_sf}+{DV.gnomad_constraints}+{DV.dbnsfp}+{DV.hpo}+{DV.orphapacket}+{DV.today}+{PV.annonars}/rocksdb/IDENTITY",
+        f"output/full/annonars/genes-{DV.acmg_sf}+{DV.gnomad_constraints}+{DV.dbnsfp}+{DV.hpo}+{DV.today}+{PV.annonars}/rocksdb/IDENTITY",
         # -- worker data
         f"output/full/worker/genes-regions-grch37-{DV.refseq_37}+{PV.worker}/refseq_genes.bin",
         f"output/full/worker/genes-regions-grch37-{DV.ensembl_37}+{PV.worker}/ensembl_genes.bin",
@@ -341,6 +343,9 @@ include: "rules/work/misc/hpo.smk"
 include: "rules/work/genes/alphamissense.smk"
 include: "rules/work/genes/dbnsfp.smk"
 include: "rules/work/genes/clingen.smk"
+include: "rules/work/genes/conditions.smk"
+include: "rules/work/genes/ctd.smk"
+include: "rules/work/genes/do.smk"
 include: "rules/work/genes/decipher.smk"
 include: "rules/work/genes/ensembl.smk"
 include: "rules/work/genes/gnomad.smk"
@@ -350,11 +355,11 @@ include: "rules/work/genes/mehari_data_tx.smk"
 include: "rules/work/genes/ncbi.smk"
 include: "rules/work/genes/omim.smk"
 include: "rules/work/genes/panelapp.smk"
-include: "rules/work/genes/orphapacket.smk"
+include: "rules/work/genes/mondo.smk"
+include: "rules/work/genes/orphadata.smk"
 include: "rules/work/genes/rcnv.smk"
 include: "rules/work/genes/shet.smk"
 include: "rules/work/genes/domino.smk"
-include: "rules/work/genes/clingen.smk"
 # Reference sequence--related rules.
 include: "rules/work/reference/human.smk"
 # Features (position and not variant specific).

download_urls.yml

@@ -1,3 +1,48 @@
+# Note that this just tests the availability of the OrphaData API.  We have a file in
+# excerpts/__orphadata__ that is used by `genes-orpha-diseases.py` in CI=true mode.
+- url: https://api.orphadata.com/rd-cross-referencing/orphacodes
+  excerpt_strategy:
+    strategy: no-excerpt
+    count: null
+- url: https://api.orphadata.com/rd-cross-referencing/orphacodes/20?lang=en
+  excerpt_strategy:
+    strategy: no-excerpt
+    count: null
+- url: https://api.orphadata.com/rd-associated-genes/orphacodes/20
+  excerpt_strategy:
+    strategy: no-excerpt
+    count: null
+
+- url: https://raw.githubusercontent.com/monarch-initiative/mondo-ingest/main/src/ontology/reports/omim_unmapped_terms.tsv
+  excerpt_strategy:
+    strategy: no-excerpt
+    count: null
+
+- url: https://github.com/DiseaseOntology/HumanDiseaseOntology/raw/main/src/deprecated/DO_NON_Production_Files/omim_import.obo
+  excerpt_strategy:
+    strategy: no-excerpt
+    count: null
+
+- url: https://raw.githubusercontent.com/DiseaseOntology/HumanDiseaseOntology/main/DOreports/OMIMinDO.tsv
+  excerpt_strategy:
+    strategy: no-excerpt
+    count: null
+
+- url: https://github.com/DiseaseOntology/HumanDiseaseOntology/raw/main/src/deprecated/reports/omim-unmapped.csv
+  excerpt_strategy:
+    strategy: no-excerpt
+    count: null
+
+- url: https://ctdbase.org/reports/CTD_diseases.tsv.gz
+  excerpt_strategy:
+    strategy: no-excerpt
+    count: null
+
+- url: http://purl.obolibrary.org/obo/mondo.obo
+  excerpt_strategy:
+    strategy: no-excerpt
+    count: null
+
 - url: https://panelapp.genomicsengland.co.uk/api/v1/entities/
   excerpt_strategy:
     strategy: no-excerpt
@@ -101,7 +146,7 @@
     strategy: manual
     count: null
 
-- url: https://github.com/Orphanet/orphapacket/archive/refs/tags/v10.1.tar.gz
+- url: https://data.bioontology.org/ontologies/ORDO/download?apikey=8b5b7825-538d-40e0-9e9e-5ab9274a9aeb&download_format=csv
   excerpt_strategy:
     strategy: no-excerpt
     count: null
@@ -130,19 +175,19 @@
     strategy: no-excerpt
     count: null
 
-- url: https://github.com/obophenotype/human-phenotype-ontology/releases/download/v2023-06-06/hp.obo
+- url: https://github.com/obophenotype/human-phenotype-ontology/releases/download/v2024-01-16/hp.obo
   excerpt_strategy:
     strategy: no-excerpt
     count: null
-- url: https://github.com/obophenotype/human-phenotype-ontology/releases/download/v2023-06-06/phenotype.hpoa
+- url: https://github.com/obophenotype/human-phenotype-ontology/releases/download/v2024-01-16/phenotype.hpoa
   excerpt_strategy:
     strategy: no-excerpt
     count: null
-- url: https://github.com/obophenotype/human-phenotype-ontology/releases/download/v2023-06-06/phenotype_to_genes.txt
+- url: https://github.com/obophenotype/human-phenotype-ontology/releases/download/v2024-01-16/phenotype_to_genes.txt
   excerpt_strategy:
     strategy: no-excerpt
     count: null
-- url: https://github.com/obophenotype/human-phenotype-ontology/releases/download/v2023-06-06/genes_to_phenotype.txt
+- url: https://github.com/obophenotype/human-phenotype-ontology/releases/download/v2024-01-16/genes_to_phenotype.txt
   excerpt_strategy:
     strategy: no-excerpt
     count: null

environment.yml

@@ -12,6 +12,8 @@ dependencies:
   - click
   - loguru
   - numpy
+  - pydantic
+  - pronto >=2.5,<3.0
   - pyyaml
   - requests
   - requests-ftp
@@ -41,9 +43,13 @@ dependencies:
   # Parallel (de)compression.
   - pigz
   # Varfish related
-  - annonars =0.33.0
+  - annonars =0.34.0
   - viguno =0.2.0
   - mehari =0.21.1
   - varfish-server-worker =0.10.2
   # S3 uploads
   - s5cmd =2.1.0
+  # async HTTP requests
+  - httpx =0.25.0
+  - httpcore =0.18.0
+  - trio

excerpt-data/111d8c6e08038f62/20

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b610d631a11a2f6f4b93b8f7f7458447fa1a974ade2a18b1b9c9b5047dac516c
+size 3338

excerpt-data/111d8c6e08038f62/url.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bda92b632aaeacd032c57d87e6420ccac3945bcd78bf51bafc117da4eb63a601
+size 69

excerpt-data/1963f3c58ea066be/omim_unmapped_terms.tsv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:384eccb1d84e8a5036027cd340d2b4b18ce41170de28407f3022d7f3dd2392dc
+size 4779

excerpt-data/1963f3c58ea066be/url.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2153c0f0281a0399c77b0964d96c1286f6716eac34304f68f4e838fe41a02bdd
+size 116

excerpt-data/32c97f6adaf88f01/hp.obo

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:508bb0326603671327e9f4af05beffd1800e053557c7c1a0d137fd1f458bb0f7
+size 9719364

excerpt-data/32c97f6adaf88f01/url.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b057fd342e531569cd4c2af122986569a44bc55348bcbc8104866b9ea696acfb
+size 94