feat: adding clingen curation data (#55)

holtgrewe · web-flow · commit 51435dfd86bc · 2023-08-25T11:18:19.000+02:00
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -31,7 +31,7 @@ jobs:
           environment-file: environment.yml
           init-shell: bash
           cache-environment: true
-          post-cleanup: 'all'
+          post-cleanup: none  # breaks otherwise
 
       - name: Check formating
         run: |
@@ -65,7 +65,7 @@ jobs:
           environment-file: environment.yml
           init-shell: bash
           cache-environment: true
-          post-cleanup: 'all'
+          post-cleanup: none  # breaks otherwise
 
       - name: Install python package
         run: |
diff --git a/Snakefile b/Snakefile
@@ -85,6 +85,7 @@ rule all:
         # == work directory =====================================================================
         #
         # genes
+        f"work/download/genes/clingen/{DV.clingen_gene}/clingen.csv",
         f"work/genes/dbnsfp/{DV.dbnsfp}/genes.tsv.gz",
         f"work/genes/ensembl/{DV.ensembl}/ensembl_xlink.tsv",
         f"work/genes/enst_ensg/grch37/{DV.ensembl_37}/enst_ensg.tsv",
@@ -317,6 +318,7 @@ rule all:
 include: "rules/work/misc/hpo.smk"
 # Gene-related rules.
 include: "rules/work/genes/dbnsfp.smk"
+include: "rules/work/genes/clingen.smk"
 include: "rules/work/genes/ensembl.smk"
 include: "rules/work/genes/gnomad.smk"
 include: "rules/work/genes/hgnc.smk"
diff --git a/download_urls.yml b/download_urls.yml
@@ -1,3 +1,9 @@
+- comment: The curation activity summary report is built in real-time.
+  url: https://search.clinicalgenome.org/kb/reports/curation-activity-summary-report
+
+- comment: ClinGen variant summary is built in real-time
+  url: http://erepo.clinicalgenome.org/evrepo/api/classifications/all?format=tabbed
+
 - url: https://github.com/bihealth/annonars-data-clinvar/releases/download/clinvar-weekly-20230625/clinvar-strucvar-grch37-2023-0625+0.6.3.tar.gz
   excerpt_strategy:
     strategy: no-excerpt
diff --git a/excerpt-data/0a27656c7f2ba08a/all b/excerpt-data/0a27656c7f2ba08a/all
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8dbeb7d6578a59b07d5296ba8f8d0f729f51dfd005cd7c3466a4c683e7a4d91c
+size 156424
diff --git a/excerpt-data/0a27656c7f2ba08a/url.txt b/excerpt-data/0a27656c7f2ba08a/url.txt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cce44490e7ce19c99ae46d09281eca6645ca05c05bfe32c8ff87b88a25ca856c
+size 77
diff --git a/excerpt-data/e847ec57405dca9c/hgnc_complete_set.json b/excerpt-data/e847ec57405dca9c/hgnc_complete_set.json
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d76098d2d3712096592a4a0f78d2aad039cae1e68b60ea802edd52d4edc4ae85
-size 6733
+oid sha256:85689d642c36284f0bb17567eb9b10f7b001bed8e6d912ed97f728fcacd5b316
+size 34848332
diff --git a/excerpt-data/f668bbde1014a975/curation-activity-summary-report b/excerpt-data/f668bbde1014a975/curation-activity-summary-report
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:56ea2a537b3b1afdd7426268eec5cbb3154a38101015aff61592689472d36f52
+size 1810278
diff --git a/excerpt-data/f668bbde1014a975/url.txt b/excerpt-data/f668bbde1014a975/url.txt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:25fee8c5e5004815bdf7fee7462711e0a9bb7b42b547f437dd58767fd640a6f2
+size 78
diff --git a/rules/output/annonars/genes.smk b/rules/output/annonars/genes.smk
@@ -4,6 +4,7 @@
 rule output_annonars_genes:  # -- build annonars genes RocksDB file
     input:
         acmg_sf="data/acmg_sf/{v_acmg_sf}/acmg_sf.tsv",
+        clingen="work/download/genes/clingen/{date}/clingen.csv",
         gnomad_constraints="work/genes/gnomad/{v_gnomad_constraints}/gnomad_constraints.tsv",
         dbnsfp="work/genes/dbnsfp/{v_dbnsfp}/genes.tsv.gz",
         hgnc="work/genes/hgnc/{date}/hgnc_info.jsonl",
@@ -25,9 +26,15 @@ rule output_annonars_genes:  # -- build annonars genes RocksDB file
         v_annonars=RE_VERSION,
     shell:
         r"""
+        export TMPDIR=$(mktemp -d)
+        trap "rm -rf $TMPDIR" EXIT
+
+        tail -n +4 {input.clingen} > $TMPDIR/clingen.csv
+
         annonars gene import \
             --path-out-rocksdb $(dirname {output.rocksdb_identity}) \
             --path-in-acmg {input.acmg_sf} \
+            --path-in-clingen $TMPDIR/clingen.csv \
             --path-in-gnomad-constraints {input.gnomad_constraints} \
             --path-in-dbnsfp {input.dbnsfp} \
             --path-in-hgnc {input.hgnc} \
diff --git a/rules/output/annonars/genes.spec.yaml b/rules/output/annonars/genes.spec.yaml
@@ -22,6 +22,8 @@ dc.source:
 x-created-from:
   - name: ACMG SF Gene List
     version: {{ v_acmg_sf }}
+  - name: ClinGen Gene Curation
+    version: {{ today }}
   - name: gnomAD constraints
     version: {{ v_gnomad_constraints }}
   - name: dbNSFP
diff --git a/rules/work/genes/clingen.smk b/rules/work/genes/clingen.smk
@@ -0,0 +1,20 @@
+## Rules related to ClinGen curation download.
+
+
+rule genes_clingen_download:  # -- download ClinGen curations
+    output:
+        csv="work/download/genes/clingen/{date}/clingen.csv",
+        csv_md5="work/download/genes/clingen/{date}/clingen.csv.md5",
+    shell:
+        r"""
+        if [[ "$(date +%Y%m%d)" != "{wildcards.date}" ]] && [[ "{FORCE_TODAY}" != "True" ]]; then
+            >&2 echo "{wildcards.date} is not today"
+            exit 1
+        fi
+
+        wget --no-check-certificate \
+            -O {output.csv} \
+            https://search.clinicalgenome.org/kb/reports/curation-activity-summary-report
+
+        md5sum {output.csv} > {output.csv_md5}
+        """
diff --git a/varfish_db_downloader/versions.py b/varfish_db_downloader/versions.py
@@ -24,6 +24,10 @@
 class DataVersions:
     """Container with data versions."""
 
+    #: String to use for ClinGen gene curation version.
+    clingen_gene: str
+    #: String to use for ClinGen variant curation version.
+    clingen_variant: str
     #: String to use for GRCh37 ENSEMBL version.
     ensembl_37: str
     #: String to use for GRCh38 ENSEMBL version.
@@ -105,6 +109,8 @@ class DataVersions:
 
 #: The data versions to use.
 DATA_VERSIONS = DataVersions(
+    clingen_gene=TODAY,
+    clingen_variant=TODAY,
     ensembl_37="87",
     ensembl_38="109",
     ensembl="109",
diff --git a/varfish_db_downloader/wget.py b/varfish_db_downloader/wget.py
@@ -230,7 +230,7 @@ def download_excerpt(url: UrlEntry, data_dir: str, force: bool):
 
     excerpt_fun = STRATEGY_MAP[url.excerpt_strategy.strategy]
     parsed = urllib.parse.urlparse(url.url)
-    basename = parsed.path.split("/")[-1]
+    basename = parsed.path.split("/")[-1] or "__index__"
     out_path_data = str(out_path / basename)
     logger.info("    getting excerpt to {}", out_path_data)
     excerpt_fun(url.url, str(out_path_data), url.excerpt_strategy.count)

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+version https://git-lfs.github.com/spec/v1`
	`2`	`+oid sha256:8dbeb7d6578a59b07d5296ba8f8d0f729f51dfd005cd7c3466a4c683e7a4d91c`
	`3`	`+size 156424`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+version https://git-lfs.github.com/spec/v1`
	`2`	`+oid sha256:cce44490e7ce19c99ae46d09281eca6645ca05c05bfe32c8ff87b88a25ca856c`
	`3`	`+size 77`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+version https://git-lfs.github.com/spec/v1`
	`2`	`+oid sha256:56ea2a537b3b1afdd7426268eec5cbb3154a38101015aff61592689472d36f52`
	`3`	`+size 1810278`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+version https://git-lfs.github.com/spec/v1`
	`2`	`+oid sha256:25fee8c5e5004815bdf7fee7462711e0a9bb7b42b547f437dd58767fd640a6f2`
	`3`	`+size 78`