feat: updating annonars genes with ClinGen dosage & DOMINO (#65)

holtgrewe · web-flow · commit 0797f2dcfd3e · 2023-11-22T07:45:17.000Z
diff --git a/Snakefile b/Snakefile
@@ -85,7 +85,6 @@ rule all:
         # == work directory =====================================================================
         #
         # genes
-        f"work/download/genes/clingen/{DV.clingen_gene}/clingen.csv",
         f"work/download/genes/rcnv/2022/Collins_rCNV_2022.dosage_sensitivity_scores.tsv.gz",
         f"work/download/genes/orphapacket/{DV.orphapacket}/orphapacket.tar.gz",
         f"work/genes/dbnsfp/{DV.dbnsfp}/genes.tsv.gz",
@@ -99,6 +98,9 @@ rule all:
         f"work/genes/orphapacket/{DV.orphapacket}+{DV.today}/orpha_diseases.tsv",
         "work/genes/rcnv/2022/rcnv_collins_2022.tsv",
         "work/genes/shet/2019/shet_weghorn_2019.tsv",
+        f"work/genes/clingen/{DV.today}/ClinGen_gene_curation_list_GRCh37.tsv",
+        f"work/genes/clingen/{DV.today}/ClinGen_gene_curation_list_GRCh38.tsv",
+        "work/genes/domino/20190219/domino.tsv",
         # reference-specific annotations
         # -- background/population sequence variants and annotations thereof
         # ---- GRCh37
@@ -132,7 +134,6 @@ rule all:
         f"work/annos/grch38/features/cons/{DV.ucsc_cons_38}/ucsc_conservation.tsv",
         f"work/annos/grch38/features/ensembl/{DV.ensembl_38}/ensembl_genes.bed.gz",
         f"work/annos/grch38/features/refseq/{DV.refseq_38}/refseq_genes.bed.gz",
-        # f"work/annos/grch38/features/clingen_dosage/{DV.today}/clingen_dosage_sensitivity_regions.bed.gz",
         #
         # == output directory ===================================================================
         #
@@ -163,9 +164,6 @@ rule all:
         # ----- conservation
         f"output/full/annonars/cons-grch37-{DV.ucsc_cons_37}+{PV.annonars}/rocksdb/IDENTITY",
         f"output/full/annonars/cons-grch38-{DV.ucsc_cons_38}+{PV.annonars}/rocksdb/IDENTITY",
-        # ----- features
-        f"output/full/annonars/clingen-dosage-grch37/{DV.today}/clingen_region_curation_list.bed.gz",
-        f"output/full/annonars/clingen-dosage-grch38/{DV.today}/clingen_region_curation_list.bed.gz",
         # ----- genes
         f"output/full/annonars/genes-{DV.acmg_sf}+{DV.gnomad_constraints}+{DV.dbnsfp}+{DV.hpo}+{DV.orphapacket}+{DV.today}+{PV.annonars}/rocksdb/IDENTITY",
         # -- worker data
@@ -341,11 +339,12 @@ include: "rules/work/genes/omim.smk"
 include: "rules/work/genes/orphapacket.smk"
 include: "rules/work/genes/rcnv.smk"
 include: "rules/work/genes/shet.smk"
+include: "rules/work/genes/domino.smk"
+include: "rules/work/genes/clingen.smk"
 # Reference sequence--related rules.
 include: "rules/work/reference/human.smk"
 # Features (position and not variant specific).
 include: "rules/work/annos/features/cons.smk"
-include: "rules/work/annos/features/clingen_dosage.smk"
 include: "rules/work/annos/features/ensembl.smk"
 include: "rules/work/annos/features/refseq.smk"
 include: "rules/work/annos/features/tads.smk"
@@ -381,8 +380,6 @@ include: "rules/output/annonars/gnomad_genomes.smk"
 include: "rules/output/annonars/gnomad_mtdna.smk"
 include: "rules/output/annonars/helix.smk"
 include: "rules/output/annonars/genes.smk"
-# ------ features
-include: "rules/output/annonars/clingen_dosage.smk"
 # ---- worker
 include: "rules/output/worker/patho_mms.smk"
 include: "rules/output/worker/clinvar.smk"
diff --git a/bundled-data/domino/get.sh b/bundled-data/domino/get.sh
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:48bd163d94781c41cb31a33bbad006f5233202758346037685fda4a018ed8a75
+size 108
diff --git a/bundled-data/domino/score_all_final_19.02.19.txt b/bundled-data/domino/score_all_final_19.02.19.txt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0e8559bcc8078f0312d69981813a2dc6968664e0ce4392984e93e76e30c47200
+size 2379134
diff --git a/download_urls.yml b/download_urls.yml
@@ -2,6 +2,8 @@
 
 - url: ftp://ftp.clinicalgenome.org/ClinGen_region_curation_list_GRCh37.tsv
 - url: ftp://ftp.clinicalgenome.org/ClinGen_region_curation_list_GRCh38.tsv
+- url: ftp://ftp.clinicalgenome.org/ClinGen_gene_curation_list_GRCh37.tsv
+- url: ftp://ftp.clinicalgenome.org/ClinGen_gene_curation_list_GRCh38.tsv
 
 - url: https://storage.googleapis.com/gtex_analysis_v8/annotations/GTEx_Analysis_v8_Annotations_SampleAttributesDS.txt
   excerpt_strategy:
@@ -23,9 +25,6 @@
 - comment: The curation activity summary report is built in real-time.
   url: https://search.clinicalgenome.org/kb/reports/curation-activity-summary-report
 
-- comment: ClinGen variant summary is built in real-time
-  url: http://erepo.clinicalgenome.org/evrepo/api/classifications/all?format=tabbed
-
 - url: https://github.com/bihealth/annonars-data-clinvar/releases/download/clinvar-weekly-20230625/clinvar-strucvar-grch37-2023-0625+0.6.3.tar.gz
   excerpt_strategy:
     strategy: no-excerpt
diff --git a/environment.yml b/environment.yml
@@ -41,7 +41,7 @@ dependencies:
   # Parallel (de)compression.
   - pigz
   # Varfish related
-  - annonars =0.18.0
+  - annonars =0.27.0
   - viguno =0.1.6
   - mehari =0.18.1
   - varfish-server-worker =0.10.1
diff --git a/excerpt-data/0a27656c7f2ba08a/all b/excerpt-data/0a27656c7f2ba08a/all
diff --git a/excerpt-data/0a27656c7f2ba08a/url.txt b/excerpt-data/0a27656c7f2ba08a/url.txt
diff --git a/excerpt-data/0e7eb7069eb4d354/ClinGen_gene_curation_list_GRCh37.tsv b/excerpt-data/0e7eb7069eb4d354/ClinGen_gene_curation_list_GRCh37.tsv
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e818e7ae751c36013377640bf29093a77d08a53eefd5adb6c3e769b982351dfc
+size 15862
diff --git a/excerpt-data/0e7eb7069eb4d354/url.txt b/excerpt-data/0e7eb7069eb4d354/url.txt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f6cde2620cd50f150d2e0974eae1676e0ae4fe6691c86721a89fa5cab8c5f256
+size 67
diff --git a/excerpt-data/cb35d21dd121ab4b/ClinGen_gene_curation_list_GRCh38.tsv b/excerpt-data/cb35d21dd121ab4b/ClinGen_gene_curation_list_GRCh38.tsv
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2c401cc2cb6aeda93c1d1b633603ffb6ea15efc049694e18bdfd21894308e4f6
+size 15860
diff --git a/excerpt-data/cb35d21dd121ab4b/url.txt b/excerpt-data/cb35d21dd121ab4b/url.txt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5507c499a3f71a87d1f2d8bd1b32331c399baf65b41f0e773b6ac80f92ed2d7d
+size 67
diff --git a/rules/output/annonars/clingen_dosage.smk b/rules/output/annonars/clingen_dosage.smk
diff --git a/rules/output/annonars/clingen_dosage.spec.yaml b/rules/output/annonars/clingen_dosage.spec.yaml
diff --git a/rules/output/annonars/genes.smk b/rules/output/annonars/genes.smk
@@ -4,7 +4,8 @@
 rule output_annonars_genes:  # -- build annonars genes RocksDB file
     input:
         acmg_sf="data/acmg_sf/{v_acmg_sf}/acmg_sf.tsv",
-        clingen="work/download/genes/clingen/{date}/clingen.csv",
+        clingen_37="work/genes/clingen/{date}/ClinGen_gene_curation_list_GRCh37.tsv",
+        clingen_38="work/genes/clingen/{date}/ClinGen_gene_curation_list_GRCh38.tsv",
         gnomad_constraints="work/genes/gnomad/{v_gnomad_constraints}/gnomad_constraints.tsv",
         dbnsfp="work/genes/dbnsfp/{v_dbnsfp}/genes.tsv.gz",
         hgnc="work/genes/hgnc/{date}/hgnc_info.jsonl",
@@ -14,6 +15,7 @@ rule output_annonars_genes:  # -- build annonars genes RocksDB file
         rcnv="work/genes/rcnv/2022/rcnv_collins_2022.tsv",
         shet="work/genes/shet/2019/shet_weghorn_2019.tsv",
         gtex="work/genes/annonars/gtex_v8/genes_tpm.jsonl.gz",
+        domino="work/genes/domino/20190219/domino.tsv",
     output:
         rocksdb_identity=(
             "output/full/annonars/genes-{v_acmg_sf}+{v_gnomad_constraints}+{v_dbnsfp}+{v_hpo}+{v_orpha}+{date}+{v_annonars}/"
@@ -31,15 +33,11 @@ rule output_annonars_genes:  # -- build annonars genes RocksDB file
         v_annonars=RE_VERSION,
     shell:
         r"""
-        export TMPDIR=$(mktemp -d)
-        trap "rm -rf $TMPDIR" EXIT
-
-        tail -n +4 {input.clingen} > $TMPDIR/clingen.csv
-
         annonars gene import \
             --path-out-rocksdb $(dirname {output.rocksdb_identity}) \
             --path-in-acmg {input.acmg_sf} \
-            --path-in-clingen $TMPDIR/clingen.csv \
+            --path-in-clingen-37 {input.clingen_37} \
+            --path-in-clingen-38 {input.clingen_38} \
             --path-in-gnomad-constraints {input.gnomad_constraints} \
             --path-in-dbnsfp {input.dbnsfp} \
             --path-in-hgnc {input.hgnc} \
@@ -48,7 +46,8 @@ rule output_annonars_genes:  # -- build annonars genes RocksDB file
             --path-in-ncbi {input.ncbi} \
             --path-in-rcnv {input.rcnv} \
             --path-in-shet {input.shet} \
-            --path-in-gtex {input.gtex}
+            --path-in-gtex {input.gtex} \
+            --path-in-domino {input.domino}
 
         varfish-db-downloader tpl \
             --template rules/output/annonars/genes.spec.yaml \
diff --git a/rules/output/annonars/genes.spec.yaml b/rules/output/annonars/genes.spec.yaml
@@ -13,16 +13,19 @@ dc.description: |
   - dbNSFP v{{ v_dbnsfp }}
   - NCBI gene database downloaded at {{ today }}
   - HGNC database downloaded at {{ today }}
+  - ClinGen Gene Dosage pathogenicity {{ today }}
+  - DOMINO data v2019-02-19
 dc.source:
   - PMID:35802134
   - PMID:32461654
   - PMID:33261662
+  - PMID:28985496
   - https://www.ncbi.nlm.nih.gov/gene/
   - https://www.genenames.org/
 x-created-from:
   - name: ACMG SF Gene List
     version: {{ v_acmg_sf }}
-  - name: ClinGen Gene Curation
+  - name: ClinGen Gene Dosage pathog
     version: {{ today }}
   - name: gnomAD constraints
     version: {{ v_gnomad_constraints }}
@@ -42,3 +45,5 @@ x-created-from:
     version: 2019-Weghorn-et-a.
   - name: GTex data
     version: v8
+  - name: DOMINO data
+    version: "2019-02-19"
diff --git a/rules/work/annos/features/clingen_dosage.smk b/rules/work/annos/features/clingen_dosage.smk
diff --git a/rules/work/genes/clingen.smk b/rules/work/genes/clingen.smk
@@ -1,20 +1,19 @@
-## Rules related to ClinGen curation download.
+## Rules related to ClinGen gene dosage pathogenicity annotation.
 
 
-rule genes_clingen_download:  # -- download ClinGen curations
+rule clingen_gene_download:  # -- download files
     output:
-        csv="work/download/genes/clingen/{date}/clingen.csv",
-        csv_md5="work/download/genes/clingen/{date}/clingen.csv.md5",
+        tsv="work/genes/clingen/{date}/ClinGen_gene_curation_list_{genome_release}.tsv",
+        tsv_md5="work/genes/clingen/{date}/ClinGen_gene_curation_list_{genome_release}.tsv.md5",
     shell:
         r"""
         if [[ "$(date +%Y%m%d)" != "{wildcards.date}" ]] && [[ "{FORCE_TODAY}" != "True" ]]; then
             >&2 echo "{wildcards.date} is not today"
             exit 1
         fi
 
-        wget --no-check-certificate \
-            -O {output.csv} \
-            https://search.clinicalgenome.org/kb/reports/curation-activity-summary-report
+        wget -O {output.tsv} \
+            ftp://ftp.clinicalgenome.org/ClinGen_gene_curation_list_{wildcards.genome_release}.tsv
 
-        md5sum {output.csv} > {output.csv_md5}
+        md5sum {output.tsv} > {output.tsv}.md5
         """
diff --git a/rules/work/genes/domino.smk b/rules/work/genes/domino.smk
@@ -0,0 +1,16 @@
+## Rules related to DOMINO gene annotation.
+
+
+rule genes_domino:  # -- postprocess file for import
+    input:
+        tsv="bundled-data/domino/score_all_final_19.02.19.txt",
+    output:
+        tsv="work/genes/domino/20190219/domino.tsv",
+        tsv_md5="work/genes/domino/20190219/domino.tsv.md5",
+    shell:
+        """
+        cut -f 1-2 {input.tsv} \
+        > {output.tsv}
+
+        md5sum {output.tsv} > {output.tsv}.md5
+        """

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+version https://git-lfs.github.com/spec/v1`
	`2`	`+oid sha256:48bd163d94781c41cb31a33bbad006f5233202758346037685fda4a018ed8a75`
	`3`	`+size 108`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+version https://git-lfs.github.com/spec/v1`
	`2`	`+oid sha256:0e8559bcc8078f0312d69981813a2dc6968664e0ce4392984e93e76e30c47200`
	`3`	`+size 2379134`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+version https://git-lfs.github.com/spec/v1`
	`2`	`+oid sha256:e818e7ae751c36013377640bf29093a77d08a53eefd5adb6c3e769b982351dfc`
	`3`	`+size 15862`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+version https://git-lfs.github.com/spec/v1`
	`2`	`+oid sha256:f6cde2620cd50f150d2e0974eae1676e0ae4fe6691c86721a89fa5cab8c5f256`
	`3`	`+size 67`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+version https://git-lfs.github.com/spec/v1`
	`2`	`+oid sha256:2c401cc2cb6aeda93c1d1b633603ffb6ea15efc049694e18bdfd21894308e4f6`
	`3`	`+size 15860`