feat: binary conversion of sequence dbs with annonars and worker (#35) (#40)

Author: holtgrewe

.gitignore

@@ -1,6 +1,6 @@
 # Ignore the workflow directories.
-work/
-output/
+/work/
+/output/
 
 # Python
 __pycache__

Makefile

@@ -50,7 +50,7 @@ check-black:
 .PHONY: check-snakefmt
 check-snakefmt:
 	snakefmt --check --diff --line-length 100 Snakefile
-	snakefmt --check --diff --line-length 100 rules/*/*.smk rules/*/*/*.smk
+	snakefmt --check --diff --line-length 100 rules/*/*/*.smk rules/*/*/*.smk
 
 # Run Python linting with flake8.
 .PHONY: flake8
@@ -86,4 +86,4 @@ black:
 .PHONY: run-snakefmt
 run-snakefmt:
 	snakefmt --line-length 100 Snakefile
-	snakefmt --line-length 100 rules/*/*.smk rules/*/*/*.smk
+	snakefmt --line-length 100 rules/*/*/*.smk rules/*/*/*.smk

Snakefile

@@ -6,13 +6,17 @@
 # ``varfish-server-worker`` and is used in the backend for filtering and/or exposed to the
 # user via a REST API.
 
-from varfish_db_downloader.data_versions import DATA_VERSIONS as DV
+from varfish_db_downloader.versions import DATA_VERSIONS as DV, PACKAGE_VERSIONS as PV
 
 # The prefix to use for all shell commands.
 SHELL_PREFIX = "export LC_ALL=C; set -x -euo pipefail;"
 # Setup the shell prefix by default.
 shell.prefix(SHELL_PREFIX)
 
+# Regular expression for genome release.
+RE_GENOME = r"grch(37|38)"
+# Regular expression for versions.
+RE_VERSION = r"\w+(\.\w+)*"
 
 # ===============================================================================================
 # Test Mode
@@ -58,6 +62,8 @@ rule help:
 ## all -- run all rules
 rule all:
     input:
+        # == work directory =====================================================================
+        #
         # genes
         f"work/genes/dbnsfp/{DV.dbnsfp}/genes.tsv.gz",
         f"work/genes/ensembl/{DV.ensembl}/ensembl_xlink.tsv",
@@ -76,20 +82,20 @@ rule all:
         f"work/download/annos/grch37/seqvars/dbnsfp/{DV.dbnsfp}c/LICENSE.txt",
         f"work/download/annos/grch37/seqvars/dbscsnv/{DV.dbscsnv}/dbscSNV{DV.dbscsnv}.chr1",
         f"work/download/annos/grch37/seqvars/dbsnp/{DV.dbsnp}/dbsnp.vcf.gz",
-        "work/annos/grch37/seqvars/helixmtdb/20200327/helixmtdb.vcf.gz",
+        f"work/annos/grch37/seqvars/helixmtdb/{DV.helixmtdb}/helixmtdb.vcf.gz",
         f"work/annos/grch37/seqvars/gnomad_mtdna/{DV.gnomad_mtdna}/gnomad_mtdna.vcf.gz",
-        f"work/annos/grch37/seqvars/gnomad_exomes/{DV.gnomad_v2}/.done",
-        f"work/annos/grch37/seqvars/gnomad_genomes/{DV.gnomad_v2}/.done",
+        f"work/download/annos/grch37/seqvars/gnomad_exomes/{DV.gnomad_v2}/.done",
+        f"work/download/annos/grch37/seqvars/gnomad_genomes/{DV.gnomad_v2}/.done",
         # ---- GRCh38
         f"work/download/annos/grch38/seqvars/cadd/{DV.cadd}/whole_genome_SNVs_inclAnno.tsv.gz",
         f"work/download/annos/grch38/seqvars/cadd/{DV.cadd}/gnomad.genomes.r3.0.indel_inclAnno.tsv.gz",
         # NB: dbNSFP is dual reference (for download)
         # NB: dbscSNV is dual reference (for download)
         f"work/download/annos/grch37/seqvars/dbsnp/{DV.dbsnp}/dbsnp.vcf.gz",
-        "work/annos/grch38/seqvars/helixmtdb/20200327/helixmtdb.vcf.gz",
+        f"work/annos/grch38/seqvars/helixmtdb/{DV.helixmtdb}/helixmtdb.vcf.gz",
         f"work/annos/grch38/seqvars/gnomad_mtdna/{DV.gnomad_mtdna}/gnomad_mtdna.vcf.gz",
-        f"work/annos/grch38/seqvars/gnomad_exomes/{DV.gnomad_v2}/.done",
-        f"work/annos/grch38/seqvars/gnomad_genomes/{DV.gnomad_v3}/.done",
+        f"work/download/annos/grch38/seqvars/gnomad_exomes/{DV.gnomad_v2}/.done",
+        f"work/download/annos/grch38/seqvars/gnomad_genomes/{DV.gnomad_v3}/.done",
         # -- background/population structural variants and annoations thereof
         # ---- GRCh37
         f"work/annos/grch37/strucvars/dbvar/{DV.dbvar}/dbvar.bed.gz",
@@ -122,38 +128,87 @@ rule all:
         f"work/annos/grch38/features/ucsc/{DV.ucsc_rmsk_38}/rmsk.bed.gz",
         f"work/annos/grch38/features/ucsc/{DV.ucsc_alt_seq_liftover_38}/altSeqLiftOverPsl.bed.gz",
         f"work/annos/grch38/features/ucsc/{DV.ucsc_fix_seq_liftover_38}/fixSeqLiftOverPsl.bed.gz",
+        #
+        # == output directory ===================================================================
+        #
+        # -- mehari data
+        # ---- frequencies (via annonars)
+        f"output/mehari/freqs-grch37-{DV.gnomad_v2}+{DV.gnomad_v2}+{DV.gnomad_mtdna}+{DV.helixmtdb}+{PV.annonars}/rocksdb/IDENTITY",
+        f"output/mehari/freqs-grch38-{DV.gnomad_v3}+{DV.gnomad_v2}+{DV.gnomad_mtdna}+{DV.helixmtdb}+{PV.annonars}/rocksdb/IDENTITY",
+        # -- varfish-server-worker data
+        # ---- CADD
+        f"output/worker/annos/seqvars/cadd-grch37-{DV.cadd}+{PV.annonars}/rocksdb/IDENTITY",
+        f"output/worker/annos/seqvars/cadd-grch38-{DV.cadd}+{PV.annonars}/rocksdb/IDENTITY",
+        # ---- dbSNP
+        f"output/worker/annos/seqvars/dbsnp-grch37-{DV.dbsnp}+{PV.annonars}/rocksdb/IDENTITY",
+        f"output/worker/annos/seqvars/dbsnp-grch38-{DV.dbsnp}+{PV.annonars}/rocksdb/IDENTITY",
+        # ---- dbNSFP
+        f"output/worker/annos/seqvars/dbnsfp-grch37-{DV.dbnsfp}a+{PV.annonars}/rocksdb/IDENTITY",
+        f"output/worker/annos/seqvars/dbnsfp-grch38-{DV.dbnsfp}a+{PV.annonars}/rocksdb/IDENTITY",
+        f"output/worker/annos/seqvars/dbnsfp-grch37-{DV.dbnsfp}c+{PV.annonars}/rocksdb/IDENTITY",
+        f"output/worker/annos/seqvars/dbnsfp-grch38-{DV.dbnsfp}c+{PV.annonars}/rocksdb/IDENTITY",
+        # ---- dbscSNV
+        f"output/worker/annos/seqvars/dbscsnv-grch37-{DV.dbscsnv}+{PV.annonars}/rocksdb/IDENTITY",
+        f"output/worker/annos/seqvars/dbscsnv-grch38-{DV.dbscsnv}+{PV.annonars}/rocksdb/IDENTITY",
+        # ---- gnomAD mtDNA
+        f"output/worker/annos/seqvars/gnomad-mtdna-grch37-{DV.gnomad_mtdna}+{PV.annonars}/rocksdb/IDENTITY",
+        f"output/worker/annos/seqvars/gnomad-mtdna-grch38-{DV.gnomad_mtdna}+{PV.annonars}/rocksdb/IDENTITY",
+        # ---- gnomAD exomes
+        f"output/worker/annos/seqvars/gnomad-exomes-grch37-{DV.gnomad_v2}+{PV.annonars}/rocksdb/IDENTITY",
+        f"output/worker/annos/seqvars/gnomad-exomes-grch38-{DV.gnomad_v2}+{PV.annonars}/rocksdb/IDENTITY",
+        # ---- gnomAD genomes
+        f"output/worker/annos/seqvars/gnomad-genomes-grch37-{DV.gnomad_v2}+{PV.annonars}/rocksdb/IDENTITY",
+        f"output/worker/annos/seqvars/gnomad-genomes-grch38-{DV.gnomad_v3}+{PV.annonars}/rocksdb/IDENTITY",
+        # ---- HelixMtDb
+        f"output/worker/annos/seqvars/helixmtdb-grch37-{DV.helixmtdb}+{PV.annonars}/rocksdb/IDENTITY",
+        f"output/worker/annos/seqvars/helixmtdb-grch38-{DV.helixmtdb}+{PV.annonars}/rocksdb/IDENTITY",
+        # ---- UCSC conservation
+        f"output/worker/annos/seqvars/cons-grch37-{DV.ucsc_cons_37}+{PV.annonars}/rocksdb/IDENTITY",
+        f"output/worker/annos/seqvars/cons-grch38-{DV.ucsc_cons_38}+{PV.annonars}/rocksdb/IDENTITY",
 
 
 # ===============================================================================================
 # Modular Snakefile Includes
 # ===============================================================================================
 
 
+# -- work directory -----------------------------------------------------------------------------
 # Gene-related rules.
-include: "rules/genes/dbnsfp.smk"
-include: "rules/genes/ensembl.smk"
-include: "rules/genes/gnomad.smk"
-include: "rules/genes/hgnc.smk"
-include: "rules/genes/ncbi.smk"
+include: "rules/work/genes/dbnsfp.smk"
+include: "rules/work/genes/ensembl.smk"
+include: "rules/work/genes/gnomad.smk"
+include: "rules/work/genes/hgnc.smk"
+include: "rules/work/genes/ncbi.smk"
 # Reference sequence--related rules.
-include: "rules/reference/human.smk"
+include: "rules/work/reference/human.smk"
 # Features (position and not variant specific).
-include: "rules/annos/features/cons.smk"
-include: "rules/annos/features/ensembl.smk"
-include: "rules/annos/features/refseq.smk"
-include: "rules/annos/features/tads.smk"
-include: "rules/annos/features/ucsc.smk"
+include: "rules/work/annos/features/cons.smk"
+include: "rules/work/annos/features/ensembl.smk"
+include: "rules/work/annos/features/refseq.smk"
+include: "rules/work/annos/features/tads.smk"
+include: "rules/work/annos/features/ucsc.smk"
 # Sequence variants and annotations.
-include: "rules/annos/seqvars/cadd.smk"
-include: "rules/annos/seqvars/dbnsfp.smk"
-include: "rules/annos/seqvars/dbscsnv.smk"
-include: "rules/annos/seqvars/dbsnp.smk"
-include: "rules/annos/seqvars/gnomad_mtdna.smk"
-include: "rules/annos/seqvars/gnomad_nuclear.smk"
-include: "rules/annos/seqvars/helix.smk"
+include: "rules/work/annos/seqvars/cadd.smk"
+include: "rules/work/annos/seqvars/dbnsfp.smk"
+include: "rules/work/annos/seqvars/dbscsnv.smk"
+include: "rules/work/annos/seqvars/dbsnp.smk"
+include: "rules/work/annos/seqvars/gnomad_mtdna.smk"
+include: "rules/work/annos/seqvars/gnomad_nuclear.smk"
+include: "rules/work/annos/seqvars/helix.smk"
 # Structural variant related.
-include: "rules/annos/strucvars/dbvar.smk"
-include: "rules/annos/strucvars/dgv.smk"
-include: "rules/annos/strucvars/exac.smk"
-include: "rules/annos/strucvars/g1k.smk"
-include: "rules/annos/strucvars/gnomad.smk"
+include: "rules/work/annos/strucvars/dbvar.smk"
+include: "rules/work/annos/strucvars/dgv.smk"
+include: "rules/work/annos/strucvars/exac.smk"
+include: "rules/work/annos/strucvars/g1k.smk"
+include: "rules/work/annos/strucvars/gnomad.smk"
+# -- output directory ---------------------------------------------------------------------------
+include: "rules/output/mehari/freqs.smk"
+include: "rules/output/worker/cadd.smk"
+include: "rules/output/worker/dbsnp.smk"
+include: "rules/output/worker/dbnsfp.smk"
+include: "rules/output/worker/dbscsnv.smk"
+include: "rules/output/worker/gnomad_mtdna.smk"
+include: "rules/output/worker/gnomad_exomes.smk"
+include: "rules/output/worker/gnomad_genomes.smk"
+include: "rules/output/worker/helix.smk"
+include: "rules/output/worker/cons.smk"

environment.yml

@@ -39,3 +39,6 @@ dependencies:
   - samtools =1.16
   # Parallel (de)compression.
   - pigz
+  # Varfish/Mehari/Annonars related
+  - annonars =0.7.0
+  - varfish-server-worker

excerpt-data/04eb6ba4c80b77b7/gnomad.genomes.v3.1.2.sites.chr15.vcf.bgz.tbi

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1c2de33ed9b4a0a8733ec18c7626583fb888fced90555001ad86db1e578f2deb
-size 89527
+oid sha256:3f8010eef34130590d7aa24055fcf7910e2182a06ae87daa2b0cf950a45c5c19
+size 152

excerpt-data/0766243104b0c0b5/gnomad.genomes.v3.1.2.sites.chr7.vcf.bgz.tbi

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:50001e32b9c77a7bad932c8aff89d3adc99419a338fc10e04a97d5e44a87e61d
-size 169578
+oid sha256:6660483ea6f7e90eb11af1cdf113b8520fa49e1006cc1d9e1c518eb135845eca
+size 115

excerpt-data/0874528fb71739a2/gnomad.genomes.r2.1.1.sites.15.vcf.bgz.tbi

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e334ca666a53dbe51cdf3df5e9239a94485712797d622b6d6084afcff2ff5736
-size 83137
+oid sha256:09d83646240ed8a3957e3a34621164bcc176879dd85de9952676be55decec157
+size 149

excerpt-data/0bc17cc34dd0b330/gnomad.exomes.r2.1.1.sites.1.liftover_grch38.vcf.bgz.tbi

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4203f3ef9596b1cf06af46cd7d0afe828de8bf2fd9bed39a857125de40d4d13d
-size 90933
+oid sha256:fe9a926b7c12e334c3febb6197319e8181935b855778aa9ec413059674ceb6c9
+size 115

excerpt-data/0c82ec0601b93b32/gnomad.genomes.r2.1.1.sites.1.vcf.bgz.tbi

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5e2861d6931677098408d2d78d85bcd99470b5647b05b1993a46b888e3f20613
-size 226696
+oid sha256:94b0b74f3e2d977f140523432b5b888ceec9997d80b5b7cf0624acd83dde93f7
+size 115

excerpt-data/15476a18eb31d5b1/whole_genome_SNVs_inclAnno.tsv.gz

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0061fc2d3d16fd4f27d6c02853b6b8244f59b26b619740cba0725e318332f587
-size 2237
+oid sha256:db2ac0adfea9cfd8b0595e3a153a7933adc66e830662e80fcefd56e3dee47361
+size 2175