varfish-org · stolpeo · Apr 15, 2025 · May 9, 2025 · May 9, 2025 · May 16, 2025
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -80,9 +80,6 @@ jobs:
            unzip qsv-0.112.0-x86_64-unknown-linux-musl.zip
            cp qsv_musl-1.2.3 /usr/local/bin/qsv
 
-      - name: Install annonars
-        run: |
-          sudo bash ./utils/install-annonars.sh
       - name: Install python package
         run: |
           pip install -e .

diff --git a/Snakefile b/Snakefile
diff --git a/download_urls.yml b/download_urls.yml
@@ -57,16 +57,16 @@
     count: null
 
 # dbNSFP v4.5a
-- url: https://dbnsfp.s3.amazonaws.com/dbNSFP4.5a.zip
+- url: https://usf.box.com/shared/static/2hzcx5s6p1xui7oen16xqzndfrkt8l9l
   excerpt_strategy:
     strategy: manual
     count: null
 # dbNSFP v4.5c
-- url: https://dbnsfp.s3.amazonaws.com/dbNSFP4.5c.zip
+- url: https://usf.box.com/shared/static/03xsrpna0nzgrytfo2pzk326t8jad4oc
   excerpt_strategy:
     strategy: manual
     count: null
-- url: ftp://dbnsfp:[email protected].com/dbscSNV1.1.zip
+- url: https://usf.box.com/shared/static/ffwlywsat3q5ijypvunno3rg6steqfs8
   skip_upstream_check: true  # does not work reliably in tests
   excerpt_strategy:
     strategy: manual
@@ -141,16 +141,34 @@
   url: https://search.clinicalgenome.org/kb/reports/curation-activity-summary-report
   skip_upstream_check: true  # does not work reliably in tests
 
-- url: https://github.com/varfish-org/clinvar-data-jsonl/releases/download/clinvar-weekly-20240612/clinvar-data-extract-vars-20240612+0.17.0.tar.gz
+- url: https://github.com/varfish-org/clinvar-data-jsonl/releases/download/clinvar-weekly-20250410/clinvar-data-extract-vars-20250410+0.18.5.tar.gz
   excerpt_strategy:
     strategy: manual
     count: null
 
-- url: https://github.com/bihealth/mehari-data-tx/releases/download/v0.4.4/mehari-data-txs-grch37-0.4.4.bin.zst
+- url: https://github.com/bihealth/mehari-data-tx/releases/download/v0.10.3/mehari-data-txs-grch37-ensembl-0.10.3.bin.zst
   excerpt_strategy:
     strategy: no-excerpt
     count: null
-- url: https://github.com/bihealth/mehari-data-tx/releases/download/v0.4.4/mehari-data-txs-grch38-0.4.4.bin.zst
+- url: https://github.com/bihealth/mehari-data-tx/releases/download/v0.10.3/mehari-data-txs-grch38-ensembl-0.10.3.bin.zst
+  excerpt_strategy:
+    strategy: no-excerpt
+    count: null
+
+- url: https://github.com/bihealth/mehari-data-tx/releases/download/v0.10.3/mehari-data-txs-grch37-refseq-0.10.3.bin.zst
+  excerpt_strategy:
+    strategy: no-excerpt
+    count: null
+- url: https://github.com/bihealth/mehari-data-tx/releases/download/v0.10.3/mehari-data-txs-grch38-refseq-0.10.3.bin.zst
+  excerpt_strategy:
+    strategy: no-excerpt
+    count: null
+
+- url: https://github.com/bihealth/mehari-data-tx/releases/download/v0.10.3/mehari-data-txs-grch37-ensembl-0.10.3.bin.zst
+  excerpt_strategy:
+    strategy: no-excerpt
+    count: null
+- url: https://github.com/bihealth/mehari-data-tx/releases/download/v0.10.3/mehari-data-txs-grch38-ensembl-0.10.3.bin.zst
   excerpt_strategy:
     strategy: no-excerpt
     count: null
@@ -251,7 +269,7 @@
     count: 10000
 
 - url: 'https://ensembl.org/biomart/martservice?query=<?xml version="1.0" encoding="UTF-8"?><!DOCTYPE Query><Query  virtualSchemaName = "default" formatter = "TSV" header = "0" uniqueRows = "0" count = "" datasetConfigVersion = "0.6" ><Dataset name = "hsapiens_gene_ensembl" interface = "default" ><Attribute name = "ensembl_gene_id" /><Attribute name = "ensembl_transcript_id" /><Attribute name = "entrezgene_id" /><Attribute name = "external_gene_name" /></Dataset></Query>'
-- url: 'https://ftp.ebi.ac.uk/pub/databases/genenames/hgnc/json/hgnc_complete_set.json'
+- url: 'https://storage.googleapis.com/public-download-files/hgnc/json/json/hgnc_complete_set.json'
   skip_upstream_check: true  # does not work reliably in tests
   excerpt_strategy:
     strategy: manual

diff --git a/environment.yml b/environment.yml
@@ -33,23 +33,23 @@ dependencies:
   - jq
   # Tools for file downloads.
   - aria2 >=1.36.0
-  - wget
   # Tool for processing BED files.
   - bedops =2
   # VCF/BCF/HTSlib/Samtools.
-  - bcftools =1.17
-  - htslib =1.17
-  - samtools =1.17
+  - bcftools =1.21
+  - htslib =1.21
+  - samtools =1.21
   # Parallel (de)compression.
   - pigz
   # Varfish related
-  # - annonars =0.41.3  # current versions not on bioconda due to build issue, but docker images are available
-  - viguno =0.3.1
-  - mehari =0.25.5
-  - varfish-server-worker =0.13.0
+  - annonars =0.44.0
+  - viguno =0.4.0
+  - mehari =0.35.1
+  - varfish-server-worker =0.17.2
   # S3 uploads
   - s5cmd =2.1.0
   # async HTTP requests
   - httpx =0.25.0
   - httpcore =0.18.0
   - trio
+  - qsv
diff --git a/excerpt-data/0ccc4915e7ecfd38/mehari-data-txs-GRCh37-refseq-0.10.3.bin.zst b/excerpt-data/0ccc4915e7ecfd38/mehari-data-txs-GRCh37-refseq-0.10.3.bin.zst
diff --git a/excerpt-data/0ccc4915e7ecfd38/url.txt b/excerpt-data/0ccc4915e7ecfd38/url.txt
diff --git a/excerpt-data/111d8c6e08038f62/20 b/excerpt-data/111d8c6e08038f62/20
diff --git a/excerpt-data/18c67f3b60f24667/mehari-data-txs-GRCh38-refseq-0.10.3.bin.zst b/excerpt-data/18c67f3b60f24667/mehari-data-txs-GRCh38-refseq-0.10.3.bin.zst
diff --git a/excerpt-data/18c67f3b60f24667/url.txt b/excerpt-data/18c67f3b60f24667/url.txt
diff --git a/excerpt-data/1963f3c58ea066be/omim_unmapped_terms.tsv b/excerpt-data/1963f3c58ea066be/omim_unmapped_terms.tsv
diff --git a/excerpt-data/5bcb7090fe0e881b/mehari-data-txs-GRCh37-ensembl-0.10.3.bin.zst b/excerpt-data/5bcb7090fe0e881b/mehari-data-txs-GRCh37-ensembl-0.10.3.bin.zst
diff --git a/excerpt-data/5bcb7090fe0e881b/url.txt b/excerpt-data/5bcb7090fe0e881b/url.txt
diff --git a/excerpt-data/615312ce3f5fc1bf/OMIMinDO.tsv b/excerpt-data/615312ce3f5fc1bf/OMIMinDO.tsv
diff --git a/excerpt-data/617bebe58c82f24e/CTD_diseases.tsv.gz b/excerpt-data/617bebe58c82f24e/CTD_diseases.tsv.gz
diff --git a/excerpt-data/649dbe83da5f50d9/mehari-data-txs-GRCh38-ensembl-0.10.3.bin.zst b/excerpt-data/649dbe83da5f50d9/mehari-data-txs-GRCh38-ensembl-0.10.3.bin.zst
diff --git a/excerpt-data/649dbe83da5f50d9/url.txt b/excerpt-data/649dbe83da5f50d9/url.txt
diff --git a/excerpt-data/652646c24140df2a/mondo.obo b/excerpt-data/652646c24140df2a/mondo.obo
diff --git a/excerpt-data/6f378db589a4bbb9/orphacodes b/excerpt-data/6f378db589a4bbb9/orphacodes
diff --git a/excerpt-data/91f964d1aa8367a5/20 b/excerpt-data/91f964d1aa8367a5/20
diff --git a/excerpt-data/b31e9b26b7aeae3f/dbNSFP4.5c.zip b/excerpt-data/b31e9b26b7aeae3f/dbNSFP4.5c.zip
diff --git a/excerpt-data/b31e9b26b7aeae3f/url.txt b/excerpt-data/b31e9b26b7aeae3f/url.txt
diff --git a/excerpt-data/bdc69c1e4cafdfaa/clinvar-data-extract-vars-20240612+0.17.0.tar.gz b/excerpt-data/bdc69c1e4cafdfaa/clinvar-data-extract-vars-20240612+0.17.0.tar.gz
diff --git a/excerpt-data/bdc69c1e4cafdfaa/url.txt b/excerpt-data/bdc69c1e4cafdfaa/url.txt
diff --git a/excerpt-data/c243f918f5fc10d2/dbNSFP4.5a.zip b/excerpt-data/c243f918f5fc10d2/dbNSFP4.5a.zip
diff --git a/excerpt-data/c243f918f5fc10d2/url.txt b/excerpt-data/c243f918f5fc10d2/url.txt
diff --git a/excerpt-data/c9c7d6df0e24b954/__index__ b/excerpt-data/c9c7d6df0e24b954/__index__
diff --git a/excerpt-data/caba5539b2dac784/clinvar-data-extract-vars-20250410+0.18.5.tar.gz b/excerpt-data/caba5539b2dac784/clinvar-data-extract-vars-20250410+0.18.5.tar.gz
diff --git a/excerpt-data/caba5539b2dac784/url.txt b/excerpt-data/caba5539b2dac784/url.txt
diff --git a/excerpt-data/cbc00fe1a5b22fe8/dbscSNV1.1.zip b/excerpt-data/cbc00fe1a5b22fe8/dbscSNV1.1.zip
diff --git a/excerpt-data/cbc00fe1a5b22fe8/url.txt b/excerpt-data/cbc00fe1a5b22fe8/url.txt
diff --git a/excerpt-data/cdaaf7a3f7595d3d/__index__ b/excerpt-data/cdaaf7a3f7595d3d/__index__
diff --git a/excerpt-data/e8a29ff831bf8ee3/hgnc_complete_set.json b/excerpt-data/e8a29ff831bf8ee3/hgnc_complete_set.json
diff --git a/excerpt-data/e8a29ff831bf8ee3/url.txt b/excerpt-data/e8a29ff831bf8ee3/url.txt
diff --git a/excerpt-data/ebc07f725c64907d/__index__ b/excerpt-data/ebc07f725c64907d/__index__
diff --git a/rules/output/annonars/alphamissense.smk b/rules/output/annonars/alphamissense.smk
@@ -23,10 +23,10 @@ rule output_annonars_alphamissense:  # -- build AlphaMissense RocksDB with annon
         manifest=(
             "output/full/annonars/alphamissense-{genome_release}-{v_alphamissense}+{v_annonars}/MANIFEST.txt"
         ),
-    threads: int(os.environ.get("THREADS_ANNONARS_IMPORT", "96"))
+    threads: THREADS
     resources:
         runtime=os.environ.get("RUNTIME_ANNONARS_IMPORT", "48h"),
-        mem_mb_per_cpu=2000,
+        mem_mb=MEMORY,
     wildcard_constraints:
         genome_release=RE_GENOME,
         v_alphamissense=RE_VERSION,

diff --git a/rules/output/annonars/cadd.smk b/rules/output/annonars/cadd.smk
@@ -54,10 +54,10 @@ rule output_annonars_cadd:  # -- build CADD RocksDB with annonars
         ),
         spec_yaml=("output/full/annonars/cadd-{genome_release}-{v_cadd}+{v_annonars}/spec.yaml"),
         manifest=("output/full/annonars/cadd-{genome_release}-{v_cadd}+{v_annonars}/MANIFEST.txt"),
-    threads: int(os.environ.get("THREADS_ANNONARS_IMPORT", "96"))
+    threads: THREADS
     resources:
         runtime=os.environ.get("RUNTIME_ANNONARS_IMPORT", "48h"),
-        mem_mb_per_cpu=2000,
+        mem_mb=MEMORY,
     wildcard_constraints:
         genome_release=RE_GENOME,
         v_cadd=RE_VERSION,

diff --git a/rules/output/annonars/cons.smk b/rules/output/annonars/cons.smk
@@ -12,10 +12,10 @@ rule output_annonars_cons:  # -- build UCSC conservation track RocksDB with anno
         ),
         spec_yaml=("output/full/annonars/cons-{genome_release}-{v_cons}+{v_annonars}/spec.yaml"),
         manifest=("output/full/annonars/cons-{genome_release}-{v_cons}+{v_annonars}/MANIFEST.txt"),
-    threads: int(os.environ.get("THREADS_ANNONARS_IMPORT", "96"))
+    threads: THREADS
     resources:
         runtime=os.environ.get("RUNTIME_ANNONARS_IMPORT", "48h"),
-        mem_mb_per_cpu=2000,
+        mem_mb=MEMORY,
     wildcard_constraints:
         genome_release=RE_GENOME,
         v_cons=RE_VERSION,

diff --git a/rules/output/annonars/dbnsfp.smk b/rules/output/annonars/dbnsfp.smk
@@ -19,10 +19,10 @@ rule output_annonars_dbnsfp:  # -- build dbNSFP RocksDB with annonars
         manifest=(
             "output/full/annonars/dbnsfp-{genome_release}-{v_dbnsfp}+{v_annonars}/MANIFEST.txt"
         ),
-    threads: int(os.environ.get("THREADS_ANNONARS_IMPORT", "96"))
+    threads: THREADS
     resources:
         runtime=os.environ.get("RUNTIME_ANNONARS_IMPORT", "48h"),
-        mem_mb_per_cpu=2000,
+        mem_mb=MEMORY,
     wildcard_constraints:
         genome_release=RE_GENOME,
         v_dbnsfp=RE_VERSION,

diff --git a/rules/output/annonars/dbscsnv.smk b/rules/output/annonars/dbscsnv.smk
@@ -21,10 +21,10 @@ rule output_annonars_dbscsnv:  # -- build dbscSNV RocksDB with annonars
         manifest=(
             "output/full/annonars/dbscsnv-{genome_release}-{v_dbscsnv}+{v_annonars}/MANIFEST.txt"
         ),
-    threads: int(os.environ.get("THREADS_ANNONARS_IMPORT", "96"))
+    threads: THREADS
     resources:
         runtime=os.environ.get("RUNTIME_ANNONARS_IMPORT", "48h"),
-        mem_mb_per_cpu=2000,
+        mem_mb=MEMORY,
     wildcard_constraints:
         genome_release=RE_GENOME,
         v_dbscsnv=RE_VERSION,

diff --git a/rules/output/annonars/dbsnp.smk b/rules/output/annonars/dbsnp.smk
@@ -12,10 +12,10 @@ rule output_annonars_dbsnp:  # -- build dbSNP RocksDB with annonars
         ),
         spec_yaml=("output/full/annonars/dbsnp-{genome_release}-{v_dbsnp}+{v_annonars}/spec.yaml"),
         manifest=("output/full/annonars/dbsnp-{genome_release}-{v_dbsnp}+{v_annonars}/MANIFEST.txt"),
-    threads: int(os.environ.get("THREADS_ANNONARS_IMPORT", "96"))
+    threads: THREADS
     resources:
         runtime=os.environ.get("RUNTIME_ANNONARS_IMPORT", "48h"),
-        mem_mb_per_cpu=2000,
+        mem_mb=MEMORY,
     wildcard_constraints:
         genome_release=RE_GENOME,
         v_dbsnp=RE_VERSION,

diff --git a/rules/output/annonars/gnomad_exomes.smk b/rules/output/annonars/gnomad_exomes.smk
@@ -16,10 +16,10 @@ rule output_annonars_gnomad_exomes:  # -- build gnomAD-exomes RocksDB with annon
         manifest=(
             "output/full/annonars/gnomad-exomes-{genome_release}-{v_gnomad}+{v_annonars}/MANIFEST.txt"
         ),
-    threads: int(os.environ.get("THREADS_ANNONARS_IMPORT", "96"))
+    threads: THREADS
     resources:
         runtime=os.environ.get("RUNTIME_ANNONARS_IMPORT", "48h"),
-        mem_mb_per_cpu=2000,
+        mem_mb=MEMORY,
     wildcard_constraints:
         genome_release=RE_GENOME,
         v_gnomad=RE_VERSION,

diff --git a/rules/output/annonars/gnomad_genomes.smk b/rules/output/annonars/gnomad_genomes.smk
@@ -16,10 +16,10 @@ rule output_annonars_gnomad_genomes:  # -- build gnomAD-genomes RocksDB with ann
         manifest=(
             "output/full/annonars/gnomad-genomes-{genome_release}-{v_gnomad}+{v_annonars}/MANIFEST.txt"
         ),
-    threads: int(os.environ.get("THREADS_ANNONARS_IMPORT", "96"))
+    threads: THREADS
     resources:
         runtime=os.environ.get("RUNTIME_ANNONARS_IMPORT", "48h"),
-        mem_mb_per_cpu=2000,
+        mem_mb=MEMORY,
     wildcard_constraints:
         genome_release=RE_GENOME,
         v_gnomad=RE_VERSION,

diff --git a/rules/output/annonars/gnomad_mtdna.smk b/rules/output/annonars/gnomad_mtdna.smk
@@ -16,10 +16,10 @@ rule output_annonars_gnomad_mtdna:  # -- build gnomAD-mtDNA RocksDB with annonar
         manifest=(
             "output/full/annonars/gnomad-mtdna-{genome_release}-{v_gnomad}+{v_annonars}/MANIFEST.txt"
         ),
-    threads: int(os.environ.get("THREADS_ANNONARS_IMPORT", "96"))
+    threads: THREADS
     resources:
         runtime=os.environ.get("RUNTIME_ANNONARS_IMPORT", "48h"),
-        mem_mb_per_cpu=2000,
+        mem_mb=MEMORY,
     wildcard_constraints:
         genome_release=RE_GENOME,
         v_gnomad=RE_VERSION,