feat: add missing background data for GRCh38 (#5) (#37)

Author: holtgrewe

Snakefile

@@ -75,7 +75,7 @@ rule all:
         f"work/download/annos/grch37/seqvars/dbnsfp/{DV.dbnsfp}a/LICENSE.txt",
         f"work/download/annos/grch37/seqvars/dbnsfp/{DV.dbnsfp}c/LICENSE.txt",
         f"work/download/annos/grch37/seqvars/dbscsnv/{DV.dbscsnv}/dbscSNV{DV.dbscsnv}.chr1",
-        f"work/download/annos/grch37/seqvars/dbsnp/{DV.dbsnp_37}/dbsnp.vcf.gz",
+        f"work/download/annos/grch37/seqvars/dbsnp/{DV.dbsnp}/dbsnp.vcf.gz",
         "work/annos/grch37/seqvars/helixmtdb/20200327/helixmtdb.vcf.gz"
         f"work/annos/grch37/seqvars/gnomad_mtdna/{DV.gnomad_mtdna}/gnomad_mtdna.vcf.gz",
         f"work/annos/grch37/seqvars/gnomad_exomes/{DV.gnomad_v2}/.done",
@@ -85,7 +85,7 @@ rule all:
         f"work/download/annos/grch38/seqvars/cadd/{DV.cadd}/gnomad.genomes.r3.0.indel_inclAnno.tsv.gz",
         # NB: dbNSFP is dual reference (for download)
         # NB: dbscSNV is dual reference (for download)
-        # TODO: "work/download/annos/grch38/seqvars/dbsnp/dbsnp.vcf.gz",
+        f"work/download/annos/grch37/seqvars/dbsnp/{DV.dbsnp}/dbsnp.vcf.gz",
         "work/annos/grch38/seqvars/helixmtdb/20200327/helixmtdb.vcf.gz"
         f"work/annos/grch38/seqvars/gnomad_mtdna/{DV.gnomad_mtdna}/gnomad_mtdna.vcf.gz",
         f"work/annos/grch38/seqvars/gnomad_exomes/{DV.gnomad_v2}/.done",
@@ -99,31 +99,29 @@ rule all:
         f"work/annos/grch37/strucvars/g1k/{DV.g1k_svs}/g1k.bed.gz",
         f"work/annos/grch37/strucvars/gnomad/{DV.gnomad_sv}/gnomad_sv.bed.gz",
         # ---- GRCh38
-        # TODO: "work/annos/grch38/strucvars/dbvar/dbvar.bed.gz",
-        # TODO: f"work/annos/grch38/strucvars/dgv/{DV.dgv}/dgv.bed.gz",
-        # TODO: f"work/annos/grch38/strucvars/dgv_gs/{DV.dgv_gs}/dgv_gs.bed.gz",
-        # TODO: "work/annos/grch38/strucvars/gnomad/gnomad_sv.bed.gz",
+        f"work/annos/grch38/strucvars/dbvar/{DV.dbvar}/dbvar.bed.gz",
+        f"work/annos/grch38/strucvars/dgv/{DV.dgv}/dgv.bed.gz",
+        f"work/annos/grch38/strucvars/dgv_gs/{DV.dgv_gs}/dgv_gs.bed.gz",
+        # NB: gnomAD-SV GRCh38 was announced end of 2020 but not released yet
         # -- genome browser "features" (position-specific)
         # ---- GRCh37
-        f"work/annos/grch37/features/cons/{DV.ucsc_cons}/ucsc_conservation.tsv",
+        f"work/annos/grch37/features/cons/{DV.ucsc_cons_37}/ucsc_conservation.tsv",
         f"work/annos/grch37/features/ensembl/{DV.ensembl_37}/ensembl_genes.bed.gz",
         f"work/annos/grch37/features/refseq/{DV.refseq_37}/refseq_genes.bed.gz",
-        "work/annos/grch37/features/tads/dixon2015/imr90.bed",
         "work/annos/grch37/features/tads/dixon2015/hesc.bed",
-        f"work/annos/grch37/features/ucsc/{DV.ucsc_genomic_super_dups}/genomicSuperDups.bed.gz",
-        f"work/annos/grch37/features/ucsc/{DV.ucsc_rmsk}/rmsk.bed.gz",
-        f"work/annos/grch37/features/ucsc/{DV.ucsc_alt_seq_liftover}/altSeqLiftOverPsl.bed.gz",
-        f"work/annos/grch37/features/ucsc/{DV.ucsc_fix_seq_liftover}/fixSeqLiftOverPsl.bed.gz",
+        f"work/annos/grch37/features/ucsc/{DV.ucsc_genomic_super_dups_37}/genomicSuperDups.bed.gz",
+        f"work/annos/grch37/features/ucsc/{DV.ucsc_rmsk_37}/rmsk.bed.gz",
+        f"work/annos/grch37/features/ucsc/{DV.ucsc_alt_seq_liftover_37}/altSeqLiftOverPsl.bed.gz",
+        f"work/annos/grch37/features/ucsc/{DV.ucsc_fix_seq_liftover_37}/fixSeqLiftOverPsl.bed.gz",
         # ---- GRCh38
-        # TODO: "work/annos/grch38/features/cons/ucsc_conservation.tsv",
-        # TODO: "work/annos/grch38/features/ensembl/ensembl_genes.bed.gz",
-        # TODO: "work/annos/grch38/features/refseq/refseq_genes.bed.gz",
-        # TODO: "work/annos/grch38/features/tads/imr90.bed",
-        # TODO: "work/annos/grch38/features/tads/hesc.bed",
-        # TODO: "work/annos/grch38/features/ucsc/genomicSuperDups.bed.gz",
-        # TODO: "work/annos/grch38/features/ucsc/rmsk.bed.gz",
-        # TODO: "work/annos/grch38/features/ucsc/altSeqLiftOverPsl.bed.gz",
-        # TODO: "work/annos/grch38/features/ucsc/fixSeqLiftOverPsl.bed.gz",
+        f"work/annos/grch38/features/cons/{DV.ucsc_cons_38}/ucsc_conservation.tsv",
+        f"work/annos/grch38/features/ensembl/{DV.ensembl_38}/ensembl_genes.bed.gz",
+        f"work/annos/grch38/features/refseq/{DV.refseq_38}/refseq_genes.bed.gz",
+        "work/annos/grch38/features/tads/dixon2015/hesc.bed",
+        f"work/annos/grch38/features/ucsc/{DV.ucsc_genomic_super_dups_38}/genomicSuperDups.bed.gz",
+        f"work/annos/grch38/features/ucsc/{DV.ucsc_rmsk_38}/rmsk.bed.gz",
+        f"work/annos/grch38/features/ucsc/{DV.ucsc_alt_seq_liftover_38}/altSeqLiftOverPsl.bed.gz",
+        f"work/annos/grch38/features/ucsc/{DV.ucsc_fix_seq_liftover_38}/fixSeqLiftOverPsl.bed.gz",
 
 
 # ===============================================================================================

download_urls.yml

@@ -1,13 +1,12 @@
 - url: https://ftp.ensembl.org/pub/current_README
 
-- comment: dbSNP listing for checking the version.
-  url: https://ftp.ncbi.nih.gov/snp/organisms/human_9606_b151_GRCh37p13/VCF
+- comment: The UCSC listing is used for checking the versions for GRCh37.
+  url: https://hgdownload.cse.ucsc.edu/goldenpath/hg19/database
   excerpt_strategy:
     strategy: no-excerpt
     count: null
-
-- comment: The UCSC listing is used for checking the versions.
-  url: https://hgdownload.cse.ucsc.edu/goldenpath/hg19/database
+- comment: The UCSC listing is used for checking the versions for GRCh38.
+  url: https://hgdownload.cse.ucsc.edu/goldenpath/hg38/database
   excerpt_strategy:
     strategy: no-excerpt
     count: null
@@ -26,12 +25,19 @@
     count: 10000
 - url: https://hgdownload.cse.ucsc.edu/goldenpath/hg38/multiz100way/alignments/knownGene.exonAA.fa.gz
 
-- url: https://compbio.med.harvard.edu/modencode/webpage/hic/IMR90_domains_hg19.bed
-- url: https://compbio.med.harvard.edu/modencode/webpage/hic/hESC_domains_hg19.bed
+- url: http://3dgenome.fsm.northwestern.edu/downloads/hg19.TADs.zip
+  excerpt_strategy:
+    strategy: no-excerpt
+    count: null
+- url: http://3dgenome.fsm.northwestern.edu/downloads/hg38.TADs.zip
+  excerpt_strategy:
+    strategy: no-excerpt
+    count: null
 
 - url: https://helix-research-public.s3.amazonaws.com/mito/HelixMTdb_20200327.tsv
 
 - url: https://ftp.ncbi.nih.gov/snp/organisms/human_9606_b151_GRCh37p13/VCF/00-All.vcf.gz
+- url: https://ftp.ncbi.nih.gov/snp/organisms/human_9606_b151_GRCh38p7/VCF/00-All.vcf.gz
 
 - url: https://kircherlab.bihealth.org/download/CADD/v1.6/GRCh37/whole_genome_SNVs_inclAnno.tsv.gz
 - url: https://kircherlab.bihealth.org/download/CADD/v1.6/GRCh37/whole_genome_SNVs_inclAnno.tsv.gz.tbi
@@ -65,11 +71,25 @@
     strategy: head
     count: 10000
 
+- url: https://ftp.ncbi.nlm.nih.gov/genomes/refseq/vertebrate_mammalian/Homo_sapiens/all_assembly_versions/GCF_000001405.40_GRCh38.p14/GCF_000001405.40_GRCh38.p14_assembly_report.txt
+  excerpt_strategy:
+    strategy: no-excerpt
+    count: null
+- url: https://ftp.ncbi.nlm.nih.gov/genomes/refseq/vertebrate_mammalian/Homo_sapiens/all_assembly_versions/GCF_000001405.40_GRCh38.p14/GCF_000001405.40_GRCh38.p14_genomic.gtf.gz
+  excerpt_strategy:
+    strategy: head
+    count: 10000
+
 - url: https://ftp.ensembl.org/pub/grch37/current/gtf/homo_sapiens/Homo_sapiens.GRCh37.87.gtf.gz
   excerpt_strategy:
     strategy: head
     count: 10000
 
+- url: https://ftp.ensembl.org/pub/release-109/gtf/homo_sapiens/Homo_sapiens.GRCh38.109.gtf.gz
+  excerpt_strategy:
+    strategy: head
+    count: 10000
+
 - url: 'https://ensembl.org/biomart/martservice?query=<?xml version="1.0" encoding="UTF-8"?><!DOCTYPE Query><Query  virtualSchemaName = "default" formatter = "TSV" header = "0" uniqueRows = "0" count = "" datasetConfigVersion = "0.6" ><Dataset name = "hsapiens_gene_ensembl" interface = "default" ><Attribute name = "ensembl_gene_id" /><Attribute name = "ensembl_transcript_id" /><Attribute name = "entrezgene_id" /><Attribute name = "external_gene_name" /></Dataset></Query>'
 - url: 'https://ftp.ebi.ac.uk/pub/databases/genenames/hgnc/json/hgnc_complete_set.json'
   skip_upstream_check: true  # does not work reliably in tests
@@ -100,6 +120,10 @@
 - url: https://hgdownload.cse.ucsc.edu/goldenpath/hg19/database/rmsk.txt.gz
 - url: https://hgdownload.cse.ucsc.edu/goldenpath/hg19/database/altSeqLiftOverPsl.txt.gz
 - url: https://hgdownload.cse.ucsc.edu/goldenpath/hg19/database/fixSeqLiftOverPsl.txt.gz
+- url: https://hgdownload.cse.ucsc.edu/goldenpath/hg38/database/genomicSuperDups.txt.gz
+- url: https://hgdownload.cse.ucsc.edu/goldenpath/hg38/database/rmsk.txt.gz
+- url: https://hgdownload.cse.ucsc.edu/goldenpath/hg38/database/altSeqLiftOverPsl.txt.gz
+- url: https://hgdownload.cse.ucsc.edu/goldenpath/hg38/database/fixSeqLiftOverPsl.txt.gz
 
 - comment: The dbVar release notes index is pulled for checking for latest release.
   url: https://ftp.ncbi.nlm.nih.gov/pub/dbVar/sandbox/sv_datasets/nonredundant/release_notes
@@ -109,8 +133,13 @@
 - url: https://ftp.ncbi.nlm.nih.gov/pub/dbVar/sandbox/sv_datasets/nonredundant/deletions/GRCh37.nr_deletions.tsv.gz
 - url: https://ftp.ncbi.nlm.nih.gov/pub/dbVar/sandbox/sv_datasets/nonredundant/duplications/GRCh37.nr_duplications.tsv.gz
 - url: https://ftp.ncbi.nlm.nih.gov/pub/dbVar/sandbox/sv_datasets/nonredundant/insertions/GRCh37.nr_insertions.tsv.gz
+- url: https://ftp.ncbi.nlm.nih.gov/pub/dbVar/sandbox/sv_datasets/nonredundant/deletions/GRCh38.nr_deletions.tsv.gz
+- url: https://ftp.ncbi.nlm.nih.gov/pub/dbVar/sandbox/sv_datasets/nonredundant/duplications/GRCh38.nr_duplications.tsv.gz
+- url: https://ftp.ncbi.nlm.nih.gov/pub/dbVar/sandbox/sv_datasets/nonredundant/insertions/GRCh38.nr_insertions.tsv.gz
 - url: http://dgv.tcag.ca/dgv/docs/GRCh37_hg19_variants_2020-02-25.txt
+- url: http://dgv.tcag.ca/dgv/docs/GRCh38_hg38_variants_2020-02-25.txt
 - url: http://dgv.tcag.ca/dgv/docs/DGV.GS.March2016.50percent.GainLossSep.Final.hg19.gff3
+- url: http://dgv.tcag.ca/dgv/docs/DGV.GS.hg38.gff3
 - url: ftp://ftp.broadinstitute.org/pub/ExAC_release/release0.3.1/cnv/exac-final.autosome-1pct-sq60-qc-prot-coding.cnv.bed
 - url: https://ftp-trace.ncbi.nih.gov/1000genomes/ftp/phase3/integrated_sv_map/ALL.wgs.integrated_sv_map_v2.20130502.svs.genotypes.vcf.gz
 - url: https://storage.googleapis.com/gcp-public-data--gnomad/papers/2019-sv/gnomad_v2.1_sv.sites.vcf.gz

excerpt-data/17f0d5f9c4671d95/hg19.TADs.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:80255d58ff142d5fdb2376e2af242af9ec61a8c46c792cdf32255f753e2ac683
+size 435707

excerpt-data/17f0d5f9c4671d95/url.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f0926751798217d37145c4902acfda3faebac9a3545605e6751ad6d36609903a
+size 61

excerpt-data/1a5f86b027a18653/IMR90_domains_hg19.bed

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f9e1b1fd2f685f294ba08eb29d3b8ab3f3173601345e68feb574d09a42354cb1
+size 80454

excerpt-data/1a5f86b027a18653/url.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ba7faa26ba4ff7d478408256659502ae661f6ec80207c967dcd5a9b263dea85b
+size 176

excerpt-data/3a9b37b0518ee213/GCF_000001405.40_GRCh38.p14_assembly_report.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8a92ff0a04acf8b1e29273a79c10929c623a35e31d81868c2371d44de19dd47c
+size 38785

excerpt-data/3a9b37b0518ee213/url.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:41054fa686d5102a56d0145b5caf89ec020add570a7b0d9cbd940392135e328d
+size 64