From 01b784977ceaca2cd478a80832a3a8a9aba3393a Mon Sep 17 00:00:00 2001 From: fgvieira <1151762+fgvieira@users.noreply.github.com> Date: Tue, 7 May 2024 16:23:28 +0200 Subject: [PATCH 1/9] Fix issue #366 --- bio/vep/cache/meta.yaml | 7 +++++++ bio/vep/cache/wrapper.py | 5 ++--- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/bio/vep/cache/meta.yaml b/bio/vep/cache/meta.yaml index a743fdd138d..aca17f92770 100644 --- a/bio/vep/cache/meta.yaml +++ b/bio/vep/cache/meta.yaml @@ -3,3 +3,10 @@ description: Download VEP cache for given species, build and release. url: http://www.ensembl.org/info/docs/tools/vep/index.html authors: - Johannes Köster +output: + - directory to store the VEP cache +params: + - url: URL from where to download cache data (optional; by default: `ftp://ftp.ensembl.org/pub`) + - species: species to download cache data + - build: build to download cache data + - release: release to download cache data diff --git a/bio/vep/cache/wrapper.py b/bio/vep/cache/wrapper.py index 543f1b26169..f85078b0d7e 100644 --- a/bio/vep/cache/wrapper.py +++ b/bio/vep/cache/wrapper.py @@ -23,10 +23,9 @@ f"{snakemake.params.species}_vep_{release}_{snakemake.params.build}.tar.gz" ) log = snakemake.log_fmt_shell(stdout=True, stderr=True) + url = snakemake.params.get("url", "ftp://ftp.ensembl.org/pub") shell( - "curl -L ftp://ftp.ensembl.org/pub/release-{snakemake.params.release}/" - "variation/{vep_dir}/{cache_tarball} " - "-o {tmpdir}/{cache_tarball} {log}" + "curl -L {url}/release-{snakemake.params.release}/variation/{vep_dir}/{cache_tarball} -o {tmpdir}/{cache_tarball} {log}" ) log = snakemake.log_fmt_shell(stdout=True, stderr=True, append=True) From e9d410074110af73d9200617fc4fa41a73557301 Mon Sep 17 00:00:00 2001 From: fgvieira <1151762+fgvieira@users.noreply.github.com> Date: Tue, 7 May 2024 16:27:17 +0200 Subject: [PATCH 2/9] Fix URL in docs --- bio/vep/cache/meta.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bio/vep/cache/meta.yaml b/bio/vep/cache/meta.yaml index aca17f92770..fc37726ab50 100644 --- a/bio/vep/cache/meta.yaml +++ b/bio/vep/cache/meta.yaml @@ -6,7 +6,7 @@ authors: output: - directory to store the VEP cache params: - - url: URL from where to download cache data (optional; by default: `ftp://ftp.ensembl.org/pub`) + - url: URL from where to download cache data (optional; by default from `Ensembl FTP `_) - species: species to download cache data - build: build to download cache data - release: release to download cache data From 5fe45d03d3530204e58bc536a3020c22af23d111 Mon Sep 17 00:00:00 2001 From: fgvieira <1151762+fgvieira@users.noreply.github.com> Date: Tue, 7 May 2024 16:53:42 +0200 Subject: [PATCH 3/9] Add test with custum URL --- bio/vep/cache/test/Snakefile | 15 +++++++++++++++ test.py | 5 +++++ 2 files changed, 20 insertions(+) diff --git a/bio/vep/cache/test/Snakefile b/bio/vep/cache/test/Snakefile index 4f6285893c1..81ae08f7ebd 100644 --- a/bio/vep/cache/test/Snakefile +++ b/bio/vep/cache/test/Snakefile @@ -10,3 +10,18 @@ rule get_vep_cache: cache: "omit-software" # save space and time with between workflow caching (see docs) wrapper: "master/bio/vep/cache" + + +rule get_vep_cache_ebi: + output: + directory("resources/vep/cache_ebi"), + params: + url="ftp://ftp.ebi.ac.uk/ensemblgenomes/pub/plants/current", + species="arabidopsis_thaliana", + build="TAIR10", + release="104", + log: + "logs/vep/cache_ebi.log", + cache: "omit-software" # save space and time with between workflow caching (see docs) + wrapper: + "master/bio/vep/cache" diff --git a/test.py b/test.py index e4b7709e6c0..1810fed4e42 100644 --- a/test.py +++ b/test.py @@ -5915,6 +5915,11 @@ def test_vep_cache(): ["snakemake", "--cores", "1", "resources/vep/cache", "--use-conda", "-F"], ) + run( + "bio/vep/cache", + ["snakemake", "--cores", "1", "resources/vep/cache_ebi", "--use-conda", "-F"], + ) + @skip_if_not_modified def test_vep_plugins(): From 74a53aad138ad62873436787cb287842b2452de5 Mon Sep 17 00:00:00 2001 From: fgvieira <1151762+fgvieira@users.noreply.github.com> Date: Tue, 7 May 2024 19:51:57 +0200 Subject: [PATCH 4/9] Fix URL --- bio/vep/cache/meta.yaml | 2 +- bio/vep/cache/wrapper.py | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/bio/vep/cache/meta.yaml b/bio/vep/cache/meta.yaml index fc37726ab50..e784df9a94f 100644 --- a/bio/vep/cache/meta.yaml +++ b/bio/vep/cache/meta.yaml @@ -6,7 +6,7 @@ authors: output: - directory to store the VEP cache params: - - url: URL from where to download cache data (optional; by default from `Ensembl FTP `_) + - url: URL from where to download cache data (optional; by default is ``ftp://ftp.ensembl.org/pub/release-{release}``) - species: species to download cache data - build: build to download cache data - release: release to download cache data diff --git a/bio/vep/cache/wrapper.py b/bio/vep/cache/wrapper.py index f85078b0d7e..f28b832736f 100644 --- a/bio/vep/cache/wrapper.py +++ b/bio/vep/cache/wrapper.py @@ -23,9 +23,12 @@ f"{snakemake.params.species}_vep_{release}_{snakemake.params.build}.tar.gz" ) log = snakemake.log_fmt_shell(stdout=True, stderr=True) - url = snakemake.params.get("url", "ftp://ftp.ensembl.org/pub") + cache_url = snakemake.params.get( + "url", "ftp://ftp.ensembl.org/pub/release-{release}" + ) + cache_url = eval(f"f'{cache_url}'") shell( - "curl -L {url}/release-{snakemake.params.release}/variation/{vep_dir}/{cache_tarball} -o {tmpdir}/{cache_tarball} {log}" + "curl -L {cache_url}/variation/{vep_dir}/{cache_tarball} -o {tmpdir}/{cache_tarball} {log}" ) log = snakemake.log_fmt_shell(stdout=True, stderr=True, append=True) From 3c02803e96ac1fde84e6e9b23ad38339eb5c8cb1 Mon Sep 17 00:00:00 2001 From: fgvieira <1151762+fgvieira@users.noreply.github.com> Date: Tue, 7 May 2024 20:16:40 +0200 Subject: [PATCH 5/9] Fix URL issues --- bio/vep/cache/test/Snakefile | 8 ++++---- bio/vep/cache/wrapper.py | 13 ++++++------- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/bio/vep/cache/test/Snakefile b/bio/vep/cache/test/Snakefile index 81ae08f7ebd..c2666c9547b 100644 --- a/bio/vep/cache/test/Snakefile +++ b/bio/vep/cache/test/Snakefile @@ -16,10 +16,10 @@ rule get_vep_cache_ebi: output: directory("resources/vep/cache_ebi"), params: - url="ftp://ftp.ebi.ac.uk/ensemblgenomes/pub/plants/current", - species="arabidopsis_thaliana", - build="TAIR10", - release="104", + url="ftp://ftp.ebi.ac.uk/ensemblgenomes/pub/plants", + species="cyanidioschyzon_merolae", + build="ASM9120v1", + release="58", log: "logs/vep/cache_ebi.log", cache: "omit-software" # save space and time with between workflow caching (see docs) diff --git a/bio/vep/cache/wrapper.py b/bio/vep/cache/wrapper.py index f28b832736f..291d8161210 100644 --- a/bio/vep/cache/wrapper.py +++ b/bio/vep/cache/wrapper.py @@ -9,26 +9,25 @@ extra = snakemake.params.get("extra", "") +log = snakemake.log_fmt_shell(stdout=True, stderr=True) + try: release = int(snakemake.params.release) except ValueError: raise ValueError("The parameter release is supposed to be an integer.") + with tempfile.TemporaryDirectory() as tmpdir: # We download the cache tarball manually because vep_install does not consider proxy settings (in contrast to curl). # See https://github.com/bcbio/bcbio-nextgen/issues/1080 - vep_dir = "vep" if release >= 97 else "VEP" + cache_url = snakemake.params.get("url", "ftp://ftp.ensembl.org/pub") cache_tarball = ( f"{snakemake.params.species}_vep_{release}_{snakemake.params.build}.tar.gz" ) - log = snakemake.log_fmt_shell(stdout=True, stderr=True) - cache_url = snakemake.params.get( - "url", "ftp://ftp.ensembl.org/pub/release-{release}" - ) - cache_url = eval(f"f'{cache_url}'") + vep_dir = "vep" if snakemake.params.get("url") or release >= 97 else "VEP" shell( - "curl -L {cache_url}/variation/{vep_dir}/{cache_tarball} -o {tmpdir}/{cache_tarball} {log}" + "curl -L {cache_url}/release-{release}/variation/{vep_dir}/{cache_tarball} -o {tmpdir}/{cache_tarball} {log}" ) log = snakemake.log_fmt_shell(stdout=True, stderr=True, append=True) From 552eca7c937eea7402d16aaa98ecaafb6a7276f7 Mon Sep 17 00:00:00 2001 From: "Filipe G. Vieira" <1151762+fgvieira@users.noreply.github.com> Date: Mon, 13 May 2024 12:27:29 +0200 Subject: [PATCH 6/9] Fix default URL --- bio/vep/cache/meta.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bio/vep/cache/meta.yaml b/bio/vep/cache/meta.yaml index e784df9a94f..c2638655906 100644 --- a/bio/vep/cache/meta.yaml +++ b/bio/vep/cache/meta.yaml @@ -6,7 +6,7 @@ authors: output: - directory to store the VEP cache params: - - url: URL from where to download cache data (optional; by default is ``ftp://ftp.ensembl.org/pub/release-{release}``) + - url: URL from where to download cache data (optional; by default is ``ftp://ftp.ensembl.org/pub/``) - species: species to download cache data - build: build to download cache data - release: release to download cache data From e33a597af398fd42e66e44f5b78cd6723c2b2d0f Mon Sep 17 00:00:00 2001 From: fgvieira <1151762+fgvieira@users.noreply.github.com> Date: Mon, 13 May 2024 12:41:21 +0200 Subject: [PATCH 7/9] Allow for custom URL on Ensembl reference data --- bio/reference/ensembl-annotation/meta.yaml | 2 ++ bio/reference/ensembl-annotation/wrapper.py | 13 ++----------- bio/reference/ensembl-sequence/meta.yaml | 4 ++++ bio/reference/ensembl-sequence/wrapper.py | 3 ++- bio/reference/ensembl-variation/meta.yaml | 4 ++++ bio/reference/ensembl-variation/wrapper.py | 10 +++------- bio/vep/cache/meta.yaml | 2 +- 7 files changed, 18 insertions(+), 20 deletions(-) diff --git a/bio/reference/ensembl-annotation/meta.yaml b/bio/reference/ensembl-annotation/meta.yaml index be0a0bb69be..b8fd0924a79 100644 --- a/bio/reference/ensembl-annotation/meta.yaml +++ b/bio/reference/ensembl-annotation/meta.yaml @@ -4,3 +4,5 @@ authors: - Johannes Köster output: - Ensemble GTF or GFF3 anotation file +params: + - url: URL from where to download cache data (optional; by default is ``ftp://ftp.ensembl.org/pub``) diff --git a/bio/reference/ensembl-annotation/wrapper.py b/bio/reference/ensembl-annotation/wrapper.py index 2f1b78c2a6a..a59ec46051e 100644 --- a/bio/reference/ensembl-annotation/wrapper.py +++ b/bio/reference/ensembl-annotation/wrapper.py @@ -48,17 +48,8 @@ ) -url = "ftp://ftp.ensembl.org/pub/{branch}release-{release}/{out_fmt}/{species}/{species_cap}.{build}.{gtf_release}.{flavor}{suffix}".format( - release=release, - gtf_release=gtf_release, - build=build, - species=species, - out_fmt=out_fmt, - species_cap=species.capitalize(), - suffix=suffix, - flavor=flavor, - branch=branch, -) +url = snakemake.params.get("url", "ftp://ftp.ensembl.org/pub") +url = f"{url}/{branch}release-{release}/{out_fmt}/{species}/{species_cap}.{build}.{gtf_release}.{flavor}{suffix}" try: diff --git a/bio/reference/ensembl-sequence/meta.yaml b/bio/reference/ensembl-sequence/meta.yaml index 189912a20e5..20c769a0d1d 100644 --- a/bio/reference/ensembl-sequence/meta.yaml +++ b/bio/reference/ensembl-sequence/meta.yaml @@ -2,3 +2,7 @@ name: ensembl-sequence description: Download sequences (e.g. genome) from ENSEMBL FTP servers, and store them in a single .fasta file. authors: - Johannes Köster +output: + - fasta file +params: + - url: URL from where to download cache data (optional; by default is ``ftp://ftp.ensembl.org/pub``) diff --git a/bio/reference/ensembl-sequence/wrapper.py b/bio/reference/ensembl-sequence/wrapper.py index df9a6eef693..cb2956a6c04 100644 --- a/bio/reference/ensembl-sequence/wrapper.py +++ b/bio/reference/ensembl-sequence/wrapper.py @@ -50,8 +50,9 @@ "invalid datatype, to select a single chromosome the datatype must be dna" ) +url = snakemake.params.get("url", "ftp://ftp.ensembl.org/pub") spec = spec.format(build=build, release=release) -url_prefix = f"ftp://ftp.ensembl.org/pub/{branch}release-{release}/fasta/{species}/{datatype}/{species.capitalize()}.{spec}" +url_prefix = f"{url}/{branch}release-{release}/fasta/{species}/{datatype}/{species.capitalize()}.{spec}" success = False for suffix in suffixes: diff --git a/bio/reference/ensembl-variation/meta.yaml b/bio/reference/ensembl-variation/meta.yaml index b562872809c..3f1a261ae4b 100644 --- a/bio/reference/ensembl-variation/meta.yaml +++ b/bio/reference/ensembl-variation/meta.yaml @@ -2,3 +2,7 @@ name: ensembl-variation description: Download known genomic variants from ENSEMBL FTP servers, and store them in a single .vcf.gz file. authors: - Johannes Köster +output: + - VCF file +params: + - url: URL from where to download cache data (optional; by default is ``ftp://ftp.ensembl.org/pub``) diff --git a/bio/reference/ensembl-variation/wrapper.py b/bio/reference/ensembl-variation/wrapper.py index ee179c51c1d..4e21292886a 100644 --- a/bio/reference/ensembl-variation/wrapper.py +++ b/bio/reference/ensembl-variation/wrapper.py @@ -62,16 +62,12 @@ species_filename = species if release >= 91 else species.capitalize() +url = snakemake.params.get("url", "ftp://ftp.ensembl.org/pub") urls = [ - "ftp://ftp.ensembl.org/pub/{branch}release-{release}/variation/vcf/{species}/{species_filename}{suffix}.vcf.gz".format( - release=release, - species=species, - suffix=suffix, - species_filename=species_filename, - branch=branch, - ) + f"{url}/{branch}release-{release}/variation/vcf/{species}/{species_filename}{suffix}.vcf.gz" for suffix in suffixes ] + names = [os.path.basename(url) for url in urls] try: diff --git a/bio/vep/cache/meta.yaml b/bio/vep/cache/meta.yaml index c2638655906..220cec2c1f6 100644 --- a/bio/vep/cache/meta.yaml +++ b/bio/vep/cache/meta.yaml @@ -6,7 +6,7 @@ authors: output: - directory to store the VEP cache params: - - url: URL from where to download cache data (optional; by default is ``ftp://ftp.ensembl.org/pub/``) + - url: URL from where to download cache data (optional; by default is ``ftp://ftp.ensembl.org/pub``) - species: species to download cache data - build: build to download cache data - release: release to download cache data From 578a7b568038153f8ed3e5191e22634d9ac53b52 Mon Sep 17 00:00:00 2001 From: fgvieira <1151762+fgvieira@users.noreply.github.com> Date: Mon, 13 May 2024 12:49:19 +0200 Subject: [PATCH 8/9] Fix species capitalize --- bio/reference/ensembl-annotation/wrapper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bio/reference/ensembl-annotation/wrapper.py b/bio/reference/ensembl-annotation/wrapper.py index a59ec46051e..c3d655cbb25 100644 --- a/bio/reference/ensembl-annotation/wrapper.py +++ b/bio/reference/ensembl-annotation/wrapper.py @@ -49,7 +49,7 @@ url = snakemake.params.get("url", "ftp://ftp.ensembl.org/pub") -url = f"{url}/{branch}release-{release}/{out_fmt}/{species}/{species_cap}.{build}.{gtf_release}.{flavor}{suffix}" +url = f"{url}/{branch}release-{release}/{out_fmt}/{species}/{species.capitalize()}.{build}.{gtf_release}.{flavor}{suffix}" try: From dcb5a5991a3591a1efca9e2ba01c270dbca33b1d Mon Sep 17 00:00:00 2001 From: fgvieira <1151762+fgvieira@users.noreply.github.com> Date: Mon, 13 May 2024 12:51:28 +0200 Subject: [PATCH 9/9] Add test using a different protocol --- bio/reference/ensembl-annotation/test/Snakefile | 2 ++ bio/reference/ensembl-sequence/test/Snakefile | 2 ++ bio/reference/ensembl-variation/test/Snakefile | 2 ++ 3 files changed, 6 insertions(+) diff --git a/bio/reference/ensembl-annotation/test/Snakefile b/bio/reference/ensembl-annotation/test/Snakefile index fed87b2ee64..3a30ca70bde 100644 --- a/bio/reference/ensembl-annotation/test/Snakefile +++ b/bio/reference/ensembl-annotation/test/Snakefile @@ -25,6 +25,8 @@ rule get_annotation_gz: # branch="plants", # optional: specify branch log: "logs/get_annotation.log", + params: + url="http://ftp.ensembl.org/pub", cache: "omit-software" # save space and time with between workflow caching (see docs) wrapper: "master/bio/reference/ensembl-annotation" diff --git a/bio/reference/ensembl-sequence/test/Snakefile b/bio/reference/ensembl-sequence/test/Snakefile index a8227f4cdd1..fec1c746a4c 100644 --- a/bio/reference/ensembl-sequence/test/Snakefile +++ b/bio/reference/ensembl-sequence/test/Snakefile @@ -25,6 +25,8 @@ rule get_single_chromosome: # branch="plants", # optional: specify branch log: "logs/get_genome.log", + params: + url="http://ftp.ensembl.org/pub", cache: "omit-software" # save space and time with between workflow caching (see docs) wrapper: "master/bio/reference/ensembl-sequence" diff --git a/bio/reference/ensembl-variation/test/Snakefile b/bio/reference/ensembl-variation/test/Snakefile index 27594273e57..9189f7de194 100644 --- a/bio/reference/ensembl-variation/test/Snakefile +++ b/bio/reference/ensembl-variation/test/Snakefile @@ -12,6 +12,8 @@ rule get_variation: type="all", # one of "all", "somatic", "structural_variation" # chromosome="21", # optionally constrain to chromosome, only supported for homo_sapiens # branch="plants", # optional: specify branch + params: + url="http://ftp.ensembl.org/pub", log: "logs/get_variation.log", cache: "omit-software" # save space and time with between workflow caching (see docs)