add to_parquet rule, env, doc

Author: deborahzani

Dockerfile

@@ -84,7 +84,7 @@ COPY workflow/envs/filtering.yaml /conda-envs/31fc19a9498faffb09aa18f9246db95e/e
 
 # Conda environment:
 #   source: workflow/envs/gwaspipe.yaml
-#   prefix: /conda-envs/724b79486d0cc8b8ba5722413b3608b2
+#   prefix: /conda-envs/b8ce59d968b60f65eb89656a355281b5
 #   name: gwaspipe
 #   channels:
 #     - conda-forge
@@ -94,9 +94,9 @@ COPY workflow/envs/filtering.yaml /conda-envs/31fc19a9498faffb09aa18f9246db95e/e
 #     - pip==24
 #     - mscorefonts
 #     - pip:
-#         - git+https://github.com/ht-diva/gwaspipe.git@aee66f1
-RUN mkdir -p /conda-envs/724b79486d0cc8b8ba5722413b3608b2
-COPY workflow/envs/gwaspipe.yaml /conda-envs/724b79486d0cc8b8ba5722413b3608b2/environment.yaml
+#         - git+https://github.com/ht-diva/gwaspipe.git@712944e
+RUN mkdir -p /conda-envs/b8ce59d968b60f65eb89656a355281b5
+COPY workflow/envs/gwaspipe.yaml /conda-envs/b8ce59d968b60f65eb89656a355281b5/environment.yaml
 
 # Conda environment:
 #   source: workflow/envs/liftover_bcftools.yaml
@@ -119,6 +119,6 @@ RUN mamba env create --prefix /conda-envs/6e056d31662ab0bd2fd3fba49416042f --fil
     mamba env create --prefix /conda-envs/20b7f0f77b859d9ac85875e0e8e2c471 --file /conda-envs/20b7f0f77b859d9ac85875e0e8e2c471/environment.yaml && \
     mamba env create --prefix /conda-envs/1952d8a40f9d550db08b42e8de561992 --file /conda-envs/1952d8a40f9d550db08b42e8de561992/environment.yaml && \
     mamba env create --prefix /conda-envs/31fc19a9498faffb09aa18f9246db95e --file /conda-envs/31fc19a9498faffb09aa18f9246db95e/environment.yaml && \
-    mamba env create --prefix /conda-envs/724b79486d0cc8b8ba5722413b3608b2 --file /conda-envs/724b79486d0cc8b8ba5722413b3608b2/environment.yaml && \
+    mamba env create --prefix /conda-envs/b8ce59d968b60f65eb89656a355281b5 --file /conda-envs/b8ce59d968b60f65eb89656a355281b5/environment.yaml && \
     mamba env create --prefix /conda-envs/bb7d3ca556579c4e816225676dfd5175 --file /conda-envs/bb7d3ca556579c4e816225676dfd5175/environment.yaml && \
     mamba clean --all -y

README.md

@@ -25,13 +25,15 @@ run:
   gwascatalog: False
   summarize: True
   delivery: False
+  to_parquet: False
  ```
 
 * _harmonization_: If set to True, the harmonization rules will be executed.
 * _liftoverbcf_harmonization_: If set to False, the harmonization rules will not be executed after BCFtools liftover.
 * _gwascatalog_: If set to False, the summary statistics will not be downloaded by GWAS Catalog.
 * _summarize_: If set to True, the summarization rules will be executed.
 * _delivery_: If set to False, the delivery rules will not be executed.
+* _to_parquet_: If set to True, the harmonized summary statistics will be available in parquet format, along with .tsv.gz.
 
 **Paths**
 
@@ -167,6 +169,10 @@ Examples of configuration files for *BELIEVE*, *Decode*, *FinnGen*, *Genes & Hea
 *Purpose*: Creates a region-based index (CHR and POS columns) of GWAS harmonized data for fast queries.<br />
 *Output*: *{sumstat_id}.gwaslab.tsv.gz.tbi*: Index of GWAS harmonized data.<br />
 
+* **to_parquet** (`to_parquet: True`): <br />
+*Purpose:*  Convert harmonized summary statistics from *.tsv.gz* to *parquet* format.<br />
+*Output*: *{sumstat_id}.gwaslab.parquet*: Standardized and aligned GWAS summary statistics in *parquet* format.<br />
+
 * **create_snp_mapping_table** (`harmonization: True`): <br />
 *Purpose:* Creates mapping table to match input data and harmonized summary statistics.<br />
 *Output*: *table.snp_mapping.tsv.gz*: Table that links input SNPID (and rsID when available) to harmonized SNPID. Note that in case `gwascatalog: True`, SNPID mapping is generated for each summary statistics.<br />

config/config.yaml

@@ -1,23 +1,17 @@
-# Rule execution flags
 run:
-  harmonization: False
-  liftoverbcf_harmonization: False
-  gwascatalog: True
+  harmonization: True
   summarize: True
   delivery: True
-  pre_filtering_and_harmonization: True
+  to_parquet: True
 
 
 # Paths
-sumstats_path: config/sumstats_from_gwascatalog.tsv
-sumstats_url: config/urls_gwascatalog_test.tsv
-sumstats_suffix: ".gwas.regenie.gz" # Suffix of input summary statistics files (check it in the sumstats_path list)
+sumstats_path: config/local.txt
+sumstats_suffix: ".gwas.regenie.gz" # Suffix of input summary statistics files (check in sumstats_path)c
+dest_path: "/project/cdh/interval/result"
+workspace_path: ../results
 sumstats_sep: "\t"
-dest_path: "../destination"
-workspace_path: "../results"
-hg37_fasta_file_path: '/group/diangelantonio/public_data/liftOver/human_g1k_v37.fasta'
-hg38_fasta_file_path: '/group/diangelantonio/public_data/liftOver/hg38.fa'
-chain_file_path: '/group/diangelantonio/public_data/liftOver/hg19ToHg38.over.chain.gz'
+
 
 # Common parameters
 input_format: &iformat "regenie" # Input format of summary
@@ -26,16 +20,9 @@ params:
   harmonize_sumstats:
     input_format: *iformat
     config_file: "config/config_harmonize_sumstats.yml"
-  snp_mapping:
-    input_format: *iformat
-    config_file: "config/config_snp_mapping.yml"
   summarize_sumstats:
     input_format: "gwaslab"
     config_file: "config/config_summarize_sumstats.yml"
-  to_vcf:
+  snp_mapping:
     input_format: *iformat
-    config_file: "config/config_to_vcf.yaml"
-  harmonize_sumstats_pre_filtering:
-    config_file: "config/config_harmonize_sumstats_pre_filtering.yml"
-  harmonize_sumstats_post_filtering:
-    config_file: "config/config_harmonize_sumstats_post_filtering.yml"
+    config_file: "config/config_snp_mapping.yml"

config/config_harmonize_sumstats.yml

@@ -29,7 +29,7 @@ steps:
     params:
       run: True
     gl_params:
-      to_fill: ['MLOG10P', 'Z' ]
+      to_fill: ['MLOG10P', 'Z', 'CHISQ', 'EAF']
       overwrite: False
       extreme: True
   harmonize: # see https://cloufield.github.io/gwaslab/Harmonization/
@@ -50,14 +50,15 @@ steps:
       workspace_subfolder: True
     gl_params:
       fmt: "gwaslab"
+      xymt_number: True
       to_csvargs:
         compression:
           method: "gzip"
           compresslevel: 1
           mtime: 1
 
 
-# Filename transformation, e.g.: seq.3007.7.gwas.regenie.gz
+# Filename transformation, e.g.: seq.3007.7.liftover.vcf.gz
 filename_mask: [ True, True, True, False, False, False]
 filename_sep: '.'
 

config/config_harmonize_sumstats_post_filtering.yml

@@ -50,7 +50,7 @@ steps:
     gl_params:
       fmt: "snp_mapping"
 
-# Filename transformation, e.g.: seq.3007.7.gwas.regenie.gz
+# Filename transformation, e.g.: seq.3007.7.liftover.vcf.gz
 filename_mask: [ True, True, True, False, False, False]
 filename_sep: '.'
 

config/config_harmonize_sumstats_pre_filtering.yml

@@ -47,7 +47,7 @@ steps:
       sig_level: 5e-8
 
 
-# Filename transformation, e.g.: seq.3007.7.gwas.regenie.gz
+# Filename transformation, e.g.: seq.3007.7.gwaslab.tsv.gz
 filename_mask: [ True, True, True, False, False, False]
 filename_sep: '.'