From d8d281c62dd67b585d345984f7ac5834f390e9e2 Mon Sep 17 00:00:00 2001 From: SantiDu Date: Sat, 23 May 2026 11:58:20 +0200 Subject: [PATCH 01/19] substitute bgen for bgen-reader --- software/conda_env.yaml | 2 +- software/metax/genotype/BGENGenotype.py | 23 ++++++++++------------- 2 files changed, 11 insertions(+), 14 deletions(-) diff --git a/software/conda_env.yaml b/software/conda_env.yaml index c0add1c..3257fa6 100644 --- a/software/conda_env.yaml +++ b/software/conda_env.yaml @@ -16,5 +16,5 @@ dependencies: - pip=25.0 - pip: - cyvcf2==0.30.28 - - bgen-reader==4.0.8 + - bgen==1.9.9 diff --git a/software/metax/genotype/BGENGenotype.py b/software/metax/genotype/BGENGenotype.py index abc625f..f85923d 100644 --- a/software/metax/genotype/BGENGenotype.py +++ b/software/metax/genotype/BGENGenotype.py @@ -1,25 +1,24 @@ -import bgen_reader import numpy import pandas import logging +from bgen import BgenReader from ..misc import Genomics def bgen_file_geno_lines(file, variant_mapping = None, force_colon = False, use_rsid=False, whitelist=None, skip_palindromic=False, liftover_conversion=None): logging.log(9, "Processing bgen %s", file) - bgen = bgen_reader.read_bgen(file) - variants = bgen["variants"] + bfile = BgenReader(file) dict_mapping = variant_mapping is not None and type(variant_mapping) == dict - for variant in variants.itertuples(): + for variant in bfile: if use_rsid: varid = variant.rsid else: - varid = variant.id + varid = variant.varid if force_colon: varid = varid.replace("_", ":") - alleles = variant.allele_ids.split(",") + alleles = variant.alleles if len(alleles) > 2: logging.info("variant %s is multiallelic, skipping", varid) continue @@ -51,11 +50,10 @@ def bgen_file_geno_lines(file, variant_mapping = None, force_colon = False, use_ # the alleles in the genotype might be swapped respect the variant in the mapping # You should verify if you must match it - v = bgen["genotype"][variant.Index].compute() - if v["phased"]: - d = numpy.apply_along_axis(lambda x: x[1] + x[3], 1, numpy.array(v["probs"], dtype=float)) + if variant.is_phased: + d = numpy.apply_along_axis(lambda x: x[1] + x[3], 1, variant.probabilities) else: - d = numpy.apply_along_axis(lambda x: x[1] + x[2] * 2, 1, numpy.array(v["probs"], dtype=float)) + d = numpy.apply_along_axis(lambda x: x[1] + x[2] * 2, 1, variant.probabilities) #e = bgen_reader.allele_expectation(bgen, variant.Index) #d2 = bgen_reader.compute_dosage(e, alt=1) @@ -70,7 +68,6 @@ def bgen_files_geno_lines(files, variant_mapping = None, force_colon = False, us def get_samples(path): logging.info("Opening bgen to get samples") - bgen = bgen_reader.read_bgen(path, verbose=False) - samples = bgen["samples"].values - samples = pandas.DataFrame({"FID":samples, "IID":samples})[["FID", "IID"]] + bfile = BgenReader(path) + samples = pandas.DataFrame({"FID":bfile.samples, "IID":bfile.samples})[["FID", "IID"]] return samples From cad228c5e4e465665ffa0611963a04c5806c39db Mon Sep 17 00:00:00 2001 From: SantiDu Date: Sat, 23 May 2026 13:36:08 +0200 Subject: [PATCH 02/19] bgen and fix numpy compatibility issue --- software/conda_env.yaml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/software/conda_env.yaml b/software/conda_env.yaml index 3257fa6..a0aa12b 100644 --- a/software/conda_env.yaml +++ b/software/conda_env.yaml @@ -7,8 +7,6 @@ channels: dependencies: - python=3.9 - pandas=2.2 - - scipy=1.12 - - numpy=1.26 - pyliftover=0.4 - statsmodels=0.14 - h5py=3.12 @@ -17,4 +15,5 @@ dependencies: - pip: - cyvcf2==0.30.28 - bgen==1.9.9 - + - scipy==1.12 + - numpy==1.26 From bcbb38e9f274ef3ac208b5a5734b3b20fc831e55 Mon Sep 17 00:00:00 2001 From: Jinrui Du <37691563+SantiDu@users.noreply.github.com> Date: Tue, 26 May 2026 21:02:56 +0200 Subject: [PATCH 03/19] Update bgen package version to 1.9.10 --- software/conda_env.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/software/conda_env.yaml b/software/conda_env.yaml index a0aa12b..539ab3e 100644 --- a/software/conda_env.yaml +++ b/software/conda_env.yaml @@ -14,6 +14,6 @@ dependencies: - pip=25.0 - pip: - cyvcf2==0.30.28 - - bgen==1.9.9 + - bgen==1.9.10 - scipy==1.12 - numpy==1.26 From e3fc1b53bd75b73b7de7ea87551f69bd24c2943a Mon Sep 17 00:00:00 2001 From: Jinrui Du <37691563+SantiDu@users.noreply.github.com> Date: Thu, 4 Jun 2026 21:54:53 +0200 Subject: [PATCH 04/19] Change bgen_reader requirement to bgen Updated bgen_reader to bgen in the README. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 5f66d2f..d165c70 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,7 @@ To run PrediXcan Associations and MulTiXcan, you also need: * [h5py-cache (>=1.0.0)](https://pypi.python.org/pypi/h5py-cache/1.0) *Now folded into h5py To run prediction of biological mechanisms on individual-level data, you will also need: -* [bgen_reader (>=3.0.3)](https://pypi.org/project/bgen-reader/) +* [bgen (>=1.9.10)](https://pypi.org/project/bgen/) * [cyvcf2 (>=0.8.0)](https://brentp.github.io/cyvcf2) [R](https://www.r-project.org/) with [ggplot](http://ggplot2.org/) and [dplyr](https://cran.r-project.org/web/packages/dplyr/index.html) From cd2701256621f83e18684076f31e9109f9b45ca7 Mon Sep 17 00:00:00 2001 From: Jinrui Du <37691563+SantiDu@users.noreply.github.com> Date: Thu, 4 Jun 2026 21:55:45 +0200 Subject: [PATCH 05/19] Update bgen_reader dependency to bgen version 1.9.10 in setup.py --- software/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/software/setup.py b/software/setup.py index 1236974..ab46e30 100644 --- a/software/setup.py +++ b/software/setup.py @@ -30,7 +30,7 @@ def read(fname): 'SMulTiXcan.py'], description=["TBD"], install_requires=['scipy>=1.2.2', 'numpy>=1.14.2', 'pandas>=0.22.0', 'patsy>=0.5.0', - 'statsmodels>=0.10.0', 'h5py>=2.7.1', 'h5py-cache>=1.0', 'bgen_reader>=3.0.3', 'cyvcf2>=0.8.0'], + 'statsmodels>=0.10.0', 'h5py>=2.7.1', 'h5py-cache>=1.0', 'bgen>=1.9.10', 'cyvcf2>=0.8.0'], extras_require={"test": ["sqlalchemy"]}, long_description=read('Readme.md'), keywords=['TBD'], From ad2db755c60247df5498ffa724d50cf62aabc9a0 Mon Sep 17 00:00:00 2001 From: Jinrui Du <37691563+SantiDu@users.noreply.github.com> Date: Thu, 4 Jun 2026 22:19:06 +0200 Subject: [PATCH 06/19] Migrate exception handling python 2 syntax to 3 in MainScreen.py --- software/metax/MainScreen.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/software/metax/MainScreen.py b/software/metax/MainScreen.py index 940deed..0cad09c 100644 --- a/software/metax/MainScreen.py +++ b/software/metax/MainScreen.py @@ -349,7 +349,7 @@ def __init__(self,args): def run(self): try: MetaXcan.run(self.args) - except Exceptions.ReportableException, e: + except Exceptions.ReportableException as e: logging.error(e.msg) except Exception as e: logging.info("Exception when running task: %s", str(e)) @@ -368,7 +368,7 @@ def run(self): work = self.works[i] work.run() del self.works[i] - except Exceptions.ReportableException, e: + except Exceptions.ReportableException as e: logging.error(e.msg) except Exception as e: logging.info("Exception when running task: %s", str(e)) @@ -376,4 +376,4 @@ def run(self): pass work = MetaxcanWorkWrapper(beta_args) - return work \ No newline at end of file + return work From e5c8bcdddc5d7f6e543b2fea71c05871291d9999 Mon Sep 17 00:00:00 2001 From: Jinrui Du <37691563+SantiDu@users.noreply.github.com> Date: Fri, 5 Jun 2026 10:01:55 +0100 Subject: [PATCH 07/19] Bump version from 0.7.5 to 0.8.2 --- software/metax/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/software/metax/__init__.py b/software/metax/__init__.py index e1f1a74..185cae0 100644 --- a/software/metax/__init__.py +++ b/software/metax/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.7.5" +__version__ = "0.8.2" def exitIf(doExit, Exception, msg): if doExit: From 53e961482978c439fb3ea370cb0519482280de6b Mon Sep 17 00:00:00 2001 From: Jinrui Du <37691563+SantiDu@users.noreply.github.com> Date: Fri, 5 Jun 2026 15:46:33 +0100 Subject: [PATCH 08/19] Add pyliftover to install_requires in setup.py --- software/setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/software/setup.py b/software/setup.py index ab46e30..0e99bca 100644 --- a/software/setup.py +++ b/software/setup.py @@ -30,7 +30,8 @@ def read(fname): 'SMulTiXcan.py'], description=["TBD"], install_requires=['scipy>=1.2.2', 'numpy>=1.14.2', 'pandas>=0.22.0', 'patsy>=0.5.0', - 'statsmodels>=0.10.0', 'h5py>=2.7.1', 'h5py-cache>=1.0', 'bgen>=1.9.10', 'cyvcf2>=0.8.0'], + 'statsmodels>=0.10.0', 'h5py>=2.7.1', 'pyliftover>=0.4', + 'bgen>=1.9.10', 'cyvcf2>=0.8.0'], extras_require={"test": ["sqlalchemy"]}, long_description=read('Readme.md'), keywords=['TBD'], From bd0a8d35acad0a24e244f6daf4e427176ccc8885 Mon Sep 17 00:00:00 2001 From: Jinrui Du <37691563+SantiDu@users.noreply.github.com> Date: Fri, 5 Jun 2026 15:53:23 +0100 Subject: [PATCH 09/19] Remove pyarrow from conda_env.yaml Removed pyarrow version 19.0 from conda environment. --- software/conda_env.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/software/conda_env.yaml b/software/conda_env.yaml index 539ab3e..3f43ea6 100644 --- a/software/conda_env.yaml +++ b/software/conda_env.yaml @@ -10,7 +10,6 @@ dependencies: - pyliftover=0.4 - statsmodels=0.14 - h5py=3.12 - - pyarrow=19.0 - pip=25.0 - pip: - cyvcf2==0.30.28 From 12dbe9663972e091db93cf6bfc07bc500138b493 Mon Sep 17 00:00:00 2001 From: Jinrui Du <37691563+SantiDu@users.noreply.github.com> Date: Fri, 5 Jun 2026 16:07:32 +0100 Subject: [PATCH 10/19] Replace h5py_cache with h5py for file handling --- software/ToHDF5.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/software/ToHDF5.py b/software/ToHDF5.py index ae52485..8a0aa74 100755 --- a/software/ToHDF5.py +++ b/software/ToHDF5.py @@ -4,7 +4,7 @@ from metax import Logging, Utilities import numpy as np -import h5py_cache +import h5py import pandas def run(args): @@ -18,7 +18,7 @@ def run(args): data = pandas.read_table(args.input) logging.info("Opening output") - f = h5py_cache.File(args.output, 'w', chunk_cache_mem_size=int(50 * (1024 ** 2))) + f = h5py.File(args.output, 'w', rdcc_nbytes=int(50 * (1024 ** 2))) n_genes = data.shape[1]-2 n_samples = data.shape[0] @@ -58,4 +58,4 @@ def run(args): args = parser.parse_args() Logging.configureLogging(args.verbosity) - run(args) \ No newline at end of file + run(args) From 634f4809725153ca93399d38cd6d3b1bbf112a4b Mon Sep 17 00:00:00 2001 From: Jinrui Du <37691563+SantiDu@users.noreply.github.com> Date: Sat, 6 Jun 2026 19:43:39 +0200 Subject: [PATCH 11/19] Update package dependencies and Python version requirement --- software/setup.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/software/setup.py b/software/setup.py index 0e99bca..a6747ee 100644 --- a/software/setup.py +++ b/software/setup.py @@ -29,15 +29,15 @@ def read(fname): 'MulTiXcan.py', 'SMulTiXcan.py'], description=["TBD"], - install_requires=['scipy>=1.2.2', 'numpy>=1.14.2', 'pandas>=0.22.0', 'patsy>=0.5.0', - 'statsmodels>=0.10.0', 'h5py>=2.7.1', 'pyliftover>=0.4', - 'bgen>=1.9.10', 'cyvcf2>=0.8.0'], + install_requires=['scipy==1.2.2', 'numpy==1.26.*', 'pandas=2.2.*', 'patsy>=0.5.0', + 'statsmodels==0.14.*', 'h5py==3.12', 'pyliftover==0.4', + 'bgen==1.9.10', 'cyvcf2==0.30.28'], extras_require={"test": ["sqlalchemy"]}, long_description=read('Readme.md'), keywords=['TBD'], test_suite='tests', package_data={'tests/files':['*']}, - python_requires='>=3.5', + python_requires='>=3.9', classifiers=[ "Development Status :: 4 - Beta", "Topic :: Utilities", From aa20263c8f6ce961cb3939048c8b3d165b89c0e2 Mon Sep 17 00:00:00 2001 From: Jinrui Du <37691563+SantiDu@users.noreply.github.com> Date: Sat, 6 Jun 2026 19:46:07 +0200 Subject: [PATCH 12/19] Fix pandas version specification in setup.py --- software/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/software/setup.py b/software/setup.py index a6747ee..ac22d3a 100644 --- a/software/setup.py +++ b/software/setup.py @@ -29,7 +29,7 @@ def read(fname): 'MulTiXcan.py', 'SMulTiXcan.py'], description=["TBD"], - install_requires=['scipy==1.2.2', 'numpy==1.26.*', 'pandas=2.2.*', 'patsy>=0.5.0', + install_requires=['scipy==1.2.2', 'numpy==1.26.*', 'pandas==2.2.*', 'patsy==0.5.0', 'statsmodels==0.14.*', 'h5py==3.12', 'pyliftover==0.4', 'bgen==1.9.10', 'cyvcf2==0.30.28'], extras_require={"test": ["sqlalchemy"]}, From 783330c1f396c17128947c0aa9eedcf6e7955f7b Mon Sep 17 00:00:00 2001 From: Jinrui Du <37691563+SantiDu@users.noreply.github.com> Date: Sat, 6 Jun 2026 19:47:38 +0200 Subject: [PATCH 13/19] Update setup.py --- software/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/software/setup.py b/software/setup.py index ac22d3a..c230564 100644 --- a/software/setup.py +++ b/software/setup.py @@ -29,7 +29,7 @@ def read(fname): 'MulTiXcan.py', 'SMulTiXcan.py'], description=["TBD"], - install_requires=['scipy==1.2.2', 'numpy==1.26.*', 'pandas==2.2.*', 'patsy==0.5.0', + install_requires=['scipy==1.2.2', 'numpy==1.26.*', 'pandas==2.2.*', 'patsy==0.5.6', 'statsmodels==0.14.*', 'h5py==3.12', 'pyliftover==0.4', 'bgen==1.9.10', 'cyvcf2==0.30.28'], extras_require={"test": ["sqlalchemy"]}, From 8899c05a58edc69b4c63a7a8c5ef16f3328adb83 Mon Sep 17 00:00:00 2001 From: Jinrui Du <37691563+SantiDu@users.noreply.github.com> Date: Sat, 6 Jun 2026 19:49:03 +0200 Subject: [PATCH 14/19] Update scipy version in install_requires --- software/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/software/setup.py b/software/setup.py index c230564..c6c30ba 100644 --- a/software/setup.py +++ b/software/setup.py @@ -29,7 +29,7 @@ def read(fname): 'MulTiXcan.py', 'SMulTiXcan.py'], description=["TBD"], - install_requires=['scipy==1.2.2', 'numpy==1.26.*', 'pandas==2.2.*', 'patsy==0.5.6', + install_requires=['scipy==1.9.2', 'numpy==1.26.*', 'pandas==2.2.*', 'patsy==0.5.6', 'statsmodels==0.14.*', 'h5py==3.12', 'pyliftover==0.4', 'bgen==1.9.10', 'cyvcf2==0.30.28'], extras_require={"test": ["sqlalchemy"]}, From 63f2a39f7331635328fcf9814748509c99f50d3a Mon Sep 17 00:00:00 2001 From: Jinrui Du <37691563+SantiDu@users.noreply.github.com> Date: Sat, 6 Jun 2026 19:50:40 +0200 Subject: [PATCH 15/19] Update numpy version in install_requires --- software/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/software/setup.py b/software/setup.py index c6c30ba..ce3f74a 100644 --- a/software/setup.py +++ b/software/setup.py @@ -29,7 +29,7 @@ def read(fname): 'MulTiXcan.py', 'SMulTiXcan.py'], description=["TBD"], - install_requires=['scipy==1.9.2', 'numpy==1.26.*', 'pandas==2.2.*', 'patsy==0.5.6', + install_requires=['scipy==1.9.2', 'numpy==1.25.*', 'pandas==2.2.*', 'patsy==0.5.6', 'statsmodels==0.14.*', 'h5py==3.12', 'pyliftover==0.4', 'bgen==1.9.10', 'cyvcf2==0.30.28'], extras_require={"test": ["sqlalchemy"]}, From 77bed2da729dfe423ce2f19b3eed0c57c5a66c01 Mon Sep 17 00:00:00 2001 From: Jinrui Du <37691563+SantiDu@users.noreply.github.com> Date: Sat, 6 Jun 2026 19:51:46 +0200 Subject: [PATCH 16/19] Update scipy version in setup.py --- software/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/software/setup.py b/software/setup.py index ce3f74a..5836d87 100644 --- a/software/setup.py +++ b/software/setup.py @@ -29,7 +29,7 @@ def read(fname): 'MulTiXcan.py', 'SMulTiXcan.py'], description=["TBD"], - install_requires=['scipy==1.9.2', 'numpy==1.25.*', 'pandas==2.2.*', 'patsy==0.5.6', + install_requires=['scipy==1.8.*', 'numpy==1.25.*', 'pandas==2.2.*', 'patsy==0.5.6', 'statsmodels==0.14.*', 'h5py==3.12', 'pyliftover==0.4', 'bgen==1.9.10', 'cyvcf2==0.30.28'], extras_require={"test": ["sqlalchemy"]}, From 832a4de0362eaac680aec49fd45a03b01a2a291d Mon Sep 17 00:00:00 2001 From: Jinrui Du <37691563+SantiDu@users.noreply.github.com> Date: Sat, 6 Jun 2026 20:03:38 +0200 Subject: [PATCH 17/19] Update install_requires for package dependencies --- software/setup.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/software/setup.py b/software/setup.py index 5836d87..4ec34c9 100644 --- a/software/setup.py +++ b/software/setup.py @@ -29,9 +29,16 @@ def read(fname): 'MulTiXcan.py', 'SMulTiXcan.py'], description=["TBD"], - install_requires=['scipy==1.8.*', 'numpy==1.25.*', 'pandas==2.2.*', 'patsy==0.5.6', - 'statsmodels==0.14.*', 'h5py==3.12', 'pyliftover==0.4', - 'bgen==1.9.10', 'cyvcf2==0.30.28'], + install_requires=[python==3.9.*, + numpy==1.26.4, + scipy==1.11.4, + pandas==2.2.3, + patsy==0.5.6, + statsmodels==0.14.4, + h5py==3.11.0, + pyliftover==0.4, + bgen==1.9.10, + cyvcf2==0.30.28], extras_require={"test": ["sqlalchemy"]}, long_description=read('Readme.md'), keywords=['TBD'], From 2be1db3644e47d97bbb864fec2d5c50e70e47a4e Mon Sep 17 00:00:00 2001 From: Jinrui Du <37691563+SantiDu@users.noreply.github.com> Date: Sat, 6 Jun 2026 20:07:39 +0200 Subject: [PATCH 18/19] Update install_requires and python_requires in setup.py --- software/setup.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/software/setup.py b/software/setup.py index 4ec34c9..1647d4d 100644 --- a/software/setup.py +++ b/software/setup.py @@ -29,16 +29,18 @@ def read(fname): 'MulTiXcan.py', 'SMulTiXcan.py'], description=["TBD"], - install_requires=[python==3.9.*, - numpy==1.26.4, - scipy==1.11.4, - pandas==2.2.3, - patsy==0.5.6, - statsmodels==0.14.4, - h5py==3.11.0, - pyliftover==0.4, - bgen==1.9.10, - cyvcf2==0.30.28], + install_requires=[ + 'numpy>=1.23,<2', + 'scipy>=1.10,<2', + 'pandas>=2.0,<3', + 'patsy>=0.5.6', + 'statsmodels>=0.14,<0.15', + 'h5py>=3.10,<4', + 'pyliftover>=0.4', + 'bgen==1.9.10', + 'cyvcf2>=0.30', + ], + python_requires='>=3.9, <3.10', extras_require={"test": ["sqlalchemy"]}, long_description=read('Readme.md'), keywords=['TBD'], From a3f34e28765745a3647ec97a9ac9489d6b855f82 Mon Sep 17 00:00:00 2001 From: Jinrui Du <37691563+SantiDu@users.noreply.github.com> Date: Sat, 6 Jun 2026 20:08:52 +0200 Subject: [PATCH 19/19] Update setup.py --- software/setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/software/setup.py b/software/setup.py index 1647d4d..13e70d5 100644 --- a/software/setup.py +++ b/software/setup.py @@ -40,7 +40,6 @@ def read(fname): 'bgen==1.9.10', 'cyvcf2>=0.30', ], - python_requires='>=3.9, <3.10', extras_require={"test": ["sqlalchemy"]}, long_description=read('Readme.md'), keywords=['TBD'],