From c724de7fa120b5196dad6bf07e729f3399f9d902 Mon Sep 17 00:00:00 2001 From: gorloffslava Date: Mon, 29 Jul 2024 21:14:14 +0500 Subject: [PATCH 1/6] + Added NUMCODECS_USE_SYSTEM_LIBS capability to build numcodecs against system libraries --- pyproject.toml | 1 + setup.py | 156 +++++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 151 insertions(+), 6 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index af8f040a..910ca0d0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,6 +5,7 @@ requires = [ "Cython", "py-cpuinfo", "numpy", + "pkgconfig" ] build-backend = "setuptools.build_meta" diff --git a/setup.py b/setup.py index e594a203..7e7bf409 100644 --- a/setup.py +++ b/setup.py @@ -8,6 +8,7 @@ from setuptools.errors import CCompilerError, ExecError, PlatformError from distutils import ccompiler from distutils.command.clean import clean +import pkgconfig # determine CPU support for SSE2 and AVX2 cpu_info = cpuinfo.get_cpu_info() @@ -16,6 +17,7 @@ have_avx2 = 'avx2' in flags disable_sse2 = 'DISABLE_NUMCODECS_SSE2' in os.environ disable_avx2 = 'DISABLE_NUMCODECS_AVX2' in os.environ +use_system_libraries = 'NUMCODECS_USE_SYSTEM_LIBS' in os.environ # setup common compile arguments have_cflags = 'CFLAGS' in os.environ @@ -48,8 +50,8 @@ def error(*msg): print('[numcodecs]', *msg, **kwargs) -def blosc_extension(): - info('setting up Blosc extension') +def _blosc_extension_with_vendored_libs(): + info('setting up Blosc extension from vendored sources') extra_compile_args = base_compile_args.copy() define_macros = [] @@ -124,8 +126,73 @@ def blosc_extension(): return extensions -def zstd_extension(): - info('setting up Zstandard extension') +def _blosc_extension_with_system_libs(): + info('setting up Blosc extension with system libraries') + + extra_compile_args = base_compile_args.copy() + + blosc_package_configuration = pkgconfig.parse("blosc") + + define_macros = blosc_package_configuration["define_macros"] + include_dirs = blosc_package_configuration["include_dirs"] + libraries = blosc_package_configuration["libraries"] + library_dirs = blosc_package_configuration["library_dirs"] + + # remove minizip because Python.h 3.8 tries to include crypt.h + include_dirs = [d for d in include_dirs if 'minizip' not in d] + define_macros += [ + ('HAVE_LZ4', 1), + # ('HAVE_SNAPPY', 1), + ('HAVE_ZLIB', 1), + ('HAVE_ZSTD', 1), + ] + # define_macros += [('CYTHON_TRACE', '1')] + + # SSE2 + if have_sse2 and not disable_sse2: + info('compiling Blosc extension with SSE2 support') + extra_compile_args.append('-DSHUFFLE_SSE2_ENABLED') + if os.name == 'nt': + define_macros += [('__SSE2__', 1)] + else: + info('compiling Blosc extension without SSE2 support') + + # AVX2 + if have_avx2 and not disable_avx2: + info('compiling Blosc extension with AVX2 support') + extra_compile_args.append('-DSHUFFLE_AVX2_ENABLED') + if os.name == 'nt': + define_macros += [('__AVX2__', 1)] + else: + info('compiling Blosc extension without AVX2 support') + + sources = ['numcodecs/blosc.pyx'] + + # define extension module + extensions = [ + Extension( + 'numcodecs.blosc', + sources=sources, + include_dirs=include_dirs, + define_macros=define_macros, + extra_compile_args=extra_compile_args, + libraries=libraries, + library_dirs=library_dirs, + ), + ] + + return extensions + + +def blosc_extension(): + if use_system_libraries: + return _blosc_extension_with_system_libs() + else: + return _blosc_extension_with_vendored_libs() + + +def _zstd_extension_with_vendored_sources(): + info('setting up Zstandard extension from vendored sources') zstd_sources = [] extra_compile_args = base_compile_args.copy() @@ -166,8 +233,46 @@ def zstd_extension(): return extensions -def lz4_extension(): - info('setting up LZ4 extension') +def _zstd_extension_with_system_libs(): + info('setting up Zstandard extension with system libraries') + + extra_compile_args = base_compile_args.copy() + + zstd_package_configuration = pkgconfig.parse("libzstd") + include_dirs = zstd_package_configuration["include_dirs"] + define_macros = zstd_package_configuration["define_macros"] + libraries = zstd_package_configuration["libraries"] + library_dirs = zstd_package_configuration["library_dirs"] + + # define_macros += [('CYTHON_TRACE', '1')] + + sources = ['numcodecs/zstd.pyx'] + + # define extension module + extensions = [ + Extension( + 'numcodecs.zstd', + sources=sources, + include_dirs=include_dirs, + define_macros=define_macros, + extra_compile_args=extra_compile_args, + libraries=libraries, + library_dirs=library_dirs, + ), + ] + + return extensions + + +def zstd_extension(): + if use_system_libraries: + return _zstd_extension_with_system_libs() + else: + return _zstd_extension_with_vendored_sources() + + +def _lz4_extension_with_vendored_sources(): + info('setting up LZ4 extension from vendored sources') extra_compile_args = base_compile_args.copy() define_macros = [] @@ -194,6 +299,45 @@ def lz4_extension(): return extensions +def _lz4_extension_with_system_libs(): + info('setting up LZ4 extension with system libraries') + + extra_compile_args = base_compile_args.copy() + + lz4_package_configuration = pkgconfig.parse("liblz4") + include_dirs = lz4_package_configuration["include_dirs"] + define_macros = lz4_package_configuration["define_macros"] + libraries = lz4_package_configuration["libraries"] + library_dirs = lz4_package_configuration["library_dirs"] + + include_dirs += ['numcodecs'] + # define_macros += [('CYTHON_TRACE', '1')] + + sources = ['numcodecs/lz4.pyx'] + + # define extension module + extensions = [ + Extension( + 'numcodecs.lz4', + sources=sources, + include_dirs=include_dirs, + define_macros=define_macros, + extra_compile_args=extra_compile_args, + libraries=libraries, + library_dirs=library_dirs, + ), + ] + + return extensions + + +def lz4_extension(): + if use_system_libraries: + return _lz4_extension_with_system_libs() + else: + return _lz4_extension_with_vendored_sources() + + def vlen_extension(): info('setting up vlen extension') import numpy From f925fdb3b79faa9bf0e9ca087b84785ef6568aa4 Mon Sep 17 00:00:00 2001 From: Benjamin Leff Date: Tue, 6 Aug 2024 12:53:48 -0400 Subject: [PATCH 2/6] docs(deps): added readme system libs instructions --- README.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/README.rst b/README.rst index c9e58f52..89244ad1 100644 --- a/README.rst +++ b/README.rst @@ -18,3 +18,12 @@ codecs for use in data storage and communication applications. .. image:: https://codecov.io/gh/zarr-developers/numcodecs/branch/main/graph/badge.svg :target: https://codecov.io/gh/zarr-developers/numcodecs + +--- +If you already have native Blosc, Zstd, and LZ4 installed on your system and want to use these system libraries instead of the vendored sources, you +should set the `NUMCODECS_USE_SYSTEM_LIBS=1` environment variable when building the wheel, like this: + + $ NUMCODECS_USE_SYSTEM_LIBS=1 pip install numcodecs --no-binary numcodecs + +Blosc, Zstd, and LZ4 are found via the `pkg-config` utility. Moreover, you must build all 3 `blosc`, `libzstd`, and `liblz4` +components. C-Blosc comes with full sources for LZ4, LZ4HC, Snappy, Zlib and Zstd and in general, you should not worry about not having (or CMake not finding) the libraries in your system because by default the included sources will be automatically compiled and included in the C-Blosc library. This effectively means that you can be confident in having a complete support for all the codecs in all the Blosc deployments (unless you are explicitly excluding support for some of them). To compile blosc, see these [instructions](https://github.com/Blosc/c-blosc?tab=readme-ov-file#compiling-the-blosc-library). \ No newline at end of file From 45fdddb1b818a4e796b4e83bafc8e2d92f54360b Mon Sep 17 00:00:00 2001 From: Benjamin Leff Date: Fri, 23 Aug 2024 13:05:49 -0400 Subject: [PATCH 3/6] Cleaned define_macros from blosc system --- setup.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/setup.py b/setup.py index 7e7bf409..e5a0b985 100644 --- a/setup.py +++ b/setup.py @@ -140,12 +140,7 @@ def _blosc_extension_with_system_libs(): # remove minizip because Python.h 3.8 tries to include crypt.h include_dirs = [d for d in include_dirs if 'minizip' not in d] - define_macros += [ - ('HAVE_LZ4', 1), - # ('HAVE_SNAPPY', 1), - ('HAVE_ZLIB', 1), - ('HAVE_ZSTD', 1), - ] + # define_macros += [('CYTHON_TRACE', '1')] # SSE2 From 32ec6abac4176f9c2a816ed18e622f64d663df6f Mon Sep 17 00:00:00 2001 From: gorloffslava Date: Wed, 16 Oct 2024 10:56:45 +0500 Subject: [PATCH 4/6] + Merged upstream --- setup.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index ee95c0bc..58abd563 100644 --- a/setup.py +++ b/setup.py @@ -507,7 +507,10 @@ def run_setup(with_extensions): + jenkins_extension() ) - cmdclass = dict(build_ext=ve_build_ext, clean=Sclean) + if use_system_libraries: + cmdclass = dict(build_ext=build_ext, clean=clean) + else: + cmdclass = dict(build_ext=ve_build_ext, clean=Sclean) else: ext_modules = [] cmdclass = {} From 08e91f274f8aedf975d61b956ff11820a1c42b4d Mon Sep 17 00:00:00 2001 From: gorloffslava Date: Mon, 4 Nov 2024 14:57:10 +0500 Subject: [PATCH 5/6] + Merged upstream --- pyproject.toml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index c85df8af..0f77cb21 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ requires = [ "setuptools-scm[toml]>=6.2", "Cython", "py-cpuinfo", - "numpy", + "numpy>2", "pkgconfig" ] build-backend = "setuptools.build_meta" @@ -16,9 +16,9 @@ A Python package providing buffer compression and transformation codecs \ for use in data storage and communication applications.""" readme = "README.rst" dependencies = [ - "numpy>=1.7", + "numpy>=1.23", ] -requires-python = ">=3.10" +requires-python = ">=3.11" dynamic = [ "version", ] @@ -69,7 +69,7 @@ zfpy = [ "numpy<2.0.0", ] pcodec = [ - "pcodec>=0.2.0", + "pcodec>=0.2,<0.3", ] crc32c = [ "crc32c>=2.7", From 130e0e7d9035b19a7d4f51de41b4c6a93fdb3bf6 Mon Sep 17 00:00:00 2001 From: Josh Moore Date: Tue, 25 Feb 2025 09:16:52 +0100 Subject: [PATCH 6/6] pre-commit fixes --- setup.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/setup.py b/setup.py index 58abd563..429c7166 100644 --- a/setup.py +++ b/setup.py @@ -1,15 +1,15 @@ import os import sys +from distutils import ccompiler +from distutils.command.clean import clean +from distutils.sysconfig import customize_compiler from glob import glob import cpuinfo +import pkgconfig from Cython.Distutils.build_ext import new_build_ext as build_ext from setuptools import Extension, setup from setuptools.errors import CCompilerError, ExecError, PlatformError -from distutils import ccompiler -from distutils.command.clean import clean -from distutils.sysconfig import customize_compiler -import pkgconfig # determine CPU support for SSE2 and AVX2 cpu_info = cpuinfo.get_cpu_info() @@ -472,14 +472,14 @@ def run(self): build_ext.run(self) except PlatformError as e: error(e) - raise BuildFailed() + raise BuildFailed def build_extension(self, ext): try: build_ext.build_extension(self, ext) except ext_errors as e: error(e) - raise BuildFailed() + raise BuildFailed class Sclean(clean):