diff --git a/CHANGELOG.md b/CHANGELOG.md index 1e7441beab..213bfc46ac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -104,6 +104,14 @@ END_UNRELEASED_TEMPLATE * (toolchains) use "command -v" to find interpreter in `$PATH` ([#3150](https://github.com/bazel-contrib/rules_python/pull/3150)). * (pypi) `bazel vendor` now works in `bzlmod` ({gh-issue}`3079`). +* (toolchains) `local_runtime_repo` Improvements in handling variations across python installations + for Linux, Windows and Mac. See ([#3148](https://github.com/bazel-contrib/rules_python/pull/3148)). +* (toolchains) `local_runtime_repo` now works on Windows + ([#3055](https://github.com/bazel-contrib/rules_python/issues/3055)). +* (toolchains) `local_runtime_repo` supports more types of Python + installations (Mac frameworks, missing dynamic libraries, and other + esoteric cases, see + [#3148](https://github.com/bazel-contrib/rules_python/pull/3148) for details). {#v0-0-0-added} ### Added diff --git a/examples/local_python/.bazelrc b/examples/local_python/.bazelrc new file mode 100644 index 0000000000..2ed86f591e --- /dev/null +++ b/examples/local_python/.bazelrc @@ -0,0 +1,4 @@ +# The equivalent bzlmod behavior is covered by examples/bzlmod/py_proto_library +common --noenable_bzlmod +common --enable_workspace +common --incompatible_python_disallow_native_rules diff --git a/examples/local_python/.gitignore b/examples/local_python/.gitignore new file mode 100644 index 0000000000..e5ae073b3c --- /dev/null +++ b/examples/local_python/.gitignore @@ -0,0 +1,4 @@ +# git ignore patterns + +/bazel-* +user.bazelrc diff --git a/examples/local_python/BUILD.bazel b/examples/local_python/BUILD.bazel new file mode 100644 index 0000000000..1a6d5cbe8a --- /dev/null +++ b/examples/local_python/BUILD.bazel @@ -0,0 +1,6 @@ +load("@rules_python//python:py_binary.bzl", "py_binary") + +py_binary( + name = "main", + srcs = ["main.py"], +) diff --git a/examples/local_python/WORKSPACE b/examples/local_python/WORKSPACE new file mode 100644 index 0000000000..f39bf14b86 --- /dev/null +++ b/examples/local_python/WORKSPACE @@ -0,0 +1,31 @@ +workspace( + name = "local_python_example", +) + +local_repository( + name = "rules_python", + path = "../..", +) + +load("@rules_python//python:repositories.bzl", "py_repositories") + +py_repositories() + +load("@rules_python//python/local_toolchains:repos.bzl", "local_runtime_repo", "local_runtime_toolchains_repo") + +# Step 1: Define the python runtime. +local_runtime_repo( + name = "local_python3", + interpreter_path = "python3", + on_failure = "fail", + # or interpreter_path = "C:\\path\\to\\python.exe" +) + +# Step 2: Create toolchains for the runtimes +local_runtime_toolchains_repo( + name = "local_toolchains", + runtimes = ["local_python3"], +) + +# Step 3: Register the toolchains +register_toolchains("@local_toolchains//:all") diff --git a/examples/local_python/main.py b/examples/local_python/main.py new file mode 100644 index 0000000000..c3e261dadd --- /dev/null +++ b/examples/local_python/main.py @@ -0,0 +1,21 @@ +# Copyright 2025 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def main(): + print(42) + + +if __name__ == "__main__": + main() diff --git a/python/private/get_local_runtime_info.py b/python/private/get_local_runtime_info.py index c8371357c2..ff3b0aeb01 100644 --- a/python/private/get_local_runtime_info.py +++ b/python/private/get_local_runtime_info.py @@ -12,47 +12,188 @@ # See the License for the specific language governing permissions and # limitations under the License. +"""Returns information about the local Python runtime as JSON.""" + import json +import os import sys import sysconfig -data = { - "major": sys.version_info.major, - "minor": sys.version_info.minor, - "micro": sys.version_info.micro, - "include": sysconfig.get_path("include"), - "implementation_name": sys.implementation.name, - "base_executable": sys._base_executable, -} +_IS_WINDOWS = sys.platform == "win32" +_IS_DARWIN = sys.platform == "darwin" + -config_vars = [ - # The libpythonX.Y.so file. Usually? - # It might be a static archive (.a) file instead. - "LDLIBRARY", - # The directory with library files. Supposedly. - # It's not entirely clear how to get the directory with libraries. +def _search_directories(get_config): + """Returns a list of library directories to search for shared libraries.""" # There's several types of libraries with different names and a plethora - # of settings. + # of settings, and many different config variables to check: + # + # LIBPL is used in python-config when shared library is not enabled: + # https://github.com/python/cpython/blob/v3.12.0/Misc/python-config.in#L63 + # + # LIBDIR may also be the python directory with library files. # https://stackoverflow.com/questions/47423246/get-pythons-lib-path - # For now, it seems LIBDIR has what is needed, so just use that. # See also: MULTIARCH - "LIBDIR", + # + # On MacOS, the LDLIBRARY may be a relative path under /Library/Frameworks, + # such as "Python.framework/Versions/3.12/Python", not a file under the + # LIBDIR/LIBPL directory, so include PYTHONFRAMEWORKPREFIX. + lib_dirs = [get_config(x) for x in ("PYTHONFRAMEWORKPREFIX", "LIBPL", "LIBDIR")] + # On Debian, with multiarch enabled, prior to Python 3.10, `LIBDIR` didn't # tell the location of the libs, just the base directory. The `MULTIARCH` # sysconfig variable tells the subdirectory within it with the libs. # See: # https://wiki.debian.org/Python/MultiArch # https://git.launchpad.net/ubuntu/+source/python3.12/tree/debian/changelog#n842 - "MULTIARCH", - # The versioned libpythonX.Y.so.N file. Usually? - # It might be a static archive (.a) file instead. - "INSTSONAME", - # The libpythonX.so file. Usually? - # It might be a static archive (a.) file instead. - "PY3LIBRARY", - # The platform-specific filename suffix for library files. - # Includes the dot, e.g. `.so` - "SHLIB_SUFFIX", -] -data.update(zip(config_vars, sysconfig.get_config_vars(*config_vars))) + multiarch = get_config("MULTIARCH") + if multiarch: + for x in ("LIBPL", "LIBDIR"): + config_value = get_config(x) + if config_value and not config_value.endswith(multiarch): + lib_dirs.append(os.path.join(config_value, multiarch)) + + if _IS_WINDOWS: + # On Windows DLLs go in the same directory as the executable, while .lib + # files live in the lib/ or libs/ subdirectory. + lib_dirs.append(get_config("BINDIR")) + lib_dirs.append(os.path.join(os.path.dirname(sys.executable))) + lib_dirs.append(os.path.join(os.path.dirname(sys.executable), "lib")) + lib_dirs.append(os.path.join(os.path.dirname(sys.executable), "libs")) + elif not _IS_DARWIN: + # On most systems the executable is in a bin/ directory and the libraries + # are in a sibling lib/ directory. + lib_dirs.append( + os.path.join(os.path.dirname(os.path.dirname(sys.executable)), "lib") + ) + + # Dedup and remove empty values, keeping the order. + lib_dirs = [v for v in lib_dirs if v] + return {k: None for k in lib_dirs}.keys() + + +def _search_library_names(get_config): + """Returns a list of library files to search for shared libraries.""" + # Quoting configure.ac in the cpython code base: + # "INSTSONAME is the name of the shared library that will be use to install + # on the system - some systems like version suffix, others don't."" + # + # A typical INSTSONAME is 'libpython3.8.so.1.0' on Linux, or + # 'Python.framework/Versions/3.9/Python' on MacOS. + # + # A typical LDLIBRARY is 'libpythonX.Y.so' on Linux, or 'pythonXY.dll' on + # Windows, or 'Python.framework/Versions/3.9/Python' on MacOS. + # + # A typical LIBRARY is 'libpythonX.Y.a' on Linux. + lib_names = [ + get_config(x) + for x in ( + "LDLIBRARY", + "INSTSONAME", + "PY3LIBRARY", + "LIBRARY", + "DLLLIBRARY", + ) + ] + + # Set the prefix and suffix to construct the library name used for linking. + # The suffix and version are set here to the default values for the OS, + # since they are used below to construct "default" library names. + if _IS_DARWIN: + suffix = ".dylib" + prefix = "lib" + elif _IS_WINDOWS: + suffix = ".dll" + prefix = "" + else: + suffix = get_config("SHLIB_SUFFIX") + prefix = "lib" + if not suffix: + suffix = ".so" + + version = get_config("VERSION") + + # Ensure that the pythonXY.dll files are included in the search. + lib_names.append(f"{prefix}python{version}{suffix}") + + # If there are ABIFLAGS, also add them to the python version lib search. + abiflags = get_config("ABIFLAGS") or get_config("abiflags") or "" + if abiflags: + lib_names.append(f"{prefix}python{version}{abiflags}{suffix}") + + # Dedup and remove empty values, keeping the order. + lib_names = [v for v in lib_names if v] + return {k: None for k in lib_names}.keys() + + +def _get_python_library_info(): + """Returns a dictionary with the static and dynamic python libraries.""" + config_vars = sysconfig.get_config_vars() + + # VERSION is X.Y in Linux/macOS and XY in Windows. This is used to + # construct library paths such as python3.12, so ensure it exists. + if not config_vars.get("VERSION"): + if sys.platform == "win32": + config_vars["VERSION"] = f"{sys.version_info.major}{sys.version_info.minor}" + else: + config_vars["VERSION"] = ( + f"{sys.version_info.major}.{sys.version_info.minor}" + ) + + search_directories = _search_directories(config_vars.get) + search_libnames = _search_library_names(config_vars.get) + + def _add_if_exists(target, path): + if os.path.exists(path) or os.path.isdir(path): + target[path] = None + + interface_libraries = {} + dynamic_libraries = {} + static_libraries = {} + for root_dir in search_directories: + for libname in search_libnames: + composed_path = os.path.join(root_dir, libname) + if libname.endswith(".a"): + _add_if_exists(static_libraries, composed_path) + continue + + _add_if_exists(dynamic_libraries, composed_path) + if libname.endswith(".dll"): + # On windows a .lib file may be an "import library" or a static library. + # The file could be inspected to determine which it is; typically python + # is used as a shared library. + # + # On Windows, extensions should link with the pythonXY.lib interface + # libraries. + # + # See: https://docs.python.org/3/extending/windows.html + # https://learn.microsoft.com/en-us/windows/win32/dlls/dynamic-link-library-creation + _add_if_exists( + interface_libraries, os.path.join(root_dir, libname[:-3] + "lib") + ) + elif libname.endswith(".so"): + # It's possible, though unlikely, that interface stubs (.ifso) exist. + _add_if_exists( + interface_libraries, os.path.join(root_dir, libname[:-2] + "ifso") + ) + + # When no libraries are found it's likely that the python interpreter is not + # configured to use shared or static libraries (minilinux). If this seems + # suspicious try running `uv tool run find_libpython --list-all -v` + return { + "dynamic_libraries": list(dynamic_libraries.keys()), + "static_libraries": list(static_libraries.keys()), + "interface_libraries": list(interface_libraries.keys()), + } + + +data = { + "major": sys.version_info.major, + "minor": sys.version_info.minor, + "micro": sys.version_info.micro, + "include": sysconfig.get_path("include"), + "implementation_name": sys.implementation.name, + "base_executable": sys._base_executable, +} +data.update(_get_python_library_info()) print(json.dumps(data)) diff --git a/python/private/local_runtime_repo.bzl b/python/private/local_runtime_repo.bzl index b8b7164b54..311fedc6d7 100644 --- a/python/private/local_runtime_repo.bzl +++ b/python/private/local_runtime_repo.bzl @@ -31,27 +31,64 @@ load("@rules_python//python/private:local_runtime_repo_setup.bzl", "define_local define_local_runtime_toolchain_impl( name = "local_runtime", - lib_ext = "{lib_ext}", major = "{major}", minor = "{minor}", micro = "{micro}", interpreter_path = "{interpreter_path}", + interface_library = {interface_library}, + shared_library = {shared_library}, implementation_name = "{implementation_name}", os = "{os}", ) """ +def _norm_path(path): + """Returns a path using '/' separators and no trailing slash.""" + path = path.replace("\\", "/") + if path[-1] == "/": + path = path[:-1] + return path + +def _symlink_first_library(rctx, logger, libraries): + """Symlinks the shared libraries into the lib/ directory. + + Args: + rctx: A repository_ctx object + logger: A repo_utils.logger object + libraries: A list of static library paths to potentially symlink. + Returns: + A single library path linked by the action. + """ + linked = None + for target in libraries: + origin = rctx.path(target) + if not origin.exists: + # The reported names don't always exist; it depends on the particulars + # of the runtime installation. + continue + linked = "lib/{}".format(origin.basename) + logger.debug("Symlinking {} to {}".format(origin, linked)) + repo_utils.watch(rctx, origin) + rctx.symlink(origin, linked) + break + + return linked + def _local_runtime_repo_impl(rctx): logger = repo_utils.logger(rctx) on_failure = rctx.attr.on_failure - result = _resolve_interpreter_path(rctx) - if not result.resolved_path: + def _emit_log(msg): if on_failure == "fail": - fail("interpreter not found: {}".format(result.describe_failure())) + logger.fail(msg) + elif on_failure == "warn": + logger.warn(msg) + else: + logger.debug(msg) - if on_failure == "warn": - logger.warn(lambda: "interpreter not found: {}".format(result.describe_failure())) + result = _resolve_interpreter_path(rctx) + if not result.resolved_path: + _emit_log(lambda: "interpreter not found: {}".format(result.describe_failure())) # else, on_failure must be skip rctx.file("BUILD.bazel", _expand_incompatible_template()) @@ -72,10 +109,7 @@ def _local_runtime_repo_impl(rctx): logger = logger, ) if exec_result.return_code != 0: - if on_failure == "fail": - fail("GetPythonInfo failed: {}".format(exec_result.describe_failure())) - if on_failure == "warn": - logger.warn(lambda: "GetPythonInfo failed: {}".format(exec_result.describe_failure())) + _emit_log(lambda: "GetPythonInfo failed: {}".format(exec_result.describe_failure())) # else, on_failure must be skip rctx.file("BUILD.bazel", _expand_incompatible_template()) @@ -114,51 +148,29 @@ def _local_runtime_repo_impl(rctx): # appear as part of this repo. rctx.symlink(info["include"], "include") - shared_lib_names = [ - info["PY3LIBRARY"], - info["LDLIBRARY"], - info["INSTSONAME"], - ] - - # In some cases, the value may be empty. Not clear why. - shared_lib_names = [v for v in shared_lib_names if v] - - # In some cases, the same value is returned for multiple keys. Not clear why. - shared_lib_names = {v: None for v in shared_lib_names}.keys() - shared_lib_dir = info["LIBDIR"] - multiarch = info["MULTIARCH"] - - # The specific files are symlinked instead of the whole directory - # because it can point to a directory that has more than just - # the Python runtime shared libraries, e.g. /usr/lib, or a Python - # specific directory with pip-installed shared libraries. rctx.report_progress("Symlinking external Python shared libraries") - for name in shared_lib_names: - origin = rctx.path("{}/{}".format(shared_lib_dir, name)) + interface_library = _symlink_first_library(rctx, logger, info["interface_libraries"]) + shared_library = _symlink_first_library(rctx, logger, info["dynamic_libraries"]) - # If the origin doesn't exist, try the multiarch location, in case - # it's an older Python / Debian release. - if not origin.exists and multiarch: - origin = rctx.path("{}/{}/{}".format(shared_lib_dir, multiarch, name)) + if not interface_library and not shared_library: + logger.warn("No external python libraries found.") - # The reported names don't always exist; it depends on the particulars - # of the runtime installation. - if origin.exists: - repo_utils.watch(rctx, origin) - rctx.symlink(origin, "lib/" + name) - - rctx.file("WORKSPACE", "") - rctx.file("MODULE.bazel", "") - rctx.file("REPO.bazel", "") - rctx.file("BUILD.bazel", _TOOLCHAIN_IMPL_TEMPLATE.format( + build_bazel = _TOOLCHAIN_IMPL_TEMPLATE.format( major = info["major"], minor = info["minor"], micro = info["micro"], - interpreter_path = interpreter_path, - lib_ext = info["SHLIB_SUFFIX"], + interpreter_path = _norm_path(interpreter_path), + interface_library = repr(interface_library), + shared_library = repr(shared_library), implementation_name = info["implementation_name"], os = "@platforms//os:{}".format(repo_utils.get_platforms_os_name(rctx)), - )) + ) + logger.debug("BUILD.bazel\n{}".format(build_bazel)) + + rctx.file("WORKSPACE", "") + rctx.file("MODULE.bazel", "") + rctx.file("REPO.bazel", "") + rctx.file("BUILD.bazel", build_bazel) local_runtime_repo = repository_rule( implementation = _local_runtime_repo_impl, @@ -218,7 +230,8 @@ def _expand_incompatible_template(): return _TOOLCHAIN_IMPL_TEMPLATE.format( interpreter_path = "/incompatible", implementation_name = "incompatible", - lib_ext = "incompatible", + interface_library = "None", + shared_library = "None", major = "0", minor = "0", micro = "0", diff --git a/python/private/local_runtime_repo_setup.bzl b/python/private/local_runtime_repo_setup.bzl index 37eab59575..37c1afbc72 100644 --- a/python/private/local_runtime_repo_setup.bzl +++ b/python/private/local_runtime_repo_setup.bzl @@ -15,6 +15,7 @@ """Setup code called by the code generated by `local_runtime_repo`.""" load("@bazel_skylib//lib:selects.bzl", "selects") +load("@rules_cc//cc:cc_import.bzl", "cc_import") load("@rules_cc//cc:cc_library.bzl", "cc_library") load("@rules_python//python:py_runtime.bzl", "py_runtime") load("@rules_python//python:py_runtime_pair.bzl", "py_runtime_pair") @@ -25,11 +26,12 @@ _PYTHON_VERSION_FLAG = Label("@rules_python//python/config_settings:python_versi def define_local_runtime_toolchain_impl( name, - lib_ext, major, minor, micro, interpreter_path, + interface_library, + shared_library, implementation_name, os): """Defines a toolchain implementation for a local Python runtime. @@ -45,11 +47,14 @@ def define_local_runtime_toolchain_impl( Args: name: `str` Only present to satisfy tooling - lib_ext: `str` The file extension for the `libpython` shared libraries major: `str` The major Python version, e.g. `3` of `3.9.1`. minor: `str` The minor Python version, e.g. `9` of `3.9.1`. micro: `str` The micro Python version, e.g. "1" of `3.9.1`. interpreter_path: `str` Absolute path to the interpreter. + interface_library: `str` Path to the interface library. + e.g. "lib/python312.lib" + shared_library: `str` Path to the dynamic library. + e.g. "lib/python312.dll" or "lib/python312.so" implementation_name: `str` The implementation name, as returned by `sys.implementation.name`. os: `str` A label to the OS constraint (e.g. `@platforms//os:linux`) for @@ -58,6 +63,19 @@ def define_local_runtime_toolchain_impl( major_minor = "{}.{}".format(major, minor) major_minor_micro = "{}.{}".format(major_minor, micro) + # To build Python C/C++ extension on Windows, we need to link to python import library pythonXY.lib + # See https://docs.python.org/3/extending/windows.html + # However not all python installations (such as manylinux) include shared or static libraries, + # so only create the import library when interface_library is set. + import_deps = [] + if interface_library: + cc_import( + name = "_python_interface_library", + interface_library = interface_library, + system_provided = 1, + ) + import_deps = [":_python_interface_library"] + cc_library( name = "_python_headers", # NOTE: Keep in sync with watch_tree() called in local_runtime_repo @@ -66,22 +84,15 @@ def define_local_runtime_toolchain_impl( # A Python install may not have C headers allow_empty = True, ), + deps = import_deps, includes = ["include"], ) cc_library( name = "_libpython", - # Don't use a recursive glob because the lib/ directory usually contains - # a subdirectory of the stdlib -- lots of unrelated files - srcs = native.glob( - [ - "lib/*{}".format(lib_ext), # Match libpython*.so - "lib/*{}*".format(lib_ext), # Also match libpython*.so.1.0 - ], - # A Python install may not have shared libraries. - allow_empty = True, - ), hdrs = [":_python_headers"], + srcs = [shared_library] if shared_library else [], + deps = import_deps, ) py_runtime(