From 76926acb51a52076381003ca5a5c7dceb64e4847 Mon Sep 17 00:00:00 2001 From: Laramie Leavitt Date: Tue, 5 Aug 2025 20:51:49 -0700 Subject: [PATCH 01/26] fix(local_runtime): Improve local_runtime usability in macos / windows local_runtime fails to handle many variations of python install on windows and MacOS, such as: * LDLIBRARY on MacOS may refer to a file under PYTHONFRAMEWORKPREFIX, not LIBDIR * LDLIBRARY on Windows refers to pythonXY.dll, not the linkable pythonXY.lib * LIBDIR may not be correctly set on Windows. * On windows, interpreter_path needs to be normalized. Other paths also require this. * SHLIB_SUFFIX does not indicate the correct suffix. For examples, see See: https://github.com/bazel-contrib/rules_python/issues/3055 See: https://docs.python.org/3/extending/windows.html Example get_local_runtime_info.py outputs: rules_python:local_runtime_repo(@@local_python3) INFO: GetPythonInfo result: INSTSONAME: Python.framework/Versions/3.12/Python LDLIBRARY: Python.framework/Versions/3.12/Python LIBDIR: /Library/Frameworks/Python.framework/Versions/3.12/lib MULTIARCH: darwin PY3LIBRARY: PYTHONFRAMEWORKPREFIX: /Library/Frameworks base_executable: /Library/Frameworks/Python.framework/Versions/3.12/bin/python3.12 implementation_name: cpython include: /Library/Frameworks/Python.framework/Versions/3.12/include/python3.12 major: 3 micro: 10 minor: 12 rules_python:local_runtime_repo(@@local_python3) INFO: GetPythonInfo result: INSTSONAME: None LDLIBRARY: python313.dll LIBDIR: T:\build_temp\home\python_4379bdec28bdff81a567a01c9b8cf10e3856c8c966e4fe53945bedea6338b416\tools\libs MULTIARCH: None PY3LIBRARY: python313.lib base_executable: T:\build_temp\home\python_4379bdec28bdff81a567a01c9b8cf10e3856c8c966e4fe53945bedea6338b416\tools\python.exe implementation_name: cpython include: T:\build_temp\home\python_4379bdec28bdff81a567a01c9b8cf10e3856c8c966e4fe53945bedea6338b416\tools\Include major: 3 micro: 1 minor: 13 On windows and macos, since SHLIB_SUFFIX does not always indicate the filenames needed searching, this has been removed from local_runtime_repo_setup and replaced with an explicit list of files. In addition, target libraries are searched in multiple locations to handle variations in macos and windows file locations. On windows the interpreter_path and other search paths are now normalized ( \ converted to /). On windows the get_local_runtime_info now returns the pythonXY.lib library Additional logging added to local_runtime_repo. :1 --- examples/local_python/.bazelrc | 4 + examples/local_python/.gitignore | 4 + examples/local_python/BUILD.bazel | 6 + examples/local_python/WORKSPACE | 35 ++++++ examples/local_python/main.py | 22 ++++ python/private/get_local_runtime_info.py | 14 ++- python/private/local_runtime_repo.bzl | 115 ++++++++++++++------ python/private/local_runtime_repo_setup.bzl | 11 +- 8 files changed, 164 insertions(+), 47 deletions(-) create mode 100644 examples/local_python/.bazelrc create mode 100644 examples/local_python/.gitignore create mode 100644 examples/local_python/BUILD.bazel create mode 100644 examples/local_python/WORKSPACE create mode 100644 examples/local_python/main.py diff --git a/examples/local_python/.bazelrc b/examples/local_python/.bazelrc new file mode 100644 index 0000000000..2ed86f591e --- /dev/null +++ b/examples/local_python/.bazelrc @@ -0,0 +1,4 @@ +# The equivalent bzlmod behavior is covered by examples/bzlmod/py_proto_library +common --noenable_bzlmod +common --enable_workspace +common --incompatible_python_disallow_native_rules diff --git a/examples/local_python/.gitignore b/examples/local_python/.gitignore new file mode 100644 index 0000000000..e5ae073b3c --- /dev/null +++ b/examples/local_python/.gitignore @@ -0,0 +1,4 @@ +# git ignore patterns + +/bazel-* +user.bazelrc diff --git a/examples/local_python/BUILD.bazel b/examples/local_python/BUILD.bazel new file mode 100644 index 0000000000..1a6d5cbe8a --- /dev/null +++ b/examples/local_python/BUILD.bazel @@ -0,0 +1,6 @@ +load("@rules_python//python:py_binary.bzl", "py_binary") + +py_binary( + name = "main", + srcs = ["main.py"], +) diff --git a/examples/local_python/WORKSPACE b/examples/local_python/WORKSPACE new file mode 100644 index 0000000000..d476bccdb4 --- /dev/null +++ b/examples/local_python/WORKSPACE @@ -0,0 +1,35 @@ +workspace( + name = "local_python_example", +) + +local_repository( + name = "rules_python", + path = "../..", +) + +load("@rules_python//python:repositories.bzl", "py_repositories") + +py_repositories() + + +load("@rules_python//python/local_toolchains:repos.bzl", "local_runtime_repo", "local_runtime_toolchains_repo") + + +# Step 1: Define the python runtime. +local_runtime_repo( + name = "local_python3", + on_failure = "fail", + interpreter_path = "python3" + # or interpreter_path = "C:\\path\\to\\python.exe" +) + +# Step 2: Create toolchains for the runtimes +local_runtime_toolchains_repo( + name = "local_toolchains", + runtimes = ["local_python3"], +) + + +# Step 3: Register the toolchains +register_toolchains("@local_toolchains//:all") + diff --git a/examples/local_python/main.py b/examples/local_python/main.py new file mode 100644 index 0000000000..236ceca3d5 --- /dev/null +++ b/examples/local_python/main.py @@ -0,0 +1,22 @@ +# Copyright 2025 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + + +def main(): + print(42) + + +if __name__ == "__main__": + main() diff --git a/python/private/get_local_runtime_info.py b/python/private/get_local_runtime_info.py index c8371357c2..3041955f2b 100644 --- a/python/private/get_local_runtime_info.py +++ b/python/private/get_local_runtime_info.py @@ -50,9 +50,17 @@ # The libpythonX.so file. Usually? # It might be a static archive (a.) file instead. "PY3LIBRARY", - # The platform-specific filename suffix for library files. - # Includes the dot, e.g. `.so` - "SHLIB_SUFFIX", + # On MacOS, the LDLIBRARY may be a relative path rooted under /Library/Frameworks, + # such as "Python.framework/Versions/3.12/Python", not a file under the LIBDIR directory. + "PYTHONFRAMEWORKPREFIX", ] data.update(zip(config_vars, sysconfig.get_config_vars(*config_vars))) + +# On windows, the SHLIB_SUFFIX is .lib, and the toolchain needs to link with pythonXY.lib. +# See: https://docs.python.org/3/extending/windows.html +if sys.platform == "win32": + data["LDLIBRARY"] = "python{}{}.lib".format( + sys.version_info.major, sys.version_info.minor + ) + print(json.dumps(data)) diff --git a/python/private/local_runtime_repo.bzl b/python/private/local_runtime_repo.bzl index b8b7164b54..407dca6856 100644 --- a/python/private/local_runtime_repo.bzl +++ b/python/private/local_runtime_repo.bzl @@ -31,27 +31,76 @@ load("@rules_python//python/private:local_runtime_repo_setup.bzl", "define_local define_local_runtime_toolchain_impl( name = "local_runtime", - lib_ext = "{lib_ext}", major = "{major}", minor = "{minor}", micro = "{micro}", interpreter_path = "{interpreter_path}", + library_srcs = {library_srcs}, implementation_name = "{implementation_name}", os = "{os}", ) """ +def _norm_path(path): + """Returns a path using '/' separators and no trailing slash.""" + path = path.replace("\\", "/") + if path[-1] == "/": + path = path[:-1] + return path + +def _symlink_libs(rctx, logger, shared_lib_dirs, shared_lib_names): + """Symlinks the shared libraries into the lib/ directory. + + Args: + rctx: A repository_ctx object + logger: A repo_utils.logger object + shared_lib_dirs: The search directories for shared libraries + shared_lib_names: The individual shared libraries to attempt to link in the directories. + + Returns: + A list of src libraries linked by the action. + + The specific files are symlinked instead of the whole directory because + shared_lib_dirs contains multiple search paths for the shared libraries, + and the python files may be missing from any of those directories, and + any of those directories may include non-python runtime libraries, + as would be the case if LIBDIR were, for example, /usr/lib. + """ + found = [] + for shared_lib_dir in shared_lib_dirs: + for name in shared_lib_names: + origin = rctx.path("{}/{}".format(shared_lib_dir, name)) + if not origin.exists: + # The reported names don't always exist; it depends on the particulars + # of the runtime installation. + continue + + found.append("lib/{}".format(origin.basename)) + logger.debug("Symlinking {} to lib/{}".format(origin, origin.basename)) + repo_utils.watch(rctx, origin) + rctx.symlink(origin, "lib/{}".format(origin.basename)) + + # Libraries will only be linked from the same directory. + if found: + break + + return found + def _local_runtime_repo_impl(rctx): logger = repo_utils.logger(rctx) on_failure = rctx.attr.on_failure - result = _resolve_interpreter_path(rctx) - if not result.resolved_path: + def _emit_log(msg): if on_failure == "fail": - fail("interpreter not found: {}".format(result.describe_failure())) + logger.fail(msg) + elif on_failure == "warn": + logger.warn(msg) + else: + logger.debug(msg) - if on_failure == "warn": - logger.warn(lambda: "interpreter not found: {}".format(result.describe_failure())) + result = _resolve_interpreter_path(rctx) + if not result.resolved_path: + _emit_log(lambda: "interpreter not found: {}".format(result.describe_failure())) # else, on_failure must be skip rctx.file("BUILD.bazel", _expand_incompatible_template()) @@ -72,10 +121,7 @@ def _local_runtime_repo_impl(rctx): logger = logger, ) if exec_result.return_code != 0: - if on_failure == "fail": - fail("GetPythonInfo failed: {}".format(exec_result.describe_failure())) - if on_failure == "warn": - logger.warn(lambda: "GetPythonInfo failed: {}".format(exec_result.describe_failure())) + _emit_log(lambda: "GetPythonInfo failed: {}".format(exec_result.describe_failure())) # else, on_failure must be skip rctx.file("BUILD.bazel", _expand_incompatible_template()) @@ -120,45 +166,44 @@ def _local_runtime_repo_impl(rctx): info["INSTSONAME"], ] - # In some cases, the value may be empty. Not clear why. + # Not all config fields exist; nor are they necessarily distinct. + # Dedup and remove empty values. shared_lib_names = [v for v in shared_lib_names if v] - - # In some cases, the same value is returned for multiple keys. Not clear why. shared_lib_names = {v: None for v in shared_lib_names}.keys() - shared_lib_dir = info["LIBDIR"] - multiarch = info["MULTIARCH"] + + shared_lib_dirs = [] + if info["LIBDIR"]: + libdir = _norm_path(info["LIBDIR"]) + shared_lib_dirs.append(libdir) + if info["MULTIARCH"]: + shared_lib_dirs.append("{}/{}".format(libdir, info["MULTIARCH"])) + if info["PYTHONFRAMEWORKPREFIX"]: + shared_lib_dirs.append(info["PYTHONFRAMEWORKPREFIX"]) # The specific files are symlinked instead of the whole directory # because it can point to a directory that has more than just # the Python runtime shared libraries, e.g. /usr/lib, or a Python # specific directory with pip-installed shared libraries. rctx.report_progress("Symlinking external Python shared libraries") - for name in shared_lib_names: - origin = rctx.path("{}/{}".format(shared_lib_dir, name)) + library_srcs = _symlink_libs(rctx, logger, shared_lib_dirs, shared_lib_names) + if not library_srcs: + logger.info("No external python libraries found in {}".format(shared_lib_dirs)) - # If the origin doesn't exist, try the multiarch location, in case - # it's an older Python / Debian release. - if not origin.exists and multiarch: - origin = rctx.path("{}/{}/{}".format(shared_lib_dir, multiarch, name)) - - # The reported names don't always exist; it depends on the particulars - # of the runtime installation. - if origin.exists: - repo_utils.watch(rctx, origin) - rctx.symlink(origin, "lib/" + name) - - rctx.file("WORKSPACE", "") - rctx.file("MODULE.bazel", "") - rctx.file("REPO.bazel", "") - rctx.file("BUILD.bazel", _TOOLCHAIN_IMPL_TEMPLATE.format( + build_bazel = _TOOLCHAIN_IMPL_TEMPLATE.format( major = info["major"], minor = info["minor"], micro = info["micro"], - interpreter_path = interpreter_path, - lib_ext = info["SHLIB_SUFFIX"], + interpreter_path = _norm_path(interpreter_path), + library_srcs = repr(library_srcs), implementation_name = info["implementation_name"], os = "@platforms//os:{}".format(repo_utils.get_platforms_os_name(rctx)), - )) + ) + logger.debug("BUILD.bazel\n{}".format(build_bazel)) + + rctx.file("WORKSPACE", "") + rctx.file("MODULE.bazel", "") + rctx.file("REPO.bazel", "") + rctx.file("BUILD.bazel", build_bazel) local_runtime_repo = repository_rule( implementation = _local_runtime_repo_impl, diff --git a/python/private/local_runtime_repo_setup.bzl b/python/private/local_runtime_repo_setup.bzl index 37eab59575..bafac6c1c3 100644 --- a/python/private/local_runtime_repo_setup.bzl +++ b/python/private/local_runtime_repo_setup.bzl @@ -25,11 +25,11 @@ _PYTHON_VERSION_FLAG = Label("@rules_python//python/config_settings:python_versi def define_local_runtime_toolchain_impl( name, - lib_ext, major, minor, micro, interpreter_path, + library_srcs, implementation_name, os): """Defines a toolchain implementation for a local Python runtime. @@ -73,14 +73,7 @@ def define_local_runtime_toolchain_impl( name = "_libpython", # Don't use a recursive glob because the lib/ directory usually contains # a subdirectory of the stdlib -- lots of unrelated files - srcs = native.glob( - [ - "lib/*{}".format(lib_ext), # Match libpython*.so - "lib/*{}*".format(lib_ext), # Also match libpython*.so.1.0 - ], - # A Python install may not have shared libraries. - allow_empty = True, - ), + srcs = library_srcs, hdrs = [":_python_headers"], ) From b9cbc675694acd7cdca3380b9f562f3cd371d196 Mon Sep 17 00:00:00 2001 From: Laramie Leavitt Date: Wed, 6 Aug 2025 00:18:46 -0700 Subject: [PATCH 02/26] Correct _expand_incompatible_template --- python/private/local_runtime_repo.bzl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/private/local_runtime_repo.bzl b/python/private/local_runtime_repo.bzl index 407dca6856..95629e727c 100644 --- a/python/private/local_runtime_repo.bzl +++ b/python/private/local_runtime_repo.bzl @@ -263,7 +263,7 @@ def _expand_incompatible_template(): return _TOOLCHAIN_IMPL_TEMPLATE.format( interpreter_path = "/incompatible", implementation_name = "incompatible", - lib_ext = "incompatible", + library_srcs = "[]" major = "0", minor = "0", micro = "0", From 965a6a9d887e56a03b60e7fadd6529c167ec24cf Mon Sep 17 00:00:00 2001 From: Laramie Leavitt Date: Wed, 6 Aug 2025 00:22:24 -0700 Subject: [PATCH 03/26] Update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 422e399026..2020951773 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -101,6 +101,7 @@ END_UNRELEASED_TEMPLATE `# gazelle:python_resolve_sibling_imports true` * (pypi) Show overridden index URL of packages when downloading metadata have failed. ([#2985](https://github.com/bazel-contrib/rules_python/issues/2985)). +* (toolchains) `local_runtime_repo` better handles variants in MacOS and Windows. {#v0-0-0-added} ### Added From 8b6191537ccd2429b944c6be500dabf7359bb8b9 Mon Sep 17 00:00:00 2001 From: Laramie Leavitt Date: Wed, 6 Aug 2025 00:51:12 -0700 Subject: [PATCH 04/26] Formatting + minor cleanups --- python/private/get_local_runtime_info.py | 5 +++-- python/private/local_runtime_repo.bzl | 2 +- python/private/local_runtime_repo_setup.bzl | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/python/private/get_local_runtime_info.py b/python/private/get_local_runtime_info.py index 3041955f2b..8069aaecbd 100644 --- a/python/private/get_local_runtime_info.py +++ b/python/private/get_local_runtime_info.py @@ -50,8 +50,9 @@ # The libpythonX.so file. Usually? # It might be a static archive (a.) file instead. "PY3LIBRARY", - # On MacOS, the LDLIBRARY may be a relative path rooted under /Library/Frameworks, - # such as "Python.framework/Versions/3.12/Python", not a file under the LIBDIR directory. + # On MacOS, the LDLIBRARY may be a relative path under /Library/Frameworks, + # such as "Python.framework/Versions/3.12/Python", not a file under the + # LIBDIR directory. "PYTHONFRAMEWORKPREFIX", ] data.update(zip(config_vars, sysconfig.get_config_vars(*config_vars))) diff --git a/python/private/local_runtime_repo.bzl b/python/private/local_runtime_repo.bzl index 95629e727c..2b409caee0 100644 --- a/python/private/local_runtime_repo.bzl +++ b/python/private/local_runtime_repo.bzl @@ -263,7 +263,7 @@ def _expand_incompatible_template(): return _TOOLCHAIN_IMPL_TEMPLATE.format( interpreter_path = "/incompatible", implementation_name = "incompatible", - library_srcs = "[]" + library_srcs = "[]", major = "0", minor = "0", micro = "0", diff --git a/python/private/local_runtime_repo_setup.bzl b/python/private/local_runtime_repo_setup.bzl index bafac6c1c3..be485dc1cd 100644 --- a/python/private/local_runtime_repo_setup.bzl +++ b/python/private/local_runtime_repo_setup.bzl @@ -45,7 +45,7 @@ def define_local_runtime_toolchain_impl( Args: name: `str` Only present to satisfy tooling - lib_ext: `str` The file extension for the `libpython` shared libraries + library_srcs: `str` A list of `libpython` shared libraries formatted as a string, e.g. '[ "lib/python312.lib" ]' major: `str` The major Python version, e.g. `3` of `3.9.1`. minor: `str` The minor Python version, e.g. `9` of `3.9.1`. micro: `str` The micro Python version, e.g. "1" of `3.9.1`. From 6ac66c94141d8c9ab0a81db542b75674db446085 Mon Sep 17 00:00:00 2001 From: Laramie Leavitt Date: Wed, 6 Aug 2025 01:40:42 -0700 Subject: [PATCH 05/26] format WORKSPACE --- examples/local_python/WORKSPACE | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/examples/local_python/WORKSPACE b/examples/local_python/WORKSPACE index d476bccdb4..f39bf14b86 100644 --- a/examples/local_python/WORKSPACE +++ b/examples/local_python/WORKSPACE @@ -11,15 +11,13 @@ load("@rules_python//python:repositories.bzl", "py_repositories") py_repositories() - load("@rules_python//python/local_toolchains:repos.bzl", "local_runtime_repo", "local_runtime_toolchains_repo") - # Step 1: Define the python runtime. local_runtime_repo( name = "local_python3", + interpreter_path = "python3", on_failure = "fail", - interpreter_path = "python3" # or interpreter_path = "C:\\path\\to\\python.exe" ) @@ -29,7 +27,5 @@ local_runtime_toolchains_repo( runtimes = ["local_python3"], ) - # Step 3: Register the toolchains register_toolchains("@local_toolchains//:all") - From f4c0fed99d4d31d4e7d36f240cc498ca8da3445c Mon Sep 17 00:00:00 2001 From: Laramie Leavitt Date: Wed, 6 Aug 2025 16:21:37 -0700 Subject: [PATCH 06/26] Move the complex logic to resolve the python shared libraries from bzl to get_local_runtime_info.py --- python/private/get_local_runtime_info.py | 190 +++++++++++++++----- python/private/local_runtime_repo.bzl | 81 +++------ python/private/local_runtime_repo_setup.bzl | 15 +- 3 files changed, 190 insertions(+), 96 deletions(-) diff --git a/python/private/get_local_runtime_info.py b/python/private/get_local_runtime_info.py index 8069aaecbd..3ec9066b45 100644 --- a/python/private/get_local_runtime_info.py +++ b/python/private/get_local_runtime_info.py @@ -13,9 +13,158 @@ # limitations under the License. import json +import os import sys import sysconfig +_IS_WINDOWS = sys.platform == "win32" + + +def _search_directories(get_config): + """Returns a list of library directories to search for shared libraries.""" + # There's several types of libraries with different names and a plethora + # of settings, and many different config variables to check: + # + # LIBPL is used in python-config when shared library is not enabled: + # https://github.com/python/cpython/blob/v3.12.0/Misc/python-config.in#L63 + # + # LIBDIR may also be the python directory with library files. + # https://stackoverflow.com/questions/47423246/get-pythons-lib-path + # See also: MULTIARCH + # + # On MacOS, the LDLIBRARY may be a relative path under /Library/Frameworks, + # such as "Python.framework/Versions/3.12/Python", not a file under the + # LIBDIR/LIBPL directory, so include PYTHONFRAMEWORKPREFIX. + lib_dirs = [ + get_config(x) for x in ("LIBPL", "LIBDIR", "PYTHONFRAMEWORKPREFIX") + ] + + # On Debian, with multiarch enabled, prior to Python 3.10, `LIBDIR` didn't + # tell the location of the libs, just the base directory. The `MULTIARCH` + # sysconfig variable tells the subdirectory within it with the libs. + # See: + # https://wiki.debian.org/Python/MultiArch + # https://git.launchpad.net/ubuntu/+source/python3.12/tree/debian/changelog#n842 + multiarch = get_config("MULTIARCH") + if multiarch: + for x in ["LIBPL", "LIBDIR"]: + config_value = get_config(x) + if config_value and not config_value.endswith(multiarch): + lib_dirs.append(os.path.join(config_value, multiarch)) + + if _IS_WINDOWS: + # On Windows DLLs go in the same directory as the executable, while .lib + # files live in the lib/ or libs/ subdirectory. + lib_dirs.append(get_config("BINDIR")) + lib_dirs.append(os.path.join(os.path.dirname(sys.executable))) + lib_dirs.append(os.path.join(os.path.dirname(sys.executable), "lib")) + lib_dirs.append(os.path.join(os.path.dirname(sys.executable), "libs")) + else: + # On most systems the executable is in a bin/ directory and the libraries + # are in a sibling lib/ directory. + lib_dirs.append( + os.path.join(os.path.dirname(os.path.dirname(sys.executable)), "lib") + ) + + # Dedup and remove empty values, keeping the order. + lib_dirs = [v for v in lib_dirs if v] + return {k: None for k in lib_dirs}.keys() + + +def _search_library_names(get_config): + """Returns a list of library files to search for shared libraries.""" + # Quoting configure.ac in the cpython code base: + # "INSTSONAME is the name of the shared library that will be use to install + # on the system - some systems like version suffix, others don't."" + # + # A typical INSTSONAME is 'libpython3.8.so.1.0' on Linux, or + # 'Python.framework/Versions/3.9/Python' on MacOS. Due to the possible + # version suffix we have to find the suffix within the filename. + # + # A typical LDLIBRARY is 'libpythonX.Y.so' on Linux, or 'pythonXY.dll' on + # Windows. + # + # A typical LIBRARY is 'libpythonX.Y.a' on Linux. + lib_names = [ + get_config(x) + for x in ( + "INSTSONAME", + "LDLIBRARY", + "PY3LIBRARY", + "LIBRARY", + "DLLLIBRARY", + ) + ] + + if _IS_WINDOWS: + so_prefix = "" + else: + so_prefix = "lib" + + # Get/override the SHLIB_SUFFIX, which is typically ".so" on Linux and + # ".dylib" on macOS. + so_suffix = get_config("SHLIB_SUFFIX") + if sys.platform == "darwin": + so_suffix = ".dylib" + elif _IS_WINDOWS: + so_suffix = ".lib" + elif not so_suffix: + so_suffix = ".so" + + version = get_config("VERSION") + abiflags = get_config("ABIFLAGS") or get_config("abiflags") or "" + + # On Windows, extensions should link with the pythonXY.lib files. + # See: https://docs.python.org/3/extending/windows.html + # So ensure that the pythonXY.lib files are included in the search. + if abiflags: + lib_names.append(f"{so_prefix}python{version}{abiflags}{so_suffix}") + lib_names.append(f"{so_prefix}python{version}{so_suffix}") + + # Dedup and remove empty values, keeping the order. + lib_names = [v for v in lib_names if v] + return {k: None for k in lib_names}.keys() + + +def _get_python_library_info(): + """Returns a dictionary with the static and dynamic python libraries.""" + config_vars = sysconfig.get_config_vars() + + # VERSION is X.Y in Linux/macOS and XY in Windows. + if not config_vars.get("VERSION"): + if sys.platform == "win32": + config_vars["VERSION"] = ( + f"{sys.version_info.major}{sys.version_info.minor}" + ) + else: + config_vars["VERSION"] = ( + f"{sys.version_info.major}.{sys.version_info.minor}" + ) + + search_directories = _search_directories(config_vars.get) + search_libnames = _search_library_names(config_vars.get) + + dynamic_libraries = {} + static_libraries = {} + for root_dir in search_directories: + for libname in search_libnames: + composed_path = os.path.join(root_dir, libname) + if not os.path.exists(composed_path) or os.path.isdir(composed_path): + continue + if composed_path.endswith(".lib") or composed_path.endswith(".a"): + static_libraries[composed_path] = None + else: + dynamic_libraries[composed_path] = None + + # NOTE: It's possible that including the dynamic libraries currently loaded + # by the running python interpreter would be a useful addition. + + return { + "dynamic_libraries": list(dynamic_libraries.keys()), + "static_libraries": list(static_libraries.keys()), + } + + data = { "major": sys.version_info.major, "minor": sys.version_info.minor, @@ -24,44 +173,5 @@ "implementation_name": sys.implementation.name, "base_executable": sys._base_executable, } - -config_vars = [ - # The libpythonX.Y.so file. Usually? - # It might be a static archive (.a) file instead. - "LDLIBRARY", - # The directory with library files. Supposedly. - # It's not entirely clear how to get the directory with libraries. - # There's several types of libraries with different names and a plethora - # of settings. - # https://stackoverflow.com/questions/47423246/get-pythons-lib-path - # For now, it seems LIBDIR has what is needed, so just use that. - # See also: MULTIARCH - "LIBDIR", - # On Debian, with multiarch enabled, prior to Python 3.10, `LIBDIR` didn't - # tell the location of the libs, just the base directory. The `MULTIARCH` - # sysconfig variable tells the subdirectory within it with the libs. - # See: - # https://wiki.debian.org/Python/MultiArch - # https://git.launchpad.net/ubuntu/+source/python3.12/tree/debian/changelog#n842 - "MULTIARCH", - # The versioned libpythonX.Y.so.N file. Usually? - # It might be a static archive (.a) file instead. - "INSTSONAME", - # The libpythonX.so file. Usually? - # It might be a static archive (a.) file instead. - "PY3LIBRARY", - # On MacOS, the LDLIBRARY may be a relative path under /Library/Frameworks, - # such as "Python.framework/Versions/3.12/Python", not a file under the - # LIBDIR directory. - "PYTHONFRAMEWORKPREFIX", -] -data.update(zip(config_vars, sysconfig.get_config_vars(*config_vars))) - -# On windows, the SHLIB_SUFFIX is .lib, and the toolchain needs to link with pythonXY.lib. -# See: https://docs.python.org/3/extending/windows.html -if sys.platform == "win32": - data["LDLIBRARY"] = "python{}{}.lib".format( - sys.version_info.major, sys.version_info.minor - ) - +data.update(_get_python_library_info()) print(json.dumps(data)) diff --git a/python/private/local_runtime_repo.bzl b/python/private/local_runtime_repo.bzl index 2b409caee0..099c0bb18a 100644 --- a/python/private/local_runtime_repo.bzl +++ b/python/private/local_runtime_repo.bzl @@ -35,7 +35,7 @@ define_local_runtime_toolchain_impl( minor = "{minor}", micro = "{micro}", interpreter_path = "{interpreter_path}", - library_srcs = {library_srcs}, + interface_library = "{interface_library}", implementation_name = "{implementation_name}", os = "{os}", ) @@ -48,17 +48,15 @@ def _norm_path(path): path = path[:-1] return path -def _symlink_libs(rctx, logger, shared_lib_dirs, shared_lib_names): +def _symlink_libs(rctx, logger, library_targets): """Symlinks the shared libraries into the lib/ directory. Args: rctx: A repository_ctx object logger: A repo_utils.logger object - shared_lib_dirs: The search directories for shared libraries - shared_lib_names: The individual shared libraries to attempt to link in the directories. - + library_targets: A list of library targets to potentially symlink. Returns: - A list of src libraries linked by the action. + A library target suitable for a cc_import rule. The specific files are symlinked instead of the whole directory because shared_lib_dirs contains multiple search paths for the shared libraries, @@ -66,23 +64,18 @@ def _symlink_libs(rctx, logger, shared_lib_dirs, shared_lib_names): any of those directories may include non-python runtime libraries, as would be the case if LIBDIR were, for example, /usr/lib. """ - found = [] - for shared_lib_dir in shared_lib_dirs: - for name in shared_lib_names: - origin = rctx.path("{}/{}".format(shared_lib_dir, name)) - if not origin.exists: - # The reported names don't always exist; it depends on the particulars - # of the runtime installation. - continue - - found.append("lib/{}".format(origin.basename)) - logger.debug("Symlinking {} to lib/{}".format(origin, origin.basename)) - repo_utils.watch(rctx, origin) - rctx.symlink(origin, "lib/{}".format(origin.basename)) - - # Libraries will only be linked from the same directory. - if found: - break + found = "" + for target in library_targets: + origin = rctx.path(target) + if not origin.exists: + # The reported names don't always exist; it depends on the particulars + # of the runtime installation. + continue + found = "lib/{}".format(origin.basename) + logger.debug("Symlinking {} to {}".format(origin, found)) + repo_utils.watch(rctx, origin) + rctx.symlink(origin, found) + break return found @@ -160,41 +153,20 @@ def _local_runtime_repo_impl(rctx): # appear as part of this repo. rctx.symlink(info["include"], "include") - shared_lib_names = [ - info["PY3LIBRARY"], - info["LDLIBRARY"], - info["INSTSONAME"], - ] - - # Not all config fields exist; nor are they necessarily distinct. - # Dedup and remove empty values. - shared_lib_names = [v for v in shared_lib_names if v] - shared_lib_names = {v: None for v in shared_lib_names}.keys() - - shared_lib_dirs = [] - if info["LIBDIR"]: - libdir = _norm_path(info["LIBDIR"]) - shared_lib_dirs.append(libdir) - if info["MULTIARCH"]: - shared_lib_dirs.append("{}/{}".format(libdir, info["MULTIARCH"])) - if info["PYTHONFRAMEWORKPREFIX"]: - shared_lib_dirs.append(info["PYTHONFRAMEWORKPREFIX"]) - - # The specific files are symlinked instead of the whole directory - # because it can point to a directory that has more than just - # the Python runtime shared libraries, e.g. /usr/lib, or a Python - # specific directory with pip-installed shared libraries. + if repo_utils.get_platforms_os_name == "windows": + library_targets = info["static_libraries"] + else: + library_targets = info["dynamic_libraries"] + rctx.report_progress("Symlinking external Python shared libraries") - library_srcs = _symlink_libs(rctx, logger, shared_lib_dirs, shared_lib_names) - if not library_srcs: - logger.info("No external python libraries found in {}".format(shared_lib_dirs)) + interface_library = _symlink_libs(rctx, logger, library_targets) build_bazel = _TOOLCHAIN_IMPL_TEMPLATE.format( major = info["major"], minor = info["minor"], micro = info["micro"], interpreter_path = _norm_path(interpreter_path), - library_srcs = repr(library_srcs), + interface_library = _norm_path(interface_library), implementation_name = info["implementation_name"], os = "@platforms//os:{}".format(repo_utils.get_platforms_os_name(rctx)), ) @@ -260,10 +232,15 @@ How to handle errors when trying to automatically determine settings. ) def _expand_incompatible_template(): + if repo_utils.get_platforms_os_name == "windows": + missing = "missing.lib" + else: + missing = "missing.so" + return _TOOLCHAIN_IMPL_TEMPLATE.format( interpreter_path = "/incompatible", implementation_name = "incompatible", - library_srcs = "[]", + interface_library = missing, major = "0", minor = "0", micro = "0", diff --git a/python/private/local_runtime_repo_setup.bzl b/python/private/local_runtime_repo_setup.bzl index be485dc1cd..96923e1631 100644 --- a/python/private/local_runtime_repo_setup.bzl +++ b/python/private/local_runtime_repo_setup.bzl @@ -15,6 +15,7 @@ """Setup code called by the code generated by `local_runtime_repo`.""" load("@bazel_skylib//lib:selects.bzl", "selects") +load("@rules_cc//cc:cc_import.bzl", "cc_import") load("@rules_cc//cc:cc_library.bzl", "cc_library") load("@rules_python//python:py_runtime.bzl", "py_runtime") load("@rules_python//python:py_runtime_pair.bzl", "py_runtime_pair") @@ -29,7 +30,7 @@ def define_local_runtime_toolchain_impl( minor, micro, interpreter_path, - library_srcs, + interface_library, implementation_name, os): """Defines a toolchain implementation for a local Python runtime. @@ -58,6 +59,14 @@ def define_local_runtime_toolchain_impl( major_minor = "{}.{}".format(major, minor) major_minor_micro = "{}.{}".format(major_minor, micro) + # To build Python C/C++ extension on Windows, we need to link to python import library pythonXY.lib + # See https://docs.python.org/3/extending/windows.html + cc_import( + name = "_python_interface", + interface_library = interface_library, + system_provided = 1, + ) + cc_library( name = "_python_headers", # NOTE: Keep in sync with watch_tree() called in local_runtime_repo @@ -71,10 +80,8 @@ def define_local_runtime_toolchain_impl( cc_library( name = "_libpython", - # Don't use a recursive glob because the lib/ directory usually contains - # a subdirectory of the stdlib -- lots of unrelated files - srcs = library_srcs, hdrs = [":_python_headers"], + deps = [":_python_interface"], ) py_runtime( From 9283a3fc7302283693fcc28a5e29f3bf5faf0e62 Mon Sep 17 00:00:00 2001 From: Laramie Leavitt Date: Wed, 6 Aug 2025 16:56:29 -0700 Subject: [PATCH 07/26] Fix formatting and method docs --- python/private/local_runtime_repo.bzl | 4 ++-- python/private/local_runtime_repo_setup.bzl | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/python/private/local_runtime_repo.bzl b/python/private/local_runtime_repo.bzl index 099c0bb18a..d82137dc5f 100644 --- a/python/private/local_runtime_repo.bzl +++ b/python/private/local_runtime_repo.bzl @@ -233,9 +233,9 @@ How to handle errors when trying to automatically determine settings. def _expand_incompatible_template(): if repo_utils.get_platforms_os_name == "windows": - missing = "missing.lib" + missing = "incompatible-missing.lib" else: - missing = "missing.so" + missing = "incompatible-missing.so" return _TOOLCHAIN_IMPL_TEMPLATE.format( interpreter_path = "/incompatible", diff --git a/python/private/local_runtime_repo_setup.bzl b/python/private/local_runtime_repo_setup.bzl index 96923e1631..eac045c082 100644 --- a/python/private/local_runtime_repo_setup.bzl +++ b/python/private/local_runtime_repo_setup.bzl @@ -46,11 +46,12 @@ def define_local_runtime_toolchain_impl( Args: name: `str` Only present to satisfy tooling - library_srcs: `str` A list of `libpython` shared libraries formatted as a string, e.g. '[ "lib/python312.lib" ]' major: `str` The major Python version, e.g. `3` of `3.9.1`. minor: `str` The minor Python version, e.g. `9` of `3.9.1`. micro: `str` The micro Python version, e.g. "1" of `3.9.1`. interpreter_path: `str` Absolute path to the interpreter. + interface_library: `str` A path to a .lib or .so file to link against. + e.g. "lib/python312.lib" implementation_name: `str` The implementation name, as returned by `sys.implementation.name`. os: `str` A label to the OS constraint (e.g. `@platforms//os:linux`) for From e8997c1cf7fc5b25e7530bf1829f29c6638026a8 Mon Sep 17 00:00:00 2001 From: Laramie Leavitt Date: Wed, 6 Aug 2025 17:57:37 -0700 Subject: [PATCH 08/26] Better handle missing interface_library values. This can happen on manylinux. --- python/private/local_runtime_repo.bzl | 9 ++------- python/private/local_runtime_repo_setup.bzl | 19 ++++++++++++------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/python/private/local_runtime_repo.bzl b/python/private/local_runtime_repo.bzl index d82137dc5f..d3f051b3fc 100644 --- a/python/private/local_runtime_repo.bzl +++ b/python/private/local_runtime_repo.bzl @@ -166,7 +166,7 @@ def _local_runtime_repo_impl(rctx): minor = info["minor"], micro = info["micro"], interpreter_path = _norm_path(interpreter_path), - interface_library = _norm_path(interface_library), + interface_library = interface_library, implementation_name = info["implementation_name"], os = "@platforms//os:{}".format(repo_utils.get_platforms_os_name(rctx)), ) @@ -232,15 +232,10 @@ How to handle errors when trying to automatically determine settings. ) def _expand_incompatible_template(): - if repo_utils.get_platforms_os_name == "windows": - missing = "incompatible-missing.lib" - else: - missing = "incompatible-missing.so" - return _TOOLCHAIN_IMPL_TEMPLATE.format( interpreter_path = "/incompatible", implementation_name = "incompatible", - interface_library = missing, + interface_library = "", major = "0", minor = "0", micro = "0", diff --git a/python/private/local_runtime_repo_setup.bzl b/python/private/local_runtime_repo_setup.bzl index eac045c082..e712387b97 100644 --- a/python/private/local_runtime_repo_setup.bzl +++ b/python/private/local_runtime_repo_setup.bzl @@ -61,12 +61,17 @@ def define_local_runtime_toolchain_impl( major_minor_micro = "{}.{}".format(major_minor, micro) # To build Python C/C++ extension on Windows, we need to link to python import library pythonXY.lib - # See https://docs.python.org/3/extending/windows.html - cc_import( - name = "_python_interface", - interface_library = interface_library, - system_provided = 1, - ) + # See https://docs.python.org/3/extending/windows.html + # However not all python installations include shared or static libraries to link with (manylinux) + # So only create the import library when interface_library is present. + _libpython_deps = [] + if interface_library: + cc_import( + name = "_python_import_lib", + interface_library = interface_library, + system_provided = 1, + ) + _libpython_deps.append(":_python_import_lib") cc_library( name = "_python_headers", @@ -82,7 +87,7 @@ def define_local_runtime_toolchain_impl( cc_library( name = "_libpython", hdrs = [":_python_headers"], - deps = [":_python_interface"], + deps = _libpython_deps, ) py_runtime( From 655ea0f34db681541c35f15b85466f461ad272a8 Mon Sep 17 00:00:00 2001 From: Laramie Leavitt Date: Wed, 6 Aug 2025 18:00:12 -0700 Subject: [PATCH 09/26] Format local_runtime_repo_setup.bzl --- python/private/local_runtime_repo_setup.bzl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/python/private/local_runtime_repo_setup.bzl b/python/private/local_runtime_repo_setup.bzl index e712387b97..09afc4d94e 100644 --- a/python/private/local_runtime_repo_setup.bzl +++ b/python/private/local_runtime_repo_setup.bzl @@ -61,9 +61,9 @@ def define_local_runtime_toolchain_impl( major_minor_micro = "{}.{}".format(major_minor, micro) # To build Python C/C++ extension on Windows, we need to link to python import library pythonXY.lib - # See https://docs.python.org/3/extending/windows.html - # However not all python installations include shared or static libraries to link with (manylinux) - # So only create the import library when interface_library is present. + # See https://docs.python.org/3/extending/windows.html + # However not all python installations (such as manylinux) include shared or static libraries, + # so only create the import library when interface_library is set. _libpython_deps = [] if interface_library: cc_import( From e4eea874c6c06e59ca6910f3beee6a852959856d Mon Sep 17 00:00:00 2001 From: Laramie Leavitt Date: Wed, 6 Aug 2025 19:47:03 -0700 Subject: [PATCH 10/26] reformat get_local_runtime_info.py and main.py --- examples/local_python/main.py | 1 - python/private/get_local_runtime_info.py | 271 +++++++++++------------ 2 files changed, 135 insertions(+), 137 deletions(-) diff --git a/examples/local_python/main.py b/examples/local_python/main.py index 236ceca3d5..c3e261dadd 100644 --- a/examples/local_python/main.py +++ b/examples/local_python/main.py @@ -13,7 +13,6 @@ # limitations under the License. - def main(): print(42) diff --git a/python/private/get_local_runtime_info.py b/python/private/get_local_runtime_info.py index 3ec9066b45..3db5c97784 100644 --- a/python/private/get_local_runtime_info.py +++ b/python/private/get_local_runtime_info.py @@ -18,151 +18,150 @@ import sysconfig _IS_WINDOWS = sys.platform == "win32" +_IS_DARWIN = sys.platform == "darwin" def _search_directories(get_config): - """Returns a list of library directories to search for shared libraries.""" - # There's several types of libraries with different names and a plethora - # of settings, and many different config variables to check: - # - # LIBPL is used in python-config when shared library is not enabled: - # https://github.com/python/cpython/blob/v3.12.0/Misc/python-config.in#L63 - # - # LIBDIR may also be the python directory with library files. - # https://stackoverflow.com/questions/47423246/get-pythons-lib-path - # See also: MULTIARCH - # - # On MacOS, the LDLIBRARY may be a relative path under /Library/Frameworks, - # such as "Python.framework/Versions/3.12/Python", not a file under the - # LIBDIR/LIBPL directory, so include PYTHONFRAMEWORKPREFIX. - lib_dirs = [ - get_config(x) for x in ("LIBPL", "LIBDIR", "PYTHONFRAMEWORKPREFIX") - ] - - # On Debian, with multiarch enabled, prior to Python 3.10, `LIBDIR` didn't - # tell the location of the libs, just the base directory. The `MULTIARCH` - # sysconfig variable tells the subdirectory within it with the libs. - # See: - # https://wiki.debian.org/Python/MultiArch - # https://git.launchpad.net/ubuntu/+source/python3.12/tree/debian/changelog#n842 - multiarch = get_config("MULTIARCH") - if multiarch: - for x in ["LIBPL", "LIBDIR"]: - config_value = get_config(x) - if config_value and not config_value.endswith(multiarch): - lib_dirs.append(os.path.join(config_value, multiarch)) - - if _IS_WINDOWS: - # On Windows DLLs go in the same directory as the executable, while .lib - # files live in the lib/ or libs/ subdirectory. - lib_dirs.append(get_config("BINDIR")) - lib_dirs.append(os.path.join(os.path.dirname(sys.executable))) - lib_dirs.append(os.path.join(os.path.dirname(sys.executable), "lib")) - lib_dirs.append(os.path.join(os.path.dirname(sys.executable), "libs")) - else: - # On most systems the executable is in a bin/ directory and the libraries - # are in a sibling lib/ directory. - lib_dirs.append( - os.path.join(os.path.dirname(os.path.dirname(sys.executable)), "lib") - ) - - # Dedup and remove empty values, keeping the order. - lib_dirs = [v for v in lib_dirs if v] - return {k: None for k in lib_dirs}.keys() + """Returns a list of library directories to search for shared libraries.""" + # There's several types of libraries with different names and a plethora + # of settings, and many different config variables to check: + # + # LIBPL is used in python-config when shared library is not enabled: + # https://github.com/python/cpython/blob/v3.12.0/Misc/python-config.in#L63 + # + # LIBDIR may also be the python directory with library files. + # https://stackoverflow.com/questions/47423246/get-pythons-lib-path + # See also: MULTIARCH + # + # On MacOS, the LDLIBRARY may be a relative path under /Library/Frameworks, + # such as "Python.framework/Versions/3.12/Python", not a file under the + # LIBDIR/LIBPL directory, so include PYTHONFRAMEWORKPREFIX. + lib_dirs = [get_config(x) for x in ("PYTHONFRAMEWORKPREFIX", "LIBPL", "LIBDIR")] + + # On Debian, with multiarch enabled, prior to Python 3.10, `LIBDIR` didn't + # tell the location of the libs, just the base directory. The `MULTIARCH` + # sysconfig variable tells the subdirectory within it with the libs. + # See: + # https://wiki.debian.org/Python/MultiArch + # https://git.launchpad.net/ubuntu/+source/python3.12/tree/debian/changelog#n842 + multiarch = get_config("MULTIARCH") + if multiarch: + for x in ["LIBPL", "LIBDIR"]: + config_value = get_config(x) + if config_value and not config_value.endswith(multiarch): + lib_dirs.append(os.path.join(config_value, multiarch)) + + if _IS_WINDOWS: + # On Windows DLLs go in the same directory as the executable, while .lib + # files live in the lib/ or libs/ subdirectory. + lib_dirs.append(get_config("BINDIR")) + lib_dirs.append(os.path.join(os.path.dirname(sys.executable))) + lib_dirs.append(os.path.join(os.path.dirname(sys.executable), "lib")) + lib_dirs.append(os.path.join(os.path.dirname(sys.executable), "libs")) + elif not _IS_DARWIN: + # On most systems the executable is in a bin/ directory and the libraries + # are in a sibling lib/ directory. + lib_dirs.append( + os.path.join(os.path.dirname(os.path.dirname(sys.executable)), "lib") + ) + + # Dedup and remove empty values, keeping the order. + lib_dirs = [v for v in lib_dirs if v] + return {k: None for k in lib_dirs}.keys() def _search_library_names(get_config): - """Returns a list of library files to search for shared libraries.""" - # Quoting configure.ac in the cpython code base: - # "INSTSONAME is the name of the shared library that will be use to install - # on the system - some systems like version suffix, others don't."" - # - # A typical INSTSONAME is 'libpython3.8.so.1.0' on Linux, or - # 'Python.framework/Versions/3.9/Python' on MacOS. Due to the possible - # version suffix we have to find the suffix within the filename. - # - # A typical LDLIBRARY is 'libpythonX.Y.so' on Linux, or 'pythonXY.dll' on - # Windows. - # - # A typical LIBRARY is 'libpythonX.Y.a' on Linux. - lib_names = [ - get_config(x) - for x in ( - "INSTSONAME", - "LDLIBRARY", - "PY3LIBRARY", - "LIBRARY", - "DLLLIBRARY", - ) - ] - - if _IS_WINDOWS: - so_prefix = "" - else: - so_prefix = "lib" - - # Get/override the SHLIB_SUFFIX, which is typically ".so" on Linux and - # ".dylib" on macOS. - so_suffix = get_config("SHLIB_SUFFIX") - if sys.platform == "darwin": - so_suffix = ".dylib" - elif _IS_WINDOWS: - so_suffix = ".lib" - elif not so_suffix: - so_suffix = ".so" - - version = get_config("VERSION") - abiflags = get_config("ABIFLAGS") or get_config("abiflags") or "" - - # On Windows, extensions should link with the pythonXY.lib files. - # See: https://docs.python.org/3/extending/windows.html - # So ensure that the pythonXY.lib files are included in the search. - if abiflags: - lib_names.append(f"{so_prefix}python{version}{abiflags}{so_suffix}") - lib_names.append(f"{so_prefix}python{version}{so_suffix}") - - # Dedup and remove empty values, keeping the order. - lib_names = [v for v in lib_names if v] - return {k: None for k in lib_names}.keys() + """Returns a list of library files to search for shared libraries.""" + # Quoting configure.ac in the cpython code base: + # "INSTSONAME is the name of the shared library that will be use to install + # on the system - some systems like version suffix, others don't."" + # + # A typical INSTSONAME is 'libpython3.8.so.1.0' on Linux, or + # 'Python.framework/Versions/3.9/Python' on MacOS. Due to the possible + # version suffix we have to find the suffix within the filename. + # + # A typical LDLIBRARY is 'libpythonX.Y.so' on Linux, or 'pythonXY.dll' on + # Windows. + # + # A typical LIBRARY is 'libpythonX.Y.a' on Linux. + lib_names = [ + get_config(x) + for x in ( + "INSTSONAME", + "LDLIBRARY", + "PY3LIBRARY", + "LIBRARY", + "DLLLIBRARY", + ) + ] + + if _IS_WINDOWS: + so_prefix = "" + else: + so_prefix = "lib" + + # Get/override the SHLIB_SUFFIX, which is typically ".so" on Linux and + # ".dylib" on macOS. + so_suffix = get_config("SHLIB_SUFFIX") + if _IS_DARWIN: + # SHLIB_SUFFIX may be ".so"; always override on darwin to be ".dynlib" + so_suffix = ".dylib" + elif _IS_WINDOWS: + # While the suffix is ".dll", the compiler needs to link with the ".lib" file. + so_suffix = ".lib" + elif not so_suffix: + so_suffix = ".so" + + version = get_config("VERSION") + abiflags = get_config("ABIFLAGS") or get_config("abiflags") or "" + + # On Windows, extensions should link with the pythonXY.lib files. + # See: https://docs.python.org/3/extending/windows.html + # So ensure that the pythonXY.lib files are included in the search. + if abiflags: + lib_names.append(f"{so_prefix}python{version}{abiflags}{so_suffix}") + lib_names.append(f"{so_prefix}python{version}{so_suffix}") + + # Dedup and remove empty values, keeping the order. + lib_names = [v for v in lib_names if v] + return {k: None for k in lib_names}.keys() def _get_python_library_info(): - """Returns a dictionary with the static and dynamic python libraries.""" - config_vars = sysconfig.get_config_vars() - - # VERSION is X.Y in Linux/macOS and XY in Windows. - if not config_vars.get("VERSION"): - if sys.platform == "win32": - config_vars["VERSION"] = ( - f"{sys.version_info.major}{sys.version_info.minor}" - ) - else: - config_vars["VERSION"] = ( - f"{sys.version_info.major}.{sys.version_info.minor}" - ) - - search_directories = _search_directories(config_vars.get) - search_libnames = _search_library_names(config_vars.get) - - dynamic_libraries = {} - static_libraries = {} - for root_dir in search_directories: - for libname in search_libnames: - composed_path = os.path.join(root_dir, libname) - if not os.path.exists(composed_path) or os.path.isdir(composed_path): - continue - if composed_path.endswith(".lib") or composed_path.endswith(".a"): - static_libraries[composed_path] = None - else: - dynamic_libraries[composed_path] = None - - # NOTE: It's possible that including the dynamic libraries currently loaded - # by the running python interpreter would be a useful addition. - - return { - "dynamic_libraries": list(dynamic_libraries.keys()), - "static_libraries": list(static_libraries.keys()), - } + """Returns a dictionary with the static and dynamic python libraries.""" + config_vars = sysconfig.get_config_vars() + + # VERSION is X.Y in Linux/macOS and XY in Windows. + if not config_vars.get("VERSION"): + if sys.platform == "win32": + config_vars["VERSION"] = f"{sys.version_info.major}{sys.version_info.minor}" + else: + config_vars["VERSION"] = ( + f"{sys.version_info.major}.{sys.version_info.minor}" + ) + + search_directories = _search_directories(config_vars.get) + search_libnames = _search_library_names(config_vars.get) + + dynamic_libraries = {} + static_libraries = {} + for root_dir in search_directories: + for libname in search_libnames: + composed_path = os.path.join(root_dir, libname) + if not os.path.exists(composed_path) or os.path.isdir(composed_path): + continue + if composed_path.endswith(".lib") or composed_path.endswith(".a"): + static_libraries[composed_path] = None + else: + dynamic_libraries[composed_path] = None + + # When no libraries are found it's likely that the python interpreter is not + # configured to use shared or static libraries (minilinux). If this seems + # suspicious try running `uv tool run find_libpython --list-all -v` + return { + "dynamic_libraries": list(dynamic_libraries.keys()), + "static_libraries": list(static_libraries.keys()), + } data = { From 87d539188a9b715a0ac5b7f46d61fa943caf12f5 Mon Sep 17 00:00:00 2001 From: Laramie Leavitt Date: Wed, 6 Aug 2025 20:02:15 -0700 Subject: [PATCH 11/26] Update CHANGELOG and minor comments. --- CHANGELOG.md | 3 ++- python/private/get_local_runtime_info.py | 3 +-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2020951773..5dfaccc7d2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -101,7 +101,8 @@ END_UNRELEASED_TEMPLATE `# gazelle:python_resolve_sibling_imports true` * (pypi) Show overridden index URL of packages when downloading metadata have failed. ([#2985](https://github.com/bazel-contrib/rules_python/issues/2985)). -* (toolchains) `local_runtime_repo` better handles variants in MacOS and Windows. +* (toolchains) `local_runtime_repo` Improvements in handling variations across python installations + for Linux, Windows and Mac. See ([#3148](https://github.com/bazel-contrib/rules_python/pull/3148)). {#v0-0-0-added} ### Added diff --git a/python/private/get_local_runtime_info.py b/python/private/get_local_runtime_info.py index 3db5c97784..90cb31dbb7 100644 --- a/python/private/get_local_runtime_info.py +++ b/python/private/get_local_runtime_info.py @@ -77,8 +77,7 @@ def _search_library_names(get_config): # on the system - some systems like version suffix, others don't."" # # A typical INSTSONAME is 'libpython3.8.so.1.0' on Linux, or - # 'Python.framework/Versions/3.9/Python' on MacOS. Due to the possible - # version suffix we have to find the suffix within the filename. + # 'Python.framework/Versions/3.9/Python' on MacOS. # # A typical LDLIBRARY is 'libpythonX.Y.so' on Linux, or 'pythonXY.dll' on # Windows. From 09bfee950d347927a03e8a6833c8139a3a1904a1 Mon Sep 17 00:00:00 2001 From: Laramie Leavitt Date: Wed, 6 Aug 2025 21:16:52 -0700 Subject: [PATCH 12/26] Minor tweaks to library name genration --- python/private/get_local_runtime_info.py | 34 ++++++++++++------------ 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/python/private/get_local_runtime_info.py b/python/private/get_local_runtime_info.py index 90cb31dbb7..7ea1ee11e9 100644 --- a/python/private/get_local_runtime_info.py +++ b/python/private/get_local_runtime_info.py @@ -94,32 +94,32 @@ def _search_library_names(get_config): ) ] - if _IS_WINDOWS: - so_prefix = "" - else: - so_prefix = "lib" - - # Get/override the SHLIB_SUFFIX, which is typically ".so" on Linux and - # ".dylib" on macOS. - so_suffix = get_config("SHLIB_SUFFIX") + # Set the prefix and suffix to construct the library name used for linking. if _IS_DARWIN: # SHLIB_SUFFIX may be ".so"; always override on darwin to be ".dynlib" - so_suffix = ".dylib" + suffix = ".dylib" + prefix = "lib" elif _IS_WINDOWS: - # While the suffix is ".dll", the compiler needs to link with the ".lib" file. - so_suffix = ".lib" - elif not so_suffix: - so_suffix = ".so" + # SHLIB_SUFFIX on windows is ".dll"; however the compiler needs to + # link with the ".lib". + suffix = ".lib" + prefix = "" + else: + suffix = get_config("SHLIB_SUFFIX") + prefix = "lib" + if not suffix: + suffix = ".so" version = get_config("VERSION") - abiflags = get_config("ABIFLAGS") or get_config("abiflags") or "" - # On Windows, extensions should link with the pythonXY.lib files. # See: https://docs.python.org/3/extending/windows.html # So ensure that the pythonXY.lib files are included in the search. + lib_names.append(f"{prefix}python{version}{suffix}") + + # If there are ABIFLAGS, also add them to the python version lib search. + abiflags = get_config("ABIFLAGS") or get_config("abiflags") or "" if abiflags: - lib_names.append(f"{so_prefix}python{version}{abiflags}{so_suffix}") - lib_names.append(f"{so_prefix}python{version}{so_suffix}") + lib_names.append(f"{prefix}python{version}{abiflags}{suffix}") # Dedup and remove empty values, keeping the order. lib_names = [v for v in lib_names if v] From dda37d6adb8f1f9759af2fc62df4b0dd39313905 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Thu, 7 Aug 2025 21:20:12 -0700 Subject: [PATCH 13/26] expand changelog description --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5dfaccc7d2..9b4d43f76a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -103,6 +103,12 @@ END_UNRELEASED_TEMPLATE ([#2985](https://github.com/bazel-contrib/rules_python/issues/2985)). * (toolchains) `local_runtime_repo` Improvements in handling variations across python installations for Linux, Windows and Mac. See ([#3148](https://github.com/bazel-contrib/rules_python/pull/3148)). +* (toolchains) `local_runtime_repo` now works on Windows + ([#3055](https://github.com/bazel-contrib/rules_python/issues/3055)). +* (toolchains) `local_runtime_repo` supports more types of Python + installations (Mac frameworks, missing dynamic libraries, and other + esoteric cases, see + [#3148](https://github.com/bazel-contrib/rules_python/pull/3148) for details). {#v0-0-0-added} ### Added From 0df9bf1dc99da1bee633754b67983230c4bc59dc Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Thu, 7 Aug 2025 22:09:13 -0700 Subject: [PATCH 14/26] name loop var, add non-uv libs finding comment --- python/private/get_local_runtime_info.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/python/private/get_local_runtime_info.py b/python/private/get_local_runtime_info.py index 7ea1ee11e9..40823b9d02 100644 --- a/python/private/get_local_runtime_info.py +++ b/python/private/get_local_runtime_info.py @@ -46,8 +46,8 @@ def _search_directories(get_config): # https://git.launchpad.net/ubuntu/+source/python3.12/tree/debian/changelog#n842 multiarch = get_config("MULTIARCH") if multiarch: - for x in ["LIBPL", "LIBDIR"]: - config_value = get_config(x) + for config_var_name in ["LIBPL", "LIBDIR"]: + config_value = get_config(config_var_name) if config_value and not config_value.endswith(multiarch): lib_dirs.append(os.path.join(config_value, multiarch)) @@ -156,7 +156,8 @@ def _get_python_library_info(): # When no libraries are found it's likely that the python interpreter is not # configured to use shared or static libraries (minilinux). If this seems - # suspicious try running `uv tool run find_libpython --list-all -v` + # suspicious try running `uv tool run find_libpython --list-all -v` or + # `python-config --libs`. return { "dynamic_libraries": list(dynamic_libraries.keys()), "static_libraries": list(static_libraries.keys()), From 5874d5c4f497d0e78b568c0e2bdc630355611435 Mon Sep 17 00:00:00 2001 From: Laramie Leavitt Date: Sat, 9 Aug 2025 19:54:53 -0700 Subject: [PATCH 15/26] Distinguish between interface library and dynamic library file paths. --- python/private/get_local_runtime_info.py | 42 ++++++++++++----- python/private/local_runtime_repo.bzl | 52 +++++++++++---------- python/private/local_runtime_repo_setup.bzl | 15 ++++-- 3 files changed, 68 insertions(+), 41 deletions(-) diff --git a/python/private/get_local_runtime_info.py b/python/private/get_local_runtime_info.py index 40823b9d02..51a35cf06e 100644 --- a/python/private/get_local_runtime_info.py +++ b/python/private/get_local_runtime_info.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +"""Returns information about the local Python runtime as JSON.""" + import json import os import sys @@ -36,7 +38,9 @@ def _search_directories(get_config): # On MacOS, the LDLIBRARY may be a relative path under /Library/Frameworks, # such as "Python.framework/Versions/3.12/Python", not a file under the # LIBDIR/LIBPL directory, so include PYTHONFRAMEWORKPREFIX. - lib_dirs = [get_config(x) for x in ("PYTHONFRAMEWORKPREFIX", "LIBPL", "LIBDIR")] + lib_dirs = [ + get_config(x) for x in ("PYTHONFRAMEWORKPREFIX", "LIBPL", "LIBDIR") + ] # On Debian, with multiarch enabled, prior to Python 3.10, `LIBDIR` didn't # tell the location of the libs, just the base directory. The `MULTIARCH` @@ -86,8 +90,8 @@ def _search_library_names(get_config): lib_names = [ get_config(x) for x in ( - "INSTSONAME", "LDLIBRARY", + "INSTSONAME", "PY3LIBRARY", "LIBRARY", "DLLLIBRARY", @@ -100,9 +104,7 @@ def _search_library_names(get_config): suffix = ".dylib" prefix = "lib" elif _IS_WINDOWS: - # SHLIB_SUFFIX on windows is ".dll"; however the compiler needs to - # link with the ".lib". - suffix = ".lib" + suffix = ".dll" prefix = "" else: suffix = get_config("SHLIB_SUFFIX") @@ -111,6 +113,7 @@ def _search_library_names(get_config): suffix = ".so" version = get_config("VERSION") + # On Windows, extensions should link with the pythonXY.lib files. # See: https://docs.python.org/3/extending/windows.html # So ensure that the pythonXY.lib files are included in the search. @@ -133,7 +136,9 @@ def _get_python_library_info(): # VERSION is X.Y in Linux/macOS and XY in Windows. if not config_vars.get("VERSION"): if sys.platform == "win32": - config_vars["VERSION"] = f"{sys.version_info.major}{sys.version_info.minor}" + config_vars["VERSION"] = ( + f"{sys.version_info.major}{sys.version_info.minor}" + ) else: config_vars["VERSION"] = ( f"{sys.version_info.major}.{sys.version_info.minor}" @@ -142,17 +147,31 @@ def _get_python_library_info(): search_directories = _search_directories(config_vars.get) search_libnames = _search_library_names(config_vars.get) + def _add_if_exists(target, path): + if os.path.exists(path) or os.path.isdir(path): + target[path] = None + + interface_libraries = {} dynamic_libraries = {} static_libraries = {} for root_dir in search_directories: for libname in search_libnames: composed_path = os.path.join(root_dir, libname) - if not os.path.exists(composed_path) or os.path.isdir(composed_path): + if libname.endswith(".a"): + _add_if_exists(static_libraries, composed_path) continue - if composed_path.endswith(".lib") or composed_path.endswith(".a"): - static_libraries[composed_path] = None - else: - dynamic_libraries[composed_path] = None + _add_if_exists(dynamic_libraries, composed_path) + + # On Windows, extensions should link with the pythonXY.lib interface + # libraries. See: https://docs.python.org/3/extending/windows.html + if libname.endswith(".so"): + _add_if_exists( + interface_libraries, os.path.join(root_dir, libname[:-2] + "ifso") + ) + elif libname.endswith(".dll"): + _add_if_exists( + interface_libraries, os.path.join(root_dir, libname[:-3] + "lib") + ) # When no libraries are found it's likely that the python interpreter is not # configured to use shared or static libraries (minilinux). If this seems @@ -161,6 +180,7 @@ def _get_python_library_info(): return { "dynamic_libraries": list(dynamic_libraries.keys()), "static_libraries": list(static_libraries.keys()), + "interface_libraries": list(interface_libraries.keys()), } diff --git a/python/private/local_runtime_repo.bzl b/python/private/local_runtime_repo.bzl index d3f051b3fc..13d3a680a1 100644 --- a/python/private/local_runtime_repo.bzl +++ b/python/private/local_runtime_repo.bzl @@ -35,7 +35,8 @@ define_local_runtime_toolchain_impl( minor = "{minor}", micro = "{micro}", interpreter_path = "{interpreter_path}", - interface_library = "{interface_library}", + interface_library = {interface_library}, + shared_library = {shared_library}, implementation_name = "{implementation_name}", os = "{os}", ) @@ -48,36 +49,36 @@ def _norm_path(path): path = path[:-1] return path -def _symlink_libs(rctx, logger, library_targets): +def _symlink_first_library(rctx, logger, libraries, is_shared = False): """Symlinks the shared libraries into the lib/ directory. Args: rctx: A repository_ctx object logger: A repo_utils.logger object - library_targets: A list of library targets to potentially symlink. + libraries: A list of static library paths to potentially symlink. + is_shared: Indicates whether the library is expected to be a shared library. Returns: - A library target suitable for a cc_import rule. - - The specific files are symlinked instead of the whole directory because - shared_lib_dirs contains multiple search paths for the shared libraries, - and the python files may be missing from any of those directories, and - any of those directories may include non-python runtime libraries, - as would be the case if LIBDIR were, for example, /usr/lib. + A single library path linked by the action. """ - found = "" - for target in library_targets: + linked = None + for target in libraries: origin = rctx.path(target) if not origin.exists: # The reported names don't always exist; it depends on the particulars # of the runtime installation. continue - found = "lib/{}".format(origin.basename) - logger.debug("Symlinking {} to {}".format(origin, found)) + if is_shared and repo_utils.get_platforms_os_name(rctx) == "osx" and not origin.basename.endswith(".dylib"): + # cc_import.shared_library has Permitted file types: .so, .dll or .dylib + linked = "lib/{}.dylib".format(origin.basename) + else: + linked = "lib/{}".format(origin.basename) + + logger.debug("Symlinking {} to {}".format(origin, linked)) repo_utils.watch(rctx, origin) - rctx.symlink(origin, found) + rctx.symlink(origin, linked) break - return found + return linked def _local_runtime_repo_impl(rctx): logger = repo_utils.logger(rctx) @@ -153,20 +154,20 @@ def _local_runtime_repo_impl(rctx): # appear as part of this repo. rctx.symlink(info["include"], "include") - if repo_utils.get_platforms_os_name == "windows": - library_targets = info["static_libraries"] - else: - library_targets = info["dynamic_libraries"] - rctx.report_progress("Symlinking external Python shared libraries") - interface_library = _symlink_libs(rctx, logger, library_targets) + interface_library = _symlink_first_library(rctx, logger, info["interface_libraries"]) + shared_library = _symlink_first_library(rctx, logger, info["dynamic_libraries"], True) + + if not interface_library and not shared_library: + logger.warn("No external python libraries found.") build_bazel = _TOOLCHAIN_IMPL_TEMPLATE.format( major = info["major"], minor = info["minor"], micro = info["micro"], - interpreter_path = _norm_path(interpreter_path), - interface_library = interface_library, + interpreter_path = interpreter_path, + interface_library = repr(interface_library), + shared_library = repr(shared_library), implementation_name = info["implementation_name"], os = "@platforms//os:{}".format(repo_utils.get_platforms_os_name(rctx)), ) @@ -235,7 +236,8 @@ def _expand_incompatible_template(): return _TOOLCHAIN_IMPL_TEMPLATE.format( interpreter_path = "/incompatible", implementation_name = "incompatible", - interface_library = "", + interface_library = "None", + shared_library = "None", major = "0", minor = "0", micro = "0", diff --git a/python/private/local_runtime_repo_setup.bzl b/python/private/local_runtime_repo_setup.bzl index 09afc4d94e..37c1afbc72 100644 --- a/python/private/local_runtime_repo_setup.bzl +++ b/python/private/local_runtime_repo_setup.bzl @@ -31,6 +31,7 @@ def define_local_runtime_toolchain_impl( micro, interpreter_path, interface_library, + shared_library, implementation_name, os): """Defines a toolchain implementation for a local Python runtime. @@ -50,8 +51,10 @@ def define_local_runtime_toolchain_impl( minor: `str` The minor Python version, e.g. `9` of `3.9.1`. micro: `str` The micro Python version, e.g. "1" of `3.9.1`. interpreter_path: `str` Absolute path to the interpreter. - interface_library: `str` A path to a .lib or .so file to link against. + interface_library: `str` Path to the interface library. e.g. "lib/python312.lib" + shared_library: `str` Path to the dynamic library. + e.g. "lib/python312.dll" or "lib/python312.so" implementation_name: `str` The implementation name, as returned by `sys.implementation.name`. os: `str` A label to the OS constraint (e.g. `@platforms//os:linux`) for @@ -64,14 +67,14 @@ def define_local_runtime_toolchain_impl( # See https://docs.python.org/3/extending/windows.html # However not all python installations (such as manylinux) include shared or static libraries, # so only create the import library when interface_library is set. - _libpython_deps = [] + import_deps = [] if interface_library: cc_import( - name = "_python_import_lib", + name = "_python_interface_library", interface_library = interface_library, system_provided = 1, ) - _libpython_deps.append(":_python_import_lib") + import_deps = [":_python_interface_library"] cc_library( name = "_python_headers", @@ -81,13 +84,15 @@ def define_local_runtime_toolchain_impl( # A Python install may not have C headers allow_empty = True, ), + deps = import_deps, includes = ["include"], ) cc_library( name = "_libpython", hdrs = [":_python_headers"], - deps = _libpython_deps, + srcs = [shared_library] if shared_library else [], + deps = import_deps, ) py_runtime( From 7bafc7264354d115e9110da01bf0b971be611d9d Mon Sep 17 00:00:00 2001 From: Laramie Leavitt Date: Sun, 10 Aug 2025 12:23:52 -0700 Subject: [PATCH 16/26] Refine comments, a few other minor changes --- python/private/get_local_runtime_info.py | 46 ++++++++++++------------ python/private/local_runtime_repo.bzl | 14 +++----- 2 files changed, 28 insertions(+), 32 deletions(-) diff --git a/python/private/get_local_runtime_info.py b/python/private/get_local_runtime_info.py index 51a35cf06e..8f923e1feb 100644 --- a/python/private/get_local_runtime_info.py +++ b/python/private/get_local_runtime_info.py @@ -38,9 +38,7 @@ def _search_directories(get_config): # On MacOS, the LDLIBRARY may be a relative path under /Library/Frameworks, # such as "Python.framework/Versions/3.12/Python", not a file under the # LIBDIR/LIBPL directory, so include PYTHONFRAMEWORKPREFIX. - lib_dirs = [ - get_config(x) for x in ("PYTHONFRAMEWORKPREFIX", "LIBPL", "LIBDIR") - ] + lib_dirs = [get_config(x) for x in ("PYTHONFRAMEWORKPREFIX", "LIBPL", "LIBDIR")] # On Debian, with multiarch enabled, prior to Python 3.10, `LIBDIR` didn't # tell the location of the libs, just the base directory. The `MULTIARCH` @@ -50,8 +48,8 @@ def _search_directories(get_config): # https://git.launchpad.net/ubuntu/+source/python3.12/tree/debian/changelog#n842 multiarch = get_config("MULTIARCH") if multiarch: - for config_var_name in ["LIBPL", "LIBDIR"]: - config_value = get_config(config_var_name) + for x in ("LIBPL", "LIBDIR"): + config_value = get_config(x) if config_value and not config_value.endswith(multiarch): lib_dirs.append(os.path.join(config_value, multiarch)) @@ -84,7 +82,7 @@ def _search_library_names(get_config): # 'Python.framework/Versions/3.9/Python' on MacOS. # # A typical LDLIBRARY is 'libpythonX.Y.so' on Linux, or 'pythonXY.dll' on - # Windows. + # Windows, or 'Python.framework/Versions/3.9/Python' on MacOS. # # A typical LIBRARY is 'libpythonX.Y.a' on Linux. lib_names = [ @@ -99,8 +97,9 @@ def _search_library_names(get_config): ] # Set the prefix and suffix to construct the library name used for linking. + # The suffix and version are set here to the default values for the OS, + # since they are used below to construct "default" library names. if _IS_DARWIN: - # SHLIB_SUFFIX may be ".so"; always override on darwin to be ".dynlib" suffix = ".dylib" prefix = "lib" elif _IS_WINDOWS: @@ -114,9 +113,7 @@ def _search_library_names(get_config): version = get_config("VERSION") - # On Windows, extensions should link with the pythonXY.lib files. - # See: https://docs.python.org/3/extending/windows.html - # So ensure that the pythonXY.lib files are included in the search. + # Ensure that the pythonXY.dll files are included in the search. lib_names.append(f"{prefix}python{version}{suffix}") # If there are ABIFLAGS, also add them to the python version lib search. @@ -136,9 +133,7 @@ def _get_python_library_info(): # VERSION is X.Y in Linux/macOS and XY in Windows. if not config_vars.get("VERSION"): if sys.platform == "win32": - config_vars["VERSION"] = ( - f"{sys.version_info.major}{sys.version_info.minor}" - ) + config_vars["VERSION"] = f"{sys.version_info.major}{sys.version_info.minor}" else: config_vars["VERSION"] = ( f"{sys.version_info.major}.{sys.version_info.minor}" @@ -160,23 +155,30 @@ def _add_if_exists(target, path): if libname.endswith(".a"): _add_if_exists(static_libraries, composed_path) continue - _add_if_exists(dynamic_libraries, composed_path) - # On Windows, extensions should link with the pythonXY.lib interface - # libraries. See: https://docs.python.org/3/extending/windows.html - if libname.endswith(".so"): + _add_if_exists(dynamic_libraries, composed_path) + if libname.endswith(".dll"): + # On windows a .lib file may be an "import library" or a static library. + # The file could be inspected to determine which it is; typically python + # is used as a shared library. + # + # On Windows, extensions should link with the pythonXY.lib interface + # libraries. + # + # See: https://docs.python.org/3/extending/windows.html + # https://learn.microsoft.com/en-us/windows/win32/dlls/dynamic-link-library-creation _add_if_exists( - interface_libraries, os.path.join(root_dir, libname[:-2] + "ifso") + interface_libraries, os.path.join(root_dir, libname[:-3] + "lib") ) - elif libname.endswith(".dll"): + elif libname.endswith(".so"): + # It's possible, though unlikely, that interface stubs (.ifso) exist. _add_if_exists( - interface_libraries, os.path.join(root_dir, libname[:-3] + "lib") + interface_libraries, os.path.join(root_dir, libname[:-2] + "ifso") ) # When no libraries are found it's likely that the python interpreter is not # configured to use shared or static libraries (minilinux). If this seems - # suspicious try running `uv tool run find_libpython --list-all -v` or - # `python-config --libs`. + # suspicious try running `uv tool run find_libpython --list-all -v` return { "dynamic_libraries": list(dynamic_libraries.keys()), "static_libraries": list(static_libraries.keys()), diff --git a/python/private/local_runtime_repo.bzl b/python/private/local_runtime_repo.bzl index 13d3a680a1..311fedc6d7 100644 --- a/python/private/local_runtime_repo.bzl +++ b/python/private/local_runtime_repo.bzl @@ -49,14 +49,13 @@ def _norm_path(path): path = path[:-1] return path -def _symlink_first_library(rctx, logger, libraries, is_shared = False): +def _symlink_first_library(rctx, logger, libraries): """Symlinks the shared libraries into the lib/ directory. Args: rctx: A repository_ctx object logger: A repo_utils.logger object libraries: A list of static library paths to potentially symlink. - is_shared: Indicates whether the library is expected to be a shared library. Returns: A single library path linked by the action. """ @@ -67,12 +66,7 @@ def _symlink_first_library(rctx, logger, libraries, is_shared = False): # The reported names don't always exist; it depends on the particulars # of the runtime installation. continue - if is_shared and repo_utils.get_platforms_os_name(rctx) == "osx" and not origin.basename.endswith(".dylib"): - # cc_import.shared_library has Permitted file types: .so, .dll or .dylib - linked = "lib/{}.dylib".format(origin.basename) - else: - linked = "lib/{}".format(origin.basename) - + linked = "lib/{}".format(origin.basename) logger.debug("Symlinking {} to {}".format(origin, linked)) repo_utils.watch(rctx, origin) rctx.symlink(origin, linked) @@ -156,7 +150,7 @@ def _local_runtime_repo_impl(rctx): rctx.report_progress("Symlinking external Python shared libraries") interface_library = _symlink_first_library(rctx, logger, info["interface_libraries"]) - shared_library = _symlink_first_library(rctx, logger, info["dynamic_libraries"], True) + shared_library = _symlink_first_library(rctx, logger, info["dynamic_libraries"]) if not interface_library and not shared_library: logger.warn("No external python libraries found.") @@ -165,7 +159,7 @@ def _local_runtime_repo_impl(rctx): major = info["major"], minor = info["minor"], micro = info["micro"], - interpreter_path = interpreter_path, + interpreter_path = _norm_path(interpreter_path), interface_library = repr(interface_library), shared_library = repr(shared_library), implementation_name = info["implementation_name"], From 7c3428ee07212136cf559cead9e7f34dc404caf1 Mon Sep 17 00:00:00 2001 From: Laramie Leavitt Date: Sun, 10 Aug 2025 12:29:52 -0700 Subject: [PATCH 17/26] Update comment. --- python/private/get_local_runtime_info.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/private/get_local_runtime_info.py b/python/private/get_local_runtime_info.py index 8f923e1feb..ff3b0aeb01 100644 --- a/python/private/get_local_runtime_info.py +++ b/python/private/get_local_runtime_info.py @@ -130,7 +130,8 @@ def _get_python_library_info(): """Returns a dictionary with the static and dynamic python libraries.""" config_vars = sysconfig.get_config_vars() - # VERSION is X.Y in Linux/macOS and XY in Windows. + # VERSION is X.Y in Linux/macOS and XY in Windows. This is used to + # construct library paths such as python3.12, so ensure it exists. if not config_vars.get("VERSION"): if sys.platform == "win32": config_vars["VERSION"] = f"{sys.version_info.major}{sys.version_info.minor}" From 499ac15cdcee85b9b346e7908bc275e85e8ffee7 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Mon, 11 Aug 2025 10:21:48 -0700 Subject: [PATCH 18/26] fix incorrect merge --- CHANGELOG.md | 6 ------ 1 file changed, 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5beac109ca..3707d33c04 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -109,18 +109,12 @@ END_UNRELEASED_TEMPLATE * (toolchains) use "command -v" to find interpreter in `$PATH` ([#3150](https://github.com/bazel-contrib/rules_python/pull/3150)). * (pypi) `bazel vendor` now works in `bzlmod` ({gh-issue}`3079`). -* (toolchains) `local_runtime_repo` Improvements in handling variations across python installations - for Linux, Windows and Mac. See ([#3148](https://github.com/bazel-contrib/rules_python/pull/3148)). * (toolchains) `local_runtime_repo` now works on Windows ([#3055](https://github.com/bazel-contrib/rules_python/issues/3055)). * (toolchains) `local_runtime_repo` supports more types of Python installations (Mac frameworks, missing dynamic libraries, and other esoteric cases, see [#3148](https://github.com/bazel-contrib/rules_python/pull/3148) for details). -* (pypi) Correctly pull `sdist` distributions using `pip` - ([#3131](https://github.com/bazel-contrib/rules_python/pull/3131)). -* (core) builds work again on `7.x` `WORKSPACE` configurations - ([#3119](https://github.com/bazel-contrib/rules_python/issues/3119)). {#v0-0-0-added} ### Added From 66a16f9886bade85fac0c8877e313401c3466846 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Mon, 11 Aug 2025 11:17:30 -0700 Subject: [PATCH 19/26] Add workspace to tests/integration/local_toolchains --- tests/integration/local_toolchains/WORKSPACE | 31 ++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/tests/integration/local_toolchains/WORKSPACE b/tests/integration/local_toolchains/WORKSPACE index e69de29bb2..4f192baa2a 100644 --- a/tests/integration/local_toolchains/WORKSPACE +++ b/tests/integration/local_toolchains/WORKSPACE @@ -0,0 +1,31 @@ +workspace( + name = "module_under_test", +) + +local_repository( + name = "rules_python", + path = "../../..", +) + +load("@rules_python//python:repositories.bzl", "py_repositories") + +py_repositories() + +load("@rules_python//python/local_toolchains:repos.bzl", "local_runtime_repo", "local_runtime_toolchains_repo") + +# Step 1: Define the python runtime. +local_runtime_repo( + name = "local_python3", + interpreter_path = "python3", + on_failure = "fail", + # or interpreter_path = "C:\\path\\to\\python.exe" +) + +# Step 2: Create toolchains for the runtimes +local_runtime_toolchains_repo( + name = "local_toolchains", + runtimes = ["local_python3"], +) + +# Step 3: Register the toolchains +register_toolchains("@local_toolchains//:all") From 9454daa8987938f9698493fb8dbdc614657cb866 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Mon, 11 Aug 2025 11:18:23 -0700 Subject: [PATCH 20/26] Delete examples/local_python directory moved to tests/local_toolchains --- examples/local_python/.bazelrc | 4 ---- examples/local_python/.gitignore | 4 ---- examples/local_python/BUILD.bazel | 6 ------ examples/local_python/WORKSPACE | 31 ------------------------------- examples/local_python/main.py | 21 --------------------- 5 files changed, 66 deletions(-) delete mode 100644 examples/local_python/.bazelrc delete mode 100644 examples/local_python/.gitignore delete mode 100644 examples/local_python/BUILD.bazel delete mode 100644 examples/local_python/WORKSPACE delete mode 100644 examples/local_python/main.py diff --git a/examples/local_python/.bazelrc b/examples/local_python/.bazelrc deleted file mode 100644 index 2ed86f591e..0000000000 --- a/examples/local_python/.bazelrc +++ /dev/null @@ -1,4 +0,0 @@ -# The equivalent bzlmod behavior is covered by examples/bzlmod/py_proto_library -common --noenable_bzlmod -common --enable_workspace -common --incompatible_python_disallow_native_rules diff --git a/examples/local_python/.gitignore b/examples/local_python/.gitignore deleted file mode 100644 index e5ae073b3c..0000000000 --- a/examples/local_python/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -# git ignore patterns - -/bazel-* -user.bazelrc diff --git a/examples/local_python/BUILD.bazel b/examples/local_python/BUILD.bazel deleted file mode 100644 index 1a6d5cbe8a..0000000000 --- a/examples/local_python/BUILD.bazel +++ /dev/null @@ -1,6 +0,0 @@ -load("@rules_python//python:py_binary.bzl", "py_binary") - -py_binary( - name = "main", - srcs = ["main.py"], -) diff --git a/examples/local_python/WORKSPACE b/examples/local_python/WORKSPACE deleted file mode 100644 index f39bf14b86..0000000000 --- a/examples/local_python/WORKSPACE +++ /dev/null @@ -1,31 +0,0 @@ -workspace( - name = "local_python_example", -) - -local_repository( - name = "rules_python", - path = "../..", -) - -load("@rules_python//python:repositories.bzl", "py_repositories") - -py_repositories() - -load("@rules_python//python/local_toolchains:repos.bzl", "local_runtime_repo", "local_runtime_toolchains_repo") - -# Step 1: Define the python runtime. -local_runtime_repo( - name = "local_python3", - interpreter_path = "python3", - on_failure = "fail", - # or interpreter_path = "C:\\path\\to\\python.exe" -) - -# Step 2: Create toolchains for the runtimes -local_runtime_toolchains_repo( - name = "local_toolchains", - runtimes = ["local_python3"], -) - -# Step 3: Register the toolchains -register_toolchains("@local_toolchains//:all") diff --git a/examples/local_python/main.py b/examples/local_python/main.py deleted file mode 100644 index c3e261dadd..0000000000 --- a/examples/local_python/main.py +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright 2025 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -def main(): - print(42) - - -if __name__ == "__main__": - main() From a9f1e07ca12b3c513e9c726b24a88897de86353c Mon Sep 17 00:00:00 2001 From: Laramie Leavitt Date: Mon, 11 Aug 2025 11:21:13 -0700 Subject: [PATCH 21/26] Fallback to shared library if there is no static library --- python/private/local_runtime_repo.bzl | 16 +++++++++++----- python/private/local_runtime_repo_setup.bzl | 8 ++++---- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/python/private/local_runtime_repo.bzl b/python/private/local_runtime_repo.bzl index 311fedc6d7..95103c7873 100644 --- a/python/private/local_runtime_repo.bzl +++ b/python/private/local_runtime_repo.bzl @@ -36,7 +36,7 @@ define_local_runtime_toolchain_impl( micro = "{micro}", interpreter_path = "{interpreter_path}", interface_library = {interface_library}, - shared_library = {shared_library}, + libraries = {libraries}, implementation_name = "{implementation_name}", os = "{os}", ) @@ -151,8 +151,14 @@ def _local_runtime_repo_impl(rctx): rctx.report_progress("Symlinking external Python shared libraries") interface_library = _symlink_first_library(rctx, logger, info["interface_libraries"]) shared_library = _symlink_first_library(rctx, logger, info["dynamic_libraries"]) - - if not interface_library and not shared_library: + static_library = _symlink_first_library(rctx, logger, info["static_libraries"]) + + libraries = [] + if shared_library: + libraries.append(shared_library) + elif static_library: + libraries.append(static_library) + if not libraries: logger.warn("No external python libraries found.") build_bazel = _TOOLCHAIN_IMPL_TEMPLATE.format( @@ -161,7 +167,7 @@ def _local_runtime_repo_impl(rctx): micro = info["micro"], interpreter_path = _norm_path(interpreter_path), interface_library = repr(interface_library), - shared_library = repr(shared_library), + libraries = repr(libraries), implementation_name = info["implementation_name"], os = "@platforms//os:{}".format(repo_utils.get_platforms_os_name(rctx)), ) @@ -231,7 +237,7 @@ def _expand_incompatible_template(): interpreter_path = "/incompatible", implementation_name = "incompatible", interface_library = "None", - shared_library = "None", + libraries = "[]", major = "0", minor = "0", micro = "0", diff --git a/python/private/local_runtime_repo_setup.bzl b/python/private/local_runtime_repo_setup.bzl index 37c1afbc72..ff35562eae 100644 --- a/python/private/local_runtime_repo_setup.bzl +++ b/python/private/local_runtime_repo_setup.bzl @@ -31,7 +31,7 @@ def define_local_runtime_toolchain_impl( micro, interpreter_path, interface_library, - shared_library, + libraries, implementation_name, os): """Defines a toolchain implementation for a local Python runtime. @@ -53,8 +53,8 @@ def define_local_runtime_toolchain_impl( interpreter_path: `str` Absolute path to the interpreter. interface_library: `str` Path to the interface library. e.g. "lib/python312.lib" - shared_library: `str` Path to the dynamic library. - e.g. "lib/python312.dll" or "lib/python312.so" + libraries: `list[str]` Path[s] to the python libraries. + e.g. ["lib/python312.dll"] or ["lib/python312.so"] implementation_name: `str` The implementation name, as returned by `sys.implementation.name`. os: `str` A label to the OS constraint (e.g. `@platforms//os:linux`) for @@ -91,7 +91,7 @@ def define_local_runtime_toolchain_impl( cc_library( name = "_libpython", hdrs = [":_python_headers"], - srcs = [shared_library] if shared_library else [], + srcs = libraries, deps = import_deps, ) From 5757bcd725eb590a4c319f676d94efcb83687291 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Mon, 11 Aug 2025 11:26:52 -0700 Subject: [PATCH 22/26] add workspace test for local_toolchains --- tests/integration/BUILD.bazel | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/integration/BUILD.bazel b/tests/integration/BUILD.bazel index d178e0f01c..df7fe15444 100644 --- a/tests/integration/BUILD.bazel +++ b/tests/integration/BUILD.bazel @@ -95,6 +95,17 @@ rules_python_integration_test( ], ) +rules_python_integration_test( + name = "local_toolchains_workspace_test", + bazel_versions = [ + version + for version in bazel_binaries.versions.all + if not version.startswith("6.") + ], + bzlmod = False, + workspace_path = "local_toolchains", +) + rules_python_integration_test( name = "pip_parse_test", ) From 931a1b6b99c304a5c5a69aa5e6f7efd5c5bbf279 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Mon, 11 Aug 2025 11:27:37 -0700 Subject: [PATCH 23/26] add mac, windows bazel-in-bazel jobs --- .bazelci/presubmit.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.bazelci/presubmit.yml b/.bazelci/presubmit.yml index 6457363ccd..77376a54de 100644 --- a/.bazelci/presubmit.yml +++ b/.bazelci/presubmit.yml @@ -503,6 +503,14 @@ tasks: <<: *common_bazelinbazel_config name: "tests/integration bazel-in-bazel: Debian" platform: debian11 + integration_test_bazelinbazel_macos: + <<: *common_bazelinbazel_config + name: "tests/integration bazel-in-bazel: macOS" + platform: macos + integration_test_bazelinbazel_windows: + <<: *common_bazelinbazel_config + name: "tests/integration bazel-in-bazel: Windows" + platform: windows integration_test_compile_pip_requirements_ubuntu: <<: *reusable_build_test_all From db3de32cc5197684167df66fa289fc1a7f64633a Mon Sep 17 00:00:00 2001 From: Laramie Leavitt Date: Mon, 11 Aug 2025 11:38:14 -0700 Subject: [PATCH 24/26] buildifier WORKSPACE --- tests/integration/local_toolchains/WORKSPACE | 62 ++++++++++---------- 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/tests/integration/local_toolchains/WORKSPACE b/tests/integration/local_toolchains/WORKSPACE index 4f192baa2a..480cd2794a 100644 --- a/tests/integration/local_toolchains/WORKSPACE +++ b/tests/integration/local_toolchains/WORKSPACE @@ -1,31 +1,31 @@ -workspace( - name = "module_under_test", -) - -local_repository( - name = "rules_python", - path = "../../..", -) - -load("@rules_python//python:repositories.bzl", "py_repositories") - -py_repositories() - -load("@rules_python//python/local_toolchains:repos.bzl", "local_runtime_repo", "local_runtime_toolchains_repo") - -# Step 1: Define the python runtime. -local_runtime_repo( - name = "local_python3", - interpreter_path = "python3", - on_failure = "fail", - # or interpreter_path = "C:\\path\\to\\python.exe" -) - -# Step 2: Create toolchains for the runtimes -local_runtime_toolchains_repo( - name = "local_toolchains", - runtimes = ["local_python3"], -) - -# Step 3: Register the toolchains -register_toolchains("@local_toolchains//:all") +workspace( + name = "module_under_test", +) + +local_repository( + name = "rules_python", + path = "../../..", +) + +load("@rules_python//python:repositories.bzl", "py_repositories") + +py_repositories() + +load("@rules_python//python/local_toolchains:repos.bzl", "local_runtime_repo", "local_runtime_toolchains_repo") + +# Step 1: Define the python runtime. +local_runtime_repo( + name = "local_python3", + interpreter_path = "python3", + on_failure = "fail", + # or interpreter_path = "C:\\path\\to\\python.exe" +) + +# Step 2: Create toolchains for the runtimes +local_runtime_toolchains_repo( + name = "local_toolchains", + runtimes = ["local_python3"], +) + +# Step 3: Register the toolchains +register_toolchains("@local_toolchains//:all") From f6f31ffdca04e29c5ccbdefcb6e6b6deb6a10d96 Mon Sep 17 00:00:00 2001 From: Laramie Leavitt Date: Mon, 11 Aug 2025 16:23:05 -0700 Subject: [PATCH 25/26] Add a test of building an extension for local_runtime --- .gitignore | 2 + python/private/local_runtime_repo.bzl | 5 +- python/private/local_runtime_repo_setup.bzl | 2 +- .../integration/local_toolchains/BUILD.bazel | 17 +++ .../integration/local_toolchains/echo_ext.cc | 21 +++ .../integration/local_toolchains/echo_test.py | 11 ++ .../local_toolchains/py_extension.bzl | 133 ++++++++++++++++++ 7 files changed, 189 insertions(+), 2 deletions(-) create mode 100644 tests/integration/local_toolchains/echo_ext.cc create mode 100644 tests/integration/local_toolchains/echo_test.py create mode 100644 tests/integration/local_toolchains/py_extension.bzl diff --git a/.gitignore b/.gitignore index 863b0e9c3f..fb1b17e466 100644 --- a/.gitignore +++ b/.gitignore @@ -37,6 +37,8 @@ /bazel-genfiles /bazel-out /bazel-testlogs +**/bazel-* + user.bazelrc # vim swap files diff --git a/python/private/local_runtime_repo.bzl b/python/private/local_runtime_repo.bzl index 95103c7873..ed440ff8df 100644 --- a/python/private/local_runtime_repo.bzl +++ b/python/private/local_runtime_repo.bzl @@ -66,7 +66,10 @@ def _symlink_first_library(rctx, logger, libraries): # The reported names don't always exist; it depends on the particulars # of the runtime installation. continue - linked = "lib/{}".format(origin.basename) + if target.endswith("/Python"): + linked = "lib/{}.dylib".format(origin.basename) + else: + linked = "lib/{}".format(origin.basename) logger.debug("Symlinking {} to {}".format(origin, linked)) repo_utils.watch(rctx, origin) rctx.symlink(origin, linked) diff --git a/python/private/local_runtime_repo_setup.bzl b/python/private/local_runtime_repo_setup.bzl index ff35562eae..1890ef0a0f 100644 --- a/python/private/local_runtime_repo_setup.bzl +++ b/python/private/local_runtime_repo_setup.bzl @@ -92,7 +92,7 @@ def define_local_runtime_toolchain_impl( name = "_libpython", hdrs = [":_python_headers"], srcs = libraries, - deps = import_deps, + deps = [], ) py_runtime( diff --git a/tests/integration/local_toolchains/BUILD.bazel b/tests/integration/local_toolchains/BUILD.bazel index 6b731181a6..c942be1159 100644 --- a/tests/integration/local_toolchains/BUILD.bazel +++ b/tests/integration/local_toolchains/BUILD.bazel @@ -14,6 +14,7 @@ load("@bazel_skylib//rules:common_settings.bzl", "string_flag") load("@rules_python//python:py_test.bzl", "py_test") +load(":py_extension.bzl", "py_extension") py_test( name = "test", @@ -35,3 +36,19 @@ string_flag( name = "py", build_setting_default = "", ) + +# Build rules to generate a python extension. +py_extension( + name = "echo_ext", + srcs = ["echo_ext.cc"], + deps = [ + "@rules_python//python/cc:current_py_cc_headers", + ], + testonly = True, +) + +py_test( + name = "echo_test", + srcs = ["echo_test.py"], + deps = [":echo"], +) diff --git a/tests/integration/local_toolchains/echo_ext.cc b/tests/integration/local_toolchains/echo_ext.cc new file mode 100644 index 0000000000..74ff6a33d3 --- /dev/null +++ b/tests/integration/local_toolchains/echo_ext.cc @@ -0,0 +1,21 @@ +#include + +static PyObject *echoArgs(PyObject *self, PyObject *args) { return args; } + +static PyMethodDef echo_methods[] = { + { "echo", echoArgs, METH_VARARGS, "Returns a tuple of the input args" }, + { NULL, NULL, 0, NULL }, +}; + +extern "C" { + +PyMODINIT_FUNC PyInit_echo(void) { + static struct PyModuleDef echo_module_def = { + // Module definition + PyModuleDef_HEAD_INIT, "echo_ext", "'echo_ext' module", -1, echo_methods + }; + + return PyModule_Create(&echo_module_def); +} + +} // extern "C" diff --git a/tests/integration/local_toolchains/echo_test.py b/tests/integration/local_toolchains/echo_test.py new file mode 100644 index 0000000000..c03c5f2831 --- /dev/null +++ b/tests/integration/local_toolchains/echo_test.py @@ -0,0 +1,11 @@ + +import echo_ext + +import unittest + + + +class ExtensionTest(unittest.TestCase): + + def test_echo_extension(self): + self.assertEqual(echo.echo(42, "str"), tuple(42, "str")) \ No newline at end of file diff --git a/tests/integration/local_toolchains/py_extension.bzl b/tests/integration/local_toolchains/py_extension.bzl new file mode 100644 index 0000000000..ddea0fafed --- /dev/null +++ b/tests/integration/local_toolchains/py_extension.bzl @@ -0,0 +1,133 @@ +load("@bazel_skylib//rules:copy_file.bzl", "copy_file") + +def py_extension( + name = None, + srcs = None, + hdrs = None, + data = None, + local_defines = None, + visibility = None, + linkopts = None, + deps = None, + testonly = False, + imports = None): + """Creates a Python module implemented in C++. + + Python modules can depend on a py_extension. Other py_extensions can depend + on a generated C++ library named with "_cc" suffix. + + Args: + name: Name for this target. + srcs: C++ source files. + hdrs: C++ header files, for other py_extensions which depend on this. + data: Files needed at runtime. This may include Python libraries. + visibility: Controls which rules can depend on this. + deps: Other C++ libraries that this library depends upon. + """ + if not linkopts: + linkopts = [] + + cc_library_name = name + "_cc" + cc_binary_so_name = name + ".so" + cc_binary_dll_name = name + ".dll" + cc_binary_pyd_name = name + ".pyd" + linker_script_name = name + ".lds" + linker_script_name_rule = name + "_lds" + shared_objects_name = name + "__shared_objects" + # buildifier: disable=native-cc + native.cc_library( + name = cc_library_name, + srcs = srcs, + hdrs = hdrs, + data = data, + local_defines = local_defines, + visibility = visibility, + deps = deps, + testonly = testonly, + alwayslink = True, + ) + + # On Unix, restrict symbol visibility. + exported_symbol = "PyInit_" + name + + # Generate linker script used on non-macOS unix platforms. + native.genrule( + name = linker_script_name_rule, + outs = [linker_script_name], + cmd = "\n".join([ + "cat <<'EOF' >$@", + "{", + " global: " + exported_symbol + ";", + " local: *;", + "};", + "EOF", + ]), + ) + + for cc_binary_name in [cc_binary_dll_name, cc_binary_so_name]: + cur_linkopts = linkopts + cur_deps = [cc_library_name] + if cc_binary_name == cc_binary_so_name: + cur_linkopts = linkopts + select({ + "@platforms//os:macos": [ + # Avoid undefined symbol errors for CPython symbols that + # will be resolved at runtime. + "-undefined", + "dynamic_lookup", + # On macOS, the linker does not support version scripts. Use + # the `-exported_symbol` option instead to restrict symbol + # visibility. + "-Wl,-exported_symbol", + # On macOS, the symbol starts with an underscore. + "-Wl,_" + exported_symbol, + ], + # On non-macOS unix, use a version script to restrict symbol + # visibility. + "//conditions:default": [ + "-Wl,--version-script", + "-Wl,$(location :" + linker_script_name + ")", + ], + }) + cur_deps = cur_deps + select({ + "@platforms//os:macos": [], + "//conditions:default": [linker_script_name], + }) + # buildifier: disable=native-cc + native.cc_binary( + name = cc_binary_name, + linkshared = True, + #linkstatic = True, + visibility = ["//visibility:private"], + deps = cur_deps, + tags = ["manual"], + testonly = testonly, + linkopts = cur_linkopts, + ) + + copy_file( + name = cc_binary_pyd_name + "__pyd_copy", + src = ":" + cc_binary_dll_name, + out = cc_binary_pyd_name, + visibility = visibility, + tags = ["manual"], + testonly = testonly, + ) + + native.filegroup( + name = shared_objects_name, + data = select({ + "@platforms//os:windows": [ + ":" + cc_binary_pyd_name, + ], + "//conditions:default": [":" + cc_binary_so_name], + }), + testonly = testonly, + ) + + native.py_library( + name = name, + data = [":" + shared_objects_name], + imports = imports, + testonly = testonly, + visibility = visibility, + ) From 70509e1b3c67b4e9114e7cbc5168d8e528dfd8b5 Mon Sep 17 00:00:00 2001 From: Laramie Leavitt Date: Mon, 11 Aug 2025 23:23:41 -0700 Subject: [PATCH 26/26] Complete extension rename: echo -> echo_ext --- tests/integration/local_toolchains/BUILD.bazel | 2 +- tests/integration/local_toolchains/echo_test.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/local_toolchains/BUILD.bazel b/tests/integration/local_toolchains/BUILD.bazel index c942be1159..8a5ab07aa1 100644 --- a/tests/integration/local_toolchains/BUILD.bazel +++ b/tests/integration/local_toolchains/BUILD.bazel @@ -50,5 +50,5 @@ py_extension( py_test( name = "echo_test", srcs = ["echo_test.py"], - deps = [":echo"], + deps = [":echo_ext"], ) diff --git a/tests/integration/local_toolchains/echo_test.py b/tests/integration/local_toolchains/echo_test.py index c03c5f2831..a54b958b6c 100644 --- a/tests/integration/local_toolchains/echo_test.py +++ b/tests/integration/local_toolchains/echo_test.py @@ -8,4 +8,4 @@ class ExtensionTest(unittest.TestCase): def test_echo_extension(self): - self.assertEqual(echo.echo(42, "str"), tuple(42, "str")) \ No newline at end of file + self.assertEqual(echo_ext.echo(42, "str"), tuple(42, "str")) \ No newline at end of file