fix(pypi): pull fewer wheels

Before this we would pull all of the wheels that the user target
configuration would be compatible with and that meant that it was not
customizable. This also meant that there were a lot of footguns in the
configuration where the select statements were not really foolproof.

With this PR we select only those sources that need to be for the
declared configurations.

TODO:
- [ ] How do we continue supporting the `freethreaded` setups is a
  little bit unclear to me as the suggested design is "have a single
  source per target platform" and we need to setup freethreaded platform
  here.

Work towards #2747
Work towards #2759
Work towards #2849

fixup the unit tests

spike: use platforms with python versino

use pep508 env for selecting the wheels

move abi construction to the extension

make want_abis configurable

Author: aignas

MODULE.bazel

@@ -63,6 +63,8 @@ pip = use_extension("//python/extensions:pip.bzl", "pip")
 pip.parse(
     # NOTE @aignas 2024-10-26: We have an integration test that depends on us
     # being able to build sdists for this hub, so explicitly set this to False.
+    #
+    # how do we test sdists? Maybe just worth adding a single sdist somewhere?
     download_only = False,
     experimental_index_url = "https://pypi.org/simple",
     hub_name = "rules_python_publish_deps",
@@ -155,22 +157,19 @@ dev_pip = use_extension(
     dev_dependency = True,
 )
 dev_pip.parse(
-    download_only = True,
     experimental_index_url = "https://pypi.org/simple",
     hub_name = "dev_pip",
     parallel_download = False,
     python_version = "3.11",
     requirements_lock = "//docs:requirements.txt",
 )
 dev_pip.parse(
-    download_only = True,
     experimental_index_url = "https://pypi.org/simple",
     hub_name = "dev_pip",
     python_version = "3.13",
     requirements_lock = "//docs:requirements.txt",
 )
 dev_pip.parse(
-    download_only = True,
     experimental_index_url = "https://pypi.org/simple",
     hub_name = "pypiserver",
     python_version = "3.11",

python/private/pypi/BUILD.bazel

@@ -422,5 +422,8 @@ bzl_library(
 bzl_library(
     name = "whl_target_platforms_bzl",
     srcs = ["whl_target_platforms.bzl"],
-    deps = [":parse_whl_name_bzl"],
+    deps = [
+        ":parse_whl_name_bzl",
+        "//python/private:version_bzl",
+    ],
 )

python/private/pypi/evaluate_markers.bzl

@@ -45,7 +45,7 @@ def evaluate_markers(*, requirements, platforms):
         for platform_str in platform_strings:
             env = platforms.get(platform_str)
             if not env:
-                fail("Please define platform: '{}'".format(platform_str))
+                continue
 
             if evaluate(req.marker, env = env):
                 ret.setdefault(req_string, []).append(platform_str)

python/private/pypi/extension.bzl

@@ -72,15 +72,25 @@ def _platforms(*, python_version, minor_mapping, config):
         version = python_version,
         minor_mapping = minor_mapping,
     )
-    abi = "cp3{}".format(python_version[2:])
 
     for platform, values in config.platforms.items():
+        implementation = values.env["implementation_name"][:2].lower()
+        abi = "{}3{}".format(implementation, python_version[2:])
         key = "{}_{}".format(abi, platform)
-        platforms[key] = env(struct(
+
+        env_ = env(struct(
             abi = abi,
             os = values.os_name,
             arch = values.arch_name,
         )) | values.env
+        platforms[key] = struct(
+            env = env_,
+            want_abis = [
+                v.format(*python_version.split("."))
+                for v in values.want_abis
+            ],
+            platform_tags = values.platform_tags,
+        )
     return platforms
 
 def _create_whl_repos(
@@ -152,6 +162,8 @@ def _create_whl_repos(
             ))
         python_interpreter_target = available_interpreters[python_name]
 
+    # TODO @aignas 2025-06-29: we should not need the version in the pip_name if
+    # we are using pipstar and we are downloading the wheel using the downloader
     pip_name = "{}_{}".format(
         hub_name,
         version_label(pip_attr.python_version),
@@ -184,11 +196,15 @@ def _create_whl_repos(
     elif config.enable_pipstar:
         evaluate_markers = lambda _, requirements: evaluate_markers_star(
             requirements = requirements,
-            platforms = _platforms(
-                python_version = pip_attr.python_version,
-                minor_mapping = minor_mapping,
-                config = config,
-            ),
+            platforms = {
+                k: p.env
+                # TODO @aignas 2025-07-05: update evaluate_markers_star
+                for k, p in _platforms(
+                    python_version = pip_attr.python_version,
+                    minor_mapping = minor_mapping,
+                    config = config,
+                ).items()
+            },
         )
     else:
         # NOTE @aignas 2024-08-02: , we will execute any interpreter that we find either
@@ -230,6 +246,11 @@ def _create_whl_repos(
             ),
             logger = logger,
         ),
+        platforms = _platforms(
+            python_version = pip_attr.python_version,
+            minor_mapping = minor_mapping,
+            config = config,
+        ),
         extra_pip_args = pip_attr.extra_pip_args,
         get_index_urls = get_index_urls,
         evaluate_markers = evaluate_markers,
@@ -359,24 +380,16 @@ def _whl_repo(*, src, whl_library_args, is_multiple_versions, download_only, net
             for p in src.target_platforms
         ]
 
-    # Pure python wheels or sdists may need to have a platform here
-    target_platforms = None
-    if is_whl and not src.filename.endswith("-any.whl"):
-        pass
-    elif is_multiple_versions:
-        target_platforms = src.target_platforms
-
     return struct(
         repo_name = whl_repo_name(src.filename, src.sha256),
         args = args,
         config_setting = whl_config_setting(
             version = python_version,
-            filename = src.filename,
-            target_platforms = target_platforms,
+            target_platforms = src.target_platforms,
         ),
     )
 
-def _configure(config, *, platform, os_name, arch_name, config_settings, env = {}, override = False):
+def _configure(config, *, platform, os_name, arch_name, config_settings, env = {}, want_abis, platform_tags, override = False):
     """Set the value in the config if the value is provided"""
     config.setdefault("platforms", {})
     if platform:
@@ -393,12 +406,25 @@ def _configure(config, *, platform, os_name, arch_name, config_settings, env = {
         if not arch_name:
             fail("'arch_name' is required")
 
+        if platform_tags and "any" not in platform_tags:
+            # the lowest priority one needs to be the first one
+            platform_tags = ["any"] + platform_tags
+
         config["platforms"][platform] = struct(
             name = platform.replace("-", "_").lower(),
             os_name = os_name,
             arch_name = arch_name,
             config_settings = config_settings,
-            env = env,
+            want_abis = want_abis or [
+                "cp{0}{1}",
+                "abi3",
+                "none",
+            ],
+            platform_tags = platform_tags,
+            env = {
+                # default to this
+                "implementation_name": "cpython",
+            } | env,
         )
     else:
         config["platforms"].pop(platform)
@@ -424,28 +450,44 @@ def _create_config(defaults):
             arch_name = cpu,
             os_name = "linux",
             platform = "linux_{}".format(cpu),
+            # TODO @aignas 2025-07-05: handle the freethreaded by having separate
+            # platforms
+            want_abis = [],
             config_settings = [
                 "@platforms//os:linux",
                 "@platforms//cpu:{}".format(cpu),
             ],
-            env = {"platform_version": "0"},
+            platform_tags = [
+                "linux_*_{}".format(cpu),
+                "manylinux_*_{}".format(cpu),
+            ],
+            env = {
+                "platform_version": "0",
+            },
         )
-    for cpu in [
-        "aarch64",
-        "x86_64",
-    ]:
+    for cpu, platform_tag_cpus in {
+        "aarch64": ["universal2", "arm64"],
+        "x86_64": ["universal2", "x86_64"],
+    }.items():
         _configure(
             defaults,
             arch_name = cpu,
-            # We choose the oldest non-EOL version at the time when we release `rules_python`.
-            # See https://endoflife.date/macos
             os_name = "osx",
             platform = "osx_{}".format(cpu),
             config_settings = [
                 "@platforms//os:osx",
                 "@platforms//cpu:{}".format(cpu),
             ],
-            env = {"platform_version": "14.0"},
+            want_abis = [],
+            platform_tags = [
+                "macosx_*_{}".format(suffix)
+                for suffix in platform_tag_cpus
+            ],
+            # We choose the oldest non-EOL version at the time when we release `rules_python`.
+            # See https://endoflife.date/macos
+            env = {
+                "platform_version": "14.0",
+            },
         )
 
     _configure(
@@ -457,7 +499,11 @@ def _create_config(defaults):
             "@platforms//os:windows",
             "@platforms//cpu:x86_64",
         ],
-        env = {"platform_version": "0"},
+        want_abis = [],
+        platform_tags = ["win_amd64"],
+        env = {
+            "platform_version": "0",
+        },
     )
     return struct(**defaults)
 
@@ -527,14 +573,15 @@ You cannot use both the additive_build_content and additive_build_content_file a
                 env = tag.env,
                 os_name = tag.os_name,
                 platform = tag.platform,
+                platform_tags = tag.platform_tags,
+                want_abis = tag.want_abis,
                 override = mod.is_root,
                 # TODO @aignas 2025-05-19: add more attr groups:
                 # * for AUTH - the default `netrc` usage could be configured through a common
                 # attribute.
                 # * for index/downloader config. This includes all of those attributes for
                 # overrides, etc. Index overrides per platform could be also used here.
-                # * for whl selection - selecting preferences of which `platform_tag`s we should use
-                # for what. We could also model the `cp313t` freethreaded as separate platforms.
+                # * for whl selection - We could also model the `cp313t` freethreaded as separate platforms.
             )
 
     config = _create_config(defaults)
@@ -666,7 +713,14 @@ You cannot use both the additive_build_content and additive_build_content_file a
             for whl_name, aliases in out.extra_aliases.items():
                 extra_aliases[hub_name].setdefault(whl_name, {}).update(aliases)
             exposed_packages.setdefault(hub_name, {}).update(out.exposed_packages)
-            whl_libraries.update(out.whl_libraries)
+            for whl_name, lib in out.whl_libraries.items():
+                if enable_pipstar:
+                    whl_libraries.setdefault(whl_name, lib)
+                elif whl_name in lib:
+                    fail("'{}' already in created".format(whl_name))
+                else:
+                    # replicate whl_libraries.update(out.whl_libraries)
+                    whl_libraries[whl_name] = lib
 
             # TODO @aignas 2024-04-05: how do we support different requirement
             # cycles for different abis/oses? For now we will need the users to
@@ -829,25 +883,6 @@ The list of labels to `config_setting` targets that need to be matched for the p
 selected.
 """,
     ),
-    "os_name": attr.string(
-        doc = """\
-The OS name to be used.
-
-:::{note}
-Either this or the appropriate `env` keys should be specified.
-:::
-""",
-    ),
-    "platform": attr.string(
-        doc = """\
-A platform identifier which will be used as the unique identifier within the extension evaluation.
-If you are defining custom platforms in your project and don't want things to clash, use extension
-[isolation] feature.
-
-[isolation]: https://bazel.build/rules/lib/globals/module#use_extension.isolate
-""",
-    ),
-} | {
     "env": attr.string_dict(
         doc = """\
 The values to use for environment markers when evaluating an expression.
@@ -873,6 +908,40 @@ This is only used if the {envvar}`RULES_PYTHON_ENABLE_PIPSTAR` is enabled.
 """,
     ),
     # The values for PEP508 env marker evaluation during the lock file parsing
+    "os_name": attr.string(
+        doc = """\
+The OS name to be used.
+
+:::{note}
+Either this or the appropriate `env` keys should be specified.
+:::
+""",
+    ),
+    "platform": attr.string(
+        doc = """\
+A platform identifier which will be used as the unique identifier within the extension evaluation.
+If you are defining custom platforms in your project and don't want things to clash, use extension
+[isolation] feature.
+
+[isolation]: https://bazel.build/rules/lib/globals/module#use_extension.isolate
+""",
+    ),
+    "platform_tags": attr.string_list(
+        doc = """\
+A list of `platform_tag` matchers so that we can select the best wheel based on the user
+preference. Per platform we will select a single wheel and the last match from this list will
+take precedence.
+
+The items in this list can contain a single `*` character that is equivalent to `.*` regex match.
+""",
+    ),
+    "want_abis": attr.string_list(
+        doc = """\
+A list of ABIs to select wheels for. The values can be either strings or include template
+parameters like `{0}` which will be replaced with python version parts. e.g. `cp{0}{1}` will
+result in `cp313` given the full python version is `3.13.5`.
+""",
+    ),
 }
 
 _SUPPORTED_PEP508_KEYS = [