Skip to content

feat(pypi): incrementally build platform configuration #3112

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
113 changes: 71 additions & 42 deletions python/private/pypi/extension.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -377,26 +377,80 @@ def _whl_repo(*, src, whl_library_args, is_multiple_versions, download_only, net
),
)

def _configure(config, *, platform, os_name, arch_name, config_settings, env = {}, override = False):
"""Set the value in the config if the value is provided"""
config.setdefault("platforms", {})
if platform:
if not override and config.get("platforms", {}).get(platform):
return
def _plat(*, name, arch_name, os_name, config_settings = [], env = {}):
return struct(
name = name,
arch_name = arch_name,
os_name = os_name,
config_settings = config_settings,
env = env,
)

def _configure(config, *, override = False, **kwargs):
"""Set the value in the config if the value is provided"""
env = kwargs.get("env")
if env:
for key in env:
if key not in _SUPPORTED_PEP508_KEYS:
fail("Unsupported key in the PEP508 environment: {}".format(key))

config["platforms"][platform] = struct(
name = platform.replace("-", "_").lower(),
os_name = os_name,
arch_name = arch_name,
config_settings = config_settings,
env = env,
)
else:
config["platforms"].pop(platform)
for key, value in kwargs.items():
if value and (override or key not in config):
config[key] = value

def build_config(
*,
module_ctx,
enable_pipstar):
"""Parse 'configure' and 'default' extension tags

Args:
module_ctx: {type}`module_ctx` module context.
enable_pipstar: {type}`bool` a flag to enable dropping Python dependency for
evaluation of the extension.

Returns:
A struct with the configuration.
"""
defaults = {
"platforms": {},
}
for mod in module_ctx.modules:
if not (mod.is_root or mod.name == "rules_python"):
continue

for tag in mod.tags.default:
platform = tag.platform
if platform:
specific_config = defaults["platforms"].setdefault(platform, {})
_configure(
specific_config,
arch_name = tag.arch_name,
config_settings = tag.config_settings,
env = tag.env,
os_name = tag.os_name,
name = platform.replace("-", "_").lower(),
override = mod.is_root,
)

if platform and not (tag.arch_name or tag.config_settings or tag.env or tag.os_name):
defaults["platforms"].pop(platform)

# TODO @aignas 2025-05-19: add more attr groups:
# * for AUTH - the default `netrc` usage could be configured through a common
# attribute.
# * for index/downloader config. This includes all of those attributes for
# overrides, etc. Index overrides per platform could be also used here.
# * for whl selection - selecting preferences of which `platform_tag`s we should use
# for what. We could also model the `cp313t` freethreaded as separate platforms.

return struct(
platforms = {
name: _plat(**values)
for name, values in defaults["platforms"].items()
},
enable_pipstar = enable_pipstar,
)

def parse_modules(
module_ctx,
Expand Down Expand Up @@ -448,33 +502,7 @@ You cannot use both the additive_build_content and additive_build_content_file a
srcs_exclude_glob = whl_mod.srcs_exclude_glob,
)

defaults = {
"enable_pipstar": enable_pipstar,
"platforms": {},
}
for mod in module_ctx.modules:
if not (mod.is_root or mod.name == "rules_python"):
continue

for tag in mod.tags.default:
_configure(
defaults,
arch_name = tag.arch_name,
config_settings = tag.config_settings,
env = tag.env,
os_name = tag.os_name,
platform = tag.platform,
override = mod.is_root,
# TODO @aignas 2025-05-19: add more attr groups:
# * for AUTH - the default `netrc` usage could be configured through a common
# attribute.
# * for index/downloader config. This includes all of those attributes for
# overrides, etc. Index overrides per platform could be also used here.
# * for whl selection - selecting preferences of which `platform_tag`s we should use
# for what. We could also model the `cp313t` freethreaded as separate platforms.
)

config = struct(**defaults)
config = build_config(module_ctx = module_ctx, enable_pipstar = enable_pipstar)

# TODO @aignas 2025-06-03: Merge override API with the builder?
_overriden_whl_set = {}
Expand Down Expand Up @@ -659,6 +687,7 @@ You cannot use both the additive_build_content and additive_build_content_file a
k: dict(sorted(args.items()))
for k, args in sorted(whl_libraries.items())
},
config = config,
)

def _pip_impl(module_ctx):
Expand Down
62 changes: 61 additions & 1 deletion tests/pypi/extension/extension_tests.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@

load("@rules_testing//lib:test_suite.bzl", "test_suite")
load("@rules_testing//lib:truth.bzl", "subjects")
load("//python/private/pypi:extension.bzl", "parse_modules") # buildifier: disable=bzl-visibility
load("//python/private/pypi:extension.bzl", "build_config", "parse_modules") # buildifier: disable=bzl-visibility
load("//python/private/pypi:parse_simpleapi_html.bzl", "parse_simpleapi_html") # buildifier: disable=bzl-visibility
load("//python/private/pypi:whl_config_setting.bzl", "whl_config_setting") # buildifier: disable=bzl-visibility

Expand Down Expand Up @@ -92,6 +92,18 @@ def _parse_modules(env, enable_pipstar = 0, **kwargs):
),
)

def _build_config(env, enable_pipstar = 0, **kwargs):
return env.expect.that_struct(
build_config(
enable_pipstar = enable_pipstar,
**kwargs
),
attrs = dict(
platforms = subjects.dict,
enable_pipstar = subjects.bool,
),
)

def _default(
arch_name = None,
config_settings = None,
Expand Down Expand Up @@ -1206,6 +1218,54 @@ optimum[onnxruntime-gpu]==1.17.1 ; sys_platform == 'linux'

_tests.append(_test_pipstar_platforms)

def _test_build_pipstar_platform(env):
config = _build_config(
env,
module_ctx = _mock_mctx(
_mod(
name = "rules_python",
default = [
_default(
platform = "myplat",
os_name = "linux",
arch_name = "x86_64",
config_settings = [
"@platforms//os:linux",
"@platforms//cpu:x86_64",
],
),
_default(),
_default(
platform = "myplat2",
os_name = "linux",
arch_name = "x86_64",
config_settings = [
"@platforms//os:linux",
"@platforms//cpu:x86_64",
],
),
_default(platform = "myplat2"),
],
),
),
enable_pipstar = True,
)
config.enable_pipstar().equals(True)
config.platforms().contains_exactly({
"myplat": struct(
name = "myplat",
os_name = "linux",
arch_name = "x86_64",
config_settings = [
"@platforms//os:linux",
"@platforms//cpu:x86_64",
],
env = {},
),
})

_tests.append(_test_build_pipstar_platform)

def extension_test_suite(name):
"""Create the test suite.

Expand Down