Sort out activating extras & URL sources

Author: arrdem

pip/private/constraints/python/defs.bzl

@@ -10,7 +10,7 @@ def supported_python(python_tag):
     - IronPython
 
     Explicitly allows only the `py` (generic) and `cp` (CPython) interpreters.
-    
+
     Args:
         python_tag (str): A wheel abi tag
 

pip/private/extension.bzl

@@ -129,6 +129,14 @@ def _parse_locks(module_ctx, venv_specs):
                     for d in package["dependencies"]:
                         d["name"] = normalize_name(d["name"])
 
+                # Note that we also have to mangle the optional deps so they tie
+                # off too. We don't mangle group names because they're
+                # eliminated when we resolve the depgraph.
+                if "optional-dependencies" in package:
+                    for name, group in package["optional-dependencies"].items():
+                        for d in group:
+                            d["name"] = normalize_name(d["name"])
+
             # FIXME: Should validate the lockfile but for now just stash it
             # Validating in starlark kinda rots anyway
             lock_specs[lock.hub_name][lock.venv_name] = lockfile
@@ -146,8 +154,15 @@ def _collect_configurations(repository_ctx, lock_specs):
     for hub_name, venvs in lock_specs.items():
         for venv_name, lock in venvs.items():
             for package in lock.get("package", []):
-                if "registry" not in package["source"]:
-                    continue
+                # Remote package sources
+                # - { source = { registry = "https://some.registry/..." } }
+                # - { source = { url = "https://ton.hosting.biz/some.whl" } }
+                # FIXME: Git URLs?
+                # FIXME: Egg URLs?
+                #
+                # These seem to be used by the package itself
+                # - { source = { editable = "." } }
+                # - { source = { virtual = "." } }
 
                 for whl in package.get("wheels", []):
                     url = whl["url"]
@@ -167,7 +182,7 @@ def _collect_configurations(repository_ctx, lock_specs):
             # Ignore configurations for unsupported interpreters
             if not supported_python(python_tag):
                 continue
-            
+
             python_tags[python_tag] = 1
 
             for platform_tag in parsed_wheel.platform_tags:
@@ -208,9 +223,6 @@ def _raw_sdist_repos(module_ctx, lock_specs):
     for hub_name, venvs in lock_specs.items():
         for venv_name, lock in venvs.items():
             for package in lock.get("package", []):
-                if "registry" not in package["source"]:
-                    continue
-
                 sdist = package.get("sdist")
                 if sdist == None:
                     continue
@@ -253,9 +265,6 @@ def _raw_whl_repos(module_ctx, lock_specs):
     for hub_name, venvs in lock_specs.items():
         for venv_name, lock in venvs.items():
             for package in lock.get("package", []):
-                if "registry" not in package["source"]:
-                    continue
-
                 wheels = package.get("wheels", [])
                 for whl in wheels:
                     url = whl["url"]
@@ -296,9 +305,6 @@ def _sbuild_repos(module_ctx, lock_specs):
     for hub_name, venvs in lock_specs.items():
         for venv_name, lock in venvs.items():
             for package in lock.get("package", []):
-                if "registry" not in package["source"]:
-                    continue
-
                 if "sdist" not in package:
                     continue
 
@@ -322,9 +328,6 @@ def _whl_install_repos(module_ctx, lock_specs):
     for hub_name, venvs in lock_specs.items():
         for venv_name, lock in venvs.items():
             for package in lock.get("package", []):
-                if "registry" not in package["source"]:
-                    continue
-
                 # This is where we need to actually choose which wheel we will
                 # "install", and so this is where prebuild selection needs to
                 # happen according to constraints.
@@ -353,32 +356,57 @@ def _venv_hub_name(hub, venv):
 def _group_repos(module_ctx, lock_specs):
     # Hub -> requirement -> venv -> True
     # For building hubs we need to know what venv configurations a given
+
+    # TODO: Missing support for `marker=""` specifications in `dependencies`.
+    # Deps may be conditional on this or that predicate being satisfied.
+    # Dependencies which have markers need to be rewritten so that the
+    # dependency edge goes through a select.
+    #
+    # TODO: What happens if you have a cycle only when this or that feature is
+    # active? Probably the fix is to turn on all dependency edges
+
     package_venvs = {}
 
     for hub_name, venvs in lock_specs.items():
         package_venvs[hub_name] = {}
 
         for venv_name, lock in venvs.items():
-            # First we need to build the adjacency graph
+            # Index all the packages by name so activating extras is easy
+            packages = {
+                package["name"]: package
+                for package in lock.get("package", [])
+            }
+
+            # Build a graph {name: {dependency_name: dependency_marker_condition}}
             graph = {}
 
-            for package in lock.get("package", []):
+            for name, package in packages.items():
                 if "registry" not in package["source"]:
                     continue
 
-                deps = []
+                deps = {}
 
                 # Enter the package into the venv internal graph
-                graph[package["name"]] = deps
+                graph.setdefault(package["name"], {})
+
                 for d in package.get("dependencies", []):
-                    deps.append(d["name"])
+                    # Or in all the dep bits
+                    graph[package["name"]][d["name"]] = d.get("marker", "")
+
+                    # Activate extras
+                    extras = packages[d["name"]].get("optional-dependencies", {})
+                    for extra in d.get("extra", []):
+                        for extra_dep in extras.get(extra, []):
+                            graph.setdefault(d["name"], {})
+                            graph[d["name"]][extra_dep["name"]] = extra_dep.get("marker", "")
 
                 # Enter the package into the venv hub manifest
                 package_venvs[hub_name].setdefault(package["name"], {})
                 package_venvs[hub_name][package["name"]][venv_name] = 1
 
             # So we can find sccs/clusters which need to co-occur
-            cycle_groups = sccs(graph)
+            # Note that we're assuming ALL marker conditional deps are live.
+            cycle_groups = sccs({k: v.keys() for k, v in graph.items()})
 
             # Now we can assign names to the sccs and collect deps Note that
             # _every_ node in the graph will be in _a_ scc, those sccs are just
@@ -412,11 +440,12 @@ def _group_repos(module_ctx, lock_specs):
                         if d not in scc:
                             deps[name][d] = 1
 
+            # TODO: How do we plumb markers through here?
+
             # At this point we have mapped every package to an scc (possibly of
             # size 1) which it participates in, named those sccs, and identified
             # their direct dependencies beyond the scc. So we can just lay down
             # targets.
-
             venv_hub(
                 name = _venv_hub_name(hub_name, venv_name),
                 aliases = scc_aliases,  # String dict

pip/private/tomltool/toml.bzl

@@ -46,7 +46,7 @@ def _translate_libc(repository_ctx):
         ldd = repository_ctx.execute(["ldd", "--version"]).stdout.lower()
         if "gnu" in ldd or "glibc" in ldd:
             return "gnu"
-        
+
         elif "musl" in ldd:
             return "musl"
 

pip/private/whl_install/repository.bzl

@@ -79,7 +79,7 @@ def _whl_install_impl(repository_ctx):
             # Escape hatch for ignoring unsupported interpreters
             if not supported_python(python_tag):
                 continue
-            
+
             for platform_tag in parsed.platform_tags:
                 # Escape hatch for ignoring weird unsupported platforms
                 if not supported_platform(platform_tag):