Optimize dependency iteration

news/optimize-dependency-cache.feature.rst

@@ -0,0 +1 @@
+Optimize dependency resolution performance by caching parsed dependencies and extras to avoid redundant parsing operations during candidate evaluation. Also add support for discovering distributions from sys.meta_path finders to enable in-memory package installations.

src/pip/_internal/metadata/importlib/_envs.py

@@ -76,6 +76,28 @@ def _find_impl(self, location: str) -> Iterator[FoundResult]:
             self._found_names.add(name)
             yield dist, info_location
 
+    def find_meta_path_distributions(self) -> Iterator[FoundResult]:
+        """Find distributions from sys.meta_path finders."""
+        try:
+            # Get all distributions without specifying a path, which includes
+            # distributions from sys.meta_path finders
+            for dist in importlib.metadata.distributions():
+                info_location = get_info_location(dist)
+                try:
+                    name = get_dist_canonical_name(dist)
+                except BadMetadata as e:
+                    logger.warning("Skipping %s due to %s", info_location, e.reason)
+                    continue
+                # Only yield if we haven't seen this name before
+                if name in self._found_names:
+                    continue
+                self._found_names.add(name)
+                yield dist, info_location
+        except Exception:
+            # If there's any issue with finding meta_path distributions,
+            # don't break the entire discovery process - backwards compatibility
+            pass
+
     def find(self, location: str) -> Iterator[BaseDistribution]:
         """Find distributions in a location.
 
@@ -133,6 +155,21 @@ def _iter_distributions(self) -> Iterator[BaseDistribution]:
             yield from finder.find(location)
             yield from finder.find_legacy_editables(location)
 
+        # Also check for distributions from sys.meta_path finders
+        # This is backwards compatible - if no custom finders exist, this does nothing
+        yield from self._iter_meta_path_distributions(finder)
+
+    def _iter_meta_path_distributions(
+        self, finder: _DistributionFinder
+    ) -> Iterator[Distribution]:
+        """Yield distributions from sys.meta_path finders."""
+        for dist, info_location in finder.find_meta_path_distributions():
+            if info_location is None:
+                installed_location: BasePath | None = None
+            else:
+                installed_location = info_location.parent
+            yield Distribution(dist, info_location, installed_location)
+
     def get_distribution(self, name: str) -> BaseDistribution | None:
         canonical_name = canonicalize_name(name)
         matches = (

src/pip/_internal/resolution/resolvelib/candidates.py

@@ -5,7 +5,12 @@
 from collections.abc import Iterable
 from typing import TYPE_CHECKING, Any, Union, cast
 
-from pip._vendor.packaging.requirements import InvalidRequirement
+from pip._vendor.packaging.requirements import (
+    InvalidRequirement,
+)
+from pip._vendor.packaging.requirements import (
+    Requirement as PackagingRequirement,
+)
 from pip._vendor.packaging.utils import NormalizedName, canonicalize_name
 from pip._vendor.packaging.version import Version
 
@@ -159,8 +164,11 @@ def __init__(
         self._ireq = ireq
         self._name = name
         self._version = version
-        self.dist = self._prepare()
         self._hash: int | None = None
+        # Cache for parsed dependencies to avoid multiple iterations
+        self._cached_dependencies: list[PackagingRequirement] | None = None
+        self._cached_extras: list[NormalizedName] | None = None
+        self.dist = self._prepare()
 
     def __str__(self) -> str:
         return f"{self.name} {self.version}"
@@ -207,6 +215,22 @@ def format_for_error(self) -> str:
             f"(from {self._link.file_path if self._link.is_file else self._link})"
         )
 
+    def _get_cached_dependencies(self) -> list[PackagingRequirement]:
+        """Get cached dependencies, parsing them only once."""
+        if self._cached_dependencies is None:
+            if self._cached_extras is None:
+                self._cached_extras = list(self.dist.iter_provided_extras())
+            self._cached_dependencies = list(
+                self.dist.iter_dependencies(self._cached_extras)
+            )
+        return self._cached_dependencies
+
+    def _get_cached_extras(self) -> list[NormalizedName]:
+        """Get cached extras, parsing them only once."""
+        if self._cached_extras is None:
+            self._cached_extras = list(self.dist.iter_provided_extras())
+        return self._cached_extras
+
     def _prepare_distribution(self) -> BaseDistribution:
         raise NotImplementedError("Override in subclass")
 
@@ -227,10 +251,14 @@ def _check_metadata_consistency(self, dist: BaseDistribution) -> None:
                 str(dist.version),
             )
         # check dependencies are valid
-        # TODO performance: this means we iterate the dependencies at least twice,
-        # we may want to cache parsed Requires-Dist
+        # Parse and cache dependencies during validation to avoid re-parsing later
         try:
-            list(dist.iter_dependencies(list(dist.iter_provided_extras())))
+            if self._cached_extras is None:
+                self._cached_extras = list(dist.iter_provided_extras())
+            if self._cached_dependencies is None:
+                self._cached_dependencies = list(
+                    dist.iter_dependencies(self._cached_extras)
+                )
         except InvalidRequirement as e:
             raise MetadataInvalid(self._ireq, str(e))
 
@@ -255,9 +283,11 @@ def iter_dependencies(self, with_requires: bool) -> Iterable[Requirement | None]
         # Emit the Requires-Python requirement first to fail fast on
         # unsupported candidates and avoid pointless downloads/preparation.
         yield self._factory.make_requires_python_requirement(self.dist.requires_python)
-        requires = self.dist.iter_dependencies() if with_requires else ()
-        for r in requires:
-            yield from self._factory.make_requirements_from_spec(str(r), self._ireq)
+        if with_requires:
+            # Use cached dependencies to avoid re-parsing
+            requires = self._get_cached_dependencies()
+            for r in requires:
+                yield from self._factory.make_requirements_from_spec(str(r), self._ireq)
 
     def get_install_requirement(self) -> InstallRequirement | None:
         return self._ireq
@@ -515,8 +545,10 @@ def iter_dependencies(self, with_requires: bool) -> Iterable[Requirement | None]
 
         # The user may have specified extras that the candidate doesn't
         # support. We ignore any unsupported extras here.
-        valid_extras = self.extras.intersection(self.base.dist.iter_provided_extras())
-        invalid_extras = self.extras.difference(self.base.dist.iter_provided_extras())
+        # Cache provided_extras to avoid multiple iterations
+        provided_extras = set(self.base.dist.iter_provided_extras())
+        valid_extras = self.extras.intersection(provided_extras)
+        invalid_extras = self.extras.difference(provided_extras)
         for extra in sorted(invalid_extras):
             logger.warning(
                 "%s %s does not provide the extra '%s'",

src/pip/_internal/resolution/resolvelib/factory.py

@@ -281,7 +281,10 @@ def _get_installed_candidate() -> Candidate | None:
             try:
                 # Don't use the installed distribution if its version
                 # does not fit the current dependency graph.
-                if not specifier.contains(installed_dist.version, prereleases=True):
+                version_check = specifier.contains(
+                    installed_dist.version, prereleases=True
+                )
+                if not version_check:
                     return None
             except InvalidVersion as e:
                 raise InvalidInstalledPackage(dist=installed_dist, invalid_exc=e)

tests/functional/test_install.py

@@ -2582,8 +2582,16 @@ def find_distributions(self, context=None):
         sys.meta_path.append(CustomFinder())
         """
     )
-    with open(script.site_packages_path / "sitecustomize.py", "w") as sitecustomize:
-        sitecustomize.write(sitecustomize_text)
+    # Create a custom module and add to site-packages
+    custom_module_path = script.site_packages_path / "setup_custom_finder.py"
+    with open(custom_module_path, "w") as f:
+        f.write(sitecustomize_text)
+
+    # Create a .pth file that imports our module
+    pth_content = "import setup_custom_finder"
+    pth_file = script.site_packages_path / "setup_custom_finder.pth"
+    with open(pth_file, "w") as f:
+        f.write(pth_content)
 
     result = script.pip("install", "example")