Additional features for pip dependencies: uniform logging, private repos and comes_from info

Makefile

@@ -1,10 +1,10 @@
-# Generated by Medikit 0.8.0 on 2020-11-25.
+# Generated by Medikit 0.8.0 on 2021-04-06.
 # All changes will be overriden.
 # Edit Projectfile and run “make update” (or “medikit update”) to regenerate.
 
 
 PACKAGE ?= medikit
-PYTHON ?= $(shell which python || echo python)
+PYTHON ?= $(shell which python3 || which python || echo python3)
 PYTHON_BASENAME ?= $(shell basename $(PYTHON))
 PYTHON_DIRNAME ?= $(shell dirname $(PYTHON))
 PYTHON_REQUIREMENTS_FILE ?= requirements.txt
@@ -18,7 +18,7 @@ VERSION ?= $(shell git describe 2>/dev/null || git rev-parse --short HEAD)
 BLACK ?= $(shell which black || echo black)
 BLACK_OPTIONS ?= --line-length 120
 ISORT ?= $(PYTHON) -m isort
-ISORT_OPTIONS ?= --recursive --apply
+ISORT_OPTIONS ?= 
 PYTEST ?= $(PYTHON_DIRNAME)/pytest
 PYTEST_OPTIONS ?= --capture=no --cov=$(PACKAGE) --cov-report html
 SPHINX_BUILD ?= $(PYTHON_DIRNAME)/sphinx-build

medikit/feature/python.py

@@ -33,14 +33,17 @@
 import os
 import tempfile
 from getpass import getuser
+from types import SimpleNamespace
 
 from pip._vendor.distlib.util import parse_requirement
+from pip._internal.req.req_install import InstallRequirement
 from piptools._compat import parse_requirements
 from piptools.cache import DependencyCache
 from piptools.locations import CACHE_DIR
 from piptools.repositories import PyPIRepository
 from piptools.resolver import Resolver
 from piptools.utils import format_requirement
+from piptools.exceptions import IncompatibleRequirements
 
 import medikit
 from medikit.events import subscribe
@@ -52,11 +55,27 @@
 
 
 def _normalize_requirement(req):
-    bits = req.requirement.split()
+    """ Normalizes the requirement string. It considers the case of having or not an URL """
+
+    if not req.url:
+        bits = req.requirement.split()
+    else:
+        bits = [req.requirement, '@', req.url]
+
     if req.extras:
         bits = [bits[0] + "[{}]".format(",".join(req.extras))] + bits[1:]
+
     return " ".join(bits)
 
+def _get_valid_link_req(req):
+    """ Formats the repo based dependencies, which can be dirty in case of collision between repo based and package
+     based declarations (for similar packages) """
+
+    tokens = str(req).split("@")
+    req_name = parse_requirement(tokens[0])
+
+    return req_name.name + "@ " + "@".join(tokens[1:])
+
 
 class PythonConfig(Feature.Config):
     """ Configuration API for the «python» feature. """
@@ -73,6 +92,10 @@ def __init__(self):
         self._create_packages = True
         self.override_requirements = False
         self.use_wheelhouse = False
+        # Use the same requirement versions among all the extras, when requirements coincide.
+        self.use_uniform_requirements = False
+        # Print the information of the "parent" requirement in the requirements*.txt files.
+        self.show_comes_from_info = False
 
     @property
     def package_dir(self):
@@ -217,6 +240,66 @@ def __add_vendors(self, reqs, extra=None):
         for req in reqs:
             self._vendors[extra].append(req)
 
+    def get_requirement_info_by_name(self, req, requirements_by_name=dict()):
+        """ Given a requirement, it provides its valid information to be included in the final file """
+
+        if self.use_uniform_requirements:
+            # If it is a repo and not a package, this will be True
+            if req.link:
+                requirement_name = parse_requirement(_get_valid_link_req(req.req)).name
+            else:
+                requirement_name = parse_requirement(str(req.req)).name
+
+            # If the requirement is not in the dict, it is because it was not needed as a dependency in the original
+            # set containing all requirements
+            if requirement_name in requirements_by_name.keys():
+                # In case we want to show the source for inherited dependencies
+                if self.show_comes_from_info and type(req.comes_from) == InstallRequirement:
+                    return "{}\t\t\t# From: {}".format(requirements_by_name[requirement_name].requirement,
+                                                       str(req.comes_from.req))
+                else:
+                    return requirements_by_name[requirement_name].requirement
+
+            else:
+                return None
+        else:
+            # If not using uniform versions, we just need to provide the information based on wether it is
+            # a repository or a package
+            if self.show_comes_from_info and type(req.comes_from) == InstallRequirement:
+                return "{}\t\t\t# From: {}".format(format_requirement(req) if not req.link else str(req.req),
+                                                   str(req.comes_from.req))
+            else:
+                return format_requirement(req) if not req.link else str(req.req)
+
+    def _check_duplicate_dependencies_by_extra(self, extra, requirements_by_name):
+        """ Checks there are not duplicate dependencies, in the case of private repositories"""
+
+        for name, req in sorted(self._requirements[extra].items()):
+            requirement_str = req.requirement if not req.url else req.url.strip().replace(" ", "")
+
+            if name not in requirements_by_name.keys():
+                # This can happen if additional requirements are included from the outside, for instance with pytest
+                continue
+
+            if (req.url or requirements_by_name[name].url) and requirements_by_name[name].requirement != requirement_str:
+                raise IncompatibleRequirements(req, requirements_by_name[name].url)
+
+    def check_duplicate_dependencies_uniform(self, requirements_by_name):
+        """ Checks there are not duplicate dependencies, when use_uniform_requirements==True """
+        for extra in itertools.chain((None,), self.get_extras()):
+            self._check_duplicate_dependencies_by_extra(extra, requirements_by_name)
+
+    def check_duplicate_dependencies_nonuniform(self, extra, resolver):
+        """ Checks there are not duplicate dependencies, when use_uniform_requirements==False """
+        requirements_by_name = {}
+        for req in resolver.resolve(max_rounds=10):
+            requirements_by_name[parse_requirement(str(req.req)).name] = SimpleNamespace(
+                requirement=format_requirement(req).strip().replace(" ", ""),
+                url=req.link
+            )
+
+        self._check_duplicate_dependencies_by_extra(extra, requirements_by_name)
+
 
 class PythonFeature(Feature):
     """
@@ -414,48 +497,59 @@ def on_start(self, event):
             version = "0.0.0"
         self.render_file_inline(python_config.version_file, "__version__ = '{}'".format(version))
 
-        setup = python_config.get_setup()
-
-        context = {
-            "url": setup.pop("url", "http://example.com/"),
-            "download_url": setup.pop("download_url", "http://example.com/"),
-        }
-
-        for k, v in context.items():
-            context[k] = context[k].format(name=setup["name"], user=getuser(), version="{version}")
-
-        context.update(
-            {
-                "entry_points": setup.pop("entry_points", {}),
-                "extras_require": python_config.get("extras_require"),
-                "install_requires": python_config.get("install_requires"),
-                "python": python_config,
-                "setup": setup,
-                "banner": get_override_warning_banner(),
-            }
-        )
-
-        # Render (with overwriting) the allmighty setup.py
-        self.render_file("setup.py", "python/setup.py.j2", context, override=True)
-
     @subscribe(medikit.on_end, priority=ABSOLUTE_PRIORITY)
     def on_end(self, event):
+
         # Our config object
         python_config = event.config["python"]
 
         # Pip / PyPI
         repository = PyPIRepository([], cache_dir=CACHE_DIR)
 
+        # We just need to construct this structure if use_uniform_requirements == True
+        requirements_by_name = {}
+
+        if python_config.use_uniform_requirements:
+            tmpfile = tempfile.NamedTemporaryFile(mode="wt", delete=False)
+            for extra in itertools.chain((None,), python_config.get_extras()):
+                tmpfile.write("\n".join(python_config.get_requirements(extra=extra)) + "\n")
+                tmpfile.flush()
+
+            constraints = list(
+                parse_requirements(
+                    tmpfile.name, finder=repository.finder, session=repository.session, options=repository.options
+                )
+            )
+
+            # This resolver is able to evaluate ALL the dependencies along the extras
+            resolver = Resolver(
+                constraints,
+                repository,
+                cache=DependencyCache(CACHE_DIR),
+                # cache=DependencyCache(tempfile.tempdir),
+                prereleases=False,
+                clear_caches=False,
+                allow_unsafe=False,
+            )
+
+            for req in resolver.resolve(max_rounds=10):
+                requirements_by_name[parse_requirement(str(req.req)).name] = SimpleNamespace(
+                    requirement=format_requirement(req).strip().replace(" ", ""),
+                    url=req.link
+                )
+
+            python_config.check_duplicate_dependencies_uniform(requirements_by_name)
+
+        # Now it iterates along the versions in extras and looks for the requirements and its dependencies, using the
+        # structure created above to select the unified versions (unless the flag indicates otherwise).
         for extra in itertools.chain((None,), python_config.get_extras()):
             requirements_file = "requirements{}.txt".format("-" + extra if extra else "")
 
             if python_config.override_requirements or not os.path.exists(requirements_file):
                 tmpfile = tempfile.NamedTemporaryFile(mode="wt", delete=False)
-                if extra:
-                    tmpfile.write("\n".join(python_config.get_requirements(extra=extra)))
-                else:
-                    tmpfile.write("\n".join(python_config.get_requirements()))
+                tmpfile.write("\n".join(python_config.get_requirements(extra=extra)) + "\n")
                 tmpfile.flush()
+
                 constraints = list(
                     parse_requirements(
                         tmpfile.name, finder=repository.finder, session=repository.session, options=repository.options
@@ -470,19 +564,50 @@ def on_end(self, event):
                     allow_unsafe=False,
                 )
 
+                if not python_config.use_uniform_requirements:
+                    python_config.check_duplicate_dependencies_nonuniform(extra, resolver)
+
+                requirements_list = []
+                for req in resolver.resolve(max_rounds=10):
+                    if req.name != python_config.get("name"):
+                        requirement = python_config.get_requirement_info_by_name(req, requirements_by_name)
+                        if requirement:
+                            requirements_list.append(requirement)
+
                 self.render_file_inline(
                     requirements_file,
                     "\n".join(
                         (
                             "-e .{}".format("[" + extra + "]" if extra else ""),
                             *(("-r requirements.txt",) if extra else ()),
                             *python_config.get_vendors(extra=extra),
-                            *sorted(
-                                format_requirement(req)
-                                for req in resolver.resolve(max_rounds=10)
-                                if req.name != python_config.get("name")
-                            ),
+                            *sorted(requirements_list),
                         )
                     ),
                     override=python_config.override_requirements,
                 )
+
+        # Updates setup file
+        setup = python_config.get_setup()
+
+        context = {
+            "url": setup.pop("url", ""),
+            "download_url": setup.pop("download_url", ""),
+        }
+
+        for k, v in context.items():
+            context[k] = context[k].format(name=setup["name"], user=getuser(), version="{version}")
+
+        context.update(
+            {
+                "entry_points": setup.pop("entry_points", {}),
+                "extras_require": python_config.get("extras_require"),
+                "install_requires": python_config.get("install_requires"),
+                "python": python_config,
+                "setup": setup,
+                "banner": get_override_warning_banner(),
+            }
+        )
+
+        # Render (with overwriting) the allmighty setup.py
+        self.render_file("setup.py", "python/setup.py.j2", context, override=True)

requirements-dev.txt

@@ -5,48 +5,46 @@ appdirs==1.4.4
 attrs==20.3.0
 babel==2.9.0
 black==20.8b1
-certifi==2020.11.8
+certifi==2020.12.5
 cfgv==3.2.0
-chardet==3.0.4
+chardet==4.0.0
 click==7.1.2
-coverage==5.3
+coverage==5.5
 distlib==0.3.1
-docutils==0.16
+docutils==0.17
 filelock==3.0.12
-identify==1.5.10
+identify==2.2.2
 idna==2.10
 imagesize==1.2.0
-importlib-metadata==3.1.0
 iniconfig==1.1.1
-isort==5.6.4
-jinja2==2.11.2
+isort==5.8.0
+jinja2==2.11.3
 markupsafe==1.1.1
 mypy-extensions==0.4.3
 nodeenv==1.5.0
-packaging==20.4
+packaging==20.9
 pathspec==0.8.1
 pluggy==0.13.1
-pre-commit==2.9.0
-py==1.9.0
-pygments==2.7.2
+pre-commit==2.9.3
+py==1.10.0
+pygments==2.8.1
 pyparsing==2.4.7
-pytest-cov==2.10.1
-pytest==6.1.2
-pytz==2020.4
-pyyaml==5.3.1
-regex==2020.11.13
+pytest-cov==2.11.1
+pytest==6.2.3
+pytz==2021.1
+pyyaml==5.4.1
+regex==2021.4.4
 releases==1.6.3
-requests==2.25.0
+requests==2.25.1
 semantic-version==2.6.0
 six==1.15.0
-snowballstemmer==2.0.0
+snowballstemmer==2.1.0
 sphinx-sitemap==1.1.0
 sphinx==1.8.5
 sphinxcontrib-serializinghtml==1.1.4
 sphinxcontrib-websupport==1.2.4
 toml==0.10.2
-typed-ast==1.4.1
+typed-ast==1.4.2
 typing-extensions==3.7.4.3
-urllib3==1.26.2
-virtualenv==20.2.1
-zipp==3.4.0
+urllib3==1.26.4
+virtualenv==20.4.3

requirements.txt

@@ -1,21 +1,19 @@
 -e .
 click==7.1.2
-colorama==0.3.9
+colorama==0.4.4
 git-semver==0.3.2
-gitdb==4.0.5
-gitpython==3.1.11
-importlib-metadata==3.1.0
-jinja2==2.11.2
+gitdb==4.0.7
+gitpython==3.1.14
+jinja2==2.11.3
 markupsafe==1.1.1
-mondrian==0.8.0
-packaging==20.4
+mondrian==0.8.1
+packaging==20.9
 pbr==5.5.1
 pip-tools==4.5.1
 pyparsing==2.4.7
 semantic-version==2.6.0
 six==1.15.0
-smmap==3.0.4
-stevedore==3.2.2
+smmap==4.0.0
+stevedore==3.3.0
 whistle==1.0.1
-yapf==0.30.0
-zipp==3.4.0
+yapf==0.31.0