From 63f7d269dc7f7a1e5e673f3e4f1d14b7d8024dfb Mon Sep 17 00:00:00 2001 From: stonebig Date: Thu, 15 May 2025 21:50:40 +0200 Subject: [PATCH 01/10] add size to pylock.toml --- src/pip/_internal/models/link.py | 18 +++++++++++++++++- src/pip/_internal/models/pylock.py | 8 +++++--- tests/functional/test_lock.py | 14 ++++++++++++++ tests/unit/test_link.py | 9 +++++++++ 4 files changed, 45 insertions(+), 4 deletions(-) diff --git a/src/pip/_internal/models/link.py b/src/pip/_internal/models/link.py index 87651c76e25..7c33b77a4a3 100644 --- a/src/pip/_internal/models/link.py +++ b/src/pip/_internal/models/link.py @@ -507,7 +507,23 @@ def hash_name(self) -> str | None: @property def show_url(self) -> str: return posixpath.basename(self._url.split("#", 1)[0].split("?", 1)[0]) - + + @property + def size(self) -> int | None: + """Fetch the size of the file from HTTP headers if available.""" + from pip._internal.network.session import PipSession + try: + session = PipSession() + response = session.head(self.url, allow_redirects=True) + response.raise_for_status() + # Check if the 'Content-Length' header is present + size = response.headers.get("Content-Length") + if size: + return int(size) + except (NetworkConnectionError, ValueError) as e: + logger.warning(f"Could not fetch size for {self.url}: {e}") + return None + @property def is_file(self) -> bool: return self.scheme == "file" diff --git a/src/pip/_internal/models/pylock.py b/src/pip/_internal/models/pylock.py index e7df01a2bc3..72897f24987 100644 --- a/src/pip/_internal/models/pylock.py +++ b/src/pip/_internal/models/pylock.py @@ -47,7 +47,7 @@ class PackageDirectory: class PackageArchive: url: str | None # (not supported) path: Optional[str] - # (not supported) size: Optional[int] + size: int | None # (not supported) upload_time: Optional[datetime] hashes: dict[str, str] subdirectory: str | None @@ -59,7 +59,7 @@ class PackageSdist: # (not supported) upload_time: Optional[datetime] url: str | None # (not supported) path: Optional[str] - # (not supported) size: Optional[int] + size: int | None hashes: dict[str, str] @@ -69,7 +69,7 @@ class PackageWheel: # (not supported) upload_time: Optional[datetime] url: str | None # (not supported) path: Optional[str] - # (not supported) size: Optional[int] + size: int | None hashes: dict[str, str] @@ -142,6 +142,7 @@ def from_install_requirement(cls, ireq: InstallRequirement, base_dir: Path) -> S PackageWheel( name=link.filename, url=download_info.url, + size=link.size, hashes=download_info.info.hashes, ) ] @@ -149,6 +150,7 @@ def from_install_requirement(cls, ireq: InstallRequirement, base_dir: Path) -> S package.sdist = PackageSdist( name=link.filename, url=download_info.url, + size=link.size, hashes=download_info.info.hashes, ) else: diff --git a/tests/functional/test_lock.py b/tests/functional/test_lock.py index 727bd72c8a0..2bab10db812 100644 --- a/tests/functional/test_lock.py +++ b/tests/functional/test_lock.py @@ -234,3 +234,17 @@ def test_lock_archive(script: PipTestEnvironment, shared_data: TestData) -> None }, }, ] + +def test_lock_includes_size(script: PipTestEnvironment, shared_data: TestData) -> None: + result = script.pip( + "lock", + "simplewheel==2.0", + "--no-index", + "--find-links", + str(shared_data.root / "packages/"), + expect_stderr=True, # for the experimental warning + ) + pylock = tomllib.loads(script.scratch_path.joinpath("pylock.toml").read_text()) + assert "size" in pylock["packages"][0]["wheels"][0] + assert pylock["packages"][0]["wheels"][0]["size"] > 0 + \ No newline at end of file diff --git a/tests/unit/test_link.py b/tests/unit/test_link.py index 92e92dffa3d..08a6d10e88e 100644 --- a/tests/unit/test_link.py +++ b/tests/unit/test_link.py @@ -5,6 +5,7 @@ from pip._internal.models.link import Link, links_equivalent from pip._internal.utils.hashes import Hashes +from unittest.mock import patch class TestLink: @pytest.mark.parametrize( @@ -240,3 +241,11 @@ def test_links_equivalent(url1: str, url2: str) -> None: ) def test_links_equivalent_false(url1: str, url2: str) -> None: assert not links_equivalent(Link(url1), Link(url2)) + +def test_link_size(): + # Mock the HTTP HEAD response + with patch("pip._internal.network.session.PipSession.head") as mock_head: + mock_head.return_value.headers = {"Content-Length": "12345"} + + link = Link(url="https://example.com/package.tar.gz", filename="package.tar.gz") + assert link.size == 12345 # Verify size is correctly fetched \ No newline at end of file From 705d315f329ad78683b1c1db1410c81e9253152e Mon Sep 17 00:00:00 2001 From: stonebig Date: Thu, 15 May 2025 21:58:51 +0200 Subject: [PATCH 02/10] add news entry --- news/13395.feature.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 news/13395.feature.rst diff --git a/news/13395.feature.rst b/news/13395.feature.rst new file mode 100644 index 00000000000..86dbc0dfb2c --- /dev/null +++ b/news/13395.feature.rst @@ -0,0 +1 @@ +add size attribute to pylock.toml From 214aed718d983e97f8b9140edbbbfad00159f914 Mon Sep 17 00:00:00 2001 From: stonebig Date: Thu, 15 May 2025 22:06:32 +0200 Subject: [PATCH 03/10] fix bug on test_link --- tests/unit/test_link.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/test_link.py b/tests/unit/test_link.py index 08a6d10e88e..5f187d36e11 100644 --- a/tests/unit/test_link.py +++ b/tests/unit/test_link.py @@ -247,5 +247,5 @@ def test_link_size(): with patch("pip._internal.network.session.PipSession.head") as mock_head: mock_head.return_value.headers = {"Content-Length": "12345"} - link = Link(url="https://example.com/package.tar.gz", filename="package.tar.gz") + link = Link(url="https://example.com/package.tar.gz") assert link.size == 12345 # Verify size is correctly fetched \ No newline at end of file From 0f4de30be6b4e370c8d53dc6d7399f6b0cfce787 Mon Sep 17 00:00:00 2001 From: stonebig Date: Thu, 15 May 2025 22:24:41 +0200 Subject: [PATCH 04/10] fix tests with size pre-commit.ci autofix --- tests/functional/test_lock.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/functional/test_lock.py b/tests/functional/test_lock.py index 2bab10db812..7facf606a0a 100644 --- a/tests/functional/test_lock.py +++ b/tests/functional/test_lock.py @@ -33,6 +33,7 @@ def test_lock_wheel_from_findlinks( "wheels": [ { "name": "simplewheel-2.0-1-py2.py3-none-any.whl", + "size": 2156, "url": path_to_url( str( shared_data.root @@ -80,9 +81,11 @@ def test_lock_sdist_from_findlinks( ), }, "name": "simple-2.0.tar.gz", + "size": 673, "url": path_to_url( str(shared_data.root / "packages" / "simple-2.0.tar.gz") ), + }, "version": "2.0", }, @@ -171,6 +174,7 @@ def test_lock_local_editable_with_dep( / "simplewheel-2.0-1-py2.py3-none-any.whl" ) ), + "size": 2156, "hashes": { "sha256": ( "71e1ca6b16ae3382a698c284013f6650" From f89dbbd7a55ae41820667432a46befc0bee80222 Mon Sep 17 00:00:00 2001 From: stonebig Date: Thu, 15 May 2025 22:39:48 +0200 Subject: [PATCH 05/10] make my size --- src/pip/_internal/models/pylock.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/pip/_internal/models/pylock.py b/src/pip/_internal/models/pylock.py index 72897f24987..68eacc69f68 100644 --- a/src/pip/_internal/models/pylock.py +++ b/src/pip/_internal/models/pylock.py @@ -125,6 +125,7 @@ def from_install_requirement(cls, ireq: InstallRequirement, base_dir: Path) -> S raise NotImplementedError() package.archive = PackageArchive( url=download_info.url, + size=link.size, hashes=download_info.info.hashes, subdirectory=download_info.subdirectory, ) From 85a74150130df47c46f22699ce1dc4a0a9841249 Mon Sep 17 00:00:00 2001 From: stonebig Date: Thu, 15 May 2025 23:13:37 +0200 Subject: [PATCH 06/10] backpedal on archives --- src/pip/_internal/models/pylock.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pip/_internal/models/pylock.py b/src/pip/_internal/models/pylock.py index 68eacc69f68..474dd76dfa6 100644 --- a/src/pip/_internal/models/pylock.py +++ b/src/pip/_internal/models/pylock.py @@ -47,7 +47,7 @@ class PackageDirectory: class PackageArchive: url: str | None # (not supported) path: Optional[str] - size: int | None + # (not supported) size: int | None # (not supported) upload_time: Optional[datetime] hashes: dict[str, str] subdirectory: str | None @@ -125,7 +125,7 @@ def from_install_requirement(cls, ireq: InstallRequirement, base_dir: Path) -> S raise NotImplementedError() package.archive = PackageArchive( url=download_info.url, - size=link.size, + # size=link.size, hashes=download_info.info.hashes, subdirectory=download_info.subdirectory, ) From e263c15b7bb13fcdd8f7c941a95bc411031159a7 Mon Sep 17 00:00:00 2001 From: stonebig Date: Thu, 15 May 2025 23:44:24 +0200 Subject: [PATCH 07/10] again --- src/pip/_internal/models/link.py | 4 ++-- tests/functional/test_lock.py | 3 +-- tests/unit/test_link.py | 2 +- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/src/pip/_internal/models/link.py b/src/pip/_internal/models/link.py index 7c33b77a4a3..cab59c63c04 100644 --- a/src/pip/_internal/models/link.py +++ b/src/pip/_internal/models/link.py @@ -520,10 +520,10 @@ def size(self) -> int | None: size = response.headers.get("Content-Length") if size: return int(size) - except (NetworkConnectionError, ValueError) as e: + except (ValueError) as e: logger.warning(f"Could not fetch size for {self.url}: {e}") return None - + @property def is_file(self) -> bool: return self.scheme == "file" diff --git a/tests/functional/test_lock.py b/tests/functional/test_lock.py index 7facf606a0a..0a0148a1826 100644 --- a/tests/functional/test_lock.py +++ b/tests/functional/test_lock.py @@ -240,7 +240,7 @@ def test_lock_archive(script: PipTestEnvironment, shared_data: TestData) -> None ] def test_lock_includes_size(script: PipTestEnvironment, shared_data: TestData) -> None: - result = script.pip( + script.pip( "lock", "simplewheel==2.0", "--no-index", @@ -251,4 +251,3 @@ def test_lock_includes_size(script: PipTestEnvironment, shared_data: TestData) - pylock = tomllib.loads(script.scratch_path.joinpath("pylock.toml").read_text()) assert "size" in pylock["packages"][0]["wheels"][0] assert pylock["packages"][0]["wheels"][0]["size"] > 0 - \ No newline at end of file diff --git a/tests/unit/test_link.py b/tests/unit/test_link.py index 5f187d36e11..38e8941dc3f 100644 --- a/tests/unit/test_link.py +++ b/tests/unit/test_link.py @@ -248,4 +248,4 @@ def test_link_size(): mock_head.return_value.headers = {"Content-Length": "12345"} link = Link(url="https://example.com/package.tar.gz") - assert link.size == 12345 # Verify size is correctly fetched \ No newline at end of file + assert link.size == 12345 # Verify size is correctly fetched From 89538df956cf2977b53a91297937e6724a8f4336 Mon Sep 17 00:00:00 2001 From: stonebig Date: Thu, 15 May 2025 23:58:01 +0200 Subject: [PATCH 08/10] again --- src/pip/_internal/models/link.py | 2 +- tests/unit/test_link.py | 13 ++++++------- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/src/pip/_internal/models/link.py b/src/pip/_internal/models/link.py index cab59c63c04..03402d3aec5 100644 --- a/src/pip/_internal/models/link.py +++ b/src/pip/_internal/models/link.py @@ -521,7 +521,7 @@ def size(self) -> int | None: if size: return int(size) except (ValueError) as e: - logger.warning(f"Could not fetch size for {self.url}: {e}") + logger.warning("Could not fetch size for " + self.url + ": " + e) return None @property diff --git a/tests/unit/test_link.py b/tests/unit/test_link.py index 38e8941dc3f..647c72ba1ac 100644 --- a/tests/unit/test_link.py +++ b/tests/unit/test_link.py @@ -189,6 +189,12 @@ def test_is_vcs(self, url: str, expected: bool) -> None: link = Link(url) assert link.is_vcs is expected + def test_link_size(self) -> None: + with patch("pip._internal.network.session.PipSession.head") as mock_head: + mock_head.return_value.headers = {"Content-Length": "12345"} + link = Link(url="https://example.com/package.tar.gz") + assert link.size == 12345 + @pytest.mark.parametrize( "url1, url2", @@ -242,10 +248,3 @@ def test_links_equivalent(url1: str, url2: str) -> None: def test_links_equivalent_false(url1: str, url2: str) -> None: assert not links_equivalent(Link(url1), Link(url2)) -def test_link_size(): - # Mock the HTTP HEAD response - with patch("pip._internal.network.session.PipSession.head") as mock_head: - mock_head.return_value.headers = {"Content-Length": "12345"} - - link = Link(url="https://example.com/package.tar.gz") - assert link.size == 12345 # Verify size is correctly fetched From eee2171fef2c8f6c7b9b376c4bf91674d1318603 Mon Sep 17 00:00:00 2001 From: stonebig Date: Fri, 16 May 2025 00:02:34 +0200 Subject: [PATCH 09/10] last try --- src/pip/_internal/models/link.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pip/_internal/models/link.py b/src/pip/_internal/models/link.py index 03402d3aec5..dd88db6e2f2 100644 --- a/src/pip/_internal/models/link.py +++ b/src/pip/_internal/models/link.py @@ -521,7 +521,7 @@ def size(self) -> int | None: if size: return int(size) except (ValueError) as e: - logger.warning("Could not fetch size for " + self.url + ": " + e) + logger.warning("Could not fetch size for %s: %s" , self.url , e) return None @property From 184011e73f81fb88d0b85ce77cbb68086fa9959a Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 15 May 2025 22:26:55 +0000 Subject: [PATCH 10/10] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/pip/_internal/models/link.py | 7 ++++--- tests/functional/test_lock.py | 2 +- tests/unit/test_link.py | 4 ++-- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/pip/_internal/models/link.py b/src/pip/_internal/models/link.py index dd88db6e2f2..2a8567d717d 100644 --- a/src/pip/_internal/models/link.py +++ b/src/pip/_internal/models/link.py @@ -507,11 +507,12 @@ def hash_name(self) -> str | None: @property def show_url(self) -> str: return posixpath.basename(self._url.split("#", 1)[0].split("?", 1)[0]) - + @property def size(self) -> int | None: """Fetch the size of the file from HTTP headers if available.""" from pip._internal.network.session import PipSession + try: session = PipSession() response = session.head(self.url, allow_redirects=True) @@ -520,8 +521,8 @@ def size(self) -> int | None: size = response.headers.get("Content-Length") if size: return int(size) - except (ValueError) as e: - logger.warning("Could not fetch size for %s: %s" , self.url , e) + except ValueError as e: + logger.warning("Could not fetch size for %s: %s", self.url, e) return None @property diff --git a/tests/functional/test_lock.py b/tests/functional/test_lock.py index 0a0148a1826..a8da8e38ef9 100644 --- a/tests/functional/test_lock.py +++ b/tests/functional/test_lock.py @@ -85,7 +85,6 @@ def test_lock_sdist_from_findlinks( "url": path_to_url( str(shared_data.root / "packages" / "simple-2.0.tar.gz") ), - }, "version": "2.0", }, @@ -239,6 +238,7 @@ def test_lock_archive(script: PipTestEnvironment, shared_data: TestData) -> None }, ] + def test_lock_includes_size(script: PipTestEnvironment, shared_data: TestData) -> None: script.pip( "lock", diff --git a/tests/unit/test_link.py b/tests/unit/test_link.py index 647c72ba1ac..f82f28dc834 100644 --- a/tests/unit/test_link.py +++ b/tests/unit/test_link.py @@ -1,11 +1,12 @@ from __future__ import annotations +from unittest.mock import patch + import pytest from pip._internal.models.link import Link, links_equivalent from pip._internal.utils.hashes import Hashes -from unittest.mock import patch class TestLink: @pytest.mark.parametrize( @@ -247,4 +248,3 @@ def test_links_equivalent(url1: str, url2: str) -> None: ) def test_links_equivalent_false(url1: str, url2: str) -> None: assert not links_equivalent(Link(url1), Link(url2)) -