From 939dc42109abb60adc4fa2515d08a6cf1daa7dc2 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Thu, 13 Feb 2025 21:48:17 +0530 Subject: [PATCH 01/30] Add an `--index-url` flag for the `piplite` CLI --- packages/pyodide-kernel/py/piplite/piplite/cli.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/packages/pyodide-kernel/py/piplite/piplite/cli.py b/packages/pyodide-kernel/py/piplite/piplite/cli.py index f4da0311..4c88c481 100644 --- a/packages/pyodide-kernel/py/piplite/piplite/cli.py +++ b/packages/pyodide-kernel/py/piplite/piplite/cli.py @@ -76,6 +76,12 @@ def _get_parser() -> ArgumentParser: action="store_true", help="whether pre-release packages should be considered", ) + parser.add_argument( + "--index-url", + "-i", + type=str, + help="the index URL to use for package lookup", + ) parser.add_argument( "packages", nargs="*", From bed093c68384e279de3e57cf13c8694cc664df58 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Thu, 13 Feb 2025 23:54:39 +0530 Subject: [PATCH 02/30] Add `pipliteInstallDefaultOptions` to schema, types --- .../schema/kernel.v0.schema.json | 15 ++++++++++++++ .../pyodide-kernel-extension/src/index.ts | 8 ++++++++ packages/pyodide-kernel/src/kernel.ts | 5 +++++ packages/pyodide-kernel/src/tokens.ts | 20 +++++++++++++++++++ packages/pyodide-kernel/src/worker.ts | 2 +- 5 files changed, 49 insertions(+), 1 deletion(-) diff --git a/packages/pyodide-kernel-extension/schema/kernel.v0.schema.json b/packages/pyodide-kernel-extension/schema/kernel.v0.schema.json index 8449920f..b9616502 100644 --- a/packages/pyodide-kernel-extension/schema/kernel.v0.schema.json +++ b/packages/pyodide-kernel-extension/schema/kernel.v0.schema.json @@ -25,6 +25,21 @@ "default": [], "format": "uri" }, + "pipliteInstallDefaultOptions": { + "type": "object", + "description": "Default options to pass to piplite.install", + "default": {}, + "properties": { + "indexUrls": { + "type": "array", + "items": { + "type": "string", + "format": "uri" + }, + "description": "Base URLs of extra indices to use" + } + } + }, "loadPyodideOptions": { "type": "object", "description": "additional options to provide to `loadPyodide`, see https://pyodide.org/en/stable/usage/api/js-api.html#globalThis.loadPyodide", diff --git a/packages/pyodide-kernel-extension/src/index.ts b/packages/pyodide-kernel-extension/src/index.ts index 448143b2..1e8cf54a 100644 --- a/packages/pyodide-kernel-extension/src/index.ts +++ b/packages/pyodide-kernel-extension/src/index.ts @@ -58,6 +58,10 @@ const kernel: JupyterLiteServerPlugin = { const pipliteUrls = rawPipUrls.map((pipUrl: string) => URLExt.parse(pipUrl).href); const disablePyPIFallback = !!config.disablePyPIFallback; const loadPyodideOptions = config.loadPyodideOptions || {}; + const pipliteInstallDefaultOptions = config.pipliteInstallDefaultOptions || {}; + + // Parse any configured index URLs + const indexUrls = pipliteInstallDefaultOptions.indexUrls || []; for (const [key, value] of Object.entries(loadPyodideOptions)) { if (key.endsWith('URL') && typeof value === 'string') { @@ -99,6 +103,10 @@ const kernel: JupyterLiteServerPlugin = { mountDrive, loadPyodideOptions, contentsManager, + pipliteInstallDefaultOptions: { + indexUrls, + ...pipliteInstallDefaultOptions + } }); }, }); diff --git a/packages/pyodide-kernel/src/kernel.ts b/packages/pyodide-kernel/src/kernel.ts index 16036257..fc30c320 100644 --- a/packages/pyodide-kernel/src/kernel.ts +++ b/packages/pyodide-kernel/src/kernel.ts @@ -381,6 +381,11 @@ export namespace PyodideKernel { */ mountDrive: boolean; + /** + * Default options to pass to piplite.install + */ + pipliteInstallDefaultOptions?: IPyodideWorkerKernel.IPipliteInstallOptions; + /** * additional options to provide to `loadPyodide` * @see https://pyodide.org/en/stable/usage/api/js-api.html#globalThis.loadPyodide diff --git a/packages/pyodide-kernel/src/tokens.ts b/packages/pyodide-kernel/src/tokens.ts index eaaed75f..1c8f9e12 100644 --- a/packages/pyodide-kernel/src/tokens.ts +++ b/packages/pyodide-kernel/src/tokens.ts @@ -51,6 +51,21 @@ export type IRemotePyodideWorkerKernel = IPyodideWorkerKernel; * An namespace for Pyodide workers. */ export namespace IPyodideWorkerKernel { + /** + * Options for piplite installation. + */ + export interface IPipliteInstallOptions { + /** + * Base URLs of extra indices to use + */ + indexUrls?: string[]; + + /** + * Any additional piplite install options + */ + [key: string]: any; + } + /** * Initialization options for a worker. */ @@ -90,6 +105,11 @@ export namespace IPyodideWorkerKernel { */ mountDrive: boolean; + /** + * Default options to pass to piplite.install + */ + pipliteInstallDefaultOptions?: IPyodideWorkerKernel.IPipliteInstallOptions; + /** * additional options to provide to `loadPyodide` * @see https://pyodide.org/en/stable/usage/api/js-api.html#globalThis.loadPyodide diff --git a/packages/pyodide-kernel/src/worker.ts b/packages/pyodide-kernel/src/worker.ts index d5b91829..23bd14ee 100644 --- a/packages/pyodide-kernel/src/worker.ts +++ b/packages/pyodide-kernel/src/worker.ts @@ -65,7 +65,7 @@ export class PyodideRemoteKernel { throw new Error('Uninitialized'); } - const { pipliteWheelUrl, disablePyPIFallback, pipliteUrls, loadPyodideOptions } = + const { pipliteWheelUrl, disablePyPIFallback, pipliteUrls, loadPyodideOptions, pipliteInstallDefaultOptions } = this._options; const preloaded = (loadPyodideOptions || {}).packages || []; From 5d45a407b0a6df8a9b72e4d59d80d2091784dfd2 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Thu, 13 Feb 2025 23:54:49 +0530 Subject: [PATCH 03/30] Fix a typo --- packages/pyodide-kernel/src/tokens.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pyodide-kernel/src/tokens.ts b/packages/pyodide-kernel/src/tokens.ts index 1c8f9e12..17a80800 100644 --- a/packages/pyodide-kernel/src/tokens.ts +++ b/packages/pyodide-kernel/src/tokens.ts @@ -48,7 +48,7 @@ export interface IPyodideWorkerKernel extends IWorkerKernel { export type IRemotePyodideWorkerKernel = IPyodideWorkerKernel; /** - * An namespace for Pyodide workers. + * A namespace for Pyodide workers. */ export namespace IPyodideWorkerKernel { /** From 83757c02d5781edf436e246878cc7c9547f83bf6 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Thu, 13 Feb 2025 23:56:12 +0530 Subject: [PATCH 04/30] Add effective index URLs to pass to piplite --- .../py/piplite/piplite/piplite.py | 32 +++++++++++++------ packages/pyodide-kernel/src/worker.ts | 18 ++++++++--- 2 files changed, 35 insertions(+), 15 deletions(-) diff --git a/packages/pyodide-kernel/py/piplite/piplite/piplite.py b/packages/pyodide-kernel/py/piplite/piplite/piplite.py index e19b7e24..84c6581a 100644 --- a/packages/pyodide-kernel/py/piplite/piplite/piplite.py +++ b/packages/pyodide-kernel/py/piplite/piplite/piplite.py @@ -33,6 +33,9 @@ #: a well-known file name respected by the rest of the build chain ALL_JSON = "/all.json" +#: default index URLs to use when no specific index URLs are provided +_PIPLITE_DEFAULT_INDEX_URLS = None + class PiplitePyPIDisabled(ValueError): """An error for when PyPI is disabled at the site level, but a download was @@ -117,16 +120,25 @@ async def _install( verbose: bool | int = False, ): """Invoke micropip.install with a patch to get data from local indexes""" - with patch("micropip.package_index.query_package", _query_package): - return await micropip.install( - requirements=requirements, - keep_going=keep_going, - deps=deps, - credentials=credentials, - pre=pre, - index_urls=index_urls, - verbose=verbose, - ) + + try: + # Use default index URLs if none provided and defaults exist + effective_index_urls = index_urls if index_urls is not None else _PIPLITE_DEFAULT_INDEX_URLS + + with patch("micropip.package_index.query_package", _query_package): + return await micropip.install( + requirements=requirements, + keep_going=keep_going, + deps=deps, + credentials=credentials, + pre=pre, + index_urls=effective_index_urls, + verbose=verbose, + ) + except Exception as e: + if effective_index_urls: + logger.error(f"Failed to install using index URLs {effective_index_urls}: {e}") + raise def install( diff --git a/packages/pyodide-kernel/src/worker.ts b/packages/pyodide-kernel/src/worker.ts index 23bd14ee..708bca46 100644 --- a/packages/pyodide-kernel/src/worker.ts +++ b/packages/pyodide-kernel/src/worker.ts @@ -81,12 +81,20 @@ export class PyodideRemoteKernel { `); } + const pythonConfig = [ + 'import piplite.piplite', + `piplite.piplite._PIPLITE_DISABLE_PYPI = ${disablePyPIFallback ? 'True' : 'False'}`, + `piplite.piplite._PIPLITE_URLS = ${JSON.stringify(pipliteUrls)}` + ]; + + if (pipliteInstallDefaultOptions?.indexUrls) { + pythonConfig.push( + `piplite.piplite._PIPLITE_DEFAULT_INDEX_URLS = ${JSON.stringify(pipliteInstallDefaultOptions.indexUrls)}` + ); + } + // get piplite early enough to impact pyodide-kernel dependencies - await this._pyodide.runPythonAsync(` - import piplite.piplite - piplite.piplite._PIPLITE_DISABLE_PYPI = ${disablePyPIFallback ? 'True' : 'False'} - piplite.piplite._PIPLITE_URLS = ${JSON.stringify(pipliteUrls)} - `); + await this._pyodide.runPythonAsync(pythonConfig.join('\n')); } protected async initKernel(options: IPyodideWorkerKernel.IOptions): Promise { From a9b539d8401a5d4e67c8ab117b03723ccebc1ac6 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Thu, 13 Feb 2025 23:57:21 +0530 Subject: [PATCH 05/30] Add `--index-url`, `-i`, to CLI flags --- packages/pyodide-kernel/py/piplite/piplite/cli.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/pyodide-kernel/py/piplite/piplite/cli.py b/packages/pyodide-kernel/py/piplite/piplite/cli.py index 4c88c481..fad9099f 100644 --- a/packages/pyodide-kernel/py/piplite/piplite/cli.py +++ b/packages/pyodide-kernel/py/piplite/piplite/cli.py @@ -30,6 +30,8 @@ async def install( from pathlib import Path REQ_FILE_PREFIX = r"^(-r|--requirements)\s*=?\s*(.*)\s*" +INDEX_URL_PREFIX = r"^(--index-url|-i)\s*=?\s*(.*)\s*" + __all__ = ["get_transformed_code"] @@ -132,6 +134,8 @@ async def get_action_kwargs(argv: list[str]) -> tuple[typing.Optional[str], dict if args.verbose: kwargs["keep_going"] = True + if args.index_url: + kwargs["index_urls"] = args.index_url for req_file in args.requirements or []: kwargs["requirements"] += await _packages_from_requirements_file( Path(req_file) From 95b47005df6439bcd39c36889fe0418d5468b76d Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Thu, 13 Feb 2025 23:58:40 +0530 Subject: [PATCH 06/30] Handle index URLs with requirements files Here, we just assume one index per requirement file as this is the pip behaviour. Multiple indices will require different requirements files in subsequent commands. --- .../pyodide-kernel/py/piplite/piplite/cli.py | 83 ++++++++++++++----- 1 file changed, 61 insertions(+), 22 deletions(-) diff --git a/packages/pyodide-kernel/py/piplite/piplite/cli.py b/packages/pyodide-kernel/py/piplite/piplite/cli.py index fad9099f..ed7003b3 100644 --- a/packages/pyodide-kernel/py/piplite/piplite/cli.py +++ b/packages/pyodide-kernel/py/piplite/piplite/cli.py @@ -26,9 +26,24 @@ async def install( import re import sys import typing +from typing import Optional, List, Tuple +from dataclasses import dataclass + from argparse import ArgumentParser from pathlib import Path + +@dataclass +class RequirementsContext: + """Track state while parsing requiremets files.""" + index_url: Optional[str] = None + requirements: List[str] = None + + def __post_init__(self): + if self.requirements is None: + self.requirements = [] + + REQ_FILE_PREFIX = r"^(-r|--requirements)\s*=?\s*(.*)\s*" INDEX_URL_PREFIX = r"^(--index-url|-i)\s*=?\s*(.*)\s*" @@ -136,51 +151,75 @@ async def get_action_kwargs(argv: list[str]) -> tuple[typing.Optional[str], dict if args.index_url: kwargs["index_urls"] = args.index_url + + # Handle requirements files in case they contain index URLs + current_index_url = args.index_url for req_file in args.requirements or []: - kwargs["requirements"] += await _packages_from_requirements_file( - Path(req_file) - ) + reqs, index_url = await _packages_from_requirements_file(Path(req_file)) + kwargs["requirements"].extend(reqs) + # Update index URL if one was found in requirements file + if index_url is not None: + current_index_url = index_url + + if current_index_url: + kwargs["index_urls"] = current_index_url return action, kwargs -async def _packages_from_requirements_file(req_path: Path) -> list[str]: - """Extract (potentially nested) package requirements from a requirements file.""" +async def _packages_from_requirements_file(req_path: Path) -> Tuple[List[str], Optional[str]]: + """Extract (potentially nested) package requirements from a requirements file. + + Returns: + Tuple of (list of package requirements, optional index URL) + """ if not req_path.exists(): warn(f"piplite could not find requirements file {req_path}") return [] - requirements = [] + context = RequirementsContext() - for line_no, line in enumerate(req_path.read_text(encoding="utf").splitlines()): - requirements += await _packages_from_requirements_line( - req_path, line_no + 1, line - ) + for line_no, line in enumerate(req_path.read_text(encoding="utf-8").splitlines()): + await _packages_from_requirements_line(req_path, line_no + 1, line, context) - return requirements + return context.requirements, context.index_url async def _packages_from_requirements_line( - req_path: Path, line_no: int, line: str -) -> list[str]: + req_path: Path, line_no: int, line: str, context: RequirementsContext +) -> None: """Extract (potentially nested) package requirements from line of a requirements file. `micropip` has a sufficient pep508 implementation to handle most cases """ req = line.strip().split("#")[0].strip() - # is it another requirement file? + if not req: + return + + # Check for nested requirements file req_file_match = re.match(REQ_FILE_PREFIX, req) if req_file_match: - if req_file_match[2].startswith("/"): - sub_req = Path(req) + sub_path = req_file_match[2] + if sub_path.startswith("/"): + sub_req = Path(sub_path) else: - sub_req = req_path.parent / req_file_match[2] - return await _packages_from_requirements_file(sub_req) + sub_req = req_path.parent / sub_path + sub_reqs, sub_index = await _packages_from_requirements_file(sub_req) + # Only update index URL if one was found + if sub_index is not None: + context.index_url = sub_index + context.requirements.extend(sub_reqs) + return + + # Check for index URL specification + index_match = re.match(INDEX_URL_PREFIX, req) + if index_match: + context.index_url = index_match[2].strip() + return if req.startswith("-"): warn(f"{req_path}:{line_no}: unrecognized requirement: {req}") - req = None - if not req: - return [] - return [req] + return + + context.requirements.append(req) From 32e0f51a8e88ea5afa701c451580ed1849e93e78 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Fri, 14 Feb 2025 00:15:17 +0530 Subject: [PATCH 07/30] Fix JS prettier lint errors --- packages/pyodide-kernel-extension/src/index.ts | 4 ++-- packages/pyodide-kernel/src/worker.ts | 13 +++++++++---- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/packages/pyodide-kernel-extension/src/index.ts b/packages/pyodide-kernel-extension/src/index.ts index 1e8cf54a..09f446d3 100644 --- a/packages/pyodide-kernel-extension/src/index.ts +++ b/packages/pyodide-kernel-extension/src/index.ts @@ -105,8 +105,8 @@ const kernel: JupyterLiteServerPlugin = { contentsManager, pipliteInstallDefaultOptions: { indexUrls, - ...pipliteInstallDefaultOptions - } + ...pipliteInstallDefaultOptions, + }, }); }, }); diff --git a/packages/pyodide-kernel/src/worker.ts b/packages/pyodide-kernel/src/worker.ts index 708bca46..d20a18fa 100644 --- a/packages/pyodide-kernel/src/worker.ts +++ b/packages/pyodide-kernel/src/worker.ts @@ -65,8 +65,13 @@ export class PyodideRemoteKernel { throw new Error('Uninitialized'); } - const { pipliteWheelUrl, disablePyPIFallback, pipliteUrls, loadPyodideOptions, pipliteInstallDefaultOptions } = - this._options; + const { + pipliteWheelUrl, + disablePyPIFallback, + pipliteUrls, + loadPyodideOptions, + pipliteInstallDefaultOptions, + } = this._options; const preloaded = (loadPyodideOptions || {}).packages || []; @@ -84,12 +89,12 @@ export class PyodideRemoteKernel { const pythonConfig = [ 'import piplite.piplite', `piplite.piplite._PIPLITE_DISABLE_PYPI = ${disablePyPIFallback ? 'True' : 'False'}`, - `piplite.piplite._PIPLITE_URLS = ${JSON.stringify(pipliteUrls)}` + `piplite.piplite._PIPLITE_URLS = ${JSON.stringify(pipliteUrls)}`, ]; if (pipliteInstallDefaultOptions?.indexUrls) { pythonConfig.push( - `piplite.piplite._PIPLITE_DEFAULT_INDEX_URLS = ${JSON.stringify(pipliteInstallDefaultOptions.indexUrls)}` + `piplite.piplite._PIPLITE_DEFAULT_INDEX_URLS = ${JSON.stringify(pipliteInstallDefaultOptions.indexUrls)}`, ); } From 8014816bcaea6a9829df850e5c6a8438993af4bb Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Fri, 14 Feb 2025 00:22:28 +0530 Subject: [PATCH 08/30] Now fix Python linter errors --- jupyterlite_pyodide_kernel/tests/test_repo.py | 6 +++--- packages/pyodide-kernel/py/piplite/piplite/cli.py | 7 +++++-- packages/pyodide-kernel/py/piplite/piplite/piplite.py | 8 ++++++-- 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/jupyterlite_pyodide_kernel/tests/test_repo.py b/jupyterlite_pyodide_kernel/tests/test_repo.py index fab999f2..20e7ad2e 100644 --- a/jupyterlite_pyodide_kernel/tests/test_repo.py +++ b/jupyterlite_pyodide_kernel/tests/test_repo.py @@ -22,9 +22,9 @@ def test_pyodide_version(): kernel_pkg_data = json.loads(KERNEL_PKG_JSON.read_text(**UTF8)) - assert kernel_pkg_data["devDependencies"]["pyodide"] == PYODIDE_VERSION, ( - f"{kernel_pkg_data} pyodide devDependency is not {PYODIDE_VERSION}" - ) + assert ( + kernel_pkg_data["devDependencies"]["pyodide"] == PYODIDE_VERSION + ), f"{kernel_pkg_data} pyodide devDependency is not {PYODIDE_VERSION}" @pytest.fixture diff --git a/packages/pyodide-kernel/py/piplite/piplite/cli.py b/packages/pyodide-kernel/py/piplite/piplite/cli.py index ed7003b3..180deafd 100644 --- a/packages/pyodide-kernel/py/piplite/piplite/cli.py +++ b/packages/pyodide-kernel/py/piplite/piplite/cli.py @@ -36,6 +36,7 @@ async def install( @dataclass class RequirementsContext: """Track state while parsing requiremets files.""" + index_url: Optional[str] = None requirements: List[str] = None @@ -167,7 +168,9 @@ async def get_action_kwargs(argv: list[str]) -> tuple[typing.Optional[str], dict return action, kwargs -async def _packages_from_requirements_file(req_path: Path) -> Tuple[List[str], Optional[str]]: +async def _packages_from_requirements_file( + req_path: Path, +) -> Tuple[List[str], Optional[str]]: """Extract (potentially nested) package requirements from a requirements file. Returns: @@ -196,7 +199,7 @@ async def _packages_from_requirements_line( req = line.strip().split("#")[0].strip() if not req: return - + # Check for nested requirements file req_file_match = re.match(REQ_FILE_PREFIX, req) if req_file_match: diff --git a/packages/pyodide-kernel/py/piplite/piplite/piplite.py b/packages/pyodide-kernel/py/piplite/piplite/piplite.py index 84c6581a..2f1a3425 100644 --- a/packages/pyodide-kernel/py/piplite/piplite/piplite.py +++ b/packages/pyodide-kernel/py/piplite/piplite/piplite.py @@ -123,7 +123,9 @@ async def _install( try: # Use default index URLs if none provided and defaults exist - effective_index_urls = index_urls if index_urls is not None else _PIPLITE_DEFAULT_INDEX_URLS + effective_index_urls = ( + index_urls if index_urls is not None else _PIPLITE_DEFAULT_INDEX_URLS + ) with patch("micropip.package_index.query_package", _query_package): return await micropip.install( @@ -137,7 +139,9 @@ async def _install( ) except Exception as e: if effective_index_urls: - logger.error(f"Failed to install using index URLs {effective_index_urls}: {e}") + logger.error( + f"Failed to install using index URLs {effective_index_urls}: {e}" + ) raise From 48d19706417a96e5490f24a0d615af1de19c202c Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Fri, 14 Feb 2025 04:19:59 +0530 Subject: [PATCH 09/30] Fix typo --- packages/pyodide-kernel/py/piplite/piplite/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pyodide-kernel/py/piplite/piplite/cli.py b/packages/pyodide-kernel/py/piplite/piplite/cli.py index 180deafd..9353f595 100644 --- a/packages/pyodide-kernel/py/piplite/piplite/cli.py +++ b/packages/pyodide-kernel/py/piplite/piplite/cli.py @@ -35,7 +35,7 @@ async def install( @dataclass class RequirementsContext: - """Track state while parsing requiremets files.""" + """Track state while parsing requirements files.""" index_url: Optional[str] = None requirements: List[str] = None From 7207454e064ea0af0024b004a3e18150b4ea0105 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Fri, 14 Feb 2025 04:23:18 +0530 Subject: [PATCH 10/30] Log what index URL is being used if verbose --- packages/pyodide-kernel/py/piplite/piplite/piplite.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/pyodide-kernel/py/piplite/piplite/piplite.py b/packages/pyodide-kernel/py/piplite/piplite/piplite.py index 2f1a3425..4fcfc968 100644 --- a/packages/pyodide-kernel/py/piplite/piplite/piplite.py +++ b/packages/pyodide-kernel/py/piplite/piplite/piplite.py @@ -127,6 +127,9 @@ async def _install( index_urls if index_urls is not None else _PIPLITE_DEFAULT_INDEX_URLS ) + if verbose: + logger.info(f"Installing with index URLs: {effective_index_urls}") + with patch("micropip.package_index.query_package", _query_package): return await micropip.install( requirements=requirements, From ffe3907011ef9ee48be96c383fe014ebc1d8b138 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Fri, 14 Feb 2025 05:29:11 +0530 Subject: [PATCH 11/30] Fix, allow adding index URL inside a requirements file --- .../pyodide-kernel/py/piplite/piplite/cli.py | 48 +++++++++++-------- 1 file changed, 28 insertions(+), 20 deletions(-) diff --git a/packages/pyodide-kernel/py/piplite/piplite/cli.py b/packages/pyodide-kernel/py/piplite/piplite/cli.py index 9353f595..3e4509e1 100644 --- a/packages/pyodide-kernel/py/piplite/piplite/cli.py +++ b/packages/pyodide-kernel/py/piplite/piplite/cli.py @@ -44,6 +44,10 @@ def __post_init__(self): if self.requirements is None: self.requirements = [] + def add_requirement(self, req: str): + """Add a requirement with the currently active index URL.""" + self.requirements.append((req, self.index_url)) + REQ_FILE_PREFIX = r"^(-r|--requirements)\s*=?\s*(.*)\s*" INDEX_URL_PREFIX = r"^(--index-url|-i)\s*=?\s*(.*)\s*" @@ -135,11 +139,14 @@ async def get_action_kwargs(argv: list[str]) -> tuple[typing.Optional[str], dict return None, {} kwargs = {} - action = args.action if action == "install": - kwargs["requirements"] = args.packages + if args.index_url: + # If there's a CLI index URL, use it for command-line packages + kwargs["requirements"] = [(pkg, args.index_url) for pkg in args.packages] + else: + kwargs["requirements"] = [(pkg, None) for pkg in args.packages] if args.pre: kwargs["pre"] = True @@ -150,20 +157,23 @@ async def get_action_kwargs(argv: list[str]) -> tuple[typing.Optional[str], dict if args.verbose: kwargs["keep_going"] = True - if args.index_url: - kwargs["index_urls"] = args.index_url - - # Handle requirements files in case they contain index URLs - current_index_url = args.index_url for req_file in args.requirements or []: - reqs, index_url = await _packages_from_requirements_file(Path(req_file)) - kwargs["requirements"].extend(reqs) - # Update index URL if one was found in requirements file - if index_url is not None: - current_index_url = index_url - - if current_index_url: - kwargs["index_urls"] = current_index_url + reqs_with_indices = await _packages_from_requirements_file(Path(req_file)) + kwargs["requirements"].extend(reqs_with_indices) + + # Convert requirements to proper format for piplite.install + if kwargs.get("requirements"): + by_index = {} + for req, idx in kwargs["requirements"]: + by_index.setdefault(idx, []).append(req) + + # Install each group with its index URL + all_requirements = [] + for idx, reqs in by_index.items(): + if idx: + kwargs["index_urls"] = idx + all_requirements.extend(reqs) + kwargs["requirements"] = all_requirements return action, kwargs @@ -208,10 +218,8 @@ async def _packages_from_requirements_line( sub_req = Path(sub_path) else: sub_req = req_path.parent / sub_path - sub_reqs, sub_index = await _packages_from_requirements_file(sub_req) - # Only update index URL if one was found - if sub_index is not None: - context.index_url = sub_index + sub_reqs, _ = await _packages_from_requirements_file(sub_req) + # Use current context's index_url for nested requirements context.requirements.extend(sub_reqs) return @@ -225,4 +233,4 @@ async def _packages_from_requirements_line( warn(f"{req_path}:{line_no}: unrecognized requirement: {req}") return - context.requirements.append(req) + context.add_requirement(req) From 1c8e574f420f40e84a6fa2e3e3fadd03cc2434ba Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Fri, 14 Feb 2025 05:54:04 +0530 Subject: [PATCH 12/30] Mark CLI alias for index_urls in docstring --- packages/pyodide-kernel/py/piplite/piplite/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pyodide-kernel/py/piplite/piplite/cli.py b/packages/pyodide-kernel/py/piplite/piplite/cli.py index 3e4509e1..83adf74c 100644 --- a/packages/pyodide-kernel/py/piplite/piplite/cli.py +++ b/packages/pyodide-kernel/py/piplite/piplite/cli.py @@ -13,7 +13,7 @@ async def install( deps: bool = True, # --no-deps credentials: str | None = None, # no CLI alias pre: bool = False, # --pre - index_urls: list[str] | str | None = None, # no CLI alias + index_urls: list[str] | str | None = None, # -i and --index-url *, verbose: bool | int | None = None, ): From e7d38189b71539fa718d1e021e9cce98c108d91c Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Fri, 14 Feb 2025 06:17:57 +0530 Subject: [PATCH 13/30] Hopefully fix Python linter --- jupyterlite_pyodide_kernel/tests/test_repo.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/jupyterlite_pyodide_kernel/tests/test_repo.py b/jupyterlite_pyodide_kernel/tests/test_repo.py index 20e7ad2e..fab999f2 100644 --- a/jupyterlite_pyodide_kernel/tests/test_repo.py +++ b/jupyterlite_pyodide_kernel/tests/test_repo.py @@ -22,9 +22,9 @@ def test_pyodide_version(): kernel_pkg_data = json.loads(KERNEL_PKG_JSON.read_text(**UTF8)) - assert ( - kernel_pkg_data["devDependencies"]["pyodide"] == PYODIDE_VERSION - ), f"{kernel_pkg_data} pyodide devDependency is not {PYODIDE_VERSION}" + assert kernel_pkg_data["devDependencies"]["pyodide"] == PYODIDE_VERSION, ( + f"{kernel_pkg_data} pyodide devDependency is not {PYODIDE_VERSION}" + ) @pytest.fixture From a5e9565ac37987f0c921b523bd4d5d6585b6058d Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Fri, 14 Feb 2025 06:19:34 +0530 Subject: [PATCH 14/30] Handle tuple unpacking better --- .../pyodide-kernel/py/piplite/piplite/cli.py | 59 +++++++++++-------- 1 file changed, 36 insertions(+), 23 deletions(-) diff --git a/packages/pyodide-kernel/py/piplite/piplite/cli.py b/packages/pyodide-kernel/py/piplite/piplite/cli.py index 83adf74c..a9ec848b 100644 --- a/packages/pyodide-kernel/py/piplite/piplite/cli.py +++ b/packages/pyodide-kernel/py/piplite/piplite/cli.py @@ -142,38 +142,51 @@ async def get_action_kwargs(argv: list[str]) -> tuple[typing.Optional[str], dict action = args.action if action == "install": - if args.index_url: - # If there's a CLI index URL, use it for command-line packages - kwargs["requirements"] = [(pkg, args.index_url) for pkg in args.packages] - else: - kwargs["requirements"] = [(pkg, None) for pkg in args.packages] - - if args.pre: - kwargs["pre"] = True - - if args.no_deps: - kwargs["deps"] = False + all_requirements = [] - if args.verbose: - kwargs["keep_going"] = True + if args.packages: + all_requirements.extend((pkg, args.index_url) for pkg in args.packages) + # Process requirements files for req_file in args.requirements or []: - reqs_with_indices = await _packages_from_requirements_file(Path(req_file)) - kwargs["requirements"].extend(reqs_with_indices) - - # Convert requirements to proper format for piplite.install - if kwargs.get("requirements"): + context = RequirementsContext() + # If we have a CLI index URL, set it as the initial context + if args.index_url: + context.index_url = args.index_url + + if not Path(req_file).exists(): + warn(f"piplite could not find requirements file {req_file}") + continue + + for line_no, line in enumerate( + Path(req_file).read_text(encoding="utf-8").splitlines() + ): + await _packages_from_requirements_line( + Path(req_file), line_no + 1, line, context + ) + + all_requirements.extend(context.requirements) + + if all_requirements: + # Group requirements by index URL by_index = {} - for req, idx in kwargs["requirements"]: + for req, idx in all_requirements: by_index.setdefault(idx, []).append(req) - # Install each group with its index URL - all_requirements = [] + kwargs["requirements"] = [] for idx, reqs in by_index.items(): if idx: kwargs["index_urls"] = idx - all_requirements.extend(reqs) - kwargs["requirements"] = all_requirements + kwargs["requirements"].extend(reqs) + + if args.pre: + kwargs["pre"] = True + + if args.no_deps: + kwargs["deps"] = False + + if args.verbose: + kwargs["keep_going"] = True return action, kwargs From c04d84f81c2e643b134b5a6e4a3ee1a11ffd84bc Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Fri, 14 Feb 2025 06:58:47 +0530 Subject: [PATCH 15/30] Try to fix index URLs in requirements file --- .../pyodide-kernel/py/piplite/piplite/cli.py | 21 +++++++++---------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/packages/pyodide-kernel/py/piplite/piplite/cli.py b/packages/pyodide-kernel/py/piplite/piplite/cli.py index a9ec848b..0a13915d 100644 --- a/packages/pyodide-kernel/py/piplite/piplite/cli.py +++ b/packages/pyodide-kernel/py/piplite/piplite/cli.py @@ -150,9 +150,6 @@ async def get_action_kwargs(argv: list[str]) -> tuple[typing.Optional[str], dict # Process requirements files for req_file in args.requirements or []: context = RequirementsContext() - # If we have a CLI index URL, set it as the initial context - if args.index_url: - context.index_url = args.index_url if not Path(req_file).exists(): warn(f"piplite could not find requirements file {req_file}") @@ -168,16 +165,18 @@ async def get_action_kwargs(argv: list[str]) -> tuple[typing.Optional[str], dict all_requirements.extend(context.requirements) if all_requirements: - # Group requirements by index URL - by_index = {} - for req, idx in all_requirements: - by_index.setdefault(idx, []).append(req) - kwargs["requirements"] = [] - for idx, reqs in by_index.items(): + used_index = None + + for req, idx in all_requirements: if idx: - kwargs["index_urls"] = idx - kwargs["requirements"].extend(reqs) + used_index = idx + kwargs["requirements"].append(req) + + # Set the index URL if one was found (either passed to the CLI or + # passed within the requirements file) + if used_index: + kwargs["index_urls"] = used_index if args.pre: kwargs["pre"] = True From 23852ca4bad8e547dc636ae056696891d016c8eb Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Fri, 14 Feb 2025 20:49:20 +0530 Subject: [PATCH 16/30] Rename `indexUrls` to `index_urls` Co-Authored-By: Nicholas Bollweg --- .../pyodide-kernel-extension/schema/kernel.v0.schema.json | 2 +- packages/pyodide-kernel-extension/src/index.ts | 4 ++-- packages/pyodide-kernel/src/tokens.ts | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/pyodide-kernel-extension/schema/kernel.v0.schema.json b/packages/pyodide-kernel-extension/schema/kernel.v0.schema.json index b9616502..24264109 100644 --- a/packages/pyodide-kernel-extension/schema/kernel.v0.schema.json +++ b/packages/pyodide-kernel-extension/schema/kernel.v0.schema.json @@ -30,7 +30,7 @@ "description": "Default options to pass to piplite.install", "default": {}, "properties": { - "indexUrls": { + "index_urls": { "type": "array", "items": { "type": "string", diff --git a/packages/pyodide-kernel-extension/src/index.ts b/packages/pyodide-kernel-extension/src/index.ts index 09f446d3..2ff65f78 100644 --- a/packages/pyodide-kernel-extension/src/index.ts +++ b/packages/pyodide-kernel-extension/src/index.ts @@ -61,7 +61,7 @@ const kernel: JupyterLiteServerPlugin = { const pipliteInstallDefaultOptions = config.pipliteInstallDefaultOptions || {}; // Parse any configured index URLs - const indexUrls = pipliteInstallDefaultOptions.indexUrls || []; + const index_urls = pipliteInstallDefaultOptions.index_urls || []; for (const [key, value] of Object.entries(loadPyodideOptions)) { if (key.endsWith('URL') && typeof value === 'string') { @@ -104,7 +104,7 @@ const kernel: JupyterLiteServerPlugin = { loadPyodideOptions, contentsManager, pipliteInstallDefaultOptions: { - indexUrls, + index_urls, ...pipliteInstallDefaultOptions, }, }); diff --git a/packages/pyodide-kernel/src/tokens.ts b/packages/pyodide-kernel/src/tokens.ts index 17a80800..bd5dfd09 100644 --- a/packages/pyodide-kernel/src/tokens.ts +++ b/packages/pyodide-kernel/src/tokens.ts @@ -58,7 +58,7 @@ export namespace IPyodideWorkerKernel { /** * Base URLs of extra indices to use */ - indexUrls?: string[]; + index_urls?: string[]; /** * Any additional piplite install options From b3f780816f782892f18c44106c41a9f164e3246d Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Fri, 14 Feb 2025 20:49:52 +0530 Subject: [PATCH 17/30] Single source of truth for installation defaults Co-Authored-By: Nicholas Bollweg --- .../py/piplite/piplite/piplite.py | 56 +++++++++++-------- packages/pyodide-kernel/src/worker.ts | 13 ++--- 2 files changed, 40 insertions(+), 29 deletions(-) diff --git a/packages/pyodide-kernel/py/piplite/piplite/piplite.py b/packages/pyodide-kernel/py/piplite/piplite/piplite.py index 4fcfc968..1b0cbeca 100644 --- a/packages/pyodide-kernel/py/piplite/piplite/piplite.py +++ b/packages/pyodide-kernel/py/piplite/piplite/piplite.py @@ -27,14 +27,24 @@ #: a cache of available packages _PIPLITE_INDICES = {} -#: don't fall back to pypi.org if a package is not found in _PIPLITE_URLS -_PIPLITE_DISABLE_PYPI = False - #: a well-known file name respected by the rest of the build chain ALL_JSON = "/all.json" -#: default index URLs to use when no specific index URLs are provided -_PIPLITE_DEFAULT_INDEX_URLS = None +#: default arguments for piplite.install +_PIPLITE_DEFAULT_INSTALL_ARGS = { + "requirements": None, + "keep_going": False, + "deps": True, + "credentials": None, + "pre": False, + "index_urls": None, + "verbose": False, +} + +# Internal flags that affect package lookup behavior +_PIPLITE_INTERNAL_FLAGS = { + "disable_pypi": False, # don't fall back to pypi.org if package not found in _PIPLITE_URLS +} class PiplitePyPIDisabled(ValueError): @@ -97,7 +107,7 @@ async def _query_package( if pypi_json_from_index: return pypi_json_from_index - if _PIPLITE_DISABLE_PYPI: + if _PIPLITE_INTERNAL_FLAGS["disable_pypi"]: raise PiplitePyPIDisabled( f"{name} could not be installed: PyPI fallback is disabled" ) @@ -122,28 +132,30 @@ async def _install( """Invoke micropip.install with a patch to get data from local indexes""" try: - # Use default index URLs if none provided and defaults exist - effective_index_urls = ( - index_urls if index_urls is not None else _PIPLITE_DEFAULT_INDEX_URLS - ) + install_args = _PIPLITE_DEFAULT_INSTALL_ARGS.copy() + + provided_args = { + "requirements": requirements, + "keep_going": keep_going, + "deps": deps, + "credentials": credentials, + "pre": pre, + "index_urls": index_urls, + "verbose": verbose, + } + install_args.update({k: v for k, v in provided_args.items() if v is not None}) + if verbose: - logger.info(f"Installing with index URLs: {effective_index_urls}") + logger.info(f"Installing with arguments: {install_args}") with patch("micropip.package_index.query_package", _query_package): - return await micropip.install( - requirements=requirements, - keep_going=keep_going, - deps=deps, - credentials=credentials, - pre=pre, - index_urls=effective_index_urls, - verbose=verbose, - ) + return await micropip.install(**install_args) + except Exception as e: - if effective_index_urls: + if install_args.get("index_urls"): logger.error( - f"Failed to install using index URLs {effective_index_urls}: {e}" + f"Failed to install using index URLs {install_args['index_urls']}: {e}" ) raise diff --git a/packages/pyodide-kernel/src/worker.ts b/packages/pyodide-kernel/src/worker.ts index d20a18fa..36066a9a 100644 --- a/packages/pyodide-kernel/src/worker.ts +++ b/packages/pyodide-kernel/src/worker.ts @@ -88,16 +88,15 @@ export class PyodideRemoteKernel { const pythonConfig = [ 'import piplite.piplite', - `piplite.piplite._PIPLITE_DISABLE_PYPI = ${disablePyPIFallback ? 'True' : 'False'}`, + `piplite.piplite._PIPLITE_DEFAULT_INSTALL_ARGS = ${JSON.stringify({ + ...pipliteInstallDefaultOptions, + })}`, + `piplite.piplite._PIPLITE_INTERNAL_FLAGS = ${JSON.stringify({ + disable_pypi: disablePyPIFallback, + })}`, `piplite.piplite._PIPLITE_URLS = ${JSON.stringify(pipliteUrls)}`, ]; - if (pipliteInstallDefaultOptions?.indexUrls) { - pythonConfig.push( - `piplite.piplite._PIPLITE_DEFAULT_INDEX_URLS = ${JSON.stringify(pipliteInstallDefaultOptions.indexUrls)}`, - ); - } - // get piplite early enough to impact pyodide-kernel dependencies await this._pyodide.runPythonAsync(pythonConfig.join('\n')); } From d0fd31ae17e311e6cfef639a272769f6a1c37820 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Fri, 14 Feb 2025 21:00:14 +0530 Subject: [PATCH 18/30] Fix Python formatting --- packages/pyodide-kernel/py/piplite/piplite/piplite.py | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/pyodide-kernel/py/piplite/piplite/piplite.py b/packages/pyodide-kernel/py/piplite/piplite/piplite.py index 1b0cbeca..c9ecb6d1 100644 --- a/packages/pyodide-kernel/py/piplite/piplite/piplite.py +++ b/packages/pyodide-kernel/py/piplite/piplite/piplite.py @@ -145,7 +145,6 @@ async def _install( } install_args.update({k: v for k, v in provided_args.items() if v is not None}) - if verbose: logger.info(f"Installing with arguments: {install_args}") From a9a62b3518dadacef9f51d3939583e298adce490 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Fri, 14 Feb 2025 22:08:42 +0530 Subject: [PATCH 19/30] Revert "Fix Python formatting" This reverts commit d0fd31ae17e311e6cfef639a272769f6a1c37820. --- packages/pyodide-kernel/py/piplite/piplite/piplite.py | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/pyodide-kernel/py/piplite/piplite/piplite.py b/packages/pyodide-kernel/py/piplite/piplite/piplite.py index c9ecb6d1..1b0cbeca 100644 --- a/packages/pyodide-kernel/py/piplite/piplite/piplite.py +++ b/packages/pyodide-kernel/py/piplite/piplite/piplite.py @@ -145,6 +145,7 @@ async def _install( } install_args.update({k: v for k, v in provided_args.items() if v is not None}) + if verbose: logger.info(f"Installing with arguments: {install_args}") From 2d7aed60714362a86803a5b0928b669e98ee6187 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Fri, 14 Feb 2025 22:08:47 +0530 Subject: [PATCH 20/30] Revert "Single source of truth for installation defaults" This reverts commit b3f780816f782892f18c44106c41a9f164e3246d. --- .../py/piplite/piplite/piplite.py | 56 ++++++++----------- packages/pyodide-kernel/src/worker.ts | 13 +++-- 2 files changed, 29 insertions(+), 40 deletions(-) diff --git a/packages/pyodide-kernel/py/piplite/piplite/piplite.py b/packages/pyodide-kernel/py/piplite/piplite/piplite.py index 1b0cbeca..4fcfc968 100644 --- a/packages/pyodide-kernel/py/piplite/piplite/piplite.py +++ b/packages/pyodide-kernel/py/piplite/piplite/piplite.py @@ -27,24 +27,14 @@ #: a cache of available packages _PIPLITE_INDICES = {} +#: don't fall back to pypi.org if a package is not found in _PIPLITE_URLS +_PIPLITE_DISABLE_PYPI = False + #: a well-known file name respected by the rest of the build chain ALL_JSON = "/all.json" -#: default arguments for piplite.install -_PIPLITE_DEFAULT_INSTALL_ARGS = { - "requirements": None, - "keep_going": False, - "deps": True, - "credentials": None, - "pre": False, - "index_urls": None, - "verbose": False, -} - -# Internal flags that affect package lookup behavior -_PIPLITE_INTERNAL_FLAGS = { - "disable_pypi": False, # don't fall back to pypi.org if package not found in _PIPLITE_URLS -} +#: default index URLs to use when no specific index URLs are provided +_PIPLITE_DEFAULT_INDEX_URLS = None class PiplitePyPIDisabled(ValueError): @@ -107,7 +97,7 @@ async def _query_package( if pypi_json_from_index: return pypi_json_from_index - if _PIPLITE_INTERNAL_FLAGS["disable_pypi"]: + if _PIPLITE_DISABLE_PYPI: raise PiplitePyPIDisabled( f"{name} could not be installed: PyPI fallback is disabled" ) @@ -132,30 +122,28 @@ async def _install( """Invoke micropip.install with a patch to get data from local indexes""" try: - install_args = _PIPLITE_DEFAULT_INSTALL_ARGS.copy() - - provided_args = { - "requirements": requirements, - "keep_going": keep_going, - "deps": deps, - "credentials": credentials, - "pre": pre, - "index_urls": index_urls, - "verbose": verbose, - } - install_args.update({k: v for k, v in provided_args.items() if v is not None}) - + # Use default index URLs if none provided and defaults exist + effective_index_urls = ( + index_urls if index_urls is not None else _PIPLITE_DEFAULT_INDEX_URLS + ) if verbose: - logger.info(f"Installing with arguments: {install_args}") + logger.info(f"Installing with index URLs: {effective_index_urls}") with patch("micropip.package_index.query_package", _query_package): - return await micropip.install(**install_args) - + return await micropip.install( + requirements=requirements, + keep_going=keep_going, + deps=deps, + credentials=credentials, + pre=pre, + index_urls=effective_index_urls, + verbose=verbose, + ) except Exception as e: - if install_args.get("index_urls"): + if effective_index_urls: logger.error( - f"Failed to install using index URLs {install_args['index_urls']}: {e}" + f"Failed to install using index URLs {effective_index_urls}: {e}" ) raise diff --git a/packages/pyodide-kernel/src/worker.ts b/packages/pyodide-kernel/src/worker.ts index 36066a9a..d20a18fa 100644 --- a/packages/pyodide-kernel/src/worker.ts +++ b/packages/pyodide-kernel/src/worker.ts @@ -88,15 +88,16 @@ export class PyodideRemoteKernel { const pythonConfig = [ 'import piplite.piplite', - `piplite.piplite._PIPLITE_DEFAULT_INSTALL_ARGS = ${JSON.stringify({ - ...pipliteInstallDefaultOptions, - })}`, - `piplite.piplite._PIPLITE_INTERNAL_FLAGS = ${JSON.stringify({ - disable_pypi: disablePyPIFallback, - })}`, + `piplite.piplite._PIPLITE_DISABLE_PYPI = ${disablePyPIFallback ? 'True' : 'False'}`, `piplite.piplite._PIPLITE_URLS = ${JSON.stringify(pipliteUrls)}`, ]; + if (pipliteInstallDefaultOptions?.indexUrls) { + pythonConfig.push( + `piplite.piplite._PIPLITE_DEFAULT_INDEX_URLS = ${JSON.stringify(pipliteInstallDefaultOptions.indexUrls)}`, + ); + } + // get piplite early enough to impact pyodide-kernel dependencies await this._pyodide.runPythonAsync(pythonConfig.join('\n')); } From a1bcf66529b075ae83bc795ac3b2628aee1c1e5f Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Fri, 14 Feb 2025 22:20:17 +0530 Subject: [PATCH 21/30] Reapply "Single source of truth for installation defaults" This reverts commit 2d7aed60714362a86803a5b0928b669e98ee6187. --- .../py/piplite/piplite/piplite.py | 56 +++++++++++-------- packages/pyodide-kernel/src/worker.ts | 13 ++--- 2 files changed, 40 insertions(+), 29 deletions(-) diff --git a/packages/pyodide-kernel/py/piplite/piplite/piplite.py b/packages/pyodide-kernel/py/piplite/piplite/piplite.py index 4fcfc968..1b0cbeca 100644 --- a/packages/pyodide-kernel/py/piplite/piplite/piplite.py +++ b/packages/pyodide-kernel/py/piplite/piplite/piplite.py @@ -27,14 +27,24 @@ #: a cache of available packages _PIPLITE_INDICES = {} -#: don't fall back to pypi.org if a package is not found in _PIPLITE_URLS -_PIPLITE_DISABLE_PYPI = False - #: a well-known file name respected by the rest of the build chain ALL_JSON = "/all.json" -#: default index URLs to use when no specific index URLs are provided -_PIPLITE_DEFAULT_INDEX_URLS = None +#: default arguments for piplite.install +_PIPLITE_DEFAULT_INSTALL_ARGS = { + "requirements": None, + "keep_going": False, + "deps": True, + "credentials": None, + "pre": False, + "index_urls": None, + "verbose": False, +} + +# Internal flags that affect package lookup behavior +_PIPLITE_INTERNAL_FLAGS = { + "disable_pypi": False, # don't fall back to pypi.org if package not found in _PIPLITE_URLS +} class PiplitePyPIDisabled(ValueError): @@ -97,7 +107,7 @@ async def _query_package( if pypi_json_from_index: return pypi_json_from_index - if _PIPLITE_DISABLE_PYPI: + if _PIPLITE_INTERNAL_FLAGS["disable_pypi"]: raise PiplitePyPIDisabled( f"{name} could not be installed: PyPI fallback is disabled" ) @@ -122,28 +132,30 @@ async def _install( """Invoke micropip.install with a patch to get data from local indexes""" try: - # Use default index URLs if none provided and defaults exist - effective_index_urls = ( - index_urls if index_urls is not None else _PIPLITE_DEFAULT_INDEX_URLS - ) + install_args = _PIPLITE_DEFAULT_INSTALL_ARGS.copy() + + provided_args = { + "requirements": requirements, + "keep_going": keep_going, + "deps": deps, + "credentials": credentials, + "pre": pre, + "index_urls": index_urls, + "verbose": verbose, + } + install_args.update({k: v for k, v in provided_args.items() if v is not None}) + if verbose: - logger.info(f"Installing with index URLs: {effective_index_urls}") + logger.info(f"Installing with arguments: {install_args}") with patch("micropip.package_index.query_package", _query_package): - return await micropip.install( - requirements=requirements, - keep_going=keep_going, - deps=deps, - credentials=credentials, - pre=pre, - index_urls=effective_index_urls, - verbose=verbose, - ) + return await micropip.install(**install_args) + except Exception as e: - if effective_index_urls: + if install_args.get("index_urls"): logger.error( - f"Failed to install using index URLs {effective_index_urls}: {e}" + f"Failed to install using index URLs {install_args['index_urls']}: {e}" ) raise diff --git a/packages/pyodide-kernel/src/worker.ts b/packages/pyodide-kernel/src/worker.ts index d20a18fa..36066a9a 100644 --- a/packages/pyodide-kernel/src/worker.ts +++ b/packages/pyodide-kernel/src/worker.ts @@ -88,16 +88,15 @@ export class PyodideRemoteKernel { const pythonConfig = [ 'import piplite.piplite', - `piplite.piplite._PIPLITE_DISABLE_PYPI = ${disablePyPIFallback ? 'True' : 'False'}`, + `piplite.piplite._PIPLITE_DEFAULT_INSTALL_ARGS = ${JSON.stringify({ + ...pipliteInstallDefaultOptions, + })}`, + `piplite.piplite._PIPLITE_INTERNAL_FLAGS = ${JSON.stringify({ + disable_pypi: disablePyPIFallback, + })}`, `piplite.piplite._PIPLITE_URLS = ${JSON.stringify(pipliteUrls)}`, ]; - if (pipliteInstallDefaultOptions?.indexUrls) { - pythonConfig.push( - `piplite.piplite._PIPLITE_DEFAULT_INDEX_URLS = ${JSON.stringify(pipliteInstallDefaultOptions.indexUrls)}`, - ); - } - // get piplite early enough to impact pyodide-kernel dependencies await this._pyodide.runPythonAsync(pythonConfig.join('\n')); } From a8cf8448037427e4384b32308f192c0b0553ffbe Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Fri, 14 Feb 2025 22:20:25 +0530 Subject: [PATCH 22/30] Reapply "Fix Python formatting" This reverts commit a9a62b3518dadacef9f51d3939583e298adce490. --- packages/pyodide-kernel/py/piplite/piplite/piplite.py | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/pyodide-kernel/py/piplite/piplite/piplite.py b/packages/pyodide-kernel/py/piplite/piplite/piplite.py index 1b0cbeca..c9ecb6d1 100644 --- a/packages/pyodide-kernel/py/piplite/piplite/piplite.py +++ b/packages/pyodide-kernel/py/piplite/piplite/piplite.py @@ -145,7 +145,6 @@ async def _install( } install_args.update({k: v for k, v in provided_args.items() if v is not None}) - if verbose: logger.info(f"Installing with arguments: {install_args}") From d2192feba9cd62a98221833990149f55dbf21572 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Fri, 14 Feb 2025 22:22:59 +0530 Subject: [PATCH 23/30] Fix boolean capitalisation b/w JS/TS and Python --- packages/pyodide-kernel/src/worker.ts | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/packages/pyodide-kernel/src/worker.ts b/packages/pyodide-kernel/src/worker.ts index 36066a9a..d1c1ea86 100644 --- a/packages/pyodide-kernel/src/worker.ts +++ b/packages/pyodide-kernel/src/worker.ts @@ -88,12 +88,14 @@ export class PyodideRemoteKernel { const pythonConfig = [ 'import piplite.piplite', - `piplite.piplite._PIPLITE_DEFAULT_INSTALL_ARGS = ${JSON.stringify({ - ...pipliteInstallDefaultOptions, - })}`, - `piplite.piplite._PIPLITE_INTERNAL_FLAGS = ${JSON.stringify({ - disable_pypi: disablePyPIFallback, - })}`, + 'piplite.piplite._PIPLITE_DEFAULT_INSTALL_ARGS = {', + ' "keep_going": False,', + ' "deps": True,', + ' "credentials": None,', + ' "pre": False,', + ' "verbose": False,', + '}', + `piplite.piplite._PIPLITE_INTERNAL_FLAGS = {"disable_pypi": ${disablePyPIFallback ? 'True' : 'False'}}`, `piplite.piplite._PIPLITE_URLS = ${JSON.stringify(pipliteUrls)}`, ]; From 4a7116c354c811a4fb60f9562df440e07c5394c0 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Fri, 14 Feb 2025 22:33:01 +0530 Subject: [PATCH 24/30] Add a TS fix --- packages/pyodide-kernel/src/worker.ts | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/packages/pyodide-kernel/src/worker.ts b/packages/pyodide-kernel/src/worker.ts index d1c1ea86..c337497e 100644 --- a/packages/pyodide-kernel/src/worker.ts +++ b/packages/pyodide-kernel/src/worker.ts @@ -95,10 +95,20 @@ export class PyodideRemoteKernel { ' "pre": False,', ' "verbose": False,', '}', - `piplite.piplite._PIPLITE_INTERNAL_FLAGS = {"disable_pypi": ${disablePyPIFallback ? 'True' : 'False'}}`, `piplite.piplite._PIPLITE_URLS = ${JSON.stringify(pipliteUrls)}`, + `piplite.piplite._PIPLITE_INTERNAL_FLAGS = {"disable_pypi": ${disablePyPIFallback ? 'True' : 'False'}}`, ]; + if (pipliteInstallDefaultOptions) { + for (const [key, value] of Object.entries(pipliteInstallDefaultOptions)) { + if (value !== undefined) { + pythonConfig.push( + `piplite.piplite._PIPLITE_DEFAULT_INSTALL_ARGS["${key}"] = ${JSON.stringify(value)}`, + ); + } + } + } + // get piplite early enough to impact pyodide-kernel dependencies await this._pyodide.runPythonAsync(pythonConfig.join('\n')); } From 443c206f1f575f1e5c3847138f58e71450cb448b Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Fri, 14 Feb 2025 23:22:04 +0530 Subject: [PATCH 25/30] Fix index URLs and requirements files again --- .../pyodide-kernel/py/piplite/piplite/cli.py | 24 ++++++++++++------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/packages/pyodide-kernel/py/piplite/piplite/cli.py b/packages/pyodide-kernel/py/piplite/piplite/cli.py index 0a13915d..e348537f 100644 --- a/packages/pyodide-kernel/py/piplite/piplite/cli.py +++ b/packages/pyodide-kernel/py/piplite/piplite/cli.py @@ -151,6 +151,11 @@ async def get_action_kwargs(argv: list[str]) -> tuple[typing.Optional[str], dict for req_file in args.requirements or []: context = RequirementsContext() + # If CLI index URL is provided, it should override within-file-level + # index URL for all requirements. + if args.index_url: + context.index_url = args.index_url + if not Path(req_file).exists(): warn(f"piplite could not find requirements file {req_file}") continue @@ -165,18 +170,21 @@ async def get_action_kwargs(argv: list[str]) -> tuple[typing.Optional[str], dict all_requirements.extend(context.requirements) if all_requirements: - kwargs["requirements"] = [] - used_index = None + by_index = {} + file_index_url = None for req, idx in all_requirements: if idx: - used_index = idx - kwargs["requirements"].append(req) + file_index_url = idx + by_index.setdefault(file_index_url, []).append(req) - # Set the index URL if one was found (either passed to the CLI or - # passed within the requirements file) - if used_index: - kwargs["index_urls"] = used_index + # Build final kwargs. We set the index URL if one was found + # (either passed to the CLI or passed within the requirements file) + kwargs["requirements"] = [] + for idx, reqs in by_index.items(): + if idx: + kwargs["index_urls"] = idx + kwargs["requirements"].extend(reqs) if args.pre: kwargs["pre"] = True From e4e7a3099fa87935e92e55d9fe789898c8355fa7 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Fri, 14 Feb 2025 23:48:00 +0530 Subject: [PATCH 26/30] Some more fixes for install order precedence --- .../pyodide-kernel/py/piplite/piplite/cli.py | 37 +++++++++---------- .../py/piplite/piplite/piplite.py | 16 +++++++- 2 files changed, 32 insertions(+), 21 deletions(-) diff --git a/packages/pyodide-kernel/py/piplite/piplite/cli.py b/packages/pyodide-kernel/py/piplite/piplite/cli.py index e348537f..93187cd0 100644 --- a/packages/pyodide-kernel/py/piplite/piplite/cli.py +++ b/packages/pyodide-kernel/py/piplite/piplite/cli.py @@ -130,7 +130,6 @@ async def get_transformed_code(argv: list[str]) -> typing.Optional[str]: async def get_action_kwargs(argv: list[str]) -> tuple[typing.Optional[str], dict]: """Get the arguments to `piplite` subcommands from CLI-like tokens.""" - parser = _get_parser() try: @@ -151,15 +150,11 @@ async def get_action_kwargs(argv: list[str]) -> tuple[typing.Optional[str], dict for req_file in args.requirements or []: context = RequirementsContext() - # If CLI index URL is provided, it should override within-file-level - # index URL for all requirements. - if args.index_url: - context.index_url = args.index_url - if not Path(req_file).exists(): warn(f"piplite could not find requirements file {req_file}") continue + # First let the file be processed normally to capture any index URL for line_no, line in enumerate( Path(req_file).read_text(encoding="utf-8").splitlines() ): @@ -167,24 +162,28 @@ async def get_action_kwargs(argv: list[str]) -> tuple[typing.Optional[str], dict Path(req_file), line_no + 1, line, context ) - all_requirements.extend(context.requirements) + # If CLI provided an index URL, it should override the file's index URL + # We update all requirements to use the CLI index URL instead. Or, we use + # whatever index URL was found in the file (if any). + if args.index_url: + all_requirements.extend( + (req, args.index_url) for req, _ in context.requirements + ) + else: + all_requirements.extend(context.requirements) if all_requirements: - by_index = {} - file_index_url = None + kwargs["requirements"] = [] + active_index_url = None for req, idx in all_requirements: - if idx: - file_index_url = idx - by_index.setdefault(file_index_url, []).append(req) + if idx is not None: + active_index_url = idx + kwargs["requirements"].append(req) - # Build final kwargs. We set the index URL if one was found - # (either passed to the CLI or passed within the requirements file) - kwargs["requirements"] = [] - for idx, reqs in by_index.items(): - if idx: - kwargs["index_urls"] = idx - kwargs["requirements"].extend(reqs) + # Set the final index URL, if we found one + if active_index_url is not None: + kwargs["index_urls"] = active_index_url if args.pre: kwargs["pre"] = True diff --git a/packages/pyodide-kernel/py/piplite/piplite/piplite.py b/packages/pyodide-kernel/py/piplite/piplite/piplite.py index c9ecb6d1..fc68195b 100644 --- a/packages/pyodide-kernel/py/piplite/piplite/piplite.py +++ b/packages/pyodide-kernel/py/piplite/piplite/piplite.py @@ -129,8 +129,15 @@ async def _install( *, verbose: bool | int = False, ): - """Invoke micropip.install with a patch to get data from local indexes""" + """Invoke micropip.install with a patch to get data from local indexes. + This function handles the installation of Python packages, respecting index URLs + from various sources (CLI, requirements files, or defaults) while maintaining + precedence order provided for indices. + + Arguments maintain the same semantics as micropip.install, but with additional + handling of index URLs and installation defaults. + """ try: install_args = _PIPLITE_DEFAULT_INSTALL_ARGS.copy() @@ -140,13 +147,18 @@ async def _install( "deps": deps, "credentials": credentials, "pre": pre, - "index_urls": index_urls, "verbose": verbose, } + + if index_urls is not None: + provided_args["index_urls"] = index_urls + install_args.update({k: v for k, v in provided_args.items() if v is not None}) if verbose: logger.info(f"Installing with arguments: {install_args}") + if install_args.get("index_urls"): + logger.info(f"Using index URL: {install_args['index_urls']}") with patch("micropip.package_index.query_package", _query_package): return await micropip.install(**install_args) From fec4e1bfe1dd6a9fdbfa0b178440ef4cd3e1f669 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Sat, 15 Feb 2025 01:17:39 +0530 Subject: [PATCH 27/30] More fixes --- .../pyodide-kernel/py/piplite/piplite/cli.py | 108 +++++++++++------- 1 file changed, 64 insertions(+), 44 deletions(-) diff --git a/packages/pyodide-kernel/py/piplite/piplite/cli.py b/packages/pyodide-kernel/py/piplite/piplite/cli.py index 93187cd0..1a697836 100644 --- a/packages/pyodide-kernel/py/piplite/piplite/cli.py +++ b/packages/pyodide-kernel/py/piplite/piplite/cli.py @@ -35,18 +35,30 @@ async def install( @dataclass class RequirementsContext: - """Track state while parsing requirements files.""" + """Track state while parsing requirements files. - index_url: Optional[str] = None + This class maintains state about requirements and their associated index URLs. + Multiple index URLs can be tracked to support searching in multiple indices + in order of specification. + """ + + index_urls: List[str] = None requirements: List[str] = None def __post_init__(self): if self.requirements is None: self.requirements = [] + if self.index_urls is None: + self.index_urls = [] + + def add_index_url(self, url: str) -> None: + """Add an index URL to the list of URLs to search from.""" + if url not in self.index_urls: + self.index_urls.append(url) def add_requirement(self, req: str): - """Add a requirement with the currently active index URL.""" - self.requirements.append((req, self.index_url)) + """Add a requirement that will use the current index URLs.""" + self.requirements.append((req, self.index_urls[:] if self.index_urls else None)) REQ_FILE_PREFIX = r"^(-r|--requirements)\s*=?\s*(.*)\s*" @@ -141,49 +153,45 @@ async def get_action_kwargs(argv: list[str]) -> tuple[typing.Optional[str], dict action = args.action if action == "install": + all_index_urls = [] + if args.index_url: + all_index_urls.append(args.index_url) + all_requirements = [] if args.packages: - all_requirements.extend((pkg, args.index_url) for pkg in args.packages) + all_requirements.extend((pkg, all_index_urls[:]) for pkg in args.packages) # Process requirements files for req_file in args.requirements or []: - context = RequirementsContext() - - if not Path(req_file).exists(): - warn(f"piplite could not find requirements file {req_file}") - continue - - # First let the file be processed normally to capture any index URL - for line_no, line in enumerate( - Path(req_file).read_text(encoding="utf-8").splitlines() - ): - await _packages_from_requirements_line( - Path(req_file), line_no + 1, line, context + try: + requirements, file_index_urls = await _packages_from_requirements_file( + Path(req_file) ) - # If CLI provided an index URL, it should override the file's index URL - # We update all requirements to use the CLI index URL instead. Or, we use - # whatever index URL was found in the file (if any). - if args.index_url: - all_requirements.extend( - (req, args.index_url) for req, _ in context.requirements - ) - else: - all_requirements.extend(context.requirements) + # If CLI provided an index URL, it should override the file's index URL + # We update all requirements to use the CLI index URL instead. Or, we use + # whatever index URL was found in the file (if any). + if args.index_url: + all_requirements.extend( + (req, all_index_urls) for req, _ in requirements + ) + else: + for url in file_index_urls: + if url not in all_index_urls: + all_index_urls.append(url) + all_requirements.extend(requirements) + except Exception as e: + warn(f"Error processing requirements file {req_file}: {e}") + continue if all_requirements: kwargs["requirements"] = [] - active_index_url = None + kwargs["requirements"].extend(req for req, _ in all_requirements) - for req, idx in all_requirements: - if idx is not None: - active_index_url = idx - kwargs["requirements"].append(req) - - # Set the final index URL, if we found one - if active_index_url is not None: - kwargs["index_urls"] = active_index_url + # Set the final index URLs, if we found any + if all_index_urls: + kwargs["index_urls"] = all_index_urls if args.pre: kwargs["pre"] = True @@ -199,22 +207,29 @@ async def get_action_kwargs(argv: list[str]) -> tuple[typing.Optional[str], dict async def _packages_from_requirements_file( req_path: Path, -) -> Tuple[List[str], Optional[str]]: - """Extract (potentially nested) package requirements from a requirements file. +) -> Tuple[List[Tuple[str, Optional[List[str]]]], List[str]]: + """Extract package requirements and index URLs from a requirements file. + + This function processes a requirements file to collect both package requirements + and any index URLs specified in it (with support for nested requirements). Returns: - Tuple of (list of package requirements, optional index URL) + A tuple of: + - List of (requirement, index_urls) pairs, where index_urls is a list of URLs + that should be used for this requirement + - List of index URLs found in the file """ + if not req_path.exists(): warn(f"piplite could not find requirements file {req_path}") - return [] + return [], [] context = RequirementsContext() for line_no, line in enumerate(req_path.read_text(encoding="utf-8").splitlines()): await _packages_from_requirements_line(req_path, line_no + 1, line, context) - return context.requirements, context.index_url + return context.requirements, context.index_urls async def _packages_from_requirements_line( @@ -237,15 +252,20 @@ async def _packages_from_requirements_line( sub_req = Path(sub_path) else: sub_req = req_path.parent / sub_path - sub_reqs, _ = await _packages_from_requirements_file(sub_req) - # Use current context's index_url for nested requirements - context.requirements.extend(sub_reqs) + # Create a new context for the nested file to maintain its own index URLs. + nested_context = RequirementsContext() + nested_context.index_urls = context.index_urls[ + : + ] # i nherit parent's index URLs + await _packages_from_requirements_file(sub_req, nested_context) + # Extend our requirements with the nested ones + context.requirements.extend(nested_context.requirements) return # Check for index URL specification index_match = re.match(INDEX_URL_PREFIX, req) if index_match: - context.index_url = index_match[2].strip() + context.add_index_url(index_match[2].strip()) return if req.startswith("-"): From 98576dcb5016027ee51f2a95bd33ba5de4eef401 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Sat, 15 Feb 2025 05:39:11 +0530 Subject: [PATCH 28/30] Simplify handling yet again --- .../pyodide-kernel/py/piplite/piplite/cli.py | 98 +++++++++---------- 1 file changed, 46 insertions(+), 52 deletions(-) diff --git a/packages/pyodide-kernel/py/piplite/piplite/cli.py b/packages/pyodide-kernel/py/piplite/piplite/cli.py index 1a697836..017edda9 100644 --- a/packages/pyodide-kernel/py/piplite/piplite/cli.py +++ b/packages/pyodide-kernel/py/piplite/piplite/cli.py @@ -35,30 +35,18 @@ async def install( @dataclass class RequirementsContext: - """Track state while parsing requirements files. + """Track state while parsing requirements files.""" - This class maintains state about requirements and their associated index URLs. - Multiple index URLs can be tracked to support searching in multiple indices - in order of specification. - """ - - index_urls: List[str] = None + index_url: Optional[str] = None requirements: List[str] = None def __post_init__(self): if self.requirements is None: self.requirements = [] - if self.index_urls is None: - self.index_urls = [] - - def add_index_url(self, url: str) -> None: - """Add an index URL to the list of URLs to search from.""" - if url not in self.index_urls: - self.index_urls.append(url) def add_requirement(self, req: str): - """Add a requirement that will use the current index URLs.""" - self.requirements.append((req, self.index_urls[:] if self.index_urls else None)) + """Add a requirement with the currently active index URL.""" + self.requirements.append((req, self.index_url)) REQ_FILE_PREFIX = r"^(-r|--requirements)\s*=?\s*(.*)\s*" @@ -153,52 +141,60 @@ async def get_action_kwargs(argv: list[str]) -> tuple[typing.Optional[str], dict action = args.action if action == "install": - all_index_urls = [] - if args.index_url: - all_index_urls.append(args.index_url) - + # CLI index URL, if provided, is the only one we'll use + cli_index_url = args.index_url all_requirements = [] + last_seen_file_index = None if args.packages: - all_requirements.extend((pkg, all_index_urls[:]) for pkg in args.packages) + all_requirements.extend((pkg, cli_index_url) for pkg in args.packages) # Process requirements files for req_file in args.requirements or []: - try: - requirements, file_index_urls = await _packages_from_requirements_file( - Path(req_file) - ) + context = RequirementsContext() - # If CLI provided an index URL, it should override the file's index URL - # We update all requirements to use the CLI index URL instead. Or, we use - # whatever index URL was found in the file (if any). - if args.index_url: - all_requirements.extend( - (req, all_index_urls) for req, _ in requirements - ) - else: - for url in file_index_urls: - if url not in all_index_urls: - all_index_urls.append(url) - all_requirements.extend(requirements) - except Exception as e: - warn(f"Error processing requirements file {req_file}: {e}") + if not Path(req_file).exists(): + warn(f"piplite could not find requirements file {req_file}") continue + # Process the file and capture any index URL it contains + for line_no, line in enumerate( + Path(req_file).read_text(encoding="utf-8").splitlines() + ): + await _packages_from_requirements_line( + Path(req_file), line_no + 1, line, context + ) + + # Keep track of the last index URL we saw in any requirements file + if context.index_url is not None: + last_seen_file_index = context.index_url + + # Add requirements - if CLI provided an index URL, use that instead + if cli_index_url: + all_requirements.extend( + (req, cli_index_url) for req, _ in context.requirements + ) + else: + all_requirements.extend(context.requirements) + if all_requirements: kwargs["requirements"] = [] + + # Add all requirements kwargs["requirements"].extend(req for req, _ in all_requirements) - # Set the final index URLs, if we found any - if all_index_urls: - kwargs["index_urls"] = all_index_urls + # Use index URL with proper precedence: + # 1. CLI index URL if provided + # 2. Otherwise, last seen index URL from any requirements file + effective_index = cli_index_url or last_seen_file_index + if effective_index: + kwargs["index_urls"] = effective_index + # Other CLI flags remain unchanged if args.pre: kwargs["pre"] = True - if args.no_deps: kwargs["deps"] = False - if args.verbose: kwargs["keep_going"] = True @@ -244,7 +240,7 @@ async def _packages_from_requirements_line( if not req: return - # Check for nested requirements file + # Handle nested requirements file req_file_match = re.match(REQ_FILE_PREFIX, req) if req_file_match: sub_path = req_file_match[2] @@ -252,20 +248,18 @@ async def _packages_from_requirements_line( sub_req = Path(sub_path) else: sub_req = req_path.parent / sub_path - # Create a new context for the nested file to maintain its own index URLs. nested_context = RequirementsContext() - nested_context.index_urls = context.index_urls[ - : - ] # i nherit parent's index URLs await _packages_from_requirements_file(sub_req, nested_context) - # Extend our requirements with the nested ones + # Use the last index URL from nested file, if one was found + if nested_context.index_url: + context.index_url = nested_context.index_url context.requirements.extend(nested_context.requirements) return - # Check for index URL specification + # Check for index URL - this becomes the new active index URL. index_match = re.match(INDEX_URL_PREFIX, req) if index_match: - context.add_index_url(index_match[2].strip()) + context.index_url = index_match[2].strip() return if req.startswith("-"): From d694c004415cd1bf5c05e8c5478c87d23f3c2283 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Sat, 15 Feb 2025 06:28:15 +0530 Subject: [PATCH 29/30] Fix URL handling that can lead to silent failures --- packages/pyodide-kernel/py/piplite/piplite/cli.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/packages/pyodide-kernel/py/piplite/piplite/cli.py b/packages/pyodide-kernel/py/piplite/piplite/cli.py index 017edda9..349ea9a0 100644 --- a/packages/pyodide-kernel/py/piplite/piplite/cli.py +++ b/packages/pyodide-kernel/py/piplite/piplite/cli.py @@ -50,7 +50,11 @@ def add_requirement(self, req: str): REQ_FILE_PREFIX = r"^(-r|--requirements)\s*=?\s*(.*)\s*" -INDEX_URL_PREFIX = r"^(--index-url|-i)\s*=?\s*(.*)\s*" + +# Matches a pip-style index URL, with support for quote enclosures +INDEX_URL_PREFIX = ( + r'^(--index-url|-i)\s*=?\s*(?:"([^"]*)"|\047([^\047]*)\047|([^\s]*))\s*$' +) __all__ = ["get_transformed_code"] @@ -259,7 +263,8 @@ async def _packages_from_requirements_line( # Check for index URL - this becomes the new active index URL. index_match = re.match(INDEX_URL_PREFIX, req) if index_match: - context.index_url = index_match[2].strip() + url = next(group for group in index_match.groups()[1:] if group is not None) + context.index_url = url.strip() return if req.startswith("-"): From 3fc23810b3c0a24aafc64f8fa45133e41b2adcaf Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Sat, 15 Feb 2025 07:54:28 +0530 Subject: [PATCH 30/30] Temporarily remove NumPy, add SPNW index URL --- examples/jupyter-lite.json | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/examples/jupyter-lite.json b/examples/jupyter-lite.json index fe309fc4..8b9c6e3c 100644 --- a/examples/jupyter-lite.json +++ b/examples/jupyter-lite.json @@ -4,8 +4,11 @@ "appName": "JupyterLite Pyodide Kernel", "litePluginSettings": { "@jupyterlite/pyodide-kernel-extension:kernel": { + "pipliteInstallDefaultOptions": { + "index_urls": "https://pypi.anaconda.org/scientific-python-nightly-wheels/simple" + }, "loadPyodideOptions": { - "packages": ["matplotlib", "micropip", "numpy", "sqlite3", "ssl"], + "packages": ["micropip", "sqlite3", "ssl"], "lockFileURL": "https://cdn.jsdelivr.net/pyodide/v0.27.2/full/pyodide-lock.json?from-lite-config=1" } }