diff --git a/pyproject.toml b/pyproject.toml index 91fe54d78c36..ff4a405fb989 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,6 +23,7 @@ dependencies = [ "lfx-arxiv>=0.1.0", "lfx-ibm>=0.1.0", "lfx-docling>=0.1.0", + "lfx-bundles[all]>=1.0,<2.0", # langflow-extensions:bundle-deps-end ] @@ -84,6 +85,7 @@ lfx-duckduckgo = { workspace = true } lfx-arxiv = { workspace = true } lfx-ibm = { workspace = true } lfx-docling = { workspace = true } +lfx-bundles = { workspace = true } # langflow-extensions:bundle-sources-end torch = { index = "pytorch-cpu" } torchvision = { index = "pytorch-cpu" } @@ -100,6 +102,7 @@ members = [ "src/bundles/arxiv", "src/bundles/ibm", "src/bundles/docling", + "src/bundles/lfx-bundles", # langflow-extensions:bundle-members-end ] diff --git a/scripts/ci/sync_bundle_lfx_pin.py b/scripts/ci/sync_bundle_lfx_pin.py index f79d2a1790d3..1938530a11c4 100644 --- a/scripts/ci/sync_bundle_lfx_pin.py +++ b/scripts/ci/sync_bundle_lfx_pin.py @@ -26,11 +26,11 @@ unconditionally from ``make patch``, including patch releases within a minor line where the floor does not move). Only the bundle's ``"lfx..."`` runtime dependency is rewritten -- self-references such as -``"lfx-docling[local]"`` and the nightly ``"lfx-nightly=="`` form are left +``"lfx-docling[local]"`` and the legacy ``"lfx-nightly=="`` form are left untouched (neither has a bare version operator immediately after ``lfx``). Stdlib only, so it runs in any CI checkout (same constraint as the sibling -``scripts/ci/update_bundle_versions.py`` and ``scripts/migrate/port_bundle.py``). +``scripts/migrate/port_bundle.py``). Usage: python scripts/ci/sync_bundle_lfx_pin.py 1.10.0 @@ -47,7 +47,8 @@ # Matches a bundle's ``"lfxVERSION[,=|~=|==)[\d.]+(?:\.(?:post|dev|a|b|rc)\d+)*' r'(?:,\s*<[\d.]+(?:\.(?:post|dev|a|b|rc)\d+)*)?"' diff --git a/scripts/ci/update_bundle_versions.py b/scripts/ci/update_bundle_versions.py deleted file mode 100644 index f1be5a44d4aa..000000000000 --- a/scripts/ci/update_bundle_versions.py +++ /dev/null @@ -1,181 +0,0 @@ -"""Rename ``lfx-*`` bundle packages to their ``-nightly`` counterparts. - -Bundles under ``src/bundles/*`` follow the same package-rename convention as -``langflow``, ``langflow-base``, ``lfx``, and ``langflow-sdk``: for nightly -builds, the distribution is published as ``-nightly`` so the regular -(non-nightly) PyPI name stays untouched. - -This script (a) renames each bundle's ``[project] name`` to -``-nightly``, (b) bumps its version to ``.dev``, (c) rewrites -its ``lfx`` runtime dep so it resolves against the renamed ``lfx-nightly`` -workspace member, and (d) updates the root ``pyproject.toml`` so its bundle -deps and ``[tool.uv.sources]`` entries reference the renamed packages. - -All operations are idempotent — running twice is a no-op. -""" - -from __future__ import annotations - -import re -import sys -from pathlib import Path - -BASE_DIR = Path(__file__).parent.parent.parent - -# Matches the lfx dep specifier inside a bundle pyproject's dependencies list. -# Accepts the bundle default ("lfx>=X.Y.Z" with an optional upper bound), -# legacy ~=/==, and the already-rewritten "lfx-nightly==X.Y.Z" form (idempotent). -_LFX_DEP_PATTERN = re.compile( - r'"lfx(?:-nightly)?' - r"(?:" - r"(?:~=|==)[\d.]+(?:\.(?:post|dev|a|b|rc)\d+)*" - r"|" - r">=[\d.]+(?:\.(?:post|dev|a|b|rc)\d+)*" - r"(?:,\s*<[\d.]+(?:\.(?:post|dev|a|b|rc)\d+)*)?" - r')"' -) - -_PROJECT_NAME_PATTERN = re.compile(r'(\[project\][^\[]*?\nname = ")([^"]+)(")', re.DOTALL) -_PROJECT_VERSION_PATTERN = re.compile(r'(\[project\][^\[]*?\nversion = ")([^"]+)(")', re.DOTALL) - - -def _strip_nightly(name: str) -> str: - """Return the base name with a trailing ``-nightly`` stripped (for idempotency).""" - return name.removesuffix("-nightly") - - -def _strip_dev_suffix(version: str) -> str: - """Return the base version with any trailing PEP 440 dev segment stripped.""" - return re.sub(r"\.dev\d+$", "", version) - - -def _extract_dev_n(tag: str) -> str: - """Extract ``N`` from a tag like ``v0.5.0.dev38`` or ``0.5.0.dev38``.""" - match = re.search(r"\.dev(\d+)$", tag) - if not match: - msg = f"Tag does not end in .devN: {tag!r}" - raise ValueError(msg) - return match.group(1) - - -def rename_bundle_pyproject(pyproject_path: Path, lfx_version: str, dev_n: str) -> tuple[str, str, str] | None: - """Rewrite a single bundle ``pyproject.toml`` for nightly publication. - - - ``[project] name`` → ``-nightly`` - - ``[project] version`` → ``.dev`` - - entry-point key → ``-nightly`` - - ``"lfx>=...,<..."`` dep → ``"lfx-nightly=="`` - - Returns ``(base_name, nightly_name, nightly_version)`` so the caller can - update the root pyproject. Returns ``None`` if the file has no - ``[project]`` name/version (shouldn't happen, but we skip rather than fail). - """ - content = pyproject_path.read_text(encoding="utf-8") - - name_match = _PROJECT_NAME_PATTERN.search(content) - version_match = _PROJECT_VERSION_PATTERN.search(content) - if not name_match or not version_match: - return None - - base_name = _strip_nightly(name_match.group(2)) - nightly_name = f"{base_name}-nightly" - base_version = _strip_dev_suffix(version_match.group(2)) - nightly_version = f"{base_version}.dev{dev_n}" - - content = _PROJECT_NAME_PATTERN.sub(rf"\g<1>{nightly_name}\g<3>", content, count=1) - content = _PROJECT_VERSION_PATTERN.sub(rf"\g<1>{nightly_version}\g<3>", content, count=1) - - # Entry-point key. The key may already be the nightly form on a re-run. - entry_point_pattern = re.compile( - rf'(\[project\.entry-points\."langflow\.extensions"\]\s*\n)' - rf"{re.escape(base_name)}(?:-nightly)?" - rf'(\s*=\s*"[^"]+")' - ) - content = entry_point_pattern.sub(rf"\g<1>{nightly_name}\g<2>", content, count=1) - - # Rewrite the lfx dep regardless of which form it's in. - content = _LFX_DEP_PATTERN.sub(f'"lfx-nightly=={lfx_version}"', content) - - pyproject_path.write_text(content, encoding="utf-8") - return base_name, nightly_name, nightly_version - - -def update_root_pyproject_for_bundle( - root_pyproject: Path, - base_name: str, - nightly_name: str, - nightly_version: str, -) -> None: - """Update root ``pyproject.toml`` to reference the nightly bundle. - - - dependency line ``"[..]"`` → ``"=="`` - - uv.sources entry `` = { workspace = true }`` → `` = ...`` - - Idempotent: also matches the already-nightly form. - """ - content = root_pyproject.read_text(encoding="utf-8") - - # Dependency in [project.dependencies] (any PEP 440 specifier or range form). - dep_pattern = re.compile( - rf'"{re.escape(base_name)}(?:-nightly)?' - r"(?:" - r"(?:~=|==|>=)[\d.]+(?:\.(?:post|dev|a|b|rc)\d+)*" - r"(?:,\s*<[\d.]+(?:\.(?:post|dev|a|b|rc)\d+)*)?" - r')"' - ) - content = dep_pattern.sub(f'"{nightly_name}=={nightly_version}"', content) - - # uv.sources entry — only the workspace = true form is used by bundles today. - source_pattern = re.compile( - rf"^{re.escape(base_name)}(?:-nightly)?(\s*=\s*\{{\s*workspace\s*=\s*true\s*\}})", - re.MULTILINE, - ) - content = source_pattern.sub(rf"{nightly_name}\g<1>", content) - - root_pyproject.write_text(content, encoding="utf-8") - - -def update_bundles_for_nightly(lfx_tag: str) -> list[tuple[str, str, str]]: - """Rename every ``src/bundles/*`` package to its ``-nightly`` counterpart. - - Returns a list of ``(base_name, nightly_name, nightly_version)`` tuples - for the bundles that were rewritten. - """ - bundles_dir = BASE_DIR / "src" / "bundles" - if not bundles_dir.is_dir(): - return [] - - lfx_version = lfx_tag.lstrip("v") - dev_n = _extract_dev_n(lfx_version) - root_pyproject = BASE_DIR / "pyproject.toml" - - results: list[tuple[str, str, str]] = [] - for bundle_pyproject in sorted(bundles_dir.glob("*/pyproject.toml")): - renamed = rename_bundle_pyproject(bundle_pyproject, lfx_version, dev_n) - if renamed is None: - continue - base_name, nightly_name, nightly_version = renamed - update_root_pyproject_for_bundle(root_pyproject, base_name, nightly_name, nightly_version) - results.append(renamed) - print(f"Renamed {base_name} -> {nightly_name} ({nightly_version}) in {bundle_pyproject.relative_to(BASE_DIR)}") - return results - - -def main() -> None: - """Entry point. - - Usage: - update_bundle_versions.py - - ``lfx_tag`` is the LFX nightly tag (e.g., ``v0.5.0.dev38``); its ``.devN`` - suffix is reused for all bundles so they share a single nightly cadence. - """ - expected_args = 2 - if len(sys.argv) != expected_args: - print("Usage: update_bundle_versions.py ") - sys.exit(1) - update_bundles_for_nightly(sys.argv[1]) - - -if __name__ == "__main__": - main() diff --git a/src/backend/tests/unit/test_bundle_lfx_pin.py b/src/backend/tests/unit/test_bundle_lfx_pin.py index 4d9988a25005..69c48c514577 100644 --- a/src/backend/tests/unit/test_bundle_lfx_pin.py +++ b/src/backend/tests/unit/test_bundle_lfx_pin.py @@ -87,7 +87,8 @@ def test_leaves_self_reference_untouched(self): assert mod.rewrite_lfx_dep(self_ref, self.FLOOR) == self_ref def test_leaves_nightly_form_untouched(self): - # update_bundle_versions.py rewrites to this; sync must not clobber it. + # Legacy form from the retired nightly bundle-rename track (see + # src/bundles/NIGHTLY.md); sync must not clobber it if encountered. assert mod.rewrite_lfx_dep('"lfx-nightly==1.10.0.dev38"', self.FLOOR) == '"lfx-nightly==1.10.0.dev38"' def test_only_rewrites_runtime_dep_in_full_block(self): diff --git a/src/bundles/lfx-bundles/README.md b/src/bundles/lfx-bundles/README.md new file mode 100644 index 000000000000..02f4d720c344 --- /dev/null +++ b/src/bundles/lfx-bundles/README.md @@ -0,0 +1,65 @@ +# lfx-bundles + +The long tail of Langflow's provider components as a **single manifest-less +metapackage**, modeled on `langchain-community`. This is the destination for +every vendor/third-party provider that does not warrant its own standalone +distribution; the curated partner providers (OpenAI, Anthropic, AWS, +DataStax, Cohere) ship as separate `lfx-` packages instead. + +## How it works + +`lfx-bundles` declares the `lfx.bundles` entry point: + +```toml +[project.entry-points."lfx.bundles"] +lfx_bundles = "lfx_bundles" +``` + +At startup, lfx resolves this package and **folder-walks its immediate +subdirectories**. Each subdirectory is one bundle, registered at the +`@official` slot under its directory name — no `extension.json`, no per-provider +manifest. Adding a provider is just adding a folder. + +``` +src/lfx_bundles/ +├── __init__.py # bare namespace marker +├── / # one bundle, e.g. tavily/, pinecone/, ... +│ └── *.py # Component subclasses +└── ... +``` + +A component's identity is its **bundle name** (`ext::@official`), +which is stable whether the provider ships here or graduates to a standalone +`lfx-` package. Because a manifest-shipping package always shadows the +manifest-less metapackage, a provider can graduate with **no lockstep release**. + +## Installing + +Available today (`lfx-bundles` is an empty skeleton until the bulk move +populates it): + +```bash +pip install langflow # everything (langflow pins lfx-bundles[all]) +pip install lfx # engine only, no bundles +``` + +Once the long-tail providers move in (and the `lfx[bundles]` extra ships with +the engine-only `lfx` split), these become available: + +```bash +pip install "lfx[bundles]" # engine + this metapackage (deployment footnote) +pip install "lfx-bundles[]" # provider code + that provider's SDK deps +``` + +`lfx-bundles` itself depends only on `lfx`. Each provider's third-party SDKs are +**optional extras** (PEP 685-normalized keys, e.g. `lfx-bundles[google-genai]`); +the generated `all` extra (empty until the first provider tranche lands) pulls +every provider's deps and is what `langflow` depends on so `pip install langflow` +is unchanged. + +## Adding a provider + +Providers are moved here by `scripts/migrate/consolidate_bundles.py`, which also +maintains the per-provider extras and the generated `all` aggregate. **Do not** +hand-edit the extras block in `pyproject.toml`. Provider folder names must be +lowercase snake_case (`a-z`, `0-9`, `_`, 2–64 chars). diff --git a/src/bundles/lfx-bundles/pyproject.toml b/src/bundles/lfx-bundles/pyproject.toml new file mode 100644 index 000000000000..a1c8a92945f9 --- /dev/null +++ b/src/bundles/lfx-bundles/pyproject.toml @@ -0,0 +1,57 @@ +[project] +name = "lfx-bundles" +version = "1.0.0" +description = "Langflow's long-tail provider bundles as a single manifest-less metapackage (the langchain-community model)." +readme = "README.md" +requires-python = ">=3.10,<3.15" +license = { text = "MIT" } +authors = [ + { name = "Langflow", email = "contact@langflow.org" }, +] +keywords = ["langflow", "lfx", "extension", "bundle", "providers"] + +# Runtime: only lfx (the BUNDLE_API surface). Each provider's third-party SDK +# is an optional extra (see [project.optional-dependencies]); installing +# lfx-bundles bare gives the provider *code* but defers each provider's SDK to +# its extra, so a user opts into exactly the providers they need. The +# generated ``all`` extra pulls every provider's deps and is what ``langflow`` +# depends on (``lfx-bundles[all]``) so ``pip install langflow`` stays +# functionally identical to today. +dependencies = [ + "lfx>=1.11.0.dev0,<2.0.0", +] + +[project.optional-dependencies] +# Per-provider extras + the ``all`` aggregate are populated by the bulk move +# (scripts/migrate/consolidate_bundles.py) as the long-tail providers land +# here. Extra keys are PEP 685-normalized (lowercase, hyphen-separated). +# ``all`` is GENERATED from the per-provider keys -- never hand-edit it. +# Empty until the first provider tranche moves in. +all = [] + +[project.urls] +Homepage = "https://github.com/langflow-ai/langflow" +Documentation = "https://docs.langflow.org/extensions" +Repository = "https://github.com/langflow-ai/langflow" + +# Manifest-less discovery via the ``lfx.bundles`` entry-point group (NOT +# ``langflow.extensions``). The loader (lfx.extension.loader._bundles_root) +# resolves this package with find_spec and folder-walks its immediate +# subdirectories -- each is one bundle at the @official slot, named after the +# directory. No extension.json; exempt from ``lfx extension validate``. +[project.entry-points."lfx.bundles"] +lfx_bundles = "lfx_bundles" + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = ["src/lfx_bundles"] + +[tool.hatch.build.targets.sdist] +include = [ + "src/lfx_bundles", + "README.md", + "pyproject.toml", +] diff --git a/src/bundles/lfx-bundles/src/lfx_bundles/__init__.py b/src/bundles/lfx-bundles/src/lfx_bundles/__init__.py new file mode 100644 index 000000000000..e1c545cd1374 --- /dev/null +++ b/src/bundles/lfx-bundles/src/lfx_bundles/__init__.py @@ -0,0 +1,16 @@ +"""lfx-bundles: the manifest-less metapackage of Langflow's long-tail providers. + +This package is a bare namespace marker. Each immediate subdirectory is one +provider bundle, discovered at runtime by lfx's ``lfx.bundles`` entry-point +folder-walk (``lfx.extension.loader._bundles_root``) and registered at the +``@official`` slot under its directory name. There are intentionally no +re-exports here and no ``extension.json`` -- providers are added as folders, +the langchain-community way. + +Provider folders are lowercase snake_case (``BUNDLE_NAME_RE``); a component's +identity is its bundle name (``ext::@official``), stable +whether the provider ships here or in a graduated ``lfx-`` package. + +Providers are added by ``scripts/migrate/consolidate_bundles.py``, never by +hand. +""" diff --git a/uv.lock b/uv.lock index dbc544ec394a..12785148a88b 100644 --- a/uv.lock +++ b/uv.lock @@ -31,6 +31,7 @@ members = [ "langflow-stepflow", "lfx", "lfx-arxiv", + "lfx-bundles", "lfx-docling", "lfx-duckduckgo", "lfx-ibm", @@ -7405,6 +7406,7 @@ source = { editable = "." } dependencies = [ { name = "langflow-base", extra = ["complete"] }, { name = "lfx-arxiv" }, + { name = "lfx-bundles" }, { name = "lfx-docling" }, { name = "lfx-duckduckgo" }, { name = "lfx-ibm" }, @@ -7495,6 +7497,7 @@ requires-dist = [ { name = "ctransformers", marker = "extra == 'local'", specifier = ">=0.2.10" }, { name = "langflow-base", extras = ["complete"], editable = "src/backend/base" }, { name = "lfx-arxiv", editable = "src/bundles/arxiv" }, + { name = "lfx-bundles", extras = ["all"], editable = "src/bundles/lfx-bundles" }, { name = "lfx-docling", editable = "src/bundles/docling" }, { name = "lfx-docling", extras = ["chunking"], marker = "extra == 'docling-chunking'", editable = "src/bundles/docling" }, { name = "lfx-docling", extras = ["image-description"], marker = "extra == 'docling-image-description'", editable = "src/bundles/docling" }, @@ -9053,6 +9056,18 @@ requires-dist = [ { name = "lfx", editable = "src/lfx" }, ] +[[package]] +name = "lfx-bundles" +version = "1.0.0" +source = { editable = "src/bundles/lfx-bundles" } +dependencies = [ + { name = "lfx" }, +] + +[package.metadata] +requires-dist = [{ name = "lfx", editable = "src/lfx" }] +provides-extras = ["all"] + [[package]] name = "lfx-docling" version = "0.1.1"