Skip to content

Commit 86c27d4

Browse files
BulkBeingBulkBeing
authored
pynumaflow-lite: Upgrade dependencies, setup CI (#362)
Signed-off-by: Sreekanth <prsreekanth920@gmail.com>
1 parent 221e1b4 commit 86c27d4

75 files changed

Lines changed: 2705 additions & 1541 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/pynumaflow-lite.yml‎

Lines changed: 281 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,281 @@
1+
name: pynumaflow-lite
2+
3+
on:
4+
push:
5+
branches: ["main", "release/*"]
6+
tags: ["pynumaflow-lite-v*"]
7+
paths:
8+
- "packages/pynumaflow-lite/**"
9+
- ".github/workflows/pynumaflow-lite.yml"
10+
pull_request:
11+
branches: ["main", "release/*"]
12+
paths:
13+
- "packages/pynumaflow-lite/**"
14+
- ".github/workflows/pynumaflow-lite.yml"
15+
workflow_dispatch:
16+
inputs:
17+
publish-testpypi:
18+
description: "Publish pynumaflow-lite wheels to TestPyPI"
19+
type: boolean
20+
default: false
21+
22+
concurrency:
23+
group: ${{ github.workflow }}-${{ github.ref }}
24+
cancel-in-progress: true
25+
26+
permissions:
27+
contents: read
28+
29+
env:
30+
RUSTFLAGS: -C debuginfo=0
31+
RUST_BACKTRACE: 1
32+
PYTHONUTF8: 1
33+
34+
jobs:
35+
lint:
36+
name: Format and lint
37+
runs-on: ubuntu-24.04
38+
defaults:
39+
run:
40+
working-directory: packages/pynumaflow-lite
41+
42+
steps:
43+
- uses: actions/checkout@v7
44+
45+
- name: Set up Python
46+
uses: actions/setup-python@v6
47+
with:
48+
python-version: "3.10"
49+
50+
- name: Install uv
51+
uses: astral-sh/setup-uv@v8.1.0
52+
with:
53+
python-version: "3.10"
54+
working-directory: packages/pynumaflow-lite
55+
56+
- name: Set up Rust
57+
run: |
58+
rustup toolchain install stable --profile minimal --component clippy --component rustfmt
59+
rustup default stable
60+
61+
- name: Install dependencies
62+
run: uv sync --locked --group dev
63+
64+
- name: Check Rust formatting
65+
run: cargo fmt --all --check
66+
67+
- name: Check Python formatting
68+
run: uv run ruff format --check pynumaflow_lite/ tests/ manifests/
69+
70+
- name: Ruff lint
71+
run: uv run ruff check .
72+
73+
- name: Clippy
74+
run: cargo clippy --locked --workspace --all-targets --all-features -- -D warnings -A clippy::module_inception
75+
76+
test:
77+
name: Test ${{ matrix.platform.artifact }}
78+
needs: lint
79+
runs-on: ${{ matrix.platform.os }}
80+
defaults:
81+
run:
82+
working-directory: packages/pynumaflow-lite
83+
84+
strategy:
85+
fail-fast: false
86+
matrix:
87+
platform:
88+
- os: ubuntu-24.04
89+
artifact: linux-x86_64
90+
- os: macos-15
91+
artifact: macos-aarch64
92+
93+
steps:
94+
- uses: actions/checkout@v7
95+
96+
- name: Install uv
97+
uses: astral-sh/setup-uv@v8.1.0
98+
with:
99+
working-directory: packages/pynumaflow-lite
100+
101+
- name: Set up Rust
102+
run: |
103+
rustup toolchain install stable --profile minimal
104+
rustup default stable
105+
106+
- name: Run tests for Python versions
107+
shell: bash
108+
run: |
109+
set -euo pipefail
110+
111+
for python_version in 3.10 3.11 3.12 3.13 3.14; do
112+
echo "::group::Python ${python_version}"
113+
uv python install "${python_version}"
114+
export UV_PYTHON="${python_version}"
115+
uv sync --locked --group dev --python "${python_version}"
116+
uv run --python "${python_version}" maturin develop --locked
117+
118+
uv run --python "${python_version}" pytest -v
119+
120+
# cargo test embeds a Python interpreter via pyo3 (Python::initialize),
121+
# which requires pinning the uv-managed interpreter at three layers:
122+
# - PYO3_PYTHON: the interpreter pyo3's build script links against.
123+
# - PYTHONHOME: where the embedded interpreter finds its stdlib at
124+
# runtime; mismatch surfaces as "No module named 'encodings'".
125+
# - LD_LIBRARY_PATH: where the dynamic loader finds libpython at load
126+
# time. On Linux the test binary links libpython3.x.so, which lives
127+
# under the interpreter's lib dir and is not on the default search
128+
# path; missing it surfaces as "error while loading shared libraries:
129+
# libpython3.x.so". (Harmless on macOS, which resolves differently.)
130+
python_bin="$(uv run --python "${python_version}" python -c 'import sys; print(sys.executable)')"
131+
export PYO3_PYTHON="${python_bin}"
132+
export PYTHONHOME="$("${python_bin}" -c 'import sys; print(sys.base_prefix)')"
133+
python_libdir="$("${python_bin}" -c 'import sysconfig; print(sysconfig.get_config_var("LIBDIR"))')"
134+
export LD_LIBRARY_PATH="${python_libdir}${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}"
135+
cargo test --locked
136+
unset PYO3_PYTHON PYTHONHOME LD_LIBRARY_PATH
137+
138+
unset UV_PYTHON
139+
echo "::endgroup::"
140+
done
141+
142+
build-wheels:
143+
name: Build wheel ${{ matrix.platform.artifact }} py${{ matrix.python-version }}
144+
needs: test
145+
runs-on: ${{ matrix.platform.os }}
146+
defaults:
147+
run:
148+
working-directory: packages/pynumaflow-lite
149+
150+
strategy:
151+
fail-fast: false
152+
matrix:
153+
python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
154+
platform:
155+
- os: ubuntu-24.04
156+
target: x86_64-unknown-linux-gnu
157+
artifact: linux-x86_64
158+
manylinux: "2014"
159+
linux: true
160+
- os: macos-15
161+
target: aarch64-apple-darwin
162+
artifact: macos-aarch64
163+
manylinux: "off"
164+
linux: false
165+
166+
steps:
167+
- uses: actions/checkout@v7
168+
169+
- name: Set up Python ${{ matrix.python-version }}
170+
uses: actions/setup-python@v6
171+
with:
172+
python-version: ${{ matrix.python-version }}
173+
174+
- name: Set up Rust
175+
run: |
176+
rustup toolchain install stable --profile minimal
177+
rustup default stable
178+
179+
- name: Verify tag matches package version
180+
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/pynumaflow-lite-v')
181+
run: |
182+
set -euo pipefail
183+
184+
tag_version="${GITHUB_REF_NAME#pynumaflow-lite-v}"
185+
package_version="$(cargo metadata --locked --no-deps --format-version 1 | python -c 'import json, sys; print(json.load(sys.stdin)["packages"][0]["version"])')"
186+
187+
if [ "${package_version}" != "${tag_version}" ]; then
188+
echo "Tag version ${tag_version} does not match Cargo.toml version ${package_version}"
189+
exit 1
190+
fi
191+
192+
- name: Set manual TestPyPI version
193+
if: github.event_name == 'workflow_dispatch' && inputs['publish-testpypi']
194+
run: |
195+
set -euo pipefail
196+
197+
base_version="$(cargo metadata --locked --no-deps --format-version 1 | python -c 'import json, sys; print(json.load(sys.stdin)["packages"][0]["version"])')"
198+
if [[ "${base_version}" == *-* ]]; then
199+
manual_version="${base_version}.dev.${GITHUB_RUN_ID}${GITHUB_RUN_ATTEMPT}"
200+
else
201+
manual_version="${base_version}-dev.${GITHUB_RUN_ID}${GITHUB_RUN_ATTEMPT}"
202+
fi
203+
export MANUAL_VERSION="${manual_version}"
204+
205+
python - <<'PY'
206+
import os
207+
import re
208+
from pathlib import Path
209+
210+
version = os.environ["MANUAL_VERSION"]
211+
212+
cargo_toml = Path("Cargo.toml")
213+
cargo_toml.write_text(
214+
re.sub(
215+
r'(?m)^version = "[^"]+"',
216+
f'version = "{version}"',
217+
cargo_toml.read_text(),
218+
count=1,
219+
)
220+
)
221+
222+
cargo_lock = Path("Cargo.lock")
223+
cargo_lock.write_text(
224+
re.sub(
225+
r'(?ms)(\[\[package\]\]\nname = "pynumaflow-lite"\nversion = ")[^"]+(")',
226+
lambda match: f"{match.group(1)}{version}{match.group(2)}",
227+
cargo_lock.read_text(),
228+
count=1,
229+
)
230+
)
231+
PY
232+
233+
cargo metadata --locked --no-deps --format-version 1 >/dev/null
234+
echo "Using manual TestPyPI version ${manual_version}"
235+
236+
- name: Build wheel
237+
uses: PyO3/maturin-action@v1
238+
with:
239+
command: build
240+
working-directory: packages/pynumaflow-lite
241+
target: ${{ matrix.platform.target }}
242+
manylinux: ${{ matrix.platform.manylinux }}
243+
args: >-
244+
--release
245+
--locked
246+
--out dist
247+
-i ${{ matrix.platform.linux && format('python{0}', matrix.python-version) || 'python' }}
248+
249+
- name: Upload wheel artifact
250+
uses: actions/upload-artifact@v7
251+
with:
252+
name: pynumaflow-lite-${{ matrix.platform.artifact }}-py${{ matrix.python-version }}
253+
path: packages/pynumaflow-lite/dist/*.whl
254+
if-no-files-found: error
255+
compression-level: 0
256+
257+
publish-testpypi:
258+
name: Publish to TestPyPI
259+
needs: build-wheels
260+
runs-on: ubuntu-24.04
261+
if: >
262+
github.repository == 'numaproj/numaflow-python' &&
263+
github.run_attempt == 1 &&
264+
(
265+
(github.event_name == 'push' && startsWith(github.ref, 'refs/tags/pynumaflow-lite-v')) ||
266+
(github.event_name == 'workflow_dispatch' && inputs['publish-testpypi'])
267+
)
268+
269+
steps:
270+
- name: Install uv
271+
uses: astral-sh/setup-uv@v8.1.0
272+
273+
- name: Download wheel artifacts
274+
uses: actions/download-artifact@v7
275+
with:
276+
pattern: pynumaflow-lite-*
277+
path: dist
278+
merge-multiple: true
279+
280+
- name: Publish to TestPyPI
281+
run: uv publish --publish-url https://test.pypi.org/legacy/ --check-url https://test.pypi.org/simple/ --token ${{ secrets.TESTPYPI_PASSWORD }} dist/*

‎.gitignore‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3,11 +3,6 @@
33
debug/
44
target/
55

6-
7-
# Remove Cargo.lock from gitignore if creating an executable, leave it for libraries
8-
# More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html
9-
Cargo.lock
10-
116
# These are backup files generated by rustfmt
127
**/*.rs.bk
138

0 commit comments

Comments
 (0)